Export an app store archive, and expose it through the dev portal API. (#56)

jwise · web-flow · commit 0e3c0b6f7117 · 2025-11-24T21:07:10.000-05:00
As described in https://rebble.io/2025/11/24/rebble-in-your-own-world.html
diff --git a/appstore/ARCHIVE_LICENSE b/appstore/ARCHIVE_LICENSE
@@ -0,0 +1,12 @@
+This archive was provided to you by the folks at the Rebble Foundation
+(https://rebble.io/)!
+
+It would be EXTREMELY COOL if, anywhere you use data derived from this
+collection, you prominently cited Rebble as the source of these data.
+
+It would be EXTREMELY UNCOOL if you did not do that.
+
+Remember that apps are the intellectual property of the original authors. 
+This archive does not, in and of itself, grant you any rights to modify or
+further redistribute the contents (unless otherwise noted in the metadata
+for any given app).
diff --git a/appstore/commands.py b/appstore/commands.py
@@ -1,6 +1,11 @@
 import datetime
+import functools
 import hashlib
+import io
 import json
+from queue import *
+from threading import Thread, Lock
+import tempfile
 import yaml
 
 import flask.json
@@ -19,9 +24,9 @@
 from algoliasearch import algoliasearch
 
 from .utils import id_generator, algolia_app
-from .models import Category, db, App, Developer, Release, CompanionApp, Binary, AssetCollection, LockerEntry, UserLike
+from .models import Category, db, App, Developer, Release, CompanionApp, Binary, AssetCollection, LockerEntry, UserLike, Collection, AvailableArchive
 from .pbw import PBW, release_from_pbw
-from .s3 import upload_pbw, upload_asset
+from .s3 import upload_pbw, upload_asset, download_pbw, download_asset, upload_archive
 from .settings import config
 
 if config['ALGOLIA_ADMIN_API_KEY']:
@@ -578,6 +583,191 @@ def path(base):
         else:
             algolia_index.delete_objects([app_obj.id])
 
+def export_archive_to_zip(fn, test_only=False, n_threads=20):
+    # test-only: only output a few files, so you can run this without a fast
+    # connection to gcs
+    assets = set()
+    binaries = set()
+    
+    def _mk_release(rel):
+        binaries.add(rel.id)
+
+        return {
+            'has_pbw': rel.has_pbw,
+            'binaries': { plat: {
+                'sdk_major': b.sdk_major,
+                'sdk_minor': b.sdk_minor,
+                'process_info_flags': b.process_info_flags,
+                'icon_resource_id': b.icon_resource_id,
+            } for plat,b in rel.binaries.items() },
+            'capabilities': rel.capabilities,
+            'js_md5': rel.js_md5,
+            'published_date': rel.published_date.timestamp() if rel.published_date else None,
+            'release_notes': rel.release_notes,
+            'version': rel.version,
+            'compatibility': rel.compatibility,
+        }
+    
+    def _mk_asset_collection(ass):
+        for img in ass.headers:
+            assets.add(img)
+        assets.add(ass.banner)
+
+        return {
+            'description': ass.description,
+            'screenshots': ass.screenshots,
+            'headers': ass.headers,
+            'banner': ass.banner,
+        }
+    
+    def _mk_app(app):
+        if _mk_app.n % 1000 == 0:
+            print(f"... {_mk_app.n} / {_mk_app.ntotal} ...")
+        _mk_app.n += 1
+        assets.add(app.icon_large)
+        assets.add(app.icon_small)
+        return {
+            'uuid': app.app_uuid,
+            'asset_collections': { plat: _mk_asset_collection(ass) for plat,ass in app.asset_collections.items() },
+            'category_id': app.category_id,
+            'companion_apps': { plat:
+                {
+                    'icon': c.icon,
+                    'url': c.url,
+                    'name': c.name,
+                    'pebblekit3': c.pebblekit3,
+                } for plat,c in app.companions.items() },
+            'collection_ids': [ c.id for c in app.collections ],
+            'created_at': app.created_at.timestamp() if app.created_at else None,
+            'developer_id': app.developer_id,
+            'hearts': app.hearts,
+            'releases': { rel.id: _mk_release(rel) for rel in app.releases if rel.is_published },
+            'icon_large': app.icon_large,
+            'icon_small': app.icon_small,
+            'published_date': app.published_date.timestamp() if app.published_date else None,
+            'source': app.source,
+            'title': app.title,
+            'timeline_enabled': app.timeline_enabled,
+            'type': app.type,
+            'website': app.website,
+        }
+    _mk_app.n = 0
+
+    with zipfile.ZipFile(fn, mode='w', compression=zipfile.ZIP_DEFLATED) as zf:
+        with zf.open("metadata/apps.json", 'w') as appsf:
+            print(f"Querying apps...")
+            _mk_app.ntotal = App.query.filter(App.visible == True).count()
+            apps = { app.id: _mk_app(app) for app in App.query.filter(App.visible == True).yield_per(1000) if app.visible }
+            print(f"Exporting apps.json...")
+            json.dump(apps, io.TextIOWrapper(appsf))
+        
+        with zf.open("metadata/categories.json", 'w') as categoriesf:
+            print(f"Exporting categories.json...")
+            categories = {
+                c.id: {
+                    'name': c.name,
+                    'slug': c.slug,
+                    'colour': c.colour,
+                    'icon': c.icon,
+                    'app_type': c.app_type,
+                    'banner_apps': [ app.id for app in c.banner_apps],
+                } for c in Category.query.filter(Category.is_visible == True)
+            }
+            json.dump(categories, io.TextIOWrapper(categoriesf))
+        
+        with zf.open("metadata/collections.json", 'w') as collectionsf:
+            print(f"Exporting collections.json...")
+            collections = {
+                c.id: {
+                    'name': c.name,
+                    'slug': c.slug,
+                    'app_type': c.app_type,
+                    'platforms': c.platforms,
+                } for c in Collection.query
+            }
+            json.dump(collections, io.TextIOWrapper(collectionsf))
+        
+        with zf.open("metadata/developers.json", 'w') as developersf:
+            print(f"Exporting developers.json...")
+            developers = {
+                d.id: d.name
+                for d in Developer.query
+            }
+            
+            # XXX: at some point it might be nice to also provide a Rebble
+            # user ID for a developer?  though I guess also developers who
+            # want to provide this to a user of the archive to verify
+            # themselves could just as well give a user oauth creds to
+            # verify their developer_id
+            json.dump(developers, io.TextIOWrapper(developersf))
+        
+        zip_targets = Queue()
+        downloads_failed = {}
+
+        for ass in assets:
+            if ass == "" or ass is None:
+                continue
+            zip_targets.put((f"assets/{ass[0]}/{ass[1]}/{ass}", functools.partial(download_asset, ass)))
+        
+        for pbw in binaries:
+            if pbw == "" or pbw is None:
+                continue
+            zip_targets.put((f"binaries/{pbw[0]}/{pbw[1]}/{pbw}.pbw", functools.partial(download_pbw, pbw)))
+        
+        n = 0
+        zip_lock = Lock()
+        def download_thread():
+            nonlocal n, downloads_failed, zip_targets
+            while True:
+                try:
+                    fname, download = zip_targets.get_nowait()
+                except Empty:
+                    return
+
+                try:
+                    if test_only and n > 50:
+                        raise TimeoutError()
+                    buf = io.BytesIO()
+                    download(buf)
+                    with zip_lock, zf.open(fname, 'w') as zff:
+                        zff.write(buf.getbuffer())
+                    buf.close()
+                except Exception as e:
+                    downloads_failed[fname] = repr(e)
+
+                if (n % 100) == 0:
+                    print(f"... {n} done, {zip_targets.qsize()} to go ...")
+                n += 1
+                
+                zip_targets.task_done()
+        
+        for i in range(n_threads):
+            Thread(target=download_thread).start()
+        zip_targets.join()
+        
+        with zf.open("metadata/failed_downloads.json", "w") as failedf:
+            json.dump(downloads_failed, io.TextIOWrapper(failedf))
+
+        with open(f"{os.path.dirname(__file__)}/ARCHIVE_LICENSE", "rb") as rf, zf.open("LICENSE.txt", "w") as wf:
+            wf.write(rf.read())
+
+@apps.command('export-archive')
+@click.option('--upload', is_flag=True)
+@click.option('--output')
+@click.option('--test', is_flag=True) # only dump 100 binaries of each type
+def export_archive(output, upload, test):
+    print(f"Preparing to export archive...")
+    if not output:
+        output = tempfile.TemporaryFile()
+    export_archive_to_zip(output, test_only=test)
+    if upload:
+        now = datetime.datetime.now()
+        filename = f"appstore-archive-{now.year:04d}{now.month:02d}.zip"
+        print(f"uploading to {filename}")
+        upload_archive(filename, output)
+        db.session.add(AvailableArchive(filename=filename, created_at=datetime.datetime.now()))
+        db.session.commit()
+
 
 def init_app(app):
     app.cli.add_command(apps)
diff --git a/appstore/developer_portal_api.py b/appstore/developer_portal_api.py
@@ -3,7 +3,7 @@
 import secrets
 
 from algoliasearch import algoliasearch
-from flask import Blueprint, jsonify, abort, request
+from flask import Blueprint, jsonify, abort, request, redirect
 from flask_cors import CORS
 
 from sqlalchemy.exc import IntegrityError
@@ -12,10 +12,10 @@
 from sqlalchemy.exc import DataError
 from zipfile import BadZipFile
 
-from .utils import demand_authed_request, id_generator, validate_new_app_fields, is_valid_category, is_valid_appinfo, is_valid_platform, clone_asset_collection_without_images, is_valid_image_file, is_valid_image_size, get_max_image_dimensions, is_users_developer_id, user_is_wizard, newAppValidationException, algolia_app, first_version_is_newer
-from .models import db, App, Developer, Release, AssetCollection
+from .utils import demand_authed_request, id_generator, validate_new_app_fields, is_valid_category, is_valid_appinfo, is_valid_platform, clone_asset_collection_without_images, is_valid_image_file, is_valid_image_size, get_max_image_dimensions, is_users_developer_id, user_is_wizard, newAppValidationException, algolia_app, first_version_is_newer, get_uid
+from .models import db, App, Developer, Release, AssetCollection, AvailableArchive
 from .pbw import PBW, release_from_pbw
-from .s3 import upload_pbw, upload_asset
+from .s3 import upload_pbw, upload_asset, get_link_for_archive
 from .settings import config
 from .discord import announce_release, announce_new_app, audit_log
 
@@ -768,6 +768,13 @@ def wizard_get_s3_assets(app_id):
     return jsonify(images = images, pbws = pbws)
 
 
+@devportal_api.route('/archive/latest', methods=['GET'])
+def download_archive():
+    uid = get_uid() # unused, does auth so AI scrapers don't waste all our bandwidth, though
+    
+    archive = AvailableArchive.query.order_by(AvailableArchive.created_at.desc()).limit(1).one()
+    return redirect(get_link_for_archive(archive.filename))
+
 def init_app(app, url_prefix='/api/dp'):
     global parent_app
     parent_app = app
diff --git a/appstore/models.py b/appstore/models.py
@@ -60,7 +60,8 @@ class App(db.Model):
     collections = db.relationship('Collection',
                                   back_populates='apps',
                                   secondary=collection_apps,
-                                  passive_deletes=True)
+                                  passive_deletes=True,
+                                  lazy='selectin')
     created_at = db.Column(db.DateTime)
     developer_id = db.Column(db.String(24), db.ForeignKey('developers.id'))
     developer = db.relationship('Developer', lazy='joined')
@@ -178,6 +179,15 @@ class UserFlag(db.Model):
     app = db.relationship('App')
 db.Index('user_flag_app_user_index', UserFlag.app_id, UserFlag.user_id, unique=True)
 
+class AvailableArchive(db.Model):
+    """
+    Archives in S3 of the appstore database.
+    """
+    __tablename__ = "available_archives"
+    id = db.Column(db.Integer(), primary_key=True, index=True)
+    created_at = db.Column(db.DateTime, index=True)
+    filename = db.Column(db.String)
+
 def init_app(app):
     app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
     db.init_app(app)
diff --git a/appstore/s3.py b/appstore/s3.py
@@ -1,11 +1,15 @@
 import json
 import boto3
+import threading
+from botocore.exceptions import ClientError
+from .models import Binary
 from .settings import config
 from .utils import id_generator
 
 # Try to find a way to get S3 credentials.
 session = None
 s3_endpoint = None
+session_lock = threading.Lock()
 
 # Try loading creds from the environment.
 try:
@@ -45,19 +49,39 @@
 if not session:
     print("no session")
 
+_clients = {}
+
+def _client_for_endpoint(endpoint):
+    me = threading.current_thread()
+    if (me, endpoint) in _clients:
+        return _clients[(me, endpoint)]
+    with session_lock:
+        s3 = session.client('s3', endpoint_url=endpoint)
+    _clients[(me, endpoint)] = s3
+    return s3
+
 def upload_pbw(release, file):
     filename = f"{config['S3_PATH']}{release.id}.pbw"
 
     if isinstance(file, str):
         print(f"uploading file {file} to {config['S3_BUCKET']}:{filename}")
-        s3 = session.client('s3', endpoint_url=s3_endpoint)
+        s3 = _client_for_endpoint(s3_endpoint)
         s3.upload_file(file, config['S3_BUCKET'], filename)
     else:
         print(f"uploading file object {file.name} to {config['S3_BUCKET']}:{filename}")   
-        s3 = session.client('s3', endpoint_url=s3_endpoint)
+        s3 = _client_for_endpoint(s3_endpoint)
         file.seek(0)
         s3.upload_fileobj(file, config['S3_BUCKET'], filename, ExtraArgs = {'ContentType': 'application/zip'})   
 
+def download_pbw(id, file):
+    filename = f"{config['S3_PATH']}{id}.pbw"
+    s3 = _client_for_endpoint(s3_endpoint)
+    if isinstance(file, str):
+        s3.download_file(config['S3_BUCKET'], filename, file)
+    else:
+        s3.download_fileobj(config['S3_BUCKET'], filename, file)
+
+
 def upload_asset(file, mime_type = None):
     id = id_generator.generate()
     filename = f"{config['S3_ASSET_PATH']}{id}"
@@ -74,14 +98,41 @@ def upload_asset(file, mime_type = None):
             else:
                 raise Exception("Unknown or unsupported mime_type for file provided to update_asset")
 
-        s3 = session.client('s3', endpoint_url=s3_endpoint)
+        s3 = _client_for_endpoint(s3_endpoint)
         s3.upload_file(file, config['S3_ASSET_BUCKET'], filename, ExtraArgs = {'ContentType': mime_type})
         return id
     
     else:
         print(f"uploading file object '{file.name}' to {config['S3_ASSET_BUCKET']}:{filename}")
         file.seek(0)
-        s3 = session.client('s3', endpoint_url=s3_endpoint)
+        s3 = _client_for_endpoint(s3_endpoint)
         s3.upload_fileobj(file, config['S3_ASSET_BUCKET'], filename, ExtraArgs = {'ContentType': mime_type})
     
     return id    
+
+def download_asset(id, file):
+    filename = f"{config['S3_ASSET_PATH']}{id}"
+    s3 = _client_for_endpoint(s3_endpoint)
+    if isinstance(file, str):
+        s3.download_file(config['S3_ASSET_BUCKET'], filename, file)
+    else:
+        s3.download_fileobj(config['S3_ASSET_BUCKET'], filename, file)
+
+def upload_archive(filename, file, mime_type = 'application/zip'):
+    s3_filename = f"{config['S3_ARCHIVE_PATH']}{filename}"
+    s3 = _client_for_endpoint(s3_endpoint)
+    if isinstance(file, str):
+        s3.upload_file(file, config['S3_ARCHIVE_BUCKET'], s3_filename, ExtraArgs = { 'ContentType': mime_type })
+    else:
+        file.seek(0)
+        s3.upload_fileobj(file, config['S3_ARCHIVE_BUCKET'], s3_filename, ExtraArgs = { 'ContentType': mime_type })
+
+def get_link_for_archive(filename, expiry = 3600):
+    s3 = _client_for_endpoint(s3_endpoint)
+    return s3.generate_presigned_url('get_object',
+        Params={
+            'Bucket': config['S3_ARCHIVE_BUCKET'],
+            'Key': f"{config['S3_ARCHIVE_PATH']}{filename}"
+        },
+        ExpiresIn=expiry
+    )
diff --git a/appstore/settings.py b/appstore/settings.py
diff --git a/migrations/versions/c2c7be2e8835_create_available_archives_table.py b/migrations/versions/c2c7be2e8835_create_available_archives_table.py
