Allow fetching CDTs too (#103)

Author: jaimergp

.github/workflows/website.yml

@@ -17,10 +17,12 @@ jobs:
       - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v5
         with:
           python-version: '3.10'
+      
+      - run: python -m pip install -U requests
 
       - name: update JSON payload
         run: |
-          python scripts/all_json.py outputs/ site/feedstock-outputs.json
+          python scripts/all_json.py outputs/ site/feedstock-outputs.json --with-cdts
 
       - name: deploy
         uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v3

scripts/all_json.py

@@ -1,6 +1,8 @@
 import argparse
 import json
+import re
 import sys
+from base64 import b64decode
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 
@@ -11,12 +13,31 @@ def read_json(path):
     return path.stem, data["feedstocks"]
 
 
-def main(sources_dir, output_json):
+def fetch_cdts():
+    import requests
+
+    r = requests.get("https://api.github.com/repos/conda-forge/cdt-builds/contents/README.md")
+    r.raise_for_status()
+    data = r.json()
+    text = b64decode(data["content"]).decode()
+    seen = set()
+    for result in re.finditer(r"\(https://anaconda\.org/conda-forge/([a-z0-9_\-\.\+]+)\)", text):
+        if result and result.group(1) not in seen:
+            seen.add(result.group(1))
+            yield result.group(1)
+
+
+def main(sources_dir, output_json, with_cdts=False):
     jsons = Path(sources_dir).glob("**/*.json")
 
     with ThreadPoolExecutor(4) as executor:
         all_packages = {pkg: repos for (pkg, repos) in executor.map(read_json, jsons)}
     print(f"Processed {len(all_packages)} packages.")
+
+    if with_cdts:
+        for cdt in fetch_cdts():
+            all_packages.setdefault(cdt, []).append("cdt-builds")
+
     output_json = Path(output_json)
     output_json.parent.mkdir(exist_ok=True, parents=True)
     with open(output_json, "w") as f:
@@ -27,6 +48,7 @@ def main(sources_dir, output_json):
     parser = argparse.ArgumentParser()
     parser.add_argument("sources_dir")
     parser.add_argument("output_json")
+    parser.add_argument("--with-cdts", action="store_true")
     args = parser.parse_args()
 
-    sys.exit(main(args.sources_dir, args.output_json))
+    sys.exit(main(args.sources_dir, args.output_json, with_cdts=args.with_cdts))