Add Android to add_to_pydotorg.py

Author: mhsmith

add_to_pydotorg.py

@@ -4,11 +4,10 @@
 To use (RELEASE is something like 3.3.5rc2):
 
 * Copy this script to dl-files (it needs access to all the release files).
-  You could also download all files, then you need to adapt the "ftp_root"
-  string below.
+  You could also download all files, then you need to use the "--ftp-root"
+  argument.
 
-* Make sure all download files are in place in the correct /srv/www.python.org
-  subdirectory.
+* Make sure all download files are in place in the correct FTP subdirectory.
 
 * Create a new Release object via the Django admin (adding via API is
   currently broken), the name MUST be "Python RELEASE".
@@ -23,6 +22,7 @@
 Georg Brandl, March 2014.
 """
 
+import argparse
 import hashlib
 import json
 import os
@@ -70,8 +70,6 @@ def run_cmd(
     )
     sys.exit()
 
-base_url = "https://www.python.org/api/v1/"
-ftp_root = "/srv/www.python.org/ftp/python/"
 download_root = "https://www.python.org/ftp/python/"
 
 tag_cre = re.compile(r"(\d+)(?:\.(\d+)(?:\.(\d+))?)?(?:([ab]|rc)(\d+))?$")
@@ -98,44 +96,47 @@ def get_file_descriptions(
 ) -> list[tuple[re.Pattern[str], tuple[str, int, bool, str]]]:
     v = minor_version_tuple(release)
     rx = re.compile
-    # value is (file "name", OS id, download button, file "description").
-    # OS=0 means no ReleaseFile object. Only one matching *file* (not regex)
+    # value is (file "name", OS slug, download button, file "description").
+    # OS=None means no ReleaseFile object. Only one matching *file* (not regex)
     # per OS can have download=True.
     return [
-        (rx(r"\.tgz$"), ("Gzipped source tarball", 3, False, "")),
-        (rx(r"\.tar\.xz$"), ("XZ compressed source tarball", 3, True, "")),
+        (rx(r"\.tgz$"), ("Gzipped source tarball", "source", False, "")),
+        (rx(r"\.tar\.xz$"), ("XZ compressed source tarball", "source", True, "")),
         (
             rx(r"windows-.+\.json"),
             (
                 "Windows release manifest",
-                1,
+                "windows",
                 False,
                 f"Install with 'py install {v[0]}.{v[1]}'",
             ),
         ),
         (
             rx(r"-embed-amd64\.zip$"),
-            ("Windows embeddable package (64-bit)", 1, False, ""),
+            ("Windows embeddable package (64-bit)", "windows", False, ""),
         ),
         (
             rx(r"-embed-arm64\.zip$"),
-            ("Windows embeddable package (ARM64)", 1, False, ""),
+            ("Windows embeddable package (ARM64)", "windows", False, ""),
+        ),
+        (
+            rx(r"-arm64\.exe$"),
+            ("Windows installer (ARM64)", "windows", False, "Experimental"),
         ),
-        (rx(r"-arm64\.exe$"), ("Windows installer (ARM64)", 1, False, "Experimental")),
         (
             rx(r"-amd64\.exe$"),
-            ("Windows installer (64-bit)", 1, v >= (3, 9), "Recommended"),
+            ("Windows installer (64-bit)", "windows", v >= (3, 9), "Recommended"),
         ),
         (
             rx(r"-embed-win32\.zip$"),
-            ("Windows embeddable package (32-bit)", 1, False, ""),
+            ("Windows embeddable package (32-bit)", "windows", False, ""),
         ),
-        (rx(r"\.exe$"), ("Windows installer (32-bit)", 1, v < (3, 9), "")),
+        (rx(r"\.exe$"), ("Windows installer (32-bit)", "windows", v < (3, 9), "")),
         (
             rx(r"-macosx10\.5(_rev\d)?\.(dm|pk)g$"),
             (
                 "macOS 32-bit i386/PPC installer",
-                2,
+                "macos",
                 False,
                 "for Mac OS X 10.5 and later",
             ),
@@ -144,7 +145,7 @@ def get_file_descriptions(
             rx(r"-macosx10\.6(_rev\d)?\.(dm|pk)g$"),
             (
                 "macOS 64-bit/32-bit Intel installer",
-                2,
+                "macos",
                 False,
                 "for Mac OS X 10.6 and later",
             ),
@@ -153,7 +154,7 @@ def get_file_descriptions(
             rx(r"-macos(x)?10\.9\.(dm|pk)g$"),
             (
                 "macOS 64-bit Intel-only installer",
-                2,
+                "macos",
                 False,
                 "for macOS 10.9 and later, deprecated",
             ),
@@ -162,11 +163,19 @@ def get_file_descriptions(
             rx(r"-macos(x)?1[1-9](\.[0-9]*)?\.pkg$"),
             (
                 "macOS 64-bit universal2 installer",
-                2,
+                "macos",
                 True,
                 f"for macOS {'10.13' if v >= (3, 12, 6) else '10.9'} and later",
             ),
         ),
+        (
+            rx(r"aarch64-linux-android.tar.gz$"),
+            ("Android embeddable package (aarch64)", "android", False, ""),
+        ),
+        (
+            rx(r"x86_64-linux-android.tar.gz$"),
+            ("Android embeddable package (x86_64)", "android", False, ""),
+        ),
     ]
 
 
@@ -182,14 +191,14 @@ def sigfile_for(release: str, rfile: str) -> str:
     return download_root + f"{release}/{rfile}.asc"
 
 
-def md5sum_for(release: str, rfile: str) -> str:
+def md5sum_for(filename: str) -> str:
     return hashlib.md5(
-        open(ftp_root + base_version(release) + "/" + rfile, "rb").read()
+        open(filename, "rb").read(),
     ).hexdigest()
 
 
-def filesize_for(release: str, rfile: str) -> int:
-    return path.getsize(ftp_root + base_version(release) + "/" + rfile)
+def filesize_for(filename: str) -> int:
+    return path.getsize(filename)
 
 
 def make_slug(text: str) -> str:
@@ -215,6 +224,7 @@ def minor_version_tuple(release: str) -> tuple[int, int]:
 
 
 def build_file_dict(
+    ftp_root: str,
     release: str,
     rfile: str,
     rel_pk: int,
@@ -224,6 +234,7 @@ def build_file_dict(
     add_desc: str,
 ) -> dict[str, Any]:
     """Return a dictionary with all needed fields for a ReleaseFile object."""
+    filename = path.join(ftp_root, base_version(release), rfile)
     d = {
         "name": file_desc,
         "slug": slug_for(release) + "-" + make_slug(file_desc)[:40],
@@ -232,36 +243,38 @@ def build_file_dict(
         "description": add_desc,
         "is_source": os_pk == 3,
         "url": download_root + f"{base_version(release)}/{rfile}",
-        "md5_sum": md5sum_for(release, rfile),
-        "filesize": filesize_for(release, rfile),
+        "md5_sum": md5sum_for(filename),
+        "filesize": filesize_for(filename),
         "download_button": add_download,
     }
     # Upload GPG signature
-    if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.asc"):
+    if os.path.exists(filename + ".asc"):
         d["gpg_signature_file"] = sigfile_for(base_version(release), rfile)
     # Upload Sigstore signature
-    if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.sig"):
+    if os.path.exists(filename + ".sig"):
         d["sigstore_signature_file"] = (
             download_root + f"{base_version(release)}/{rfile}.sig"
         )
     # Upload Sigstore certificate
-    if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.crt"):
+    if os.path.exists(filename + ".crt"):
         d["sigstore_cert_file"] = download_root + f"{base_version(release)}/{rfile}.crt"
     # Upload Sigstore bundle
-    if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.sigstore"):
+    if os.path.exists(filename + ".sigstore"):
         d["sigstore_bundle_file"] = (
             download_root + f"{base_version(release)}/{rfile}.sigstore"
         )
     # Upload SPDX SBOM file
-    if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.spdx.json"):
+    if os.path.exists(filename + ".spdx.json"):
         d["sbom_spdx2_file"] = (
             download_root + f"{base_version(release)}/{rfile}.spdx.json"
         )
 
     return d
 
 
-def list_files(release: str) -> Generator[tuple[str, str, int, bool, str], None, None]:
+def list_files(
+    ftp_root: str, release: str
+) -> Generator[tuple[str, str, int, bool, str], None, None]:
     """List all of the release's download files."""
     reldir = base_version(release)
     for rfile in os.listdir(path.join(ftp_root, reldir)):
@@ -283,15 +296,14 @@ def list_files(release: str) -> Generator[tuple[str, str, int, bool, str], None,
 
         for rx, info in get_file_descriptions(release):
             if rx.search(rfile):
-                file_desc, os_pk, add_download, add_desc = info
-                yield rfile, file_desc, os_pk, add_download, add_desc
+                yield (rfile, *info)
                 break
         else:
             print(f"    File {reldir}/{rfile} not recognized")
             continue
 
 
-def query_object(objtype: str, **params: Any) -> int:
+def query_object(base_url: str, objtype: str, **params: Any) -> int:
     """Find an API object by query parameters."""
     uri = base_url + f"downloads/{objtype}/"
     uri += "?" + "&".join(f"{k}={v}" for k, v in params.items())
@@ -302,7 +314,7 @@ def query_object(objtype: str, **params: Any) -> int:
     return int(obj["resource_uri"].strip("/").split("/")[-1])
 
 
-def post_object(objtype: str, datadict: dict[str, Any]) -> int:
+def post_object(base_url: str, objtype: str, datadict: dict[str, Any]) -> int:
     """Create a new API object."""
     resp = requests.post(
         base_url + "downloads/" + objtype + "/",
@@ -324,11 +336,11 @@ def post_object(objtype: str, datadict: dict[str, Any]) -> int:
 
 
 def sign_release_files_with_sigstore(
-    release: str, release_files: list[tuple[str, str, int, bool, str]]
+    ftp_root: str, release: str, release_files: list[tuple[str, str, int, bool, str]]
 ) -> None:
     filenames = [
         ftp_root + f"{base_version(release)}/{rfile}"
-        for rfile, file_desc, os_pk, add_download, add_desc in release_files
+        for rfile, *_ in release_files
     ]
 
     def has_sigstore_signature(filename: str) -> bool:
@@ -445,34 +457,65 @@ def has_sigstore_signature(filename: str) -> bool:
             )
 
 
+def parse_args() -> argparse.Namespace:
+    def ensure_trailing_slash(s: str):
+        if not s.endswith("/"):
+            s += "/"
+        return s
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--base-url",
+        metavar="URL",
+        type=ensure_trailing_slash,
+        default="https://www.python.org/api/v1/",
+        help="API URL; defaults to %(default)s",
+    )
+    parser.add_argument(
+        "--ftp-root",
+        metavar="DIR",
+        type=ensure_trailing_slash,
+        default="/srv/www.python.org/ftp/python",
+        help="FTP root; defaults to %(default)s",
+    )
+    parser.add_argument(
+        "release",
+        help="Python version number, e.g. 3.3.5rc2",
+    )
+    return parser.parse_args()
+
+
 def main() -> None:
-    rel = sys.argv[1]
+    args = parse_args()
+    rel = args.release
     print("Querying python.org for release", rel)
-    rel_pk = query_object("release", name="Python+" + rel)
+    rel_pk = query_object(args.base_url, "release", name="Python+" + rel)
     print("Found Release object: id =", rel_pk)
-    release_files = list(list_files(rel))
-    sign_release_files_with_sigstore(rel, release_files)
+
+    release_files = list(list_files(args.ftp_root, rel))
+    sign_release_files_with_sigstore(args.ftp_root, rel, release_files)
     n = 0
     file_dicts = {}
-    for rfile, file_desc, os_pk, add_download, add_desc in release_files:
+    for rfile, file_desc, os_slug, add_download, add_desc in release_files:
+        if not os_slug:
+            continue
+        os_pk = query_object(args.base_url, "os", slug=os_slug)
         file_dict = build_file_dict(
-            rel, rfile, rel_pk, file_desc, os_pk, add_download, add_desc
+            args.ftp_root, rel, rfile, rel_pk, file_desc, os_pk, add_download, add_desc
         )
         key = file_dict["slug"]
-        if not os_pk:
-            continue
         print("Creating ReleaseFile object for", rfile, key)
         if key in file_dicts:
             raise RuntimeError(f"duplicate slug generated: {key}")
         file_dicts[key] = file_dict
     print("Deleting previous release files")
     resp = requests.delete(
-        base_url + f"downloads/release_file/?release={rel_pk}", headers=headers
+        args.base_url + f"downloads/release_file/?release={rel_pk}", headers=headers
     )
     if resp.status_code != 204:
         raise RuntimeError(f"deleting previous releases failed: {resp.status_code}")
     for file_dict in file_dicts.values():
-        file_pk = post_object("release_file", file_dict)
+        file_pk = post_object(args.base_url, "release_file", file_dict)
         if file_pk >= 0:
             print("Created as id =", file_pk)
             n += 1

tests/fake-ftp-files.txt

@@ -521,6 +521,7 @@ python-3.14.0b2.exe.spdx.json
 python-3.14.0b2t-amd64.zip
 python-3.14.0b2t-arm64.zip
 python-3.14.0b2t-win32.zip
+python-3.14.0b3-aarch64-linux-android.tar.gz
 python-3.14.0b3-amd64.exe
 python-3.14.0b3-amd64.exe.crt
 python-3.14.0b3-amd64.exe.sig
@@ -559,6 +560,7 @@ python-3.14.0b3-test-amd64.zip
 python-3.14.0b3-test-arm64.zip
 python-3.14.0b3-test-win32.zip
 python-3.14.0b3-win32.zip
+python-3.14.0b3-x86_64-linux-android.tar.gz
 python-3.14.0b3.exe
 python-3.14.0b3.exe.crt
 python-3.14.0b3.exe.sig