[WIP] toolchain: auto resolve deps

Author: T-Dynamos

pythonforandroid/build.py

@@ -2,19 +2,24 @@
 import copy
 import glob
 import os
+import json
+import tempfile
 from os import environ
 from os.path import (
-    abspath, join, realpath, dirname, expanduser, exists
+    abspath, join, realpath, dirname, expanduser, exists, basename
 )
 import re
 import shutil
 import subprocess
 
 import sh
 
+from packaging.utils import parse_wheel_filename
+from packaging.requirements import Requirement
+
 from pythonforandroid.androidndk import AndroidNDK
 from pythonforandroid.archs import ArchARM, ArchARMv7_a, ArchAarch_64, Archx86, Archx86_64
-from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint)
+from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint, Out_Style, Out_Fore)
 from pythonforandroid.pythonpackage import get_package_name
 from pythonforandroid.recipe import CythonRecipe, Recipe
 from pythonforandroid.recommendations import (
@@ -90,6 +95,8 @@ class Context:
 
     recipe_build_order = None  # Will hold the list of all built recipes
 
+    python_modules = None  # Will hold resolved pure python packages
+
     symlink_bootstrap_files = False  # If True, will symlink instead of copying during build
 
     java_build_tool = 'auto'
@@ -444,6 +451,12 @@ def has_package(self, name, arch=None):
                 # Failed to look up any meaningful name.
                 return False
 
+        # normalize name to remove version tags
+        try:
+            name = Requirement(name).name
+        except Exception:
+            pass
+
         # Try to look up recipe by name:
         try:
             recipe = Recipe.get_recipe(name, self)
@@ -649,6 +662,103 @@ def run_setuppy_install(ctx, project_dir, env=None, arch=None):
             os.remove("._tmp_p4a_recipe_constraints.txt")
 
 
+def is_wheel_platform_independent(whl_name):
+    name, version, build, tags = parse_wheel_filename(whl_name)
+    return all(tag.platform == "any" for tag in tags)
+
+
+def process_python_modules(ctx, modules):
+    """Use pip --dry-run to resolve dependencies and filter for pure-Python packages
+    """
+    modules = list(modules)
+    build_order = list(ctx.recipe_build_order)
+    _requirement_names = []
+    processed_modules = []
+
+    for module in modules+build_order:
+        try:
+            # we need to normalize names
+            # eg Requests>=2.0 becomes requests
+            _requirement_names.append(Requirement(module).name)
+        except Exception:
+            # name parsing failed; skip processing this module via pip
+            processed_modules.append(module)
+            if module in modules:
+                modules.remove(module)
+
+    if len(processed_modules) > 0:
+        warning(f'Ignored by module resolver : {processed_modules}')
+
+    # preserve the original module list
+    processed_modules.extend(modules)
+
+    # temp file for pip report
+    fd, path = tempfile.mkstemp()
+    os.close(fd)
+
+    # setup hostpython recipe
+    host_recipe = Recipe.get_recipe("hostpython3", ctx)
+
+    env = environ.copy()
+    _python_path = host_recipe.get_path_to_python()
+    libdir = glob.glob(join(_python_path, "build", "lib*"))
+    env['PYTHONPATH'] = host_recipe.site_dir + ":" + join(
+        _python_path, "Modules") + ":" + (libdir[0] if libdir else "")
+
+    shprint(
+        host_recipe.pip, 'install', *modules,
+        '--dry-run', '--break-system-packages', '--ignore-installed',
+        '--report', path, '-q', _env=env
+    )
+
+    with open(path, "r") as f:
+        report = json.load(f)
+
+    os.remove(path)
+
+    info('Extra resolved pure python dependencies :')
+
+    ignored_str = " (ignored)"
+    # did we find any non pure python package?
+    any_not_pure_python = False
+
+    # just for style
+    info(" ")
+    for module in report["install"]:
+
+        mname = module["metadata"]["name"]
+        mver = module["metadata"]["version"]
+        filename = basename(module["download_info"]["url"])
+        pure_python = True
+
+        if (filename.endswith(".whl") and not is_wheel_platform_independent(filename)):
+            any_not_pure_python = True
+            pure_python = False
+
+        # does this module matches any recipe name?
+        if mname.lower() in _requirement_names:
+            continue
+
+        color = Out_Fore.GREEN if pure_python else Out_Fore.RED
+        ignored = "" if pure_python else ignored_str
+
+        info(
+            f"  {color}{mname}{Out_Fore.WHITE} : "
+            f"{Out_Style.BRIGHT}{mver}{Out_Style.RESET_ALL}"
+            f"{ignored}"
+        )
+
+        if pure_python:
+            processed_modules.append(f"{mname}=={mver}")
+    info(" ")
+
+    if any_not_pure_python:
+        warning("Some packages were ignored because they are not pure Python.")
+        warning("To install the ignored packages, explicitly list them in your requirements file.")
+
+    return processed_modules
+
+
 def run_pymodules_install(ctx, arch, modules, project_dir=None,
                           ignore_setup_py=False):
     """ This function will take care of all non-recipe things, by:
@@ -663,6 +773,7 @@ def run_pymodules_install(ctx, arch, modules, project_dir=None,
 
     info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE FOR ARCH: {} ***'.format(arch))
 
+    modules = process_python_modules(ctx, modules)
     modules = [m for m in modules if ctx.not_has_package(m, arch)]
 
     # We change current working directory later, so this has to be an absolute

pythonforandroid/recipe.py

@@ -979,7 +979,7 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True):
         env['LANG'] = "en_GB.UTF-8"
 
         # Binaries made by packages installed by pip
-        self.patch_shebangs(self._host_recipe.local_bin, self.real_hostpython_location)
+        self.patch_shebangs(self._host_recipe.local_bin, self._host_recipe.python_exe)
         env["PATH"] = self._host_recipe.local_bin + ":" + self._host_recipe.site_bin + ":" + env["PATH"]
 
         host_env = self.get_hostrecipe_env(arch)
@@ -1022,10 +1022,9 @@ def install_python_package(self, arch, name=None, env=None, is_dir=True):
 
         info('Installing {} into site-packages'.format(self.name))
 
-        hostpython = sh.Command(self.hostpython_location)
         hpenv = env.copy()
         with current_directory(self.get_build_dir(arch.arch)):
-            shprint(hostpython, '-m', 'pip', 'install', '.',
+            shprint(self._host_recipe.pip, 'install', '.',
                     '--compile', '--target',
                     self.ctx.get_python_install_dir(arch.arch),
                     _env=hpenv, *self.setup_extra_args
@@ -1045,8 +1044,7 @@ def hostpython_site_dir(self):
 
     def install_hostpython_package(self, arch):
         env = self.get_hostrecipe_env(arch)
-        real_hostpython = sh.Command(self.real_hostpython_location)
-        shprint(real_hostpython, '-m', 'pip', 'install', '.',
+        shprint(self._host_recipe.pip, 'install', '.',
                 '--compile',
                 '--root={}'.format(self._host_recipe.site_root),
                 _env=env, *self.setup_extra_args)
@@ -1075,8 +1073,7 @@ def install_hostpython_prerequisites(self, packages=None, force_upgrade=True):
             pip_options.append("--upgrade")
         # Use system's pip
         pip_env = self.get_hostrecipe_env()
-        pip_env["HOME"] = "/tmp"
-        shprint(sh.Command(self.real_hostpython_location), "-m", "pip", *pip_options, _env=pip_env)
+        shprint(self._host_recipe.pip, *pip_options, _env=pip_env)
 
     def restore_hostpython_prerequisites(self, packages):
         _packages = []
@@ -1270,10 +1267,14 @@ def get_recipe_env(self, arch, **kwargs):
         return env
 
     def get_wheel_platform_tag(self, arch):
+        # https://peps.python.org/pep-0738/#packaging
+        # official python only supports 64 bit:
+        # android_21_arm64_v8a
+        # android_21_x86_64
         return f"android_{self.ctx.ndk_api}_" + {
-            "armeabi-v7a": "arm",
-            "arm64-v8a": "aarch64",
+            "arm64-v8a": "arm64_v8a",
             "x86_64": "x86_64",
+            "armeabi-v7a": "arm",
             "x86": "i686",
         }[arch.arch]
 

pythonforandroid/recipes/hostpython3/__init__.py

@@ -36,7 +36,7 @@ class HostPython3Recipe(Recipe):
         :class:`~pythonforandroid.python.HostPythonRecipe`
     '''
 
-    version = '3.14.0'
+    version = '3.14.2'
 
     url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz'
     '''The default url to download our host python recipe. This url will
@@ -113,6 +113,14 @@ def site_dir(self):
             f"usr/local/lib/python{p_version.major}.{p_version.minor}/site-packages/"
         )
 
+    @property
+    def _pip(self):
+        return join(self.local_bin, "pip3")
+
+    @property
+    def pip(self):
+        return sh.Command(self._pip)
+
     def build_arch(self, arch):
         env = self.get_recipe_env(arch)
 
@@ -160,10 +168,12 @@ def build_arch(self, arch):
 
         ensure_dir(self.site_root)
         self.ctx.hostpython = self.python_exe
+
         if build_configured:
+
             shprint(
                 sh.Command(self.python_exe), "-m", "ensurepip", "--root", self.site_root, "-U",
-                _env={"HOME": "/tmp"}
+                _env={"PATH": self.local_bin}
             )
 
 

pythonforandroid/recipes/python3/__init__.py

@@ -54,7 +54,7 @@ class Python3Recipe(TargetPythonRecipe):
         :class:`~pythonforandroid.python.GuestPythonRecipe`
     '''
 
-    version = '3.14.0'
+    version = '3.14.2'
     _p_version = Version(version)
     url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz'
     name = 'python3'
@@ -78,6 +78,7 @@ class Python3Recipe(TargetPythonRecipe):
 
     if _p_version.minor >= 14:
         patches.append('patches/3.14_armv7l_fix.patch')
+        patches.append('patches/3.14_fix_remote_debug.patch')
 
     if shutil.which('lld') is not None:
         if _p_version.minor == 7:

pythonforandroid/recipes/python3/patches/3.14_fix_remote_debug.patch

@@ -0,0 +1,28 @@
+diff '--color=auto' -uNr cpython-3.14.2/Modules/_remote_debugging_module.c cpython-3.14.2.mod/Modules/_remote_debugging_module.c
+--- cpython-3.14.2/Modules/_remote_debugging_module.c	2025-12-05 22:19:16.000000000 +0530
++++ cpython-3.14.2.mod/Modules/_remote_debugging_module.c	2025-12-13 20:22:44.011497868 +0530
+@@ -812,7 +812,9 @@
+         PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process.");
+         _PyErr_ChainExceptions1(exc);
+     }
+-#elif defined(__linux__)
++
++// https://github.com/python/cpython/commit/1963e701001839389cfb1b11d803b0743f4705d7
++#elif defined(__linux__) && HAVE_PROCESS_VM_READV
+     // On Linux, search for asyncio debug in executable or DLL
+     address = search_linux_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython");
+     if (address == 0) {
+diff '--color=auto' -uNr cpython-3.14.2/Python/remote_debug.h cpython-3.14.2.mod/Python/remote_debug.h
+--- cpython-3.14.2/Python/remote_debug.h	2025-12-05 22:19:16.000000000 +0530
++++ cpython-3.14.2.mod/Python/remote_debug.h	2025-12-13 20:23:27.917518543 +0530
+@@ -881,7 +881,9 @@
+             handle->pid);
+         _PyErr_ChainExceptions1(exc);
+     }
+-#elif defined(__linux__)
++
++// https://github.com/python/cpython/commit/1963e701001839389cfb1b11d803b0743f4705d7
++#elif defined(__linux__) && HAVE_PROCESS_VM_READV
+     // On Linux, search for 'python' in executable or DLL
+     address = search_linux_map_for_section(handle, "PyRuntime", "python");
+     if (address == 0) {