From 6b43f885c7feaa9dd7e158292575b5fc961c6de1 Mon Sep 17 00:00:00 2001 From: Ansh Dadwal Date: Sat, 2 Nov 2024 11:04:38 +0530 Subject: [PATCH] `python`: update to 3.13 --- pythonforandroid/artifact.py | 95 +++++++++ .../java/org/kivy/android/PythonUtil.java | 4 +- pythonforandroid/build.py | 15 +- pythonforandroid/graph.py | 6 +- pythonforandroid/pythonpackage.py | 7 +- pythonforandroid/recipe.py | 139 ++++++++------ pythonforandroid/recipebuild.py | 79 ++++++++ pythonforandroid/recipes/android/__init__.py | 12 +- pythonforandroid/recipes/android/src/setup.py | 34 +++- .../recipes/hostpython3/__init__.py | 33 +++- .../patches/pyconfig_detection.patch | 13 -- pythonforandroid/recipes/libb2/__init__.py | 37 ++++ pythonforandroid/recipes/libcurl/__init__.py | 8 +- pythonforandroid/recipes/libffi/__init__.py | 4 +- pythonforandroid/recipes/liblzma/__init__.py | 2 +- .../recipes/libsodium/__init__.py | 11 +- pythonforandroid/recipes/openssl/__init__.py | 39 ++-- pythonforandroid/recipes/pyjnius/__init__.py | 11 +- pythonforandroid/recipes/python3/__init__.py | 180 ++++++++++-------- .../cpython-313-ctypes-find-library.patch | 11 ++ pythonforandroid/recipes/sdl2/__init__.py | 2 +- .../recipes/sdl2_image/__init__.py | 27 ++- .../recipes/setuptools/__init__.py | 3 +- pythonforandroid/recipes/six/__init__.py | 10 - pythonforandroid/recipes/sqlite3/__init__.py | 6 +- .../recipes/util-linux/__init__.py | 36 ++++ pythonforandroid/toolchain.py | 29 ++- 27 files changed, 593 insertions(+), 260 deletions(-) create mode 100644 pythonforandroid/artifact.py create mode 100644 pythonforandroid/recipebuild.py delete mode 100644 pythonforandroid/recipes/hostpython3/patches/pyconfig_detection.patch create mode 100644 pythonforandroid/recipes/libb2/__init__.py create mode 100644 pythonforandroid/recipes/python3/patches/cpython-313-ctypes-find-library.patch delete mode 100644 pythonforandroid/recipes/six/__init__.py create mode 100644 pythonforandroid/recipes/util-linux/__init__.py diff --git a/pythonforandroid/artifact.py b/pythonforandroid/artifact.py new file mode 100644 index 0000000000..c49f89d18c --- /dev/null +++ b/pythonforandroid/artifact.py @@ -0,0 +1,95 @@ +import zipfile +import json +import os +import subprocess + + +class ArtifactName: + + platform = "android" + + def __init__(self, recipe, arch): + self.recipe = recipe + self._arch = arch + + @property + def stage(self): + return "master" + result = subprocess.check_output( + ["git", "branch", "--show-current"], + stderr=subprocess.PIPE, + universal_newlines=True, + ) + return result.strip() + + @property + def kind(self): + return "lib" + + @property + def arch(self): + return self._arch.arch + + @property + def native_api_level(self): + return str(self.recipe.ctx.ndk_api) + + @property + def file_props(self): + return [ + self.stage, + self.kind, + self.recipe.name, + self.arch, + self.platform + self.native_api_level, + self.recipe.version, + ] + + @property + def filename(self): + return "_".join(self.file_props) + ".zip" + + +def build_artifact( + save_path, + recipe, + arch, + lib_dependencies=[], + files_dependencies=[], + install_instructions=[], +): + # Parse artifact name + artifact_name = ArtifactName(recipe, arch) + zip_path = os.path.join(save_path, artifact_name.filename) + + # Contents of zip file + metadata_folder = "metadata/" + data_folder = "data/" + prop_file = os.path.join(metadata_folder, "properties.json") + install_file = os.path.join(metadata_folder, "install.json") + + properties = { + "stage": artifact_name.stage, + "name": recipe.name, + "arch": artifact_name.arch, + "native_api_level": artifact_name.native_api_level, + "kind": artifact_name.kind, + "version": recipe.version, + "release_version": recipe.version, + "lib_dependencies": lib_dependencies, + "files_dependencies": files_dependencies, + } + + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zipf: + zipf.writestr(metadata_folder, "") + zipf.writestr(data_folder, "") + + for file_name in lib_dependencies + files_dependencies: + with open(file_name, "rb") as file: + file_name_ = os.path.join(data_folder + os.path.basename(file_name)) + zipf.writestr(file_name_, file.read()) + file.close() + + zipf.writestr(prop_file, json.dumps(properties)) + zipf.writestr(install_file, json.dumps(install_instructions)) + zipf.close() diff --git a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java index cc04d83f6b..791ad9f467 100644 --- a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java +++ b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java @@ -56,6 +56,8 @@ protected static ArrayList getLibraries(File libsDir) { libsList.add("python3.9"); libsList.add("python3.10"); libsList.add("python3.11"); + libsList.add("python3.12"); + libsList.add("python3.13"); libsList.add("main"); return libsList; } @@ -75,7 +77,7 @@ public static void loadLibraries(File filesDir, File libsDir) { // load, and it has failed, give a more // general error Log.v(TAG, "Library loading error: " + e.getMessage()); - if (lib.startsWith("python3.11") && !foundPython) { + if (lib.startsWith("python3.13") && !foundPython) { throw new RuntimeException("Could not load any libpythonXXX.so"); } else if (lib.startsWith("python")) { continue; diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index 98e2d70b2b..edefabd3c5 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -94,6 +94,8 @@ class Context: java_build_tool = 'auto' + save_prebuilt = False + @property def packages_path(self): '''Where packages are downloaded before being unpacked''' @@ -147,11 +149,16 @@ def setup_dirs(self, storage_dir): 'specify a path with --storage-dir') self.build_dir = join(self.storage_dir, 'build') self.dist_dir = join(self.storage_dir, 'dists') + self.prebuilt_dir = join(self.storage_dir, 'output') def ensure_dirs(self): ensure_dir(self.storage_dir) ensure_dir(self.build_dir) ensure_dir(self.dist_dir) + + if self.save_prebuilt: + ensure_dir(self.prebuilt_dir) + ensure_dir(join(self.build_dir, 'bootstrap_builds')) ensure_dir(join(self.build_dir, 'other_builds')) @@ -672,6 +679,7 @@ def run_pymodules_install(ctx, arch, modules, project_dir=None, # Bail out if no python deps and no setup.py to process: if not modules and ( ignore_setup_py or + project_dir is None or not project_has_setup_py(project_dir) ): info('No Python modules and no setup.py to process, skipping') @@ -687,7 +695,8 @@ def run_pymodules_install(ctx, arch, modules, project_dir=None, "If this fails, it may mean that the module has compiled " "components and needs a recipe." ) - if project_has_setup_py(project_dir) and not ignore_setup_py: + if project_dir is not None and \ + project_has_setup_py(project_dir) and not ignore_setup_py: info( "Will process project install, if it fails then the " "project may not be compatible for Android install." @@ -759,7 +768,9 @@ def run_pymodules_install(ctx, arch, modules, project_dir=None, _env=copy.copy(env)) # Afterwards, run setup.py if present: - if project_has_setup_py(project_dir) and not ignore_setup_py: + if project_dir is not None and ( + project_has_setup_py(project_dir) and not ignore_setup_py + ): run_setuppy_install(ctx, project_dir, env, arch) elif not ignore_setup_py: info("No setup.py found in project directory: " + str(project_dir)) diff --git a/pythonforandroid/graph.py b/pythonforandroid/graph.py index 4f8866a805..5536c0a5b0 100644 --- a/pythonforandroid/graph.py +++ b/pythonforandroid/graph.py @@ -240,7 +240,7 @@ def obvious_conflict_checker(ctx, name_tuples, blacklist=None): return None -def get_recipe_order_and_bootstrap(ctx, names, bs=None, blacklist=None): +def get_recipe_order_and_bootstrap(ctx, names, bs=None, blacklist=None, should_log = False): # Get set of recipe/dependency names, clean up and add bootstrap deps: names = set(names) if bs is not None and bs.recipe_depends: @@ -311,7 +311,7 @@ def get_recipe_order_and_bootstrap(ctx, names, bs=None, blacklist=None): for order in orders: info(' {}'.format(order)) info('Using the first of these: {}'.format(chosen_order)) - else: + elif should_log: info('Found a single valid recipe set: {}'.format(chosen_order)) if bs is None: @@ -322,7 +322,7 @@ def get_recipe_order_and_bootstrap(ctx, names, bs=None, blacklist=None): "Could not find any compatible bootstrap!" ) recipes, python_modules, bs = get_recipe_order_and_bootstrap( - ctx, chosen_order, bs=bs, blacklist=blacklist + ctx, chosen_order, bs=bs, blacklist=blacklist, should_log=True ) else: # check if each requirement has a recipe diff --git a/pythonforandroid/pythonpackage.py b/pythonforandroid/pythonpackage.py index 0649d8848a..bfaacc6a96 100644 --- a/pythonforandroid/pythonpackage.py +++ b/pythonforandroid/pythonpackage.py @@ -556,10 +556,11 @@ def _extract_info_from_package(dependency, # Get build requirements from pyproject.toml if requested: requirements = [] - pyproject_toml_path = os.path.join(output_folder, 'pyproject.toml') - if os.path.exists(pyproject_toml_path) and include_build_requirements: + if os.path.exists(os.path.join(output_folder, + 'pyproject.toml') + ) and include_build_requirements: # Read build system from pyproject.toml file: (PEP518) - with open(pyproject_toml_path) as f: + with open(os.path.join(output_folder, 'pyproject.toml')) as f: build_sys = toml.load(f)['build-system'] if "requires" in build_sys: requirements += build_sys["requires"] diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 0cace3346e..a10555f5cd 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -1,7 +1,7 @@ from os.path import basename, dirname, exists, isdir, isfile, join, realpath, split import glob + import hashlib -import json from re import match import sh @@ -12,6 +12,8 @@ from urllib.request import urlretrieve from os import listdir, unlink, environ, curdir, walk from sys import stdout +from wheel.wheelfile import WheelFile +from wheel.cli.tags import tags as wheel_tags import time try: from urlparse import urlparse @@ -24,8 +26,9 @@ logger, info, warning, debug, shprint, info_main, error) from pythonforandroid.util import ( current_directory, ensure_dir, BuildInterruptingException, rmdir, move, - touch, patch_wheel_setuptools_logging) + touch) from pythonforandroid.util import load_source as import_recipe +from pythonforandroid.artifact import build_artifact url_opener = urllib.request.build_opener() @@ -381,6 +384,10 @@ def download_if_necessary(self): self.name, self.name)) return self.download() + + @property + def download_dir(self): + return self.name + "_" + self.version def download(self): if self.url is None: @@ -402,9 +409,9 @@ def download(self): if expected_digest: expected_digests[alg] = expected_digest - ensure_dir(join(self.ctx.packages_path, self.name)) + ensure_dir(join(self.ctx.packages_path, self.download_dir)) - with current_directory(join(self.ctx.packages_path, self.name)): + with current_directory(join(self.ctx.packages_path, self.download_dir)): filename = shprint(sh.basename, url).stdout[:-1].decode('utf-8') do_download = True @@ -478,7 +485,7 @@ def unpack(self, arch): if not exists(directory_name) or not isdir(directory_name): extraction_filename = join( - self.ctx.packages_path, self.name, filename) + self.ctx.packages_path, self.download_dir, filename) if isfile(extraction_filename): if extraction_filename.endswith(('.zip', '.whl')): try: @@ -587,6 +594,7 @@ def build_arch(self, arch): if hasattr(self, build): getattr(self, build)() + def install_libraries(self, arch): '''This method is always called after `build_arch`. In case that we detect a library recipe, defined by the class attribute @@ -595,9 +603,21 @@ def install_libraries(self, arch): ''' if not self.built_libraries: return + shared_libs = [ lib for lib in self.get_libraries(arch) if lib.endswith(".so") ] + + if self.ctx.save_prebuilt: + build_artifact( + self.ctx.prebuilt_dir, + self, + arch, + shared_libs, + [], + [{"LIBSCOPY": [basename(lib) for lib in shared_libs]},] + ) + self.install_libs(arch, *shared_libs) def postbuild_arch(self, arch): @@ -869,7 +889,8 @@ class PythonRecipe(Recipe): hostpython_prerequisites = [] '''List of hostpython packages required to build a recipe''' - + + _host_recipe = None def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if 'python3' not in self.depends: @@ -882,6 +903,11 @@ def __init__(self, *args, **kwargs): depends = list(set(depends)) self.depends = depends + def download_if_necessary(self): + # a little prep + self._host_recipe = Recipe.get_recipe("hostpython3", self.ctx) + return super().download_if_necessary() + def clean_build(self, arch=None): super().clean_build(arch=arch) name = self.folder_name @@ -899,8 +925,7 @@ def clean_build(self, arch=None): def real_hostpython_location(self): host_name = 'host{}'.format(self.ctx.python_recipe.name) if host_name == 'hostpython3': - python_recipe = Recipe.get_recipe(host_name, self.ctx) - return python_recipe.python_exe + return self._host_recipe.python_exe else: python_recipe = self.ctx.python_recipe return 'python{}'.format(python_recipe.version) @@ -920,13 +945,18 @@ def folder_name(self): return name def get_recipe_env(self, arch=None, with_flags_in_cc=True): + if self._host_recipe is None: + self._host_recipe = Recipe.get_recipe("hostpython3", self.ctx) + env = super().get_recipe_env(arch, with_flags_in_cc) - env['PYTHONNOUSERSITE'] = '1' # Set the LANG, this isn't usually important but is a better default # as it occasionally matters how Python e.g. reads files env['LANG'] = "en_GB.UTF-8" # Binaries made by packages installed by pip - env["PATH"] = join(self.hostpython_site_dir, "bin") + ":" + env["PATH"] + env["PATH"] = self._host_recipe.site_bin + ":" + env["PATH"] + + host_env = self.get_hostrecipe_env() + env['PYTHONPATH'] = host_env["PYTHONPATH"] if not self.call_hostpython_via_targetpython: env['CFLAGS'] += ' -I{}'.format( @@ -936,24 +966,11 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): self.ctx.python_recipe.link_root(arch.arch), self.ctx.python_recipe.link_version, ) - - hppath = [] - hppath.append(join(dirname(self.hostpython_location), 'Lib')) - hppath.append(join(hppath[0], 'site-packages')) - builddir = join(dirname(self.hostpython_location), 'build') - if exists(builddir): - hppath += [join(builddir, d) for d in listdir(builddir) - if isdir(join(builddir, d))] - if len(hppath) > 0: - if 'PYTHONPATH' in env: - env['PYTHONPATH'] = ':'.join(hppath + [env['PYTHONPATH']]) - else: - env['PYTHONPATH'] = ':'.join(hppath) return env def should_build(self, arch): name = self.folder_name - if self.ctx.has_package(name, arch): + if self.ctx.has_package(name, arch) or name in listdir(self._host_recipe.site_dir): info('Python package already exists in site-packages') return False info('{} apparently isn\'t already in site-packages'.format(name)) @@ -980,30 +997,31 @@ def install_python_package(self, arch, name=None, env=None, is_dir=True): hostpython = sh.Command(self.hostpython_location) hpenv = env.copy() with current_directory(self.get_build_dir(arch.arch)): - shprint(hostpython, 'setup.py', 'install', '-O2', - '--root={}'.format(self.ctx.get_python_install_dir(arch.arch)), - '--install-lib=.', - _env=hpenv, *self.setup_extra_args) + if isfile("setup.py"): + shprint(hostpython, 'setup.py', 'install', '-O2', - # If asked, also install in the hostpython build dir - if self.install_in_hostpython: - self.install_hostpython_package(arch) + '--root={}'.format(self.ctx.get_python_install_dir(arch.arch)), + '--install-lib=.', + _env=hpenv, *self.setup_extra_args) + + # If asked, also install in the hostpython build dir + if self.install_in_hostpython: + self.install_hostpython_package(arch) + else: + warning("`PythonRecipe.install_python_package` called without `setup.py` file!") - def get_hostrecipe_env(self, arch): + def get_hostrecipe_env(self): env = environ.copy() - env['PYTHONPATH'] = self.hostpython_site_dir + _python_path = self._host_recipe.get_path_to_python() + env['PYTHONPATH'] = self._host_recipe.site_dir + ":" + join( + _python_path, "Modules") + ":" + glob.glob(join(_python_path, "build", "lib*"))[0] return env - @property - def hostpython_site_dir(self): - return join(dirname(self.real_hostpython_location), 'Lib', 'site-packages') - def install_hostpython_package(self, arch): - env = self.get_hostrecipe_env(arch) + env = self.get_hostrecipe_env() real_hostpython = sh.Command(self.real_hostpython_location) shprint(real_hostpython, 'setup.py', 'install', '-O2', - '--root={}'.format(dirname(self.real_hostpython_location)), - '--install-lib=Lib/site-packages', + '--root={}'.format(self._host_recipe.site_root), _env=env, *self.setup_extra_args) @property @@ -1011,7 +1029,7 @@ def python_major_minor_version(self): parsed_version = packaging.version.parse(self.ctx.python_recipe.version) return f"{parsed_version.major}.{parsed_version.minor}" - def install_hostpython_prerequisites(self, packages=None, force_upgrade=True): + def install_hostpython_prerequisites(self, packages=None, force_upgrade=True, arch=None): if not packages: packages = self.hostpython_prerequisites @@ -1021,7 +1039,7 @@ def install_hostpython_prerequisites(self, packages=None, force_upgrade=True): pip_options = [ "install", *packages, - "--target", self.hostpython_site_dir, "--python-version", + "--target", self._host_recipe.site_dir, "--python-version", self.ctx.python_recipe.version, # Don't use sources, instead wheels "--only-binary=:all:", @@ -1029,7 +1047,7 @@ def install_hostpython_prerequisites(self, packages=None, force_upgrade=True): if force_upgrade: pip_options.append("--upgrade") # Use system's pip - shprint(sh.pip, *pip_options) + shprint(sh.Command(self.real_hostpython_location), "-m", "pip", *pip_options, _env=self.get_hostrecipe_env()) def restore_hostpython_prerequisites(self, packages): _packages = [] @@ -1068,13 +1086,12 @@ def build_compiled_components(self, arch): env['STRIP'], '{}', ';', _env=env) def install_hostpython_package(self, arch): - env = self.get_hostrecipe_env(arch) + env = self.get_hostrecipe_env() self.rebuild_compiled_components(arch, env) super().install_hostpython_package(arch) def rebuild_compiled_components(self, arch, env): info('Rebuilding compiled components in {}'.format(self.name)) - hostpython = sh.Command(self.real_hostpython_location) shprint(hostpython, 'setup.py', 'clean', '--all', _env=env) shprint(hostpython, 'setup.py', self.build_cmd, '-v', _env=env, @@ -1108,7 +1125,7 @@ def build_cython_components(self, arch): with current_directory(self.get_build_dir(arch.arch)): hostpython = sh.Command(self.ctx.hostpython) - shprint(hostpython, '-c', 'import sys; print(sys.path)', _env=env) + shprint(hostpython, '-c', 'import sys; print(sys.path, sys.exec_prefix, sys.prefix)', _env=env) debug('cwd is {}'.format(realpath(curdir))) info('Trying first build of {} to get cython files: this is ' 'expected to fail'.format(self.name)) @@ -1198,7 +1215,7 @@ def get_recipe_env(self, arch, with_flags_in_cc=True): class PyProjectRecipe(PythonRecipe): - """Recipe for projects which contain `pyproject.toml`""" + '''Recipe for projects which contain `pyproject.toml`''' # Extra args to pass to `python -m build ...` extra_build_args = [] @@ -1223,7 +1240,7 @@ def get_recipe_env(self, arch, **kwargs): return env def get_wheel_platform_tag(self, arch): - return "android_" + { + return f"android_{self.ctx.ndk_api}_" + { "armeabi-v7a": "arm", "arm64-v8a": "aarch64", "x86_64": "x86_64", @@ -1231,9 +1248,6 @@ def get_wheel_platform_tag(self, arch): }[arch.arch] def install_wheel(self, arch, built_wheels): - with patch_wheel_setuptools_logging(): - from wheel.cli.tags import tags as wheel_tags - from wheel.wheelfile import WheelFile _wheel = built_wheels[0] built_wheel_dir = dirname(_wheel) # Fix wheel platform tag @@ -1244,17 +1258,19 @@ def install_wheel(self, arch, built_wheels): ) selected_wheel = join(built_wheel_dir, wheel_tag) - _dev_wheel_dir = environ.get("P4A_WHEEL_DIR", False) - if _dev_wheel_dir: - ensure_dir(_dev_wheel_dir) - shprint(sh.cp, selected_wheel, _dev_wheel_dir) + if self.ctx.save_prebuilt: + shprint(sh.cp, selected_wheel, self.ctx.prebuilt_dir) + + def _(arch, wheel_tag, selected_wheel): + info(f"Installing built wheel: {wheel_tag}") + destination = self.ctx.get_python_install_dir(arch.arch) + with WheelFile(selected_wheel) as wf: + for zinfo in wf.filelist: + wf.extract(zinfo, destination) + wf.close() + + self.install_libraries = lambda arch, wheel_tag=wheel_tag, selected_wheel=selected_wheel : _(arch, wheel_tag, selected_wheel) - info(f"Installing built wheel: {wheel_tag}") - destination = self.ctx.get_python_install_dir(arch.arch) - with WheelFile(selected_wheel) as wf: - for zinfo in wf.filelist: - wf.extract(zinfo, destination) - wf.close() def build_arch(self, arch): self.install_hostpython_prerequisites( @@ -1282,6 +1298,7 @@ def build_arch(self, arch): sh.Command(self.ctx.python_recipe.python_exe), *build_args, _env=env ) built_wheels = [realpath(whl) for whl in glob.glob("dist/*.whl")] + self.install_wheel(arch, built_wheels) diff --git a/pythonforandroid/recipebuild.py b/pythonforandroid/recipebuild.py new file mode 100644 index 0000000000..055b013140 --- /dev/null +++ b/pythonforandroid/recipebuild.py @@ -0,0 +1,79 @@ +import sys +import os +from argparse import ArgumentParser + +from pythonforandroid.recipe import Recipe +from pythonforandroid.logger import setup_color, info_main, Colo_Fore, info +from pythonforandroid.build import Context +from pythonforandroid.graph import get_recipe_order_and_bootstrap +from pythonforandroid.util import ensure_dir +from pythonforandroid.bootstraps.empty import bootstrap +from pythonforandroid.distribution import Distribution +from pythonforandroid.androidndk import AndroidNDK + + +class RecipeBuilder: + def __init__(self, parsed_args): + setup_color(True) + self.build_dir = parsed_args.workdir + self.init_context() + self.build_recipes(set(parsed_args.recipes), set(parsed_args.arch)) + + def init_context(self): + self.ctx = Context() + self.ctx.save_prebuilt = True + self.ctx.setup_dirs(self.build_dir) + self.ctx.ndk_api = 24 + self.ctx.android_api = 24 + self.ctx.ndk_dir = "/home/tdynamos/.buildozer/android/platform/android-ndk-r25b" + + def build_recipes(self, recipes, archs): + info_main(f"# Requested recipes: {Colo_Fore.BLUE}{recipes}") + + _recipes, _non_recipes, bootstrap = get_recipe_order_and_bootstrap( + self.ctx, recipes + ) + self.ctx.prepare_bootstrap(bootstrap) + self.ctx.set_archs(archs) + self.ctx.bootstrap.distribution = Distribution.get_distribution( + self.ctx, name=bootstrap.name, recipes=recipes, archs=archs, + ) + self.ctx.ndk = AndroidNDK("/home/tdynamos/.buildozer/android/platform/android-ndk-r25b") + recipes = [Recipe.get_recipe(recipe, self.ctx) for recipe in _recipes] + + self.ctx.recipe_build_order = _recipes + for recipe in recipes: + recipe.download_if_necessary() + + for arch in self.ctx.archs: + info_main("# Building all recipes for arch {}".format(arch.arch)) + + info_main("# Unpacking recipes") + for recipe in recipes: + ensure_dir(recipe.get_build_container_dir(arch.arch)) + recipe.prepare_build_dir(arch.arch) + + info_main("# Prebuilding recipes") + # 2) prebuild packages + for recipe in recipes: + info_main("Prebuilding {} for {}".format(recipe.name, arch.arch)) + recipe.prebuild_arch(arch) + recipe.apply_patches(arch) + + info_main("# Building recipes") + for recipe in recipes: + info_main("Building {} for {}".format(recipe.name, arch.arch)) + if recipe.should_build(arch): + recipe.build_arch(arch) + else: + info("{} said it is already built, skipping".format(recipe.name)) + recipe.install_libraries(arch) + + # input() + +if __name__ == "__main__": + parser = ArgumentParser(description="Build and package recipes.") + parser.add_argument('-r', '--recipes', nargs='+', help='Recipes to build.', required=True) + parser.add_argument('-a', '--arch', nargs='+', help='Android arch(s) to build.', required=True) + parser.add_argument('-w', '--workdir', type=str, help="Workdir for building recipes.", required=True) + RecipeBuilder(parser.parse_args()) diff --git a/pythonforandroid/recipes/android/__init__.py b/pythonforandroid/recipes/android/__init__.py index 608d9ee738..0912d81d21 100644 --- a/pythonforandroid/recipes/android/__init__.py +++ b/pythonforandroid/recipes/android/__init__.py @@ -1,11 +1,11 @@ -from pythonforandroid.recipe import CythonRecipe, IncludedFilesBehaviour +from pythonforandroid.recipe import PyProjectRecipe, IncludedFilesBehaviour, Recipe from pythonforandroid.util import current_directory from pythonforandroid import logger from os.path import join -class AndroidRecipe(IncludedFilesBehaviour, CythonRecipe): +class AndroidRecipe(IncludedFilesBehaviour, PyProjectRecipe): # name = 'android' version = None url = None @@ -13,11 +13,12 @@ class AndroidRecipe(IncludedFilesBehaviour, CythonRecipe): src_filename = 'src' depends = [('sdl2', 'genericndkbuild'), 'pyjnius'] + hostpython_prerequisites = ["cython"] config_env = {} - def get_recipe_env(self, arch): - env = super().get_recipe_env(arch) + def get_recipe_env(self, arch, **kwargs): + env = super().get_recipe_env(arch, **kwargs) env.update(self.config_env) return env @@ -49,6 +50,9 @@ def prebuild_arch(self, arch): 'BOOTSTRAP': bootstrap, 'IS_SDL2': int(is_sdl2), 'PY2': 0, + 'ANDROID_LIBS_DIR':join( + Recipe.get_recipe("sdl2", self.ctx).get_build_dir(arch.arch), "../..", "libs", arch.arch + ), 'JAVA_NAMESPACE': java_ns, 'JNI_NAMESPACE': jni_ns, 'ACTIVITY_CLASS_NAME': self.ctx.activity_class_name, diff --git a/pythonforandroid/recipes/android/src/setup.py b/pythonforandroid/recipes/android/src/setup.py index bcd411f46b..c0f6b8d835 100755 --- a/pythonforandroid/recipes/android/src/setup.py +++ b/pythonforandroid/recipes/android/src/setup.py @@ -1,24 +1,38 @@ from distutils.core import setup, Extension +from Cython.Build import cythonize import os -library_dirs = ['libs/' + os.environ['ARCH']] +# Define the library directories +library_dirs = [os.environ['ANDROID_LIBS_DIR']] lib_dict = { 'sdl2': ['SDL2', 'SDL2_image', 'SDL2_mixer', 'SDL2_ttf'] } sdl_libs = lib_dict.get(os.environ['BOOTSTRAP'], ['main']) -modules = [Extension('android._android', - ['android/_android.c', 'android/_android_jni.c'], - libraries=sdl_libs + ['log'], - library_dirs=library_dirs), - Extension('android._android_billing', - ['android/_android_billing.c', 'android/_android_billing_jni.c'], - libraries=['log'], - library_dirs=library_dirs)] +# Define the extensions with Cython +modules = [ + Extension('android._android', + ['android/_android.pyx', 'android/_android_jni.c'], + libraries=sdl_libs + ['log'], + library_dirs=library_dirs), + Extension('android._android_billing', + ['android/_android_billing.pyx', 'android/_android_billing_jni.c'], + libraries=['log'], + library_dirs=library_dirs), + Extension('android._android_sound', + ['android/_android_sound.pyx', 'android/_android_sound_jni.c'], + libraries=['log'], + library_dirs=library_dirs) +] + +# Use cythonize to build the modules +cythonized_modules = cythonize(modules, compiler_directives={'language_level': "3"}) + +# Setup the package setup(name='android', version='1.0', packages=['android'], package_dir={'android': 'android'}, - ext_modules=modules + ext_modules=cythonized_modules ) diff --git a/pythonforandroid/recipes/hostpython3/__init__.py b/pythonforandroid/recipes/hostpython3/__init__.py index 9ba4580019..1439d9f69e 100644 --- a/pythonforandroid/recipes/hostpython3/__init__.py +++ b/pythonforandroid/recipes/hostpython3/__init__.py @@ -5,7 +5,8 @@ from pathlib import Path from os.path import join -from pythonforandroid.logger import shprint +from packaging.version import Version +from pythonforandroid.logger import shprint, info from pythonforandroid.recipe import Recipe from pythonforandroid.util import ( BuildInterruptingException, @@ -35,19 +36,17 @@ class HostPython3Recipe(Recipe): :class:`~pythonforandroid.python.HostPythonRecipe` ''' - version = '3.11.5' - name = 'hostpython3' + version = '3.13.0' + _p_version = Version(version) + url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz' build_subdir = 'native-build' '''Specify the sub build directory for the hostpython3 recipe. Defaults to ``native-build``.''' - url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz' '''The default url to download our host python recipe. This url will change depending on the python version set in attribute :attr:`version`.''' - patches = ['patches/pyconfig_detection.patch'] - @property def _exe_name(self): ''' @@ -94,6 +93,26 @@ def get_build_dir(self, arch=None): def get_path_to_python(self): return join(self.get_build_dir(), self.build_subdir) + + @property + def site_root(self): + return join(self.get_path_to_python(), "root") + + @property + def site_bin(self): + dir = None + # TODO: implement mac os + if os.name == "posix": + dir = "usr/local/bin/" + return join(self.site_root, dir) + + @property + def site_dir(self): + dir = None + # TODO: implement mac os + if os.name == "posix": + dir = f"usr/local/lib/python{self._p_version.major}.{self._p_version.minor}/site-packages/" + return join(self.site_root, dir) def build_arch(self, arch): env = self.get_recipe_env(arch) @@ -138,7 +157,9 @@ def build_arch(self, arch): shprint(sh.cp, exe, self.python_exe) break + ensure_dir(self.site_root) self.ctx.hostpython = self.python_exe + shprint(sh.Command(self.python_exe), "-m", "ensurepip", "--root", self.site_root, "-U") recipe = HostPython3Recipe() diff --git a/pythonforandroid/recipes/hostpython3/patches/pyconfig_detection.patch b/pythonforandroid/recipes/hostpython3/patches/pyconfig_detection.patch deleted file mode 100644 index 7f78b664e1..0000000000 --- a/pythonforandroid/recipes/hostpython3/patches/pyconfig_detection.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff -Nru Python-3.8.2/Lib/site.py Python-3.8.2-new/Lib/site.py ---- Python-3.8.2/Lib/site.py 2020-04-28 12:48:38.000000000 -0700 -+++ Python-3.8.2-new/Lib/site.py 2020-04-28 12:52:46.000000000 -0700 -@@ -487,7 +487,8 @@ - if key == 'include-system-site-packages': - system_site = value.lower() - elif key == 'home': -- sys._home = value -+ # this is breaking pyconfig.h path detection with venv -+ print('Ignoring "sys._home = value" override', file=sys.stderr) - - sys.prefix = sys.exec_prefix = site_prefix - diff --git a/pythonforandroid/recipes/libb2/__init__.py b/pythonforandroid/recipes/libb2/__init__.py new file mode 100644 index 0000000000..7715e78558 --- /dev/null +++ b/pythonforandroid/recipes/libb2/__init__.py @@ -0,0 +1,37 @@ +from pythonforandroid.recipe import Recipe +from pythonforandroid.toolchain import current_directory, shprint, warning +from os.path import join +import sh + + +class Libb2Recipe(Recipe): + version = '0.98.1' + url = 'https://github.com/BLAKE2/libb2/releases/download/v{version}/libb2-{version}.tar.gz' + built_libraries = {'libb2.so': './src/.libs/', "libomp.so": "./src/.libs"} + + def build_arch(self, arch): + # TODO: this build fails for x86_64 and x86 + # checking whether mmx is supported... /home/tdynamos/p4acache/build/other_builds/libb2/x86_64__ndk_target_24/libb2/configure: line 13165: 0xunknown: value too great for base (error token is "0xunknown") + if arch.arch in ["x86_64", "x86"]: + warning(f"libb2 build disabled for {arch.arch}") + self.built_libraries = {} + return + + with current_directory(self.get_build_dir(arch.arch)): + env = self.get_recipe_env(arch) + flags = [ + '--host=' + arch.command_prefix, + ] + configure = sh.Command('./configure') + shprint(configure, *flags, _env=env) + shprint(sh.make, _env=env) + arch_ = {"armeabi-v7a":"arm", "arm64-v8a": "aarch64", "x86_64":"x86_64", "x86":"i386"}[arch.arch] + # also requires libomp.so + shprint( + sh.cp, + join(self.ctx.ndk.llvm_prebuilt_dir, f"lib64/clang/14.0.6/lib/linux/{arch_}/libomp.so"), + "./src/.libs" + ) + + +recipe = Libb2Recipe() diff --git a/pythonforandroid/recipes/libcurl/__init__.py b/pythonforandroid/recipes/libcurl/__init__.py index 2971532fb1..1804a243af 100644 --- a/pythonforandroid/recipes/libcurl/__init__.py +++ b/pythonforandroid/recipes/libcurl/__init__.py @@ -7,11 +7,15 @@ class LibcurlRecipe(Recipe): - version = '7.55.1' - url = 'https://curl.haxx.se/download/curl-7.55.1.tar.gz' + version = '8.8.0' + url = 'https://github.com/curl/curl/releases/download/curl-{_version}/curl-{version}.tar.gz' built_libraries = {'libcurl.so': 'dist/lib'} depends = ['openssl'] + @property + def versioned_url(self): + return self.url.format(version=self.version, _version=self.version.replace(".", "_")) + def build_arch(self, arch): env = self.get_recipe_env(arch) diff --git a/pythonforandroid/recipes/libffi/__init__.py b/pythonforandroid/recipes/libffi/__init__.py index 767881b793..d63c173f4b 100644 --- a/pythonforandroid/recipes/libffi/__init__.py +++ b/pythonforandroid/recipes/libffi/__init__.py @@ -14,8 +14,8 @@ class LibffiRecipe(Recipe): - `libltdl-dev` which defines the `LT_SYS_SYMBOL_USCORE` macro """ name = 'libffi' - version = 'v3.4.2' - url = 'https://github.com/libffi/libffi/archive/{version}.tar.gz' + version = '3.4.6' + url = 'https://github.com/libffi/libffi/archive/v{version}.tar.gz' patches = ['remove-version-info.patch'] diff --git a/pythonforandroid/recipes/liblzma/__init__.py b/pythonforandroid/recipes/liblzma/__init__.py index 0b880bc484..8144112322 100644 --- a/pythonforandroid/recipes/liblzma/__init__.py +++ b/pythonforandroid/recipes/liblzma/__init__.py @@ -11,7 +11,7 @@ class LibLzmaRecipe(Recipe): - version = '5.2.4' + version = '5.6.2' url = 'https://tukaani.org/xz/xz-{version}.tar.gz' built_libraries = {'liblzma.so': 'p4a_install/lib'} diff --git a/pythonforandroid/recipes/libsodium/__init__.py b/pythonforandroid/recipes/libsodium/__init__.py index a8a1909588..f66fc18e7f 100644 --- a/pythonforandroid/recipes/libsodium/__init__.py +++ b/pythonforandroid/recipes/libsodium/__init__.py @@ -3,24 +3,15 @@ from pythonforandroid.logger import shprint from multiprocessing import cpu_count import sh -from packaging import version as packaging_version class LibsodiumRecipe(Recipe): version = '1.0.16' - url = 'https://github.com/jedisct1/libsodium/releases/download/{}/libsodium-{}.tar.gz' + url = 'https://github.com/jedisct1/libsodium/releases/download/{version}/libsodium-{version}.tar.gz' depends = [] patches = ['size_max_fix.patch'] built_libraries = {'libsodium.so': 'src/libsodium/.libs'} - @property - def versioned_url(self): - asked_version = packaging_version.parse(self.version) - if asked_version > packaging_version.parse('1.0.16'): - return self._url.format(self.version + '-RELEASE', self.version) - else: - return self._url.format(self.version, self.version) - def build_arch(self, arch): env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): diff --git a/pythonforandroid/recipes/openssl/__init__.py b/pythonforandroid/recipes/openssl/__init__.py index 766c10e361..e1c2c0a4d1 100644 --- a/pythonforandroid/recipes/openssl/__init__.py +++ b/pythonforandroid/recipes/openssl/__init__.py @@ -1,4 +1,5 @@ from os.path import join +from multiprocessing import cpu_count from pythonforandroid.recipe import Recipe from pythonforandroid.util import current_directory @@ -44,35 +45,24 @@ class OpenSSLRecipe(Recipe): ''' - version = '1.1' - '''the major minor version used to link our recipes''' - - url_version = '1.1.1w' - '''the version used to download our libraries''' - - url = 'https://www.openssl.org/source/openssl-{url_version}.tar.gz' + version = '3.3.1' + url = 'https://www.openssl.org/source/openssl-{version}.tar.gz' built_libraries = { - 'libcrypto{version}.so'.format(version=version): '.', - 'libssl{version}.so'.format(version=version): '.', + 'libcrypto.so': '.', + 'libssl.so': '.', } - @property - def versioned_url(self): - if self.url is None: - return None - return self.url.format(url_version=self.url_version) - def get_build_dir(self, arch): return join( - self.get_build_container_dir(arch), self.name + self.version + self.get_build_container_dir(arch), self.name + self.version[0] ) def include_flags(self, arch): '''Returns a string with the include folders''' openssl_includes = join(self.get_build_dir(arch.arch), 'include') return (' -I' + openssl_includes + - ' -I' + join(openssl_includes, 'internal') + + # ' -I' + join(openssl_includes, 'internal') + ' -I' + join(openssl_includes, 'openssl')) def link_dirs_flags(self, arch): @@ -85,7 +75,7 @@ def link_libs_flags(self): '''Returns a string with the appropriate `-l` flags to link with the openssl libs. This string is usually added to the environment variable `LIBS`''' - return ' -lcrypto{version} -lssl{version}'.format(version=self.version) + return ' -lcrypto -lssl' def link_flags(self, arch): '''Returns a string with the flags to link with the openssl libraries @@ -94,10 +84,12 @@ def link_flags(self, arch): def get_recipe_env(self, arch=None): env = super().get_recipe_env(arch) - env['OPENSSL_VERSION'] = self.version - env['MAKE'] = 'make' # This removes the '-j5', which isn't safe + env['OPENSSL_VERSION'] = self.version[0] env['CC'] = 'clang' - env['ANDROID_NDK_HOME'] = self.ctx.ndk_dir + env['ANDROID_NDK_ROOT'] = self.ctx.ndk_dir + env["PATH"] = f"{self.ctx.ndk.llvm_bin_dir}:{env['PATH']}" + env["CFLAGS"] += " -Wno-macro-redefined" + env["MAKE"]= "make" return env def select_build_arch(self, arch): @@ -125,13 +117,12 @@ def build_arch(self, arch): 'shared', 'no-dso', 'no-asm', + 'no-tests', buildarch, '-D__ANDROID_API__={}'.format(self.ctx.ndk_api), ] shprint(perl, 'Configure', *config_args, _env=env) - self.apply_patch('disable-sover.patch', arch.arch) - - shprint(sh.make, 'build_libs', _env=env) + shprint(sh.make, '-j', str(cpu_count()), _env=env) recipe = OpenSSLRecipe() diff --git a/pythonforandroid/recipes/pyjnius/__init__.py b/pythonforandroid/recipes/pyjnius/__init__.py index 0bcb74d392..06019aaee1 100644 --- a/pythonforandroid/recipes/pyjnius/__init__.py +++ b/pythonforandroid/recipes/pyjnius/__init__.py @@ -1,21 +1,20 @@ -from pythonforandroid.recipe import CythonRecipe +from pythonforandroid.recipe import PyProjectRecipe from pythonforandroid.toolchain import shprint, current_directory, info from pythonforandroid.patching import will_build import sh from os.path import join -class PyjniusRecipe(CythonRecipe): +class PyjniusRecipe(PyProjectRecipe): version = '1.6.1' url = 'https://github.com/kivy/pyjnius/archive/{version}.zip' name = 'pyjnius' - depends = [('genericndkbuild', 'sdl2'), 'six'] + depends = ['six'] site_packages_name = 'jnius' - patches = [('genericndkbuild_jnienv_getter.patch', will_build('genericndkbuild'))] - def get_recipe_env(self, arch): - env = super().get_recipe_env(arch) + def get_recipe_env(self, arch, **kwargs): + env = super().get_recipe_env(arch, **kwargs) # NDKPLATFORM is our switch for detecting Android platform, so can't be None env['NDKPLATFORM'] = "NOTNONE" return env diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index 2334db6add..245d1f30bd 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -5,8 +5,10 @@ from os import environ, utime from os.path import dirname, exists, join from pathlib import Path +from multiprocessing import cpu_count import shutil +from packaging.version import Version from pythonforandroid.logger import info, warning, shprint from pythonforandroid.patching import version_starts_with from pythonforandroid.recipe import Recipe, TargetPythonRecipe @@ -40,14 +42,13 @@ class Python3Recipe(TargetPythonRecipe): - _ctypes: you must add the recipe for ``libffi``. - _sqlite3: you must add the recipe for ``sqlite3``. - _ssl: you must add the recipe for ``openssl``. - - _bz2: you must add the recipe for ``libbz2`` (optional). - - _lzma: you must add the recipe for ``liblzma`` (optional). + - _bz2: you must add the recipe for ``libbz2``. + - _lzma: you must add the recipe for ``liblzma``. .. note:: This recipe can be built only against API 21+. .. versionchanged:: 2019.10.06.post0 - Refactored from deleted class ``python.GuestPythonRecipe`` into here - - Added optional dependencies: :mod:`~pythonforandroid.recipes.libbz2` and :mod:`~pythonforandroid.recipes.liblzma` .. versionchanged:: 0.6.0 @@ -55,58 +56,61 @@ class Python3Recipe(TargetPythonRecipe): :class:`~pythonforandroid.python.GuestPythonRecipe` ''' - version = '3.11.5' - url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz' + version = '3.13.0' + _p_version = Version(version) + url = 'https://github.com/python/cpython/archive/refs/tags/v{version}.tar.gz' name = 'python3' patches = [ - 'patches/pyconfig_detection.patch', 'patches/reproducible-buildinfo.diff', + # Python 3.7.x + *(['patches/py3.7.1_fix-ctypes-util-find-library.patch', + 'patches/py3.7.1_fix-zlib-version.patch', + 'patches/py3.7.1_fix_cortex_a8.patch' + ] if version.startswith("3.7") else []), + + # Python < 3.11 patches + *([ + 'patches/py3.8.1.patch', + 'patches/py3.8.1_fix_cortex_a8.patch' + ] if _p_version < Version("3.11") else []), + ] - # Python 3.7.1 - ('patches/py3.7.1_fix-ctypes-util-find-library.patch', version_starts_with("3.7")), - ('patches/py3.7.1_fix-zlib-version.patch', version_starts_with("3.7")), + if _p_version >= Version("3.11") and _p_version < Version("3.13"): + # for 3.12 and 3.11 + patches.append("patches/cpython-311-ctypes-find-library.patch") - # Python 3.8.1 & 3.9.X - ('patches/py3.8.1.patch', version_starts_with("3.8")), - ('patches/py3.8.1.patch', version_starts_with("3.9")), - ('patches/py3.8.1.patch', version_starts_with("3.10")), - ('patches/cpython-311-ctypes-find-library.patch', version_starts_with("3.11")), - ] + if version_starts_with("3.13"): + # for 3.13 + patches.append("patches/cpython-313-ctypes-find-library.patch") - if shutil.which('lld') is not None: - patches += [ - ("patches/py3.7.1_fix_cortex_a8.patch", version_starts_with("3.7")), - ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.8")), - ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.9")), - ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.10")), - ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.11")), - ] - - depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi'] - # those optional depends allow us to build python compression modules: - # - _bz2.so - # - _lzma.so - opt_depends = ['libbz2', 'liblzma'] - '''The optional libraries which we would like to get our python linked''' - - configure_args = ( + depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi', 'libbz2', 'libb2', 'liblzma', 'util-linux'] + + configure_args = [ '--host={android_host}', '--build={android_build}', '--enable-shared', '--enable-ipv6', - 'ac_cv_file__dev_ptmx=yes', - 'ac_cv_file__dev_ptc=no', + '--enable-loadable-sqlite-extensions', '--without-ensurepip', - 'ac_cv_little_endian_double=yes', - 'ac_cv_header_sys_eventfd_h=no', + '--without-static-libpython', + '--without-readline', + + # Android prefix '--prefix={prefix}', '--exec-prefix={exec_prefix}', - '--enable-loadable-sqlite-extensions' - ) + + # Special cross compile args + 'ac_cv_file__dev_ptmx=yes', + 'ac_cv_file__dev_ptc=no', + 'ac_cv_header_sys_eventfd_h=no', + 'ac_cv_little_endian_double=yes', + ] - if version_starts_with("3.11"): - configure_args += ('--with-build-python={python_host_bin}',) + if _p_version >= Version("3.11"): + configure_args.extend([ + '--with-build-python={python_host_bin}', + ]) '''The configure arguments needed to build the python recipe. Those are used in method :meth:`build_arch` (if not overwritten like python3's @@ -217,13 +221,10 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): ) env['LDFLAGS'] = env.get('LDFLAGS', '') - if shutil.which('lld') is not None: - # Note: The -L. is to fix a bug in python 3.7. - # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409 - env['LDFLAGS'] += ' -L. -fuse-ld=lld' - else: - warning('lld not found, linking without it. ' - 'Consider installing lld if linker errors occur.') + # Note: The -L. is to fix a bug in python 3.7. + # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409 + env["PATH"] = f"{self.ctx.ndk.llvm_bin_dir}:{env['PATH']}" # find lld + env['LDFLAGS'] += ' -L. -fuse-ld=lld' return env @@ -235,39 +236,44 @@ def add_flags(include_flags, link_dirs, link_libs): env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs env['LIBS'] = env.get('LIBS', '') + link_libs - - if 'sqlite3' in self.ctx.recipe_build_order: - info('Activating flags for sqlite3') - recipe = Recipe.get_recipe('sqlite3', self.ctx) - add_flags(' -I' + recipe.get_build_dir(arch.arch), - ' -L' + recipe.get_lib_dir(arch), ' -lsqlite3') - - if 'libffi' in self.ctx.recipe_build_order: - info('Activating flags for libffi') - recipe = Recipe.get_recipe('libffi', self.ctx) - # In order to force the correct linkage for our libffi library, we - # set the following variable to point where is our libffi.pc file, - # because the python build system uses pkg-config to configure it. - env['PKG_CONFIG_PATH'] = recipe.get_build_dir(arch.arch) - add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)), - ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'), - ' -lffi') - - if 'openssl' in self.ctx.recipe_build_order: - info('Activating flags for openssl') - recipe = Recipe.get_recipe('openssl', self.ctx) - self.configure_args += \ - ('--with-openssl=' + recipe.get_build_dir(arch.arch),) - add_flags(recipe.include_flags(arch), - recipe.link_dirs_flags(arch), recipe.link_libs_flags()) + + if self._p_version >= Version("3.11"): + # blake2 error fix + info('Activating flags for libb2 and libuuid') + add_flags( + ' -I' + join(Recipe.get_recipe('libb2', self.ctx).get_build_dir(arch.arch), "src") + + ' -I' + join(Recipe.get_recipe('util-linux', self.ctx).get_build_dir(arch.arch), "libuuid", "src"), + ' -L' + self.ctx.get_libs_dir(arch.arch), ' -lb2 -luuid' + ) + + info('Activating flags for sqlite3') + recipe = Recipe.get_recipe('sqlite3', self.ctx) + add_flags(' -I' + recipe.get_build_dir(arch.arch), + ' -L' + recipe.get_lib_dir(arch), ' -lsqlite3') + + info('Activating flags for libffi') + recipe = Recipe.get_recipe('libffi', self.ctx) + # In order to force the correct linkage for our libffi library, we + # set the following variable to point where is our libffi.pc file, + # because the python build system uses pkg-config to configure it. + env['PKG_CONFIG_PATH'] = recipe.get_build_dir(arch.arch) + add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)), + ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'), + ' -lffi') + + info('Activating flags for openssl') + recipe = Recipe.get_recipe('openssl', self.ctx) + self.configure_args += \ + ('--with-openssl=' + recipe.get_build_dir(arch.arch),) + add_flags(recipe.include_flags(arch), + recipe.link_dirs_flags(arch), recipe.link_libs_flags()) for library_name in {'libbz2', 'liblzma'}: - if library_name in self.ctx.recipe_build_order: - info(f'Activating flags for {library_name}') - recipe = Recipe.get_recipe(library_name, self.ctx) - add_flags(recipe.get_library_includes(arch), - recipe.get_library_ldflags(arch), - recipe.get_library_libs_flag()) + info(f'Activating flags for {library_name}') + recipe = Recipe.get_recipe(library_name, self.ctx) + add_flags(recipe.get_library_includes(arch), + recipe.get_library_ldflags(arch), + recipe.get_library_libs_flag()) # python build system contains hardcoded zlib version which prevents # the build of zlib module, here we search for android's zlib version @@ -312,8 +318,8 @@ def build_arch(self, arch): ensure_dir(build_dir) # TODO: Get these dynamically, like bpo-30386 does - sys_prefix = '/usr/local' - sys_exec_prefix = '/usr/local' + sys_prefix = "/usr/local/" + sys_exec_prefix = "/usr/local/" env = self.get_recipe_env(arch) env = self.set_libs_flags(env, arch) @@ -321,6 +327,11 @@ def build_arch(self, arch): android_build = sh.Command( join(recipe_build_dir, 'config.guess'))().strip() + + # disable blake2 for x86 and x86_64 + if arch.arch in ["x86_64", "x86"]: + warning(f"blake2 disabled for {arch.arch}") + self.configure_args.append("--with-builtin-hashlib-hashes=md5,sha1,sha2,sha3") with current_directory(build_dir): if not exists('config.status'): @@ -336,11 +347,10 @@ def build_arch(self, arch): exec_prefix=sys_exec_prefix)).split(' '), _env=env) - # Python build does not seem to play well with make -j option from Python 3.11 and onwards - # Before losing some time, please check issue - # https://github.com/python/cpython/issues/101295 , as the root cause looks similar shprint( sh.make, + '-j', + str(cpu_count()), 'all', 'INSTSONAME={lib_name}'.format(lib_name=self._libpython), _env=env @@ -374,7 +384,9 @@ def create_python_bundle(self, dirn, arch): self.get_build_dir(arch.arch), 'android-build', 'build', - 'lib.linux{}-{}-{}'.format( + 'lib.{}{}-{}-{}'.format( + # android is now supported platform + "android" if self._p_version >= Version("3.13") else "linux", '2' if self.version[0] == '2' else '', arch.command_prefix.split('-')[0], self.major_minor_version_string diff --git a/pythonforandroid/recipes/python3/patches/cpython-313-ctypes-find-library.patch b/pythonforandroid/recipes/python3/patches/cpython-313-ctypes-find-library.patch new file mode 100644 index 0000000000..25f34372be --- /dev/null +++ b/pythonforandroid/recipes/python3/patches/cpython-313-ctypes-find-library.patch @@ -0,0 +1,11 @@ +--- Python-3.13.0/Lib/ctypes/util.py 2024-10-07 10:32:14.000000000 +0530 ++++ Python-3.11.5.mod/Lib/ctypes/util.py 2024-11-01 12:15:54.130409172 +0530 +@@ -97,6 +97,8 @@ + + fname = f"{directory}/lib{name}.so" + return fname if os.path.isfile(fname) else None ++ from android._ctypes_library_finder import find_library as _find_lib ++ find_library = _find_lib + + elif os.name == "posix": + # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump diff --git a/pythonforandroid/recipes/sdl2/__init__.py b/pythonforandroid/recipes/sdl2/__init__.py index 8d5fbc2dc2..8f12cac0d4 100644 --- a/pythonforandroid/recipes/sdl2/__init__.py +++ b/pythonforandroid/recipes/sdl2/__init__.py @@ -12,7 +12,7 @@ class LibSDL2Recipe(BootstrapNDKRecipe): dir_name = 'SDL' - depends = ['sdl2_image', 'sdl2_mixer', 'sdl2_ttf'] + depends = ['sdl2_image', 'sdl2_mixer', 'sdl2_ttf', 'python3'] def get_recipe_env(self, arch=None, with_flags_in_cc=True, with_python=True): env = super().get_recipe_env( diff --git a/pythonforandroid/recipes/sdl2_image/__init__.py b/pythonforandroid/recipes/sdl2_image/__init__.py index b3ac504fbf..fbf560f96a 100644 --- a/pythonforandroid/recipes/sdl2_image/__init__.py +++ b/pythonforandroid/recipes/sdl2_image/__init__.py @@ -6,7 +6,7 @@ class LibSDL2Image(BootstrapNDKRecipe): - version = '2.8.0' + version = '2.8.2' url = 'https://github.com/libsdl-org/SDL_image/releases/download/release-{version}/SDL2_image-{version}.tar.gz' dir_name = 'SDL2_image' @@ -20,10 +20,27 @@ def get_include_dirs(self, arch): def prebuild_arch(self, arch): # We do not have a folder for each arch on BootstrapNDKRecipe, so we # need to skip the external deps download if we already have done it. - external_deps_dir = os.path.join(self.get_build_dir(arch.arch), "external") - if not os.path.exists(os.path.join(external_deps_dir, "libwebp")): - with current_directory(external_deps_dir): - shprint(sh.Command("./download.sh")) + + build_dir = self.get_build_dir(arch.arch) + + with open(os.path.join(build_dir, ".gitmodules"), "r") as file: + for section in file.read().split('[submodule "')[1:]: + line_split = section.split(" = ") + # Parse .gitmoulde section + clone_path, url, branch = ( + os.path.join(build_dir, line_split[1].split("\n")[0].strip()), + line_split[2].split("\n")[0].strip(), + line_split[-1].strip() + ) + # Clone if needed + if not os.path.exists(clone_path) or os.listdir(clone_path) == 0: + shprint( + sh.git, "clone", url, + "--depth", "1", "-b", + branch, clone_path, "--recursive" + ) + file.close() + super().prebuild_arch(arch) diff --git a/pythonforandroid/recipes/setuptools/__init__.py b/pythonforandroid/recipes/setuptools/__init__.py index 02b205023d..c73d7b1760 100644 --- a/pythonforandroid/recipes/setuptools/__init__.py +++ b/pythonforandroid/recipes/setuptools/__init__.py @@ -3,9 +3,10 @@ class SetuptoolsRecipe(PythonRecipe): version = '69.2.0' - url = 'https://pypi.python.org/packages/source/s/setuptools/setuptools-{version}.tar.gz' + url = '' call_hostpython_via_targetpython = False install_in_hostpython = True + hostpython_prerequisites = [f"setuptools=={version}"] recipe = SetuptoolsRecipe() diff --git a/pythonforandroid/recipes/six/__init__.py b/pythonforandroid/recipes/six/__init__.py deleted file mode 100644 index 3be8ce7578..0000000000 --- a/pythonforandroid/recipes/six/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from pythonforandroid.recipe import PythonRecipe - - -class SixRecipe(PythonRecipe): - version = '1.15.0' - url = 'https://pypi.python.org/packages/source/s/six/six-{version}.tar.gz' - depends = ['setuptools'] - - -recipe = SixRecipe() diff --git a/pythonforandroid/recipes/sqlite3/__init__.py b/pythonforandroid/recipes/sqlite3/__init__.py index 1f4292c1eb..1e72991dfb 100644 --- a/pythonforandroid/recipes/sqlite3/__init__.py +++ b/pythonforandroid/recipes/sqlite3/__init__.py @@ -6,12 +6,14 @@ class Sqlite3Recipe(NDKRecipe): - version = '3.35.5' + version = '3.46.0' # Don't forget to change the URL when changing the version - url = 'https://www.sqlite.org/2021/sqlite-amalgamation-3350500.zip' + url = 'https://sqlite.org/2024/sqlite-amalgamation-3460000.zip' generated_libraries = ['sqlite3'] + built_libraries = {"libsqlite3.so": "."} def should_build(self, arch): + self.built_libraries["libsqlite3.so"] = join(self.get_build_dir(arch.arch), 'libs', arch.arch) return not self.has_libs(arch, 'libsqlite3.so') def prebuild_arch(self, arch): diff --git a/pythonforandroid/recipes/util-linux/__init__.py b/pythonforandroid/recipes/util-linux/__init__.py new file mode 100644 index 0000000000..d0f98ddf96 --- /dev/null +++ b/pythonforandroid/recipes/util-linux/__init__.py @@ -0,0 +1,36 @@ +from pythonforandroid.recipe import Recipe +from pythonforandroid.toolchain import current_directory, shprint +from multiprocessing import cpu_count +import sh + + +class UTIL_LINUXRecipe(Recipe): + version = '2.40.2' + url = 'https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v2.40/util-linux-{version}.tar.xz' + depends = ["libpthread"] + built_libraries = {'libuuid.so': './.libs/'} + utils = ["uuidd"] # enable more utils as per requirements + + def get_recipe_env(self, arch, **kwargs): + env = super().get_recipe_env(arch, **kwargs) + if arch.arch in ["x86_64", "arm64_v8a"]: + env["ac_cv_func_prlimit"] = "yes" + return env + + def build_arch(self, arch): + with current_directory(self.get_build_dir(arch.arch)): + env = self.get_recipe_env(arch) + flags = [ + '--host=' + arch.command_prefix, + '--without-systemd', + ] + + if arch.arch in ["armeabi-v7a", "x86"]: + # Fun fact: Android 32 bit devices won't work in year 2038 + flags.append("--disable-year2038") + + configure = sh.Command('./configure') + shprint(configure, *flags, _env=env) + shprint(sh.make, "-j", str(cpu_count()), *self.utils, _env=env) + +recipe = UTIL_LINUXRecipe() diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index e05bb3a8fe..9821ba03b5 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -29,7 +29,7 @@ from pythonforandroid import __version__ from pythonforandroid.bootstrap import Bootstrap -from pythonforandroid.build import Context, build_recipes, project_has_setup_py +from pythonforandroid.build import Context, build_recipes from pythonforandroid.distribution import Distribution, pretty_log_dists from pythonforandroid.entrypoints import main from pythonforandroid.graph import get_recipe_order_and_bootstrap @@ -569,18 +569,18 @@ def add_parser(subparsers, *args, **kwargs): args, unknown = parser.parse_known_args(sys.argv[1:]) args.unknown_args = unknown - if getattr(args, "private", None) is not None: + if hasattr(args, "private") and args.private is not None: # Pass this value on to the internal bootstrap build.py: args.unknown_args += ["--private", args.private] - if getattr(args, "build_mode", None) == "release": + if hasattr(args, "build_mode") and args.build_mode == "release": args.unknown_args += ["--release"] - if getattr(args, "with_debug_symbols", False): + if hasattr(args, "with_debug_symbols") and args.with_debug_symbols: args.unknown_args += ["--with-debug-symbols"] - if getattr(args, "ignore_setup_py", False): + if hasattr(args, "ignore_setup_py") and args.ignore_setup_py: args.use_setup_py = False - if getattr(args, "activity_class_name", "org.kivy.android.PythonActivity") != 'org.kivy.android.PythonActivity': + if hasattr(args, "activity_class_name") and args.activity_class_name != 'org.kivy.android.PythonActivity': args.unknown_args += ["--activity-class-name", args.activity_class_name] - if getattr(args, "service_class_name", "org.kivy.android.PythonService") != 'org.kivy.android.PythonService': + if hasattr(args, "service_class_name") and args.service_class_name != 'org.kivy.android.PythonService': args.unknown_args += ["--service-class-name", args.service_class_name] self.args = args @@ -603,13 +603,21 @@ def add_parser(subparsers, *args, **kwargs): args, "with_debug_symbols", False ) + have_setup_py_or_similar = False + if getattr(args, "private", None) is not None: + project_dir = getattr(args, "private") + if (os.path.exists(os.path.join(project_dir, "setup.py")) or + os.path.exists(os.path.join(project_dir, + "pyproject.toml"))): + have_setup_py_or_similar = True + # Process requirements and put version in environ if hasattr(args, 'requirements'): requirements = [] # Add dependencies from setup.py, but only if they are recipes # (because otherwise, setup.py itself will install them later) - if (project_has_setup_py(getattr(args, "private", None)) and + if (have_setup_py_or_similar and getattr(args, "use_setup_py", False)): try: info("Analyzing package dependencies. MAY TAKE A WHILE.") @@ -690,7 +698,10 @@ def warn_on_deprecated_args(self, args): # Output warning if setup.py is present and neither --ignore-setup-py # nor --use-setup-py was specified. - if project_has_setup_py(getattr(args, "private", None)): + if getattr(args, "private", None) is not None and \ + (os.path.exists(os.path.join(args.private, "setup.py")) or + os.path.exists(os.path.join(args.private, "pyproject.toml")) + ): if not getattr(args, "use_setup_py", False) and \ not getattr(args, "ignore_setup_py", False): warning(" **** FUTURE BEHAVIOR CHANGE WARNING ****")