From 5ff00abad71d57c9bdba429c834696f0a6159e53 Mon Sep 17 00:00:00 2001 From: HyTurtle <81598434+HyTurtle@users.noreply.github.com> Date: Fri, 26 May 2023 07:28:40 +0100 Subject: [PATCH 01/30] Update `cffi` recipe for Python 3.10 (#2800) * Update __init__.py version bump to 1.15.1 * Update disable-pkg-config.patch adjust patch for 1.15.1 --- pythonforandroid/recipes/cffi/__init__.py | 2 +- .../recipes/cffi/disable-pkg-config.patch | 20 +++++++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/pythonforandroid/recipes/cffi/__init__.py b/pythonforandroid/recipes/cffi/__init__.py index a198a3db0d..f0c25a92c9 100644 --- a/pythonforandroid/recipes/cffi/__init__.py +++ b/pythonforandroid/recipes/cffi/__init__.py @@ -7,7 +7,7 @@ class CffiRecipe(CompiledComponentsPythonRecipe): Extra system dependencies: autoconf, automake and libtool. """ name = 'cffi' - version = '1.13.2' + version = '1.15.1' url = 'https://pypi.python.org/packages/source/c/cffi/cffi-{version}.tar.gz' depends = ['setuptools', 'pycparser', 'libffi'] diff --git a/pythonforandroid/recipes/cffi/disable-pkg-config.patch b/pythonforandroid/recipes/cffi/disable-pkg-config.patch index cf2abd5b86..b1a5ff9b4c 100644 --- a/pythonforandroid/recipes/cffi/disable-pkg-config.patch +++ b/pythonforandroid/recipes/cffi/disable-pkg-config.patch @@ -1,19 +1,19 @@ -diff --git a/setup.py b/setup.py -index c1db368..57311c3 100644 +diff --git a/setup.py b/setup copy.py +index 4ce0007..9be4a6d 100644 --- a/setup.py -+++ b/setup.py -@@ -5,8 +5,7 @@ import errno ++++ b/setup +@@ -9,8 +9,7 @@ if sys.platform == "win32": sources = ['c/_cffi_backend.c'] libraries = ['ffi'] -include_dirs = ['/usr/include/ffi', - '/usr/include/libffi'] # may be changed by pkg-config -+include_dirs = os.environ['FFI_INC'].split(",") if 'FFI_INC' in os.environ else [] - define_macros = [] ++include_dirs = os.environ['FFI_INC'].split(',') if 'FFI_INC' in os.environ else [] + define_macros = [('FFI_BUILDING', '1')] # for linking with libffi static library library_dirs = [] extra_compile_args = [] -@@ -67,14 +66,7 @@ def ask_supports_thread(): - sys.stderr.write("The above error message can be safely ignored\n") +@@ -105,14 +104,7 @@ def uses_msvc(): + return config.try_compile('#ifndef _MSC_VER\n#error "not MSVC"\n#endif') def use_pkg_config(): - if sys.platform == 'darwin' and os.path.exists('/usr/local/bin/brew'): @@ -25,6 +25,4 @@ index c1db368..57311c3 100644 - _ask_pkg_config(extra_link_args, '--libs-only-other') - _ask_pkg_config(libraries, '--libs-only-l', '-l') + pass - - def use_homebrew_for_libffi(): - # We can build by setting: + From 70debe5aaa1e090c18f4c366c32a63069d9ee724 Mon Sep 17 00:00:00 2001 From: Steve Kowalik Date: Wed, 31 May 2023 05:57:03 +1000 Subject: [PATCH 02/30] Use build rather than pep517 for building (#2784) pep517 has been renamed to pyproject-hooks, and as a consequence all of the deprecated functionality has been removed. build now provides the functionality required, and since we are only interested in the metadata, we can leverage a helper function for that. I've also removed all of the subprocess machinery for calling the wrapping function, since it appears to not be as noisy as pep517. --- pythonforandroid/pythonpackage.py | 89 ++++--------------------------- setup.py | 4 +- tests/test_pythonpackage.py | 2 +- tests/test_pythonpackage_basic.py | 13 ++--- 4 files changed, 18 insertions(+), 90 deletions(-) diff --git a/pythonforandroid/pythonpackage.py b/pythonforandroid/pythonpackage.py index 3b03a513f5..1275ff8501 100644 --- a/pythonforandroid/pythonpackage.py +++ b/pythonforandroid/pythonpackage.py @@ -40,7 +40,6 @@ import sys import tarfile import tempfile -import textwrap import time import zipfile from io import open # needed for python 2 @@ -48,8 +47,7 @@ from urllib.parse import urlparse import toml -from pep517.envbuild import BuildEnvironment -from pep517.wrappers import Pep517HookCaller +import build.util def transform_dep_for_pip(dependency): @@ -113,40 +111,7 @@ def extract_metainfo_files_from_package( ) package = os.path.join(temp_folder, "package") - # Because PEP517 can be noisy and contextlib.redirect_* fails to - # contain it, we will run the actual analysis in a separate process: - try: - subprocess.check_output([ - sys.executable, - "-c", - "import importlib\n" - "import json\n" - "import os\n" - "import sys\n" - "sys.path = [os.path.dirname(sys.argv[3])] + sys.path\n" - "m = importlib.import_module(\n" - " os.path.basename(sys.argv[3]).partition('.')[0]\n" - ")\n" - "m._extract_metainfo_files_from_package_unsafe(" - " sys.argv[1]," - " sys.argv[2]," - ")", - package, output_folder, os.path.abspath(__file__)], - stderr=subprocess.STDOUT, # make sure stderr is muted. - cwd=os.path.join(os.path.dirname(__file__), "..") - ) - except subprocess.CalledProcessError as e: - output = e.output.decode("utf-8", "replace") - if debug: - print("Got error obtaining meta info.") - print("Detail output:") - print(output) - print("End of Detail output.") - raise ValueError( - "failed to obtain meta info - " - "is '{}' a valid package? " - "Detailed output:\n{}".format(package, output) - ) + _extract_metainfo_files_from_package_unsafe(package, output_folder) finally: shutil.rmtree(temp_folder) @@ -461,51 +426,17 @@ def _extract_metainfo_files_from_package_unsafe( clean_up_path = True try: - build_requires = [] metadata_path = None if path_type != "wheel": - # We need to process this first to get the metadata. - - # Ensure pyproject.toml is available (pep517 expects it) - if not os.path.exists(os.path.join(path, "pyproject.toml")): - with open(os.path.join(path, "pyproject.toml"), "w") as f: - f.write(textwrap.dedent(u"""\ - [build-system] - requires = ["setuptools", "wheel"] - build-backend = "setuptools.build_meta" - """)) - - # Copy the pyproject.toml: - shutil.copyfile( - os.path.join(path, 'pyproject.toml'), - os.path.join(output_path, 'pyproject.toml') - ) - - # Get build backend and requirements from pyproject.toml: - with open(os.path.join(path, 'pyproject.toml')) as f: - build_sys = toml.load(f)['build-system'] - backend = build_sys["build-backend"] - build_requires.extend(build_sys["requires"]) - - # Get a virtualenv with build requirements and get all metadata: - env = BuildEnvironment() - metadata = None - with env: - hooks = Pep517HookCaller(path, backend) - env.pip_install( - [transform_dep_for_pip(req) for req in build_requires] - ) - reqs = hooks.get_requires_for_build_wheel({}) - env.pip_install([transform_dep_for_pip(req) for req in reqs]) - try: - metadata = hooks.prepare_metadata_for_build_wheel(path) - except Exception: # sadly, pep517 has no good error here - pass - if metadata is not None: - metadata_path = os.path.join( - path, metadata, "METADATA" - ) + # Use a build helper function to fetch the metadata directly + metadata = build.util.project_wheel_metadata(path) + # And write it to a file + metadata_path = os.path.join(output_path, "built_metadata") + with open(metadata_path, 'w') as f: + for key in metadata.keys(): + for value in metadata.get_all(key): + f.write("{}: {}\n".format(key, value)) else: # This is a wheel, so metadata should be in *.dist-info folder: metadata_path = os.path.join( diff --git a/setup.py b/setup.py index 57bddc2593..c0539b48c0 100644 --- a/setup.py +++ b/setup.py @@ -22,9 +22,9 @@ install_reqs = [ 'appdirs', 'colorama>=0.3.3', 'jinja2', 'sh>=1.10, <2.0; sys_platform!="nt"', - 'pep517', 'toml', 'packaging', + 'build', 'toml', 'packaging', ] -# (pep517 and toml are used by pythonpackage.py) +# (build and toml are used by pythonpackage.py) # By specifying every file manually, package_data will be able to diff --git a/tests/test_pythonpackage.py b/tests/test_pythonpackage.py index 2f88cf2aa4..21412e9258 100644 --- a/tests/test_pythonpackage.py +++ b/tests/test_pythonpackage.py @@ -42,7 +42,7 @@ def test_get_package_dependencies(): if "MarkupSafe" in dep ] # Check setuptools not being in non-recursive deps: - # (It will be in recursive ones due to p4a's pep517 dependency) + # (It will be in recursive ones due to p4a's build dependency) assert "setuptools" not in deps_nonrecursive # Check setuptools is present in non-recursive deps, # if we also add build requirements: diff --git a/tests/test_pythonpackage_basic.py b/tests/test_pythonpackage_basic.py index b05344b56b..e98a5f99b0 100644 --- a/tests/test_pythonpackage_basic.py +++ b/tests/test_pythonpackage_basic.py @@ -236,7 +236,7 @@ def run__get_system_python_executable(self, pybin): pybin, "-c", "import importlib\n" - "import json\n" + "import build.util\n" "import os\n" "import sys\n" "sys.path = [os.path.dirname(sys.argv[1])] + sys.path\n" @@ -273,8 +273,8 @@ def test_systemwide_python(self): # Some deps may not be installed, so we just avoid to raise # an exception here, as a missing dep should not make the test # fail. - if "pep517" in str(e.args): - # System python probably doesn't have pep517 available! + if "build" in str(e.args): + # System python probably doesn't have build available! pass elif "toml" in str(e.args): # System python probably doesn't have toml available! @@ -304,11 +304,8 @@ def test_venv(self): ]) subprocess.check_output([ os.path.join(test_dir, "venv", "bin", "pip"), - "install", "-U", "pep517" - ]) - subprocess.check_output([ - os.path.join(test_dir, "venv", "bin", "pip"), - "install", "-U", "toml" + "install", "-U", "build", "toml", "sh<2.0", "colorama", + "appdirs", "jinja2", "packaging" ]) sys_python_path = self.run__get_system_python_executable( os.path.join(test_dir, "venv", "bin", "python") From c86cc24ada24c5999610c2aae4c6a495900a3eb1 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Fri, 2 Jun 2023 07:59:01 +0200 Subject: [PATCH 03/30] Bump actions/setup-python and actions/checkout versions, as old ones are deprecated (#2827) --- .github/workflows/push.yml | 26 +++++++++++++------------- .github/workflows/pypi-release.yml | 6 +++--- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 1faaa18d70..bf38f43fbb 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -15,9 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout python-for-android - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python 3.x - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.x - name: Run flake8 @@ -36,9 +36,9 @@ jobs: os: [ubuntu-latest, macOs-latest] steps: - name: Checkout python-for-android - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Tox tests @@ -70,7 +70,7 @@ jobs: target: testapps-webview steps: - name: Checkout python-for-android - uses: actions/checkout@v2 + uses: actions/checkout@v3 # helps with GitHub runner getting out of space - name: Free disk space run: | @@ -116,7 +116,7 @@ jobs: ANDROID_NDK_HOME: ${HOME}/.android/android-ndk steps: - name: Checkout python-for-android - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install python-for-android run: | source ci/osx_ci.sh @@ -161,7 +161,7 @@ jobs: target: testapps-webview-aab steps: - name: Checkout python-for-android - uses: actions/checkout@v2 + uses: actions/checkout@v3 # helps with GitHub runner getting out of space - name: Free disk space run: | @@ -201,7 +201,7 @@ jobs: target: testapps-service_library-aar steps: - name: Checkout python-for-android - uses: actions/checkout@v2 + uses: actions/checkout@v3 # helps with GitHub runner getting out of space - name: Free disk space run: | @@ -248,7 +248,7 @@ jobs: ANDROID_NDK_HOME: ${HOME}/.android/android-ndk steps: - name: Checkout python-for-android - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install python-for-android run: | source ci/osx_ci.sh @@ -289,8 +289,8 @@ jobs: env: REBUILD_UPDATED_RECIPES_EXTRA_ARGS: --arch=${{ matrix.android_arch }} steps: - - name: Checkout python-for-android - uses: actions/checkout@v2 + - name: Checkout python-for-android (all-history) + uses: actions/checkout@v3 with: fetch-depth: 0 # helps with GitHub runner getting out of space @@ -325,8 +325,8 @@ jobs: ANDROID_NDK_HOME: ${HOME}/.android/android-ndk REBUILD_UPDATED_RECIPES_EXTRA_ARGS: --arch=${{ matrix.android_arch }} steps: - - name: Checkout python-for-android - uses: actions/checkout@v2 + - name: Checkout python-for-android (all-history) + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Install python-for-android diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index b5bb5fbb6c..a66a30567e 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -5,11 +5,11 @@ jobs: pypi_release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python 3.x - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: 3.x + python-version: '3.x' - name: Install dependencies run: | python -m pip install --upgrade setuptools wheel twine From 3ff192b85dafc13b723f5be187200412fa741bc9 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Fri, 2 Jun 2023 07:59:41 +0200 Subject: [PATCH 04/30] Removes `mysqldb` recipe as does not support Python 3 (#2828) --- pythonforandroid/recipes/mysqldb/__init__.py | 52 ------------------- .../recipes/mysqldb/disable-zip.patch | 8 --- .../mysqldb/override-mysql-config.patch | 21 -------- 3 files changed, 81 deletions(-) delete mode 100644 pythonforandroid/recipes/mysqldb/__init__.py delete mode 100644 pythonforandroid/recipes/mysqldb/disable-zip.patch delete mode 100644 pythonforandroid/recipes/mysqldb/override-mysql-config.patch diff --git a/pythonforandroid/recipes/mysqldb/__init__.py b/pythonforandroid/recipes/mysqldb/__init__.py deleted file mode 100644 index 768cb72af2..0000000000 --- a/pythonforandroid/recipes/mysqldb/__init__.py +++ /dev/null @@ -1,52 +0,0 @@ -from pythonforandroid.recipe import CompiledComponentsPythonRecipe -from os.path import join - - -class MysqldbRecipe(CompiledComponentsPythonRecipe): - name = 'mysqldb' - version = '1.2.5' - url = 'https://pypi.python.org/packages/source/M/MySQL-python/MySQL-python-{version}.zip' - site_packages_name = 'MySQLdb' - - depends = ['setuptools', 'libmysqlclient'] - - patches = ['override-mysql-config.patch', - 'disable-zip.patch'] - - # call_hostpython_via_targetpython = False - - def convert_newlines(self, filename): - print('converting newlines in {}'.format(filename)) - with open(filename, 'rb') as f: - data = f.read() - with open(filename, 'wb') as f: - f.write(data.replace(b'\r\n', b'\n').replace(b'\r', b'\n')) - - def prebuild_arch(self, arch): - super().prebuild_arch(arch) - setupbase = join(self.get_build_dir(arch.arch), 'setup') - self.convert_newlines(setupbase + '.py') - self.convert_newlines(setupbase + '_posix.py') - - def get_recipe_env(self, arch=None): - env = super().get_recipe_env(arch) - - hostpython = self.get_recipe('hostpython3', self.ctx) - # TODO: fix hardcoded path - env['PYTHONPATH'] = (join(hostpython.get_build_dir(arch.arch), - 'build', 'lib.linux-x86_64-2.7') + - ':' + env.get('PYTHONPATH', '')) - - libmysql = self.get_recipe('libmysqlclient', self.ctx) - mydir = join(libmysql.get_build_dir(arch.arch), 'libmysqlclient') - # env['CFLAGS'] += ' -I' + join(mydir, 'include') - # env['LDFLAGS'] += ' -L' + join(mydir) - libdir = self.ctx.get_libs_dir(arch.arch) - env['MYSQL_libs'] = env['MYSQL_libs_r'] = '-L' + libdir + ' -lmysql' - env['MYSQL_cflags'] = env['MYSQL_include'] = '-I' + join(mydir, - 'include') - - return env - - -recipe = MysqldbRecipe() diff --git a/pythonforandroid/recipes/mysqldb/disable-zip.patch b/pythonforandroid/recipes/mysqldb/disable-zip.patch deleted file mode 100644 index 51f804ea2c..0000000000 --- a/pythonforandroid/recipes/mysqldb/disable-zip.patch +++ /dev/null @@ -1,8 +0,0 @@ ---- mysqldb/setup.py 2014-01-02 13:52:50.000000000 -0600 -+++ b/setup.py 2016-01-13 15:48:36.781216443 -0600 -@@ -18,4 +18,5 @@ - metadata['ext_modules'] = [ - setuptools.Extension(sources=['_mysql.c'], **options)] - metadata['long_description'] = metadata['long_description'].replace(r'\n', '') -+metadata['zip_safe'] = False - setuptools.setup(**metadata) diff --git a/pythonforandroid/recipes/mysqldb/override-mysql-config.patch b/pythonforandroid/recipes/mysqldb/override-mysql-config.patch deleted file mode 100644 index 195ebdacbc..0000000000 --- a/pythonforandroid/recipes/mysqldb/override-mysql-config.patch +++ /dev/null @@ -1,21 +0,0 @@ ---- mysqldb/setup_posix.py 2014-01-02 13:52:50.000000000 -0600 -+++ b/setup_posix.py 2016-01-13 15:48:18.732883429 -0600 -@@ -13,17 +13,7 @@ - return "-%s" % f - - def mysql_config(what): -- from os import popen -- -- f = popen("%s --%s" % (mysql_config.path, what)) -- data = f.read().strip().split() -- ret = f.close() -- if ret: -- if ret/256: -- data = [] -- if ret/256 > 1: -- raise EnvironmentError("%s not found" % (mysql_config.path,)) -- return data -+ return os.environ['MYSQL_' + what.replace('-', '_')].strip().split() - mysql_config.path = "mysql_config" - - def get_config(): From d56cd398bb1ca554996103ae7ba0d082a5a8abbd Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Fri, 2 Jun 2023 08:00:38 +0200 Subject: [PATCH 05/30] Removes `Babel` recipe as it's not needed anymore. (#2826) --- pythonforandroid/recipes/babel/__init__.py | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 pythonforandroid/recipes/babel/__init__.py diff --git a/pythonforandroid/recipes/babel/__init__.py b/pythonforandroid/recipes/babel/__init__.py deleted file mode 100644 index fc17f8e4b0..0000000000 --- a/pythonforandroid/recipes/babel/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from pythonforandroid.recipe import PythonRecipe - - -class BabelRecipe(PythonRecipe): - name = 'babel' - version = '2.2.0' - url = 'https://pypi.python.org/packages/source/B/Babel/Babel-{version}.tar.gz' - - depends = ['setuptools', 'pytz'] - - call_hostpython_via_targetpython = False - install_in_hostpython = True - - -recipe = BabelRecipe() From 656179a449c56436218c374d6dee53704f3d98f8 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Fri, 2 Jun 2023 16:48:32 +0200 Subject: [PATCH 06/30] Remove dateutil recipe, as it's not needed anymore (#2829) --- pythonforandroid/recipes/dateutil/__init__.py | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 pythonforandroid/recipes/dateutil/__init__.py diff --git a/pythonforandroid/recipes/dateutil/__init__.py b/pythonforandroid/recipes/dateutil/__init__.py deleted file mode 100644 index 3367f8d145..0000000000 --- a/pythonforandroid/recipes/dateutil/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -from pythonforandroid.recipe import PythonRecipe - - -class DateutilRecipe(PythonRecipe): - name = 'dateutil' - version = '2.6.0' - url = 'https://pypi.python.org/packages/51/fc/39a3fbde6864942e8bb24c93663734b74e281b984d1b8c4f95d64b0c21f6/python-dateutil-2.6.0.tar.gz' - - depends = ["setuptools"] - call_hostpython_via_targetpython = False - install_in_hostpython = True - - -recipe = DateutilRecipe() From 37cb53e2629df4f5a0e40d837002df73fc771929 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Mon, 5 Jun 2023 18:44:18 +0200 Subject: [PATCH 07/30] Optimize CI runs, by avoiding unnecessary rebuilds (#2833) --- .github/workflows/push.yml | 208 ++++++++----------------------------- Makefile | 67 +++++------- 2 files changed, 72 insertions(+), 203 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index bf38f43fbb..58890b5efb 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -53,8 +53,8 @@ jobs: parallel: true flag-name: run-${{ matrix.os }}-${{ matrix.python-version }} - ubuntu_build_apk: - name: Unit test apk [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ] + ubuntu_build: + name: Build test APP [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ] needs: [flake8] runs-on: ${{ matrix.runs_on }} continue-on-error: true @@ -68,36 +68,41 @@ jobs: target: testapps-with-scipy - name: webview target: testapps-webview + - name: service_library + target: testapps-service_library-aar steps: - name: Checkout python-for-android uses: actions/checkout@v3 - # helps with GitHub runner getting out of space - - name: Free disk space - run: | - df -h - sudo swapoff -a - sudo rm -f /swapfile - sudo apt -y clean - docker rmi $(docker image ls -aq) - df -h - - name: Pull docker image - run: | - make docker/pull - - name: Build multi-arch apk Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86) - run: | - mkdir -p apks - make docker/run/make/with-artifact/apk/${{ matrix.bootstrap.target }} - - name: Rename apk artifact to include the build platform name - run: | - mv apks/${{ env.APK_ARTIFACT_FILENAME }} apks/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} - - name: Upload apk artifact - uses: actions/upload-artifact@v1 + - name: Build python-for-android docker image + run: | + docker build --tag=kivy/python-for-android . + - name: Build multi-arch ${{ matrix.bootstrap.target }} artifact with docker + run: | + docker run --name p4a-latest kivy/python-for-android make ${{ matrix.bootstrap.target }} + - name: Copy produced artifacts from docker container (*.apk, *.aab) + if: matrix.bootstrap.name != 'service_library' + run: | + mkdir -p dist + docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/${{ env.APK_ARTIFACT_FILENAME }} dist/ + docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/${{ env.AAB_ARTIFACT_FILENAME }} dist/ + - name: Copy produced artifacts from docker container (*.aar) + if: matrix.bootstrap.name == 'service_library' + run: | + mkdir -p dist + docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/${{ env.AAR_ARTIFACT_FILENAME }} dist/ + - name: Rename artifacts to include the build platform name (*.apk, *.aab, *.aar) + run: | + if [ -f dist/${{ env.APK_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.APK_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}; fi + if [ -f dist/${{ env.AAB_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.AAB_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }}; fi + if [ -f dist/${{ env.AAR_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.AAR_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAR_ARTIFACT_FILENAME }}; fi + - name: Upload artifacts + uses: actions/upload-artifact@v3 with: - name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} - path: apks/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} + name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-artifacts + path: dist - macos_build_apk: - name: Unit test apk [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ] + macos_build: + name: Build test APP [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ] needs: [flake8] runs-on: ${{ matrix.runs_on }} continue-on-error: true @@ -137,146 +142,21 @@ jobs: source ci/osx_ci.sh arm64_set_path_and_python_version 3.9.7 make ${{ matrix.bootstrap.target }} - - name: Rename apk artifact to include the build platform name - run: | - mv testapps/on_device_unit_tests/${{ env.APK_ARTIFACT_FILENAME }} ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} - - name: Upload apk artifact - uses: actions/upload-artifact@v1 - with: - name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} - path: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} - - ubuntu_build_aab: - name: Unit test aab [ ${{ matrix.runs_on }} ] - needs: [flake8] - runs-on: ${{ matrix.runs_on }} - continue-on-error: true - strategy: - matrix: - runs_on: [ubuntu-latest] - bootstrap: - - name: sdl2 - target: testapps-with-numpy-aab - - name: webview - target: testapps-webview-aab - steps: - - name: Checkout python-for-android - uses: actions/checkout@v3 - # helps with GitHub runner getting out of space - - name: Free disk space - run: | - df -h - sudo swapoff -a - sudo rm -f /swapfile - sudo apt -y clean - docker rmi $(docker image ls -aq) - df -h - - name: Pull docker image - run: | - make docker/pull - - name: Build Android App Bundle Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86) - run: | - mkdir -p aabs - make docker/run/make/with-artifact/aab/${{ matrix.bootstrap.target }} - - name: Rename artifact to include the build platform name - run: | - mv aabs/${{ env.AAB_ARTIFACT_FILENAME }} aabs/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAB_ARTIFACT_FILENAME }} - - name: Upload apk artifact - uses: actions/upload-artifact@v1 - with: - name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAB_ARTIFACT_FILENAME }} - path: aabs/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAB_ARTIFACT_FILENAME }} - - - ubuntu_build_aar: - name: Unit test aar [ ${{ matrix.runs_on }} ] - needs: [flake8] - runs-on: ${{ matrix.runs_on }} - continue-on-error: true - strategy: - matrix: - runs_on: [ubuntu-latest] - bootstrap: - - name: service_library - target: testapps-service_library-aar - steps: - - name: Checkout python-for-android - uses: actions/checkout@v3 - # helps with GitHub runner getting out of space - - name: Free disk space - run: | - df -h - sudo swapoff -a - sudo rm -f /swapfile - sudo apt -y clean - docker rmi $(docker image ls -aq) - df -h - - name: Pull docker image - run: | - make docker/pull - - name: Build Android AAR Python 3 (arm64-v8a) - run: | - mkdir -p aars - make docker/run/make/with-artifact/aar/${{ matrix.bootstrap.target }} - - name: Rename artifact to include the build platform name - run: | - mv aars/${{ env.AAR_ARTIFACT_FILENAME }} aars/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAR_ARTIFACT_FILENAME }} - - name: Upload aar artifact - uses: actions/upload-artifact@v1 - with: - name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAR_ARTIFACT_FILENAME }} - path: aars/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAR_ARTIFACT_FILENAME }} - - - macos_build_aab: - name: Unit test aab [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ] - needs: [flake8] - runs-on: ${{ matrix.runs_on }} - continue-on-error: true - strategy: - matrix: - runs_on: [macos-latest, apple-silicon-m1] - bootstrap: - - name: sdl2 - target: testapps-with-numpy-aab - - name: webview - target: testapps-webview-aab - env: - ANDROID_HOME: ${HOME}/.android - ANDROID_SDK_ROOT: ${HOME}/.android/android-sdk - ANDROID_SDK_HOME: ${HOME}/.android/android-sdk - ANDROID_NDK_HOME: ${HOME}/.android/android-ndk - steps: - - name: Checkout python-for-android - uses: actions/checkout@v3 - - name: Install python-for-android - run: | - source ci/osx_ci.sh - arm64_set_path_and_python_version 3.9.7 - python3 -m pip install -e . - - name: Install prerequisites via pythonforandroid/prerequisites.py (Experimental) - run: | - source ci/osx_ci.sh - arm64_set_path_and_python_version 3.9.7 - python3 pythonforandroid/prerequisites.py - - name: Install dependencies (Legacy) + - name: Copy produced artifacts into dist/ (*.apk, *.aab) run: | - source ci/osx_ci.sh - arm64_set_path_and_python_version 3.9.7 - make --file ci/makefiles/osx.mk - - name: Build multi-arch sdl2 aab Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86) - run: | - source ci/osx_ci.sh - arm64_set_path_and_python_version 3.9.7 - make ${{ matrix.bootstrap.target }} - - name: Rename sdl2 artifact to include the build platform name + mkdir -p dist + cp testapps/on_device_unit_tests/*.apk dist/ + cp testapps/on_device_unit_tests/*.aab dist/ + ls -l dist/ + - name: Rename artifacts to include the build platform name (*.apk, *.aab) run: | - mv testapps/on_device_unit_tests/${{ env.AAB_ARTIFACT_FILENAME }} ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }} - - name: Upload sdl2 apk artifact - uses: actions/upload-artifact@v1 + if [ -f dist/${{ env.APK_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.APK_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}; fi + if [ -f dist/${{ env.AAB_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.AAB_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }}; fi + - name: Upload artifacts + uses: actions/upload-artifact@v3 with: - name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }} - path: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }} + name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-artifacts + path: dist ubuntu_rebuild_updated_recipes: name: Test updated recipes for arch ${{ matrix.android_arch }} [ ubuntu-latest ] diff --git a/Makefile b/Makefile index ea5adadcd8..5b7166b277 100644 --- a/Makefile +++ b/Makefile @@ -36,47 +36,51 @@ rebuild_updated_recipes: virtualenv ANDROID_SDK_HOME=$(ANDROID_SDK_HOME) ANDROID_NDK_HOME=$(ANDROID_NDK_HOME) \ $(PYTHON) ci/rebuild_updated_recipes.py $(REBUILD_UPDATED_RECIPES_EXTRA_ARGS) -testapps-with-numpy: virtualenv +testapps-with-numpy: testapps-with-numpy/debug/apk testapps-with-numpy/release/aab + +# testapps-with-numpy/MODE/ARTIFACT +testapps-with-numpy/%: virtualenv + $(eval MODE := $(word 2, $(subst /, ,$@))) + $(eval ARTIFACT := $(word 3, $(subst /, ,$@))) + @echo Building testapps-with-numpy for $(MODE) mode and $(ARTIFACT) artifact . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ - python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + python setup.py $(ARTIFACT) --$(MODE) --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,urllib3,chardet,idna,sqlite3,setuptools,numpy \ --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 \ --permission "(name=android.permission.WRITE_EXTERNAL_STORAGE;maxSdkVersion=18)" --permission "(name=android.permission.INTERNET)" -testapps-with-scipy: virtualenv +testapps-with-scipy: testapps-with-scipy/debug/apk testapps-with-scipy/release/aab + +# testapps-with-scipy/MODE/ARTIFACT +testapps-with-scipy/%: virtualenv + $(eval MODE := $(word 2, $(subst /, ,$@))) + $(eval ARTIFACT := $(word 3, $(subst /, ,$@))) + @echo Building testapps-with-scipy for $(MODE) mode and $(ARTIFACT) artifact . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ export LEGACY_NDK=$(ANDROID_NDK_HOME_LEGACY) && \ - python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + python setup.py $(ARTIFACT) --$(MODE) --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ --requirements python3,scipy,kivy \ --arch=armeabi-v7a --arch=arm64-v8a -testapps-with-numpy-aab: virtualenv - . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ - python setup.py aab --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ - --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,urllib3,chardet,idna,sqlite3,setuptools,numpy \ - --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 --release \ - --permission "(name=android.permission.WRITE_EXTERNAL_STORAGE;maxSdkVersion=18)" --permission "(name=android.permission.INTERNET)" +testapps-webview: testapps-webview/debug/apk testapps-webview/release/aab -testapps-service_library-aar: virtualenv +# testapps-webview/MODE/ARTIFACT +testapps-webview/%: virtualenv + $(eval MODE := $(word 2, $(subst /, ,$@))) + $(eval ARTIFACT := $(word 3, $(subst /, ,$@))) + @echo Building testapps-webview for $(MODE) mode and $(ARTIFACT) artifact . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ - python setup.py aar --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ - --bootstrap service_library \ - --requirements python3 \ - --arch=arm64-v8a --arch=x86 --release - -testapps-webview: virtualenv - . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ - python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + python setup.py $(ARTIFACT) --$(MODE) --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ --bootstrap webview \ --requirements sqlite3,libffi,openssl,pyjnius,flask,python3,genericndkbuild \ --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 -testapps-webview-aab: virtualenv +testapps-service_library-aar: virtualenv . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ - python setup.py aab --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ - --bootstrap webview \ - --requirements sqlite3,libffi,openssl,pyjnius,flask,python3,genericndkbuild \ - --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 --release + python setup.py aar --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + --bootstrap service_library \ + --requirements python3 \ + --arch=arm64-v8a --arch=x86 --release testapps/%: virtualenv $(eval $@_APP_ARCH := $(shell basename $*)) @@ -106,21 +110,6 @@ docker/run/test: docker/build docker/run/command: docker/build docker run --rm --env-file=.env $(DOCKER_IMAGE) /bin/sh -c "$(COMMAND)" -docker/run/make/with-artifact/apk/%: docker/build - docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $* - docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-debug-1.1.apk ./apks - docker rm -fv p4a-latest - -docker/run/make/with-artifact/aar/%: docker/build - docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $* - docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-release-1.1.aar ./aars - docker rm -fv p4a-latest - -docker/run/make/with-artifact/aab/%: docker/build - docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $* - docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-release-1.1.aab ./aabs - docker rm -fv p4a-latest - docker/run/make/rebuild_updated_recipes: docker/build docker run --name p4a-latest -e REBUILD_UPDATED_RECIPES_EXTRA_ARGS --env-file=.env $(DOCKER_IMAGE) make rebuild_updated_recipes From da9a1942dd1ce178fec2b9cea2549c0dc1cd5c0f Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sat, 10 Jun 2023 08:59:34 +0200 Subject: [PATCH 08/30] Remove `pytz` recipe, as it's not needed anymore (#2830) --- pythonforandroid/recipes/pandas/__init__.py | 4 ++-- pythonforandroid/recipes/pytz/__init__.py | 15 --------------- 2 files changed, 2 insertions(+), 17 deletions(-) delete mode 100644 pythonforandroid/recipes/pytz/__init__.py diff --git a/pythonforandroid/recipes/pandas/__init__.py b/pythonforandroid/recipes/pandas/__init__.py index a217ab635a..a43209a339 100644 --- a/pythonforandroid/recipes/pandas/__init__.py +++ b/pythonforandroid/recipes/pandas/__init__.py @@ -7,9 +7,9 @@ class PandasRecipe(CppCompiledComponentsPythonRecipe): version = '1.0.3' url = 'https://github.com/pandas-dev/pandas/releases/download/v{version}/pandas-{version}.tar.gz' # noqa - depends = ['cython', 'numpy', 'pytz', 'libbz2', 'liblzma'] + depends = ['cython', 'numpy', 'libbz2', 'liblzma'] - python_depends = ['python-dateutil'] + python_depends = ['python-dateutil', 'pytz'] patches = ['fix_numpy_includes.patch'] call_hostpython_via_targetpython = False diff --git a/pythonforandroid/recipes/pytz/__init__.py b/pythonforandroid/recipes/pytz/__init__.py deleted file mode 100644 index ff9bc37573..0000000000 --- a/pythonforandroid/recipes/pytz/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from pythonforandroid.recipe import PythonRecipe - - -class PytzRecipe(PythonRecipe): - name = 'pytz' - version = '2019.3' - url = 'https://pypi.python.org/packages/source/p/pytz/pytz-{version}.tar.gz' - - depends = [] - - call_hostpython_via_targetpython = False - install_in_hostpython = True - - -recipe = PytzRecipe() From aae845d6e5e2aefea65b5a7297deb9644ac4086d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20Lindstr=C3=B6m?= Date: Sun, 2 Jul 2023 10:58:00 +0300 Subject: [PATCH 09/30] `freetype` recipe: Changed the url to use https as http doesn't work (#2846) --- pythonforandroid/recipes/freetype/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonforandroid/recipes/freetype/__init__.py b/pythonforandroid/recipes/freetype/__init__.py index 0b04c95da6..e5ddfe1424 100644 --- a/pythonforandroid/recipes/freetype/__init__.py +++ b/pythonforandroid/recipes/freetype/__init__.py @@ -25,7 +25,7 @@ class FreetypeRecipe(Recipe): """ version = '2.10.1' - url = 'http://download.savannah.gnu.org/releases/freetype/freetype-{version}.tar.gz' # noqa + url = 'https://download.savannah.gnu.org/releases/freetype/freetype-{version}.tar.gz' # noqa built_libraries = {'libfreetype.so': 'objs/.libs'} def get_recipe_env(self, arch=None, with_harfbuzz=False): From 4ebd3a87fda0b6dbe7255fc647967bf5a803ab3d Mon Sep 17 00:00:00 2001 From: Ansh Dadwal Date: Sun, 2 Jul 2023 15:59:29 +0530 Subject: [PATCH 10/30] Fix `vlc` recipe build (#2841) --- pythonforandroid/recipes/vlc/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pythonforandroid/recipes/vlc/__init__.py b/pythonforandroid/recipes/vlc/__init__.py index 490c4f2a18..0995576f5f 100644 --- a/pythonforandroid/recipes/vlc/__init__.py +++ b/pythonforandroid/recipes/vlc/__init__.py @@ -6,7 +6,7 @@ class VlcRecipe(Recipe): - version = '3.0.0' + version = '3.0.18' url = None name = 'vlc' @@ -52,7 +52,7 @@ def prebuild_arch(self, arch): def build_arch(self, arch): super().build_arch(arch) build_dir = self.get_build_dir(arch.arch) - port_dir = join(build_dir, 'vlc-port-android') + port_dir = join(build_dir, 'vlc-port-android', 'buildsystem') aar = self.aars[arch] if not isfile(aar): with current_directory(port_dir): @@ -67,7 +67,7 @@ def build_arch(self, arch): if not isfile(join('bin', 'VLC-debug.apk')): shprint(sh.Command('./compile.sh'), _env=env, _tail=50, _critical=True) - shprint(sh.Command('./compile-libvlc.sh'), _env=env, + shprint(sh.Command('./compile-medialibrary.sh'), _env=env, _tail=50, _critical=True) shprint(sh.cp, '-a', aar, self.ctx.aars_dir) From 403cf8ae792d5548e37764393899bf712bf5852c Mon Sep 17 00:00:00 2001 From: Julian Date: Sun, 16 Jul 2023 18:51:08 +1000 Subject: [PATCH 11/30] Correct sys_platform (#2852) On Window, sys.platform = "win32". I think "nt" is a reference to os.name. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c0539b48c0..9ced788ea8 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ # https://github.com/kivy/buildozer/issues/722 install_reqs = [ 'appdirs', 'colorama>=0.3.3', 'jinja2', - 'sh>=1.10, <2.0; sys_platform!="nt"', + 'sh>=1.10, <2.0; sys_platform!="win32"', 'build', 'toml', 'packaging', ] # (build and toml are used by pythonpackage.py) From d1f8cb6c2fc507b9d0abaece511fbec7323f8451 Mon Sep 17 00:00:00 2001 From: "Kulothungan U.G" Date: Thu, 20 Jul 2023 21:58:06 +0530 Subject: [PATCH 12/30] Fix code string - quickstart.rst --- doc/source/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 6797bcadc6..2e3fe21108 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -232,7 +232,7 @@ Exporting the Android App Bundle (aab) for distributing it on Google Play Starting from August 2021 for new apps and from November 2021 for updates to existings apps, Google Play Console will require the Android App Bundle instead of the long lived apk. -python-for-android handles by itself the needed work to accomplish the new requirements: +python-for-android handles by itself the needed work to accomplish the new requirements:: p4a aab --private $HOME/code/myapp --package=org.example.myapp --name="My App" --version 0.1 --bootstrap=sdl2 --requirements=python3,kivy --arch=arm64-v8a --arch=armeabi-v7a --release From f7f591bb6eea9e4dbd68e2004a22c4d839d400a1 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Thu, 20 Jul 2023 22:02:57 +0200 Subject: [PATCH 13/30] Bump `kivy` version to `2.2.1` (#2855) --- pythonforandroid/recipes/kivy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonforandroid/recipes/kivy/__init__.py b/pythonforandroid/recipes/kivy/__init__.py index c213879b64..ebf7b29e82 100644 --- a/pythonforandroid/recipes/kivy/__init__.py +++ b/pythonforandroid/recipes/kivy/__init__.py @@ -22,7 +22,7 @@ def is_kivy_affected_by_deadlock_issue(recipe=None, arch=None): class KivyRecipe(CythonRecipe): - version = '2.2.0' + version = '2.2.1' url = 'https://github.com/kivy/kivy/archive/{version}.zip' name = 'kivy' From 08b58fd5f09a126c697e5e7ab5e1475759fa9068 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sat, 22 Jul 2023 11:55:04 +0200 Subject: [PATCH 14/30] Use a pinned version of `Cython` for now, as most of the recipes are incompatible with `Cython==3.x.x` (#2862) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5b7166b277..9747c6c43d 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ all: virtualenv $(VIRTUAL_ENV): python3 -m venv $(VIRTUAL_ENV) - $(PIP) install Cython + $(PIP) install Cython==0.29.36 $(PIP) install -e . virtualenv: $(VIRTUAL_ENV) From f824d0eaff19cc26f358c03c8b7bab5e74505086 Mon Sep 17 00:00:00 2001 From: Julian Date: Sat, 22 Jul 2023 20:00:37 +1000 Subject: [PATCH 15/30] Automatically generate required pre-requisites (#2858) `get_required_prerequisites()` maintains a list of Prerequisites required by each platform. But that same information is already stored in each Prerequisite class. Rather than rather than maintaining two lists which might become inconsistent, auto-generate one. --- pythonforandroid/prerequisites.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/pythonforandroid/prerequisites.py b/pythonforandroid/prerequisites.py index d85eb0b76d..38b1cea49d 100644 --- a/pythonforandroid/prerequisites.py +++ b/pythonforandroid/prerequisites.py @@ -370,22 +370,19 @@ def darwin_installer(self): def get_required_prerequisites(platform="linux"): - DEFAULT_PREREQUISITES = dict( - darwin=[ - HomebrewPrerequisite(), - AutoconfPrerequisite(), - AutomakePrerequisite(), - LibtoolPrerequisite(), - PkgConfigPrerequisite(), - CmakePrerequisite(), - OpenSSLPrerequisite(), - JDKPrerequisite(), - ], - linux=[], - all_platforms=[], - ) - - return DEFAULT_PREREQUISITES["all_platforms"] + DEFAULT_PREREQUISITES[platform] + return [ + prerequisite_cls() + for prerequisite_cls in [ + HomebrewPrerequisite, + AutoconfPrerequisite, + AutomakePrerequisite, + LibtoolPrerequisite, + PkgConfigPrerequisite, + CmakePrerequisite, + OpenSSLPrerequisite, + JDKPrerequisite, + ] if prerequisite_cls.mandatory.get(platform, False) + ] def check_and_install_default_prerequisites(): From ad117beac2655d7e17a3c955e5cea82c6808a124 Mon Sep 17 00:00:00 2001 From: Julian Date: Sat, 22 Jul 2023 20:06:34 +1000 Subject: [PATCH 16/30] Use `platform.uname` instead of `os.uname` (#2857) Advantages: - Works cross platform, not just Unix. - Is a namedtuple, so can use meaningful fieldnames. Also snuck in correction to typo in Readme which doesn't warrant its own review. --- README.md | 2 +- pythonforandroid/util.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 269673a0b6..aef7e5e4ef 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ to install and begin creating APKs and AABs. (for the develop branch: `pip install git+https://github.com/kivy/python-for-android.git`) -Test that the install works with: +Test that theinstall works with: p4a --version diff --git a/pythonforandroid/util.py b/pythonforandroid/util.py index f290cdcb25..df4e79f810 100644 --- a/pythonforandroid/util.py +++ b/pythonforandroid/util.py @@ -1,6 +1,7 @@ import contextlib from os.path import exists, join -from os import getcwd, chdir, makedirs, walk, uname +from os import getcwd, chdir, makedirs, walk +from platform import uname import shutil from fnmatch import fnmatch from tempfile import mkdtemp @@ -8,7 +9,7 @@ build_platform = '{system}-{machine}'.format( - system=uname()[0], machine=uname()[-1]).lower() + system=uname().system, machine=uname().machine.lower()) """the build platform in the format `system-machine`. We use this string to define the right build system when compiling some recipes or to get the right path for clang compiler""" From 2a4504980de2b365e2c0d11f45697759c735e239 Mon Sep 17 00:00:00 2001 From: Julian Date: Sat, 22 Jul 2023 23:30:55 +1000 Subject: [PATCH 17/30] Fix simple typos in comments (#2863) One typo I introduced while trying to fix the other. --- README.md | 2 +- pythonforandroid/pythonpackage.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index aef7e5e4ef..269673a0b6 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ to install and begin creating APKs and AABs. (for the develop branch: `pip install git+https://github.com/kivy/python-for-android.git`) -Test that theinstall works with: +Test that the install works with: p4a --version diff --git a/pythonforandroid/pythonpackage.py b/pythonforandroid/pythonpackage.py index 1275ff8501..53b1226c55 100644 --- a/pythonforandroid/pythonpackage.py +++ b/pythonforandroid/pythonpackage.py @@ -4,17 +4,17 @@ Usage examples: # Getting package name from pip reference: - from pytonforandroid.pythonpackage import get_package_name + from pythonforandroid.pythonpackage import get_package_name print(get_package_name("pillow")) # Outputs: "Pillow" (note the spelling!) # Getting package dependencies: - from pytonforandroid.pythonpackage import get_package_dependencies + from pythonforandroid.pythonpackage import get_package_dependencies print(get_package_dependencies("pep517")) # Outputs: "['pytoml']" # Get package name from arbitrary package source: - from pytonforandroid.pythonpackage import get_package_name + from pythonforandroid.pythonpackage import get_package_name print(get_package_name("/some/local/project/folder/")) # Outputs package name From fd0377506af19b4aca51166724956f804eb116e1 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sat, 22 Jul 2023 19:10:09 +0200 Subject: [PATCH 18/30] `build_platform` should be all-lowercase (#2864) --- pythonforandroid/util.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pythonforandroid/util.py b/pythonforandroid/util.py index df4e79f810..713c2b46c0 100644 --- a/pythonforandroid/util.py +++ b/pythonforandroid/util.py @@ -8,8 +8,9 @@ from pythonforandroid.logger import (logger, Err_Fore, error, info) -build_platform = '{system}-{machine}'.format( - system=uname().system, machine=uname().machine.lower()) +build_platform = "{system}-{machine}".format( + system=uname().system, machine=uname().machine +).lower() """the build platform in the format `system-machine`. We use this string to define the right build system when compiling some recipes or to get the right path for clang compiler""" From 4755bcc36d7edf4e94f85a2cca3f67feb203be98 Mon Sep 17 00:00:00 2001 From: "Kulothungan U.G" Date: Sat, 22 Jul 2023 22:41:23 +0530 Subject: [PATCH 19/30] Docs: Fix typos and improved README quickstart (#2860) * Made p4a apk build command more general - readme * Fix twice spelled - readme * updated p4a build command --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 269673a0b6..764ea7b7a7 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ other API levels may not work. With everything installed, build an APK with SDL2 with e.g.: - p4a apk --requirements=kivy --private /home/username/devel/planewave_frozen/ --package=net.inclem.planewavessdl2 --name="planewavessdl2" --version=0.5 --bootstrap=sdl2 + p4a apk --private PATH_TO_YOUR_APP_CODE --package=org.example.myapp --name "My application" --version 0.1 --bootstrap=sdl2 --requirements=python3,kivy **If you need to deploy your app on Google Play, Android App Bundle (aab) is required since 1 August 2021:** @@ -96,7 +96,7 @@ Please refer to the LICENSE file. In 2015 these tools were rewritten to provide a new, easier-to-use and easier-to-extend interface. If you'd like to browse the old toolchain, its -status is recorded for posterity at at +status is recorded for posterity at https://github.com/kivy/python-for-android/tree/old_toolchain. In the last quarter of 2018 the python recipes were changed. The From 32817fbb4b7663ccee36efbb4061c4b4d508f813 Mon Sep 17 00:00:00 2001 From: Julian Date: Thu, 27 Jul 2023 02:25:33 +1000 Subject: [PATCH 20/30] Cleanup `patching.py` (#2868) Major changes to comments, param names, function ordering. Removed deprecated LooseVersion Move os.uname to platform.uname Added win32 check Windows fix --- pythonforandroid/patching.py | 187 ++++++++++++++++++++++++++--------- 1 file changed, 138 insertions(+), 49 deletions(-) diff --git a/pythonforandroid/patching.py b/pythonforandroid/patching.py index 96a3b273bb..1e143cef90 100644 --- a/pythonforandroid/patching.py +++ b/pythonforandroid/patching.py @@ -1,89 +1,178 @@ -from os import uname -from distutils.version import LooseVersion +""" + Helper functions for recipes. + Recipes must supply a list of patches. -def check_all(*callables): - def check(**kwargs): - return all(c(**kwargs) for c in callables) - return check + Patches consist of a filename and an optional conditional, which is + any function of the form: + def patch_check(arch: string, recipe : Recipe) -> bool + This library provides some helpful conditionals and mechanisms to + join multiple conditionals. -def check_any(*callables): - def check(**kwargs): - return any(c(**kwargs) for c in callables) - return check + Example: + patches = [ + ("linux_or_darwin_only.patch", + check_any(is_linux, is_darwin), + ("recent_android_API.patch", + is_apt_gte(27)), + ] +""" +from platform import uname +from packaging.version import Version + + +# Platform checks def is_platform(platform): - def is_x(**kwargs): - return uname()[0] == platform - return is_x + """ + Returns true if the host platform matches the parameter given. + """ + def check(arch, recipe): + return uname().system.lower() == platform.lower() + + return check -is_linux = is_platform('Linux') -is_darwin = is_platform('Darwin') + +is_linux = is_platform("Linux") +is_darwin = is_platform("Darwin") +is_windows = is_platform("Windows") def is_arch(xarch): - def is_x(arch, **kwargs): + """ + Returns true if the target architecture platform matches the parameter + given. + """ + + def check(arch): return arch.arch == xarch - return is_x + return check + + +# Android API comparisons: +# Return true if the Android API level being targeted +# is equal (or >, >=, <, <= as appropriate) the given parameter + + +def is_api(apiver: int): + def check(arch, recipe): + return recipe.ctx.android_api == apiver + + return check -def is_api_gt(apiver): - def is_x(recipe, **kwargs): + +def is_api_gt(apiver: int): + def check(arch, recipe): return recipe.ctx.android_api > apiver - return is_x + + return check -def is_api_gte(apiver): - def is_x(recipe, **kwargs): +def is_api_gte(apiver: int): + def check(arch, recipe): return recipe.ctx.android_api >= apiver - return is_x + + return check -def is_api_lt(apiver): - def is_x(recipe, **kwargs): +def is_api_lt(apiver: int): + def check(arch, recipe): return recipe.ctx.android_api < apiver - return is_x + + return check -def is_api_lte(apiver): - def is_x(recipe, **kwargs): +def is_api_lte(apiver: int): + def check(arch, recipe): return recipe.ctx.android_api <= apiver - return is_x - -def is_api(apiver): - def is_x(recipe, **kwargs): - return recipe.ctx.android_api == apiver - return is_x + return check -def will_build(recipe_name): - def will(recipe, **kwargs): - return recipe_name in recipe.ctx.recipe_build_order - return will +# Android API comparisons: def is_ndk(ndk): - def is_x(recipe, **kwargs): + """ + Return true if the Minimum Supported Android NDK level being targeted + is equal the given parameter (which should be an AndroidNDK instance) + """ + + def check(arch, recipe): return recipe.ctx.ndk == ndk - return is_x + + return check + + +# Recipe Version comparisons: +# These compare the Recipe's version with the provided string (or +# Packaging.Version). +# +# Warning: Both strings must conform to PEP 440 - e.g. "3.2.1" or "1.0rc1" def is_version_gt(version): - def is_x(recipe, **kwargs): - return LooseVersion(recipe.version) > version + """Return true if the Recipe's version is greater""" + + def check(arch, recipe): + return Version(recipe.version) > Version(version) + + return check def is_version_lt(version): - def is_x(recipe, **kwargs): - return LooseVersion(recipe.version) < version - return is_x + """Return true if the Recipe's version is less than""" + + def check(arch, recipe): + return Version(recipe.version) < Version(version) + + return check -def version_starts_with(version): - def is_x(recipe, **kwargs): - return recipe.version.startswith(version) - return is_x +def version_starts_with(version_prefix): + def check(arch, recipe): + return recipe.version.startswith(version_prefix) + + return check + + +# Will Build + + +def will_build(recipe_name): + """Return true if the recipe with this name is planned to be included in + the distribution.""" + + def check(arch, recipe): + return recipe_name in recipe.ctx.recipe_build_order + + return check + + +# Conjunctions + + +def check_all(*patch_checks): + """ + Given a collection of patch_checks as params, return if all returned true. + """ + + def check(arch, recipe): + return all(patch_check(arch, recipe) for patch_check in patch_checks) + + return check + + +def check_any(*patch_checks): + """ + Given a collection of patch_checks as params, return if any returned true. + """ + + def check(arch, recipe): + return any(patch_check(arch, recipe) for patch_check in patch_checks) + + return check From 82aa8fbaa15fc40f6673b12af2fe8481cebee975 Mon Sep 17 00:00:00 2001 From: Julian-o Date: Thu, 27 Jul 2023 22:09:32 +1000 Subject: [PATCH 21/30] Update Python versions in CI tests Python 3.11 was released October 2022. Python 3.7 went end-of-life June 2023 --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 58890b5efb..c6c5f55c2f 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -32,7 +32,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] + python-version: ['3.8', '3.9', '3.10', '3.11'] os: [ubuntu-latest, macOs-latest] steps: - name: Checkout python-for-android From ce443a9b5300aea0cab98b2a017c058e36946260 Mon Sep 17 00:00:00 2001 From: Julian-o Date: Thu, 27 Jul 2023 22:13:22 +1000 Subject: [PATCH 22/30] Update documentation to reflect Python 3.7 being end-of-life. --- doc/source/buildoptions.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/buildoptions.rst b/doc/source/buildoptions.rst index 4acefa1885..8ad067b205 100644 --- a/doc/source/buildoptions.rst +++ b/doc/source/buildoptions.rst @@ -8,8 +8,8 @@ This page contains instructions for using different build options. Python versions --------------- -python-for-android supports using Python 3.7 or higher. To explicitly select a Python -version in your requirements, use e.g. ``--requirements=python3==3.7.1,hostpython3==3.7.1``. +python-for-android supports using Python 3.8 or higher. To explicitly select a Python +version in your requirements, use e.g. ``--requirements=python3==3.10.11,hostpython3==3.10.11``. The last python-for-android version supporting Python2 was `v2019.10.06 `__ From 910df0a68e87365c850b436d578d111bae65123b Mon Sep 17 00:00:00 2001 From: Julian Date: Sat, 29 Jul 2023 22:49:07 +1000 Subject: [PATCH 23/30] Correct check for `--sdk` option (#2870) --- pythonforandroid/bootstraps/common/build/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonforandroid/bootstraps/common/build/build.py b/pythonforandroid/bootstraps/common/build/build.py index 3aaf51d009..0f5ab5d2a2 100644 --- a/pythonforandroid/bootstraps/common/build/build.py +++ b/pythonforandroid/bootstraps/common/build/build.py @@ -1006,7 +1006,7 @@ def _read_configuration(): print('Billing not yet supported!') sys.exit(1) - if args.sdk_version == -1: + if args.sdk_version != -1: print('WARNING: Received a --sdk argument, but this argument is ' 'deprecated and does nothing.') args.sdk_version = -1 # ensure it is not used From 7a0c9354b36d8d9e85cd28774560f87bfec6b524 Mon Sep 17 00:00:00 2001 From: Julian Date: Sun, 30 Jul 2023 17:14:47 +1000 Subject: [PATCH 24/30] Factor out dependency checking. Use modern version handling (#2866) LooseVersion used again Handle bad SDK versions --- pythonforandroid/checkdependencies.py | 70 +++++++++++++++ pythonforandroid/toolchain.py | 122 ++++++++------------------ 2 files changed, 109 insertions(+), 83 deletions(-) create mode 100644 pythonforandroid/checkdependencies.py diff --git a/pythonforandroid/checkdependencies.py b/pythonforandroid/checkdependencies.py new file mode 100644 index 0000000000..c53115de7a --- /dev/null +++ b/pythonforandroid/checkdependencies.py @@ -0,0 +1,70 @@ +from importlib import import_module +from os import environ +import sys + +from packaging.version import Version + +from pythonforandroid.prerequisites import ( + check_and_install_default_prerequisites, +) + + +def check_python_dependencies(): + """ + Check if the Python requirements are installed. This must appears + before other imports because otherwise they're imported elsewhere. + + Using the ok check instead of failing immediately so that all + errors are printed at once. + """ + + ok = True + + modules = [("colorama", "0.3.3"), "appdirs", ("sh", "1.10"), "jinja2"] + + for module in modules: + if isinstance(module, tuple): + module, version = module + else: + version = None + + try: + import_module(module) + except ImportError: + if version is None: + print( + "ERROR: The {} Python module could not be found, please " + "install it.".format(module) + ) + ok = False + else: + print( + "ERROR: The {} Python module could not be found, " + "please install version {} or higher".format( + module, version + ) + ) + ok = False + else: + if version is None: + continue + try: + cur_ver = sys.modules[module].__version__ + except AttributeError: # this is sometimes not available + continue + if Version(cur_ver) < Version(version): + print( + "ERROR: {} version is {}, but python-for-android needs " + "at least {}.".format(module, cur_ver, version) + ) + ok = False + + if not ok: + print("python-for-android is exiting due to the errors logged above") + exit(1) + + +def check(): + if not environ.get("SKIP_PREREQUISITES_CHECK", "0") == "1": + check_and_install_default_prerequisites() + check_python_dependencies() diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index 85404a2359..38c700740c 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -6,96 +6,43 @@ This module defines the entry point for command line and programmatic use. """ +from appdirs import user_data_dir +import argparse +from functools import wraps +import glob +import logging +import os from os import environ -from pythonforandroid import __version__ -from pythonforandroid.pythonpackage import get_dep_names_of_package -from pythonforandroid.recommendations import ( - RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API, print_recommendations) -from pythonforandroid.util import BuildInterruptingException, load_source -from pythonforandroid.entrypoints import main -from pythonforandroid.prerequisites import check_and_install_default_prerequisites - - -def check_python_dependencies(): - # Check if the Python requirements are installed. This appears - # before the imports because otherwise they're imported elsewhere. - - # Using the ok check instead of failing immediately so that all - # errors are printed at once - - from distutils.version import LooseVersion - from importlib import import_module - import sys - - ok = True - - modules = [('colorama', '0.3.3'), 'appdirs', ('sh', '1.10'), 'jinja2'] - - for module in modules: - if isinstance(module, tuple): - module, version = module - else: - version = None - - try: - import_module(module) - except ImportError: - if version is None: - print('ERROR: The {} Python module could not be found, please ' - 'install it.'.format(module)) - ok = False - else: - print('ERROR: The {} Python module could not be found, ' - 'please install version {} or higher'.format( - module, version)) - ok = False - else: - if version is None: - continue - try: - cur_ver = sys.modules[module].__version__ - except AttributeError: # this is sometimes not available - continue - if LooseVersion(cur_ver) < LooseVersion(version): - print('ERROR: {} version is {}, but python-for-android needs ' - 'at least {}.'.format(module, cur_ver, version)) - ok = False - - if not ok: - print('python-for-android is exiting due to the errors logged above') - exit(1) - - -if not environ.get('SKIP_PREREQUISITES_CHECK', '0') == '1': - check_and_install_default_prerequisites() -check_python_dependencies() - - -import sys -from sys import platform from os.path import (join, dirname, realpath, exists, expanduser, basename) -import os -import glob -import shutil import re import shlex -from functools import wraps +import shutil +import sys +from sys import platform -import argparse +# This must be imported and run before other third-party or p4a +# packages. +from pythonforandroid.checkdependencies import check +check() + +from packaging.version import Version, InvalidVersion import sh -from appdirs import user_data_dir -import logging -from distutils.version import LooseVersion -from pythonforandroid.recipe import Recipe -from pythonforandroid.logger import (logger, info, warning, setup_color, - Out_Style, Out_Fore, - info_notify, info_main, shprint) -from pythonforandroid.util import current_directory +from pythonforandroid import __version__ from pythonforandroid.bootstrap import Bootstrap +from pythonforandroid.build import Context, build_recipes from pythonforandroid.distribution import Distribution, pretty_log_dists +from pythonforandroid.entrypoints import main from pythonforandroid.graph import get_recipe_order_and_bootstrap -from pythonforandroid.build import Context, build_recipes +from pythonforandroid.logger import (logger, info, warning, setup_color, + Out_Style, Out_Fore, + info_notify, info_main, shprint) +from pythonforandroid.pythonpackage import get_dep_names_of_package +from pythonforandroid.recipe import Recipe +from pythonforandroid.recommendations import ( + RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API, print_recommendations) +from pythonforandroid.util import ( + current_directory, BuildInterruptingException, load_source) user_dir = dirname(realpath(os.path.curdir)) toolchain_dir = dirname(__file__) @@ -1068,13 +1015,22 @@ def _build_package(self, args, package_type): self.hook("before_apk_assemble") build_tools_versions = os.listdir(join(ctx.sdk_dir, 'build-tools')) - build_tools_versions = sorted(build_tools_versions, - key=LooseVersion) + + def sort_key(version_text): + try: + # Historically, Android build release candidates have had + # spaces in the version number. + return Version(version_text.replace(" ", "")) + except InvalidVersion: + # Put badly named versions at worst position. + return Version("0") + + build_tools_versions.sort(key=sort_key) build_tools_version = build_tools_versions[-1] info(('Detected highest available build tools ' 'version to be {}').format(build_tools_version)) - if build_tools_version < '25.0': + if Version(build_tools_version.replace(" ", "")) < Version('25.0'): raise BuildInterruptingException( 'build_tools >= 25 is required, but %s is installed' % build_tools_version) if not exists("gradlew"): From be650bcfa82dcc7cd3007401ddf5844c9937078a Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Sun, 6 Aug 2023 08:09:02 +0200 Subject: [PATCH 25/30] Linter fixes (#2874) The errors were: ``` pythonforandroid/bootstrap.py:136:5: F811 redefinition of unused 'name' from line 73 pythonforandroid/build.py:111:5: F811 redefinition of unused 'libs_dir' from line 82 pythonforandroid/build.py:127:5: F811 redefinition of unused 'aars_dir' from line 83 pythonforandroid/graph.py:48:12: E721 do not compare types, for exact checks use `is` pythonforandroid/graph.py:163:20: E721 do not compare types, for exact checks use `is` tests/test_build.py:39:41: E231 missing whitespace after ',' tests/test_build.py:40:58: E231 missing whitespace after ',' tests/test_build.py:41:61: E231 missing whitespace after ',' tests/test_build.py:42:71: E231 missing whitespace after ',' ``` --- pythonforandroid/bootstrap.py | 1 - pythonforandroid/build.py | 9 ++++----- pythonforandroid/graph.py | 4 ++-- tests/test_build.py | 8 ++++---- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/pythonforandroid/bootstrap.py b/pythonforandroid/bootstrap.py index 0a5225e526..0712f9081a 100755 --- a/pythonforandroid/bootstrap.py +++ b/pythonforandroid/bootstrap.py @@ -70,7 +70,6 @@ class Bootstrap: '''An Android project template, containing recipe stuff for compilation and templated fields for APK info. ''' - name = '' jni_subdir = '/jni' ctx = None diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index 645b368d00..82856608a4 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -77,11 +77,6 @@ class Context: # the Android project folder where everything ends up dist_dir = None - # where Android libs are cached after build - # but before being placed in dists - libs_dir = None - aars_dir = None - # Whether setup.py or similar should be used if present: use_setup_py = False @@ -109,6 +104,10 @@ def templates_dir(self): @property def libs_dir(self): + """ + where Android libs are cached after build + but before being placed in dists + """ # Was previously hardcoded as self.build_dir/libs directory = join(self.build_dir, 'libs_collections', self.bootstrap.distribution.name) diff --git a/pythonforandroid/graph.py b/pythonforandroid/graph.py index bdaca4349c..4edb8f4c90 100644 --- a/pythonforandroid/graph.py +++ b/pythonforandroid/graph.py @@ -45,7 +45,7 @@ def get_dependency_tuple_list_for_recipe(recipe, blacklist=None): """ if blacklist is None: blacklist = set() - assert type(blacklist) == set + assert type(blacklist) is set if recipe.depends is None: dependencies = [] else: @@ -160,7 +160,7 @@ def obvious_conflict_checker(ctx, name_tuples, blacklist=None): current_to_be_added = list(to_be_added) to_be_added = [] for (added_tuple, adding_recipe) in current_to_be_added: - assert type(added_tuple) == tuple + assert type(added_tuple) is tuple if len(added_tuple) > 1: # No obvious commitment in what to add, don't check it itself # but throw it into deps for later comparing against diff --git a/tests/test_build.py b/tests/test_build.py index f386b8410f..cf9fa7801d 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -36,10 +36,10 @@ def test_strip_if_with_debug_symbols(self): modules = ["mymodule"] project_dir = None with mock.patch('pythonforandroid.build.info'), \ - mock.patch('sh.Command'),\ - mock.patch('pythonforandroid.build.open'),\ - mock.patch('pythonforandroid.build.shprint'),\ - mock.patch('pythonforandroid.build.current_directory'),\ + mock.patch('sh.Command'), \ + mock.patch('pythonforandroid.build.open'), \ + mock.patch('pythonforandroid.build.shprint'), \ + mock.patch('pythonforandroid.build.current_directory'), \ mock.patch('pythonforandroid.build.CythonRecipe') as m_CythonRecipe, \ mock.patch('pythonforandroid.build.project_has_setup_py') as m_project_has_setup_py, \ mock.patch('pythonforandroid.build.run_setuppy_install'): From fa3e5bfe39286e4a293882a4db0728d9bf9f4968 Mon Sep 17 00:00:00 2001 From: Julian Date: Tue, 22 Aug 2023 23:29:04 +1000 Subject: [PATCH 26/30] Remove deprecated FlatDir in Gradle template (#2876) Based on lessons from https://stackoverflow.com/questions/68215302/using-flatdirs-should-be-avoided-because-it-doesnt-support-any-meta-data-format --- .../bootstraps/common/build/templates/build.tmpl.gradle | 3 --- 1 file changed, 3 deletions(-) diff --git a/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle b/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle index bb000393a4..ce105736d3 100644 --- a/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle +++ b/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle @@ -16,9 +16,6 @@ allprojects { {%- for repo in args.gradle_repositories %} {{repo}} {%- endfor %} - flatDir { - dirs 'libs' - } } } From c6f76b9f43c24e7a270a49a15c9ae1e808a31b95 Mon Sep 17 00:00:00 2001 From: Julian Date: Fri, 25 Aug 2023 23:05:17 +1000 Subject: [PATCH 27/30] Standardise `ensure_dir` and `rmdir` (#2871) * Standardise ensure_dir and rmdir * Standardise ensure_dir and rmdir * Add libmysqlclient to broken list * Libtorrent failing to be rebuilt * Add boost to broken recipes list --- ci/constants.py | 7 +++++ pythonforandroid/bdistapk.py | 18 ++++++------ pythonforandroid/bootstrap.py | 5 ++-- .../bootstraps/common/build/build.py | 17 +++++------ pythonforandroid/bootstraps/sdl2/__init__.py | 10 ++++--- .../bootstraps/service_only/__init__.py | 4 +-- .../bootstraps/webview/__init__.py | 8 ++++-- pythonforandroid/build.py | 27 +++++++++--------- pythonforandroid/distribution.py | 13 +++++---- pythonforandroid/prerequisites.py | 16 ++++------- pythonforandroid/pythonpackage.py | 18 ++++++------ pythonforandroid/recipe.py | 25 +++++++---------- pythonforandroid/recipes/ifaddrs/__init__.py | 15 +++++----- pythonforandroid/recipes/lapack/__init__.py | 5 ++-- pythonforandroid/recipes/libglob/__init__.py | 17 ++++++----- .../recipes/libmysqlclient/__init__.py | 2 +- .../recipes/libtorrent/__init__.py | 7 +++-- pythonforandroid/recipes/opencv/__init__.py | 10 ++++--- pythonforandroid/recipes/sqlite3/__init__.py | 9 +++--- pythonforandroid/recipes/twisted/__init__.py | 4 +-- .../recipes/zope_interface/__init__.py | 12 ++++---- pythonforandroid/toolchain.py | 24 ++++++---------- pythonforandroid/util.py | 28 +++++++++++++------ tests/recipes/test_libmysqlclient.py | 2 +- tests/recipes/test_openal.py | 2 +- tests/recipes/test_openssl.py | 2 +- tests/test_bootstrap.py | 15 ++++++---- tests/test_distribution.py | 6 ++-- tests/test_util.py | 10 +++---- 29 files changed, 176 insertions(+), 162 deletions(-) diff --git a/ci/constants.py b/ci/constants.py index 021e64a74a..6a81a23405 100644 --- a/ci/constants.py +++ b/ci/constants.py @@ -33,6 +33,13 @@ class TargetPython(Enum): 'twisted', # genericndkbuild is incompatible with sdl2 (which is build by default when targeting sdl2 bootstrap) 'genericndkbuild', + # libmysqlclient gives a linker failure (See issue #2808) + 'libmysqlclient', + # boost gives errors (requires numpy? syntax error in .jam?) + 'boost', + # libtorrent gives errors (requires boost. Also, see issue #2809, to start with) + 'libtorrent', + ]) BROKEN_RECIPES = { diff --git a/pythonforandroid/bdistapk.py b/pythonforandroid/bdistapk.py index 575e0e17e5..b4467b9f91 100644 --- a/pythonforandroid/bdistapk.py +++ b/pythonforandroid/bdistapk.py @@ -1,10 +1,10 @@ +from glob import glob +from os.path import realpath, join, dirname, curdir, basename, split from setuptools import Command - +from shutil import copyfile import sys -from os.path import realpath, join, exists, dirname, curdir, basename, split -from os import makedirs -from glob import glob -from shutil import rmtree, copyfile + +from pythonforandroid.util import rmdir, ensure_dir def argv_contains(t): @@ -90,9 +90,8 @@ def prepare_build_dir(self): 'that.') bdist_dir = 'build/bdist.android-{}'.format(self.arch) - if exists(bdist_dir): - rmtree(bdist_dir) - makedirs(bdist_dir) + rmdir(bdist_dir) + ensure_dir(bdist_dir) globs = [] for directory, patterns in self.distribution.package_data.items(): @@ -107,8 +106,7 @@ def prepare_build_dir(self): if not argv_contains('--launcher'): for filen in filens: new_dir = join(bdist_dir, dirname(filen)) - if not exists(new_dir): - makedirs(new_dir) + ensure_dir(new_dir) print('Including {}'.format(filen)) copyfile(filen, join(bdist_dir, filen)) if basename(filen) in ('main.py', 'main.pyc'): diff --git a/pythonforandroid/bootstrap.py b/pythonforandroid/bootstrap.py index 0712f9081a..965ee4e7bf 100755 --- a/pythonforandroid/bootstrap.py +++ b/pythonforandroid/bootstrap.py @@ -10,7 +10,8 @@ from pythonforandroid.logger import (shprint, info, logger, debug) from pythonforandroid.util import ( - current_directory, ensure_dir, temp_directory, BuildInterruptingException) + current_directory, ensure_dir, temp_directory, BuildInterruptingException, + rmdir) from pythonforandroid.recipe import Recipe @@ -396,7 +397,7 @@ def fry_eggs(self, sitepackages): files = [join(rd, f) for f in listdir(rd) if f != 'EGG-INFO'] if files: shprint(sh.mv, '-t', sitepackages, *files) - shprint(sh.rm, '-rf', d) + rmdir(d) def expand_dependencies(recipes, ctx): diff --git a/pythonforandroid/bootstraps/common/build/build.py b/pythonforandroid/bootstraps/common/build/build.py index 0f5ab5d2a2..0b6b9832f0 100644 --- a/pythonforandroid/bootstraps/common/build/build.py +++ b/pythonforandroid/bootstraps/common/build/build.py @@ -21,6 +21,8 @@ from fnmatch import fnmatch import jinja2 +from pythonforandroid.util import rmdir, ensure_dir + def get_dist_info_for(key, error_if_missing=True): try: @@ -93,11 +95,6 @@ def get_bootstrap_name(): DEFAULT_PYTHON_SERVICE_JAVA_CLASS = 'org.kivy.android.PythonService' -def ensure_dir(path): - if not exists(path): - makedirs(path) - - def render(template, dest, **kwargs): '''Using jinja2, render `template` to the filename `dest`, supplying the @@ -241,7 +238,7 @@ def make_package(args): assets_dir = "src/main/assets" # Delete the old assets. - shutil.rmtree(assets_dir, ignore_errors=True) + rmdir(assets_dir, ignore_errors=True) ensure_dir(assets_dir) # Add extra environment variable file into tar-able directory: @@ -290,7 +287,7 @@ def make_package(args): not exists( join(main_py_only_dir, dir_path) )): - os.mkdir(join(main_py_only_dir, dir_path)) + ensure_dir(join(main_py_only_dir, dir_path)) # Copy actual file: shutil.copyfile( join(args.private, variant), @@ -328,17 +325,17 @@ def make_package(args): ) finally: for directory in _temp_dirs_to_clean: - shutil.rmtree(directory) + rmdir(directory) # Remove extra env vars tar-able directory: - shutil.rmtree(env_vars_tarpath) + rmdir(env_vars_tarpath) # Prepare some variables for templating process res_dir = "src/main/res" res_dir_initial = "src/res_initial" # make res_dir stateless if exists(res_dir_initial): - shutil.rmtree(res_dir, ignore_errors=True) + rmdir(res_dir, ignore_errors=True) shutil.copytree(res_dir_initial, res_dir) else: shutil.copytree(res_dir, res_dir_initial) diff --git a/pythonforandroid/bootstraps/sdl2/__init__.py b/pythonforandroid/bootstraps/sdl2/__init__.py index 662d43c0ef..9334724a33 100644 --- a/pythonforandroid/bootstraps/sdl2/__init__.py +++ b/pythonforandroid/bootstraps/sdl2/__init__.py @@ -1,9 +1,11 @@ -from pythonforandroid.toolchain import ( - Bootstrap, shprint, current_directory, info, info_main) -from pythonforandroid.util import ensure_dir from os.path import join + import sh +from pythonforandroid.toolchain import ( + Bootstrap, shprint, current_directory, info, info_main) +from pythonforandroid.util import ensure_dir, rmdir + class SDL2GradleBootstrap(Bootstrap): name = 'sdl2' @@ -15,8 +17,8 @@ class SDL2GradleBootstrap(Bootstrap): def assemble_distribution(self): info_main("# Creating Android project ({})".format(self.name)) + rmdir(self.dist_dir) info("Copying SDL2/gradle build") - shprint(sh.rm, "-rf", self.dist_dir) shprint(sh.cp, "-r", self.build_dir, self.dist_dir) # either the build use environment variable (ANDROID_HOME) diff --git a/pythonforandroid/bootstraps/service_only/__init__.py b/pythonforandroid/bootstraps/service_only/__init__.py index b9e000c012..4f0d6cf20b 100644 --- a/pythonforandroid/bootstraps/service_only/__init__.py +++ b/pythonforandroid/bootstraps/service_only/__init__.py @@ -2,7 +2,7 @@ from os.path import join from pythonforandroid.toolchain import ( Bootstrap, current_directory, info, info_main, shprint) -from pythonforandroid.util import ensure_dir +from pythonforandroid.util import ensure_dir, rmdir class ServiceOnlyBootstrap(Bootstrap): @@ -18,7 +18,7 @@ def assemble_distribution(self): self.name)) info('This currently just copies the build stuff straight from the build dir.') - shprint(sh.rm, '-rf', self.dist_dir) + rmdir(self.dist_dir) shprint(sh.cp, '-r', self.build_dir, self.dist_dir) with current_directory(self.dist_dir): with open('local.properties', 'w') as fileh: diff --git a/pythonforandroid/bootstraps/webview/__init__.py b/pythonforandroid/bootstraps/webview/__init__.py index da33ac72d5..7604ed3b84 100644 --- a/pythonforandroid/bootstraps/webview/__init__.py +++ b/pythonforandroid/bootstraps/webview/__init__.py @@ -1,8 +1,10 @@ -from pythonforandroid.toolchain import Bootstrap, current_directory, info, info_main, shprint -from pythonforandroid.util import ensure_dir from os.path import join + import sh +from pythonforandroid.toolchain import Bootstrap, current_directory, info, info_main, shprint +from pythonforandroid.util import ensure_dir, rmdir + class WebViewBootstrap(Bootstrap): name = 'webview' @@ -15,7 +17,7 @@ def assemble_distribution(self): info_main('# Creating Android project from build and {} bootstrap'.format( self.name)) - shprint(sh.rm, '-rf', self.dist_dir) + rmdir(self.dist_dir) shprint(sh.cp, '-r', self.build_dir, self.dist_dir) with current_directory(self.dist_dir): with open('local.properties', 'w') as fileh: diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index 82856608a4..4777e2f934 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -1,28 +1,29 @@ +from contextlib import suppress +import copy +import glob +import os +from os import environ from os.path import ( abspath, join, realpath, dirname, expanduser, exists ) -from os import environ -import copy -import os -import glob import re -import sh import shutil import subprocess -from contextlib import suppress -from pythonforandroid.util import ( - current_directory, ensure_dir, - BuildInterruptingException, -) -from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint) +import sh + +from pythonforandroid.androidndk import AndroidNDK from pythonforandroid.archs import ArchARM, ArchARMv7_a, ArchAarch_64, Archx86, Archx86_64 +from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint) from pythonforandroid.pythonpackage import get_package_name from pythonforandroid.recipe import CythonRecipe, Recipe from pythonforandroid.recommendations import ( check_ndk_version, check_target_api, check_ndk_api, RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API) -from pythonforandroid.androidndk import AndroidNDK +from pythonforandroid.util import ( + current_directory, ensure_dir, + BuildInterruptingException, rmdir +) def get_targets(sdk_dir): @@ -641,7 +642,7 @@ def run_setuppy_install(ctx, project_dir, env=None, arch=None): for f in set(copied_over_contents + new_venv_additions): full_path = os.path.join(venv_site_packages_dir, f) if os.path.isdir(full_path): - shutil.rmtree(full_path) + rmdir(full_path) else: os.remove(full_path) finally: diff --git a/pythonforandroid/distribution.py b/pythonforandroid/distribution.py index 2b1f1a3f5e..c878e0ea87 100644 --- a/pythonforandroid/distribution.py +++ b/pythonforandroid/distribution.py @@ -1,10 +1,11 @@ -from os.path import exists, join -import glob import json +import glob +from os.path import exists, join -from pythonforandroid.logger import (debug, info, info_notify, warning, Err_Style, Err_Fore) -from pythonforandroid.util import current_directory, BuildInterruptingException -from shutil import rmtree +from pythonforandroid.logger import ( + debug, info, info_notify, warning, Err_Style, Err_Fore) +from pythonforandroid.util import ( + current_directory, BuildInterruptingException, rmdir) class Distribution: @@ -201,7 +202,7 @@ def folder_exists(self): return exists(self.dist_dir) def delete(self): - rmtree(self.dist_dir) + rmdir(self.dist_dir) @classmethod def get_distributions(cls, ctx, extra_dist_dirs=[]): diff --git a/pythonforandroid/prerequisites.py b/pythonforandroid/prerequisites.py index 38b1cea49d..d5e013f11d 100644 --- a/pythonforandroid/prerequisites.py +++ b/pythonforandroid/prerequisites.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import sys -import platform import os -import subprocess +import platform import shutil +import subprocess +import sys + from pythonforandroid.logger import info, warning, error +from pythonforandroid.util import ensure_dir class Prerequisite(object): @@ -247,13 +249,7 @@ def darwin_installer(self): "~/Library/Java/JavaVirtualMachines" ) info(f"Extracting {filename} to {user_library_java_path}") - subprocess.check_output( - [ - "mkdir", - "-p", - user_library_java_path, - ], - ) + ensure_dir(user_library_java_path) subprocess.check_output( ["tar", "xzf", f"/tmp/{filename}", "-C", user_library_java_path], ) diff --git a/pythonforandroid/pythonpackage.py b/pythonforandroid/pythonpackage.py index 53b1226c55..9e4c29bd81 100644 --- a/pythonforandroid/pythonpackage.py +++ b/pythonforandroid/pythonpackage.py @@ -34,6 +34,7 @@ import functools +from io import open # needed for python 2 import os import shutil import subprocess @@ -41,14 +42,15 @@ import tarfile import tempfile import time -import zipfile -from io import open # needed for python 2 from urllib.parse import unquote as urlunquote from urllib.parse import urlparse +import zipfile import toml import build.util +from pythonforandroid.util import rmdir, ensure_dir + def transform_dep_for_pip(dependency): if dependency.find("@") > 0 and ( @@ -113,7 +115,7 @@ def extract_metainfo_files_from_package( _extract_metainfo_files_from_package_unsafe(package, output_folder) finally: - shutil.rmtree(temp_folder) + rmdir(temp_folder) def _get_system_python_executable(): @@ -314,7 +316,7 @@ def get_package_as_folder(dependency): ) # Create download subfolder: - os.mkdir(os.path.join(venv_path, "download")) + ensure_dir(os.path.join(venv_path, "download")) # Write a requirements.txt with our package and download: with open(os.path.join(venv_path, "requirements.txt"), @@ -394,11 +396,11 @@ def to_unicode(s): # Needed for Python 2. # Copy result to new dedicated folder so we can throw away # our entire virtualenv nonsense after returning: result_path = tempfile.mkdtemp() - shutil.rmtree(result_path) + rmdir(result_path) shutil.copytree(result_folder_or_file, result_path) return (dl_type, result_path) finally: - shutil.rmtree(venv_parent) + rmdir(venv_parent) def _extract_metainfo_files_from_package_unsafe( @@ -458,7 +460,7 @@ def _extract_metainfo_files_from_package_unsafe( shutil.copyfile(metadata_path, os.path.join(output_path, "METADATA")) finally: if clean_up_path: - shutil.rmtree(path) + rmdir(path) def is_filesystem_path(dep): @@ -576,7 +578,7 @@ def _extract_info_from_package(dependency, return list(set(requirements)) # remove duplicates finally: - shutil.rmtree(output_folder) + rmdir(output_folder) package_name_cache = dict() diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 67c309e197..a2578e6b32 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -1,6 +1,5 @@ from os.path import basename, dirname, exists, isdir, isfile, join, realpath, split import glob -from shutil import rmtree import hashlib from re import match @@ -10,7 +9,7 @@ import fnmatch import urllib.request from urllib.request import urlretrieve -from os import listdir, unlink, environ, mkdir, curdir, walk +from os import listdir, unlink, environ, curdir, walk from sys import stdout import time try: @@ -19,7 +18,7 @@ from urllib.parse import urlparse from pythonforandroid.logger import (logger, info, warning, debug, shprint, info_main) from pythonforandroid.util import (current_directory, ensure_dir, - BuildInterruptingException) + BuildInterruptingException, rmdir) from pythonforandroid.util import load_source as import_recipe @@ -218,7 +217,7 @@ def report_hook(index, blksize, size): url = url[4:] # if 'version' is specified, do a shallow clone if self.version: - shprint(sh.mkdir, '-p', target) + ensure_dir(target) with current_directory(target): shprint(sh.git, 'init') shprint(sh.git, 'remote', 'add', 'origin', url) @@ -367,7 +366,7 @@ def download(self): if expected_digest: expected_digests[alg] = expected_digest - shprint(sh.mkdir, '-p', join(self.ctx.packages_path, self.name)) + ensure_dir(join(self.ctx.packages_path, self.name)) with current_directory(join(self.ctx.packages_path, self.name)): filename = shprint(sh.basename, url).stdout[:-1].decode('utf-8') @@ -423,9 +422,7 @@ def unpack(self, arch): self.name.lower())) if exists(self.get_build_dir(arch)): return - shprint(sh.rm, '-rf', build_dir) - shprint(sh.mkdir, '-p', build_dir) - shprint(sh.rmdir, build_dir) + rmdir(build_dir) ensure_dir(build_dir) shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch)) return @@ -473,7 +470,7 @@ def unpack(self, arch): 'Could not extract {} download, it must be .zip, ' '.tar.gz or .tar.bz2 or .tar.xz'.format(extraction_filename)) elif isdir(extraction_filename): - mkdir(directory_name) + ensure_dir(directory_name) for entry in listdir(extraction_filename): if entry not in ('.git',): shprint(sh.cp, '-Rv', @@ -614,13 +611,11 @@ def clean_build(self, arch=None): 'build dirs'.format(self.name)) for directory in dirs: - if exists(directory): - info('Deleting {}'.format(directory)) - shutil.rmtree(directory) + rmdir(directory) # Delete any Python distributions to ensure the recipe build # doesn't persist in site-packages - shutil.rmtree(self.ctx.python_installs_dir) + rmdir(self.ctx.python_installs_dir) def install_libs(self, arch, *libs): libs_dir = self.ctx.get_libs_dir(arch.arch) @@ -721,7 +716,7 @@ def prepare_build_dir(self, arch): if self.src_filename is None: raise BuildInterruptingException( 'IncludedFilesBehaviour failed: no src_filename specified') - shprint(sh.rm, '-rf', self.get_build_dir(arch)) + rmdir(self.get_build_dir(arch)) shprint(sh.cp, '-a', join(self.get_recipe_dir(), self.src_filename), self.get_build_dir(arch)) @@ -861,7 +856,7 @@ def clean_build(self, arch=None): build_dir = join(site_packages_dir[0], name) if exists(build_dir): info('Deleted {}'.format(build_dir)) - rmtree(build_dir) + rmdir(build_dir) @property def real_hostpython_location(self): diff --git a/pythonforandroid/recipes/ifaddrs/__init__.py b/pythonforandroid/recipes/ifaddrs/__init__.py index 7d44f9cd72..1317dc2556 100644 --- a/pythonforandroid/recipes/ifaddrs/__init__.py +++ b/pythonforandroid/recipes/ifaddrs/__init__.py @@ -1,10 +1,13 @@ """ ifaddrs for Android """ -from os.path import join, exists +from os.path import join + import sh -from pythonforandroid.logger import info, shprint + +from pythonforandroid.logger import shprint from pythonforandroid.recipe import CompiledComponentsPythonRecipe from pythonforandroid.toolchain import current_directory +from pythonforandroid.util import ensure_dir class IFAddrRecipe(CompiledComponentsPythonRecipe): @@ -19,9 +22,7 @@ class IFAddrRecipe(CompiledComponentsPythonRecipe): def prebuild_arch(self, arch): """Make the build and target directories""" path = self.get_build_dir(arch.arch) - if not exists(path): - info("creating {}".format(path)) - shprint(sh.mkdir, '-p', path) + ensure_dir(path) def build_arch(self, arch): """simple shared compile""" @@ -30,9 +31,7 @@ def build_arch(self, arch): self.get_build_dir(arch.arch), join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Lib'), join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Include')): - if not exists(path): - info("creating {}".format(path)) - shprint(sh.mkdir, '-p', path) + ensure_dir(path) cli = env['CC'].split()[0] # makes sure first CC command is the compiler rather than ccache, refs: # https://github.com/kivy/python-for-android/issues/1398 diff --git a/pythonforandroid/recipes/lapack/__init__.py b/pythonforandroid/recipes/lapack/__init__.py index ae20e69538..b6124dc285 100644 --- a/pythonforandroid/recipes/lapack/__init__.py +++ b/pythonforandroid/recipes/lapack/__init__.py @@ -11,7 +11,7 @@ import sh import shutil from os import environ -from pythonforandroid.util import build_platform +from pythonforandroid.util import build_platform, rmdir arch_to_sysroot = {'armeabi': 'arm', 'armeabi-v7a': 'arm', 'arm64-v8a': 'arm64'} @@ -57,7 +57,8 @@ def build_arch(self, arch): with current_directory(build_target): env = self.get_recipe_env(arch) ndk_dir = environ["LEGACY_NDK"] - shprint(sh.rm, '-rf', 'CMakeFiles/', 'CMakeCache.txt', _env=env) + rmdir('CMakeFiles') + shprint(sh.rm, '-f', 'CMakeCache.txt', _env=env) opts = [ '-DCMAKE_SYSTEM_NAME=Android', '-DCMAKE_POSITION_INDEPENDENT_CODE=1', diff --git a/pythonforandroid/recipes/libglob/__init__.py b/pythonforandroid/recipes/libglob/__init__.py index f63db42628..39a68b7ee2 100644 --- a/pythonforandroid/recipes/libglob/__init__.py +++ b/pythonforandroid/recipes/libglob/__init__.py @@ -2,11 +2,14 @@ android libglob available via '-lglob' LDFLAG """ -from os.path import exists, join +from os.path import join + +import sh + +from pythonforandroid.logger import shprint from pythonforandroid.recipe import Recipe from pythonforandroid.toolchain import current_directory -from pythonforandroid.logger import info, shprint -import sh +from pythonforandroid.util import ensure_dir class LibGlobRecipe(Recipe): @@ -32,9 +35,7 @@ def should_build(self, arch): def prebuild_arch(self, arch): """Make the build and target directories""" path = self.get_build_dir(arch.arch) - if not exists(path): - info("creating {}".format(path)) - shprint(sh.mkdir, '-p', path) + ensure_dir(path) def build_arch(self, arch): """simple shared compile""" @@ -43,9 +44,7 @@ def build_arch(self, arch): self.get_build_dir(arch.arch), join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Lib'), join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Include')): - if not exists(path): - info("creating {}".format(path)) - shprint(sh.mkdir, '-p', path) + ensure_dir(path) cli = env['CC'].split()[0] # makes sure first CC command is the compiler rather than ccache, refs: # https://github.com/kivy/python-for-android/issues/1399 diff --git a/pythonforandroid/recipes/libmysqlclient/__init__.py b/pythonforandroid/recipes/libmysqlclient/__init__.py index 31ebd3c540..84fd8d30ac 100644 --- a/pythonforandroid/recipes/libmysqlclient/__init__.py +++ b/pythonforandroid/recipes/libmysqlclient/__init__.py @@ -26,7 +26,7 @@ def build_arch(self, arch): env = self.get_recipe_env(arch) with current_directory(join(self.get_build_dir(arch.arch), 'libmysqlclient')): shprint(sh.cp, '-t', '.', join(self.get_recipe_dir(), 'p4a.cmake')) - # shprint(sh.mkdir, 'Platform') + # ensure_dir('Platform') # shprint(sh.cp, '-t', 'Platform', join(self.get_recipe_dir(), 'Linux.cmake')) shprint(sh.rm, '-f', 'CMakeCache.txt') shprint(sh.cmake, '-G', 'Unix Makefiles', diff --git a/pythonforandroid/recipes/libtorrent/__init__.py b/pythonforandroid/recipes/libtorrent/__init__.py index 24f94081c6..1086e00fcc 100644 --- a/pythonforandroid/recipes/libtorrent/__init__.py +++ b/pythonforandroid/recipes/libtorrent/__init__.py @@ -1,9 +1,12 @@ -from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory from multiprocessing import cpu_count -from os.path import join, basename from os import listdir, walk +from os.path import join, basename +import shutil + import sh +from pythonforandroid.toolchain import Recipe, shprint, current_directory + # This recipe builds libtorrent with Python bindings # It depends on Boost.Build and the source of several Boost libraries present # in BOOST_ROOT, which is all provided by the boost recipe diff --git a/pythonforandroid/recipes/opencv/__init__.py b/pythonforandroid/recipes/opencv/__init__.py index c760cbdda7..650c77e508 100644 --- a/pythonforandroid/recipes/opencv/__init__.py +++ b/pythonforandroid/recipes/opencv/__init__.py @@ -1,9 +1,11 @@ +from multiprocessing import cpu_count from os.path import join + import sh -from pythonforandroid.recipe import NDKRecipe -from pythonforandroid.util import current_directory + from pythonforandroid.logger import shprint -from multiprocessing import cpu_count +from pythonforandroid.recipe import NDKRecipe +from pythonforandroid.util import current_directory, ensure_dir class OpenCVRecipe(NDKRecipe): @@ -45,7 +47,7 @@ def get_recipe_env(self, arch): def build_arch(self, arch): build_dir = join(self.get_build_dir(arch.arch), 'build') - shprint(sh.mkdir, '-p', build_dir) + ensure_dir(build_dir) opencv_extras = [] if 'opencv_extras' in self.ctx.recipe_build_order: diff --git a/pythonforandroid/recipes/sqlite3/__init__.py b/pythonforandroid/recipes/sqlite3/__init__.py index 955d808141..1f4292c1eb 100644 --- a/pythonforandroid/recipes/sqlite3/__init__.py +++ b/pythonforandroid/recipes/sqlite3/__init__.py @@ -1,7 +1,8 @@ -from pythonforandroid.recipe import NDKRecipe -from pythonforandroid.toolchain import shutil from os.path import join -import sh +import shutil + +from pythonforandroid.recipe import NDKRecipe +from pythonforandroid.util import ensure_dir class Sqlite3Recipe(NDKRecipe): @@ -16,7 +17,7 @@ def should_build(self, arch): def prebuild_arch(self, arch): super().prebuild_arch(arch) # Copy the Android make file - sh.mkdir('-p', join(self.get_build_dir(arch.arch), 'jni')) + ensure_dir(join(self.get_build_dir(arch.arch), 'jni')) shutil.copyfile(join(self.get_recipe_dir(), 'Android.mk'), join(self.get_build_dir(arch.arch), 'jni/Android.mk')) diff --git a/pythonforandroid/recipes/twisted/__init__.py b/pythonforandroid/recipes/twisted/__init__.py index 0c390a5b14..30a7af4bb9 100644 --- a/pythonforandroid/recipes/twisted/__init__.py +++ b/pythonforandroid/recipes/twisted/__init__.py @@ -1,7 +1,7 @@ import os -import shutil from pythonforandroid.recipe import CythonRecipe +from pythonforandroid.util import rmdir class TwistedRecipe(CythonRecipe): @@ -23,7 +23,7 @@ def prebuild_arch(self, arch): for item in os.walk(source_dir): if os.path.basename(item[0]) == 'test': full_path = os.path.join(source_dir, item[0]) - shutil.rmtree(full_path, ignore_errors=True) + rmdir(full_path, ignore_errors=True) def get_recipe_env(self, arch): env = super().get_recipe_env(arch) diff --git a/pythonforandroid/recipes/zope_interface/__init__.py b/pythonforandroid/recipes/zope_interface/__init__.py index 46a1820c2b..7e7fecaea5 100644 --- a/pythonforandroid/recipes/zope_interface/__init__.py +++ b/pythonforandroid/recipes/zope_interface/__init__.py @@ -1,7 +1,8 @@ +from os.path import join + from pythonforandroid.recipe import PythonRecipe from pythonforandroid.toolchain import current_directory -from os.path import join -import sh +from pythonforandroid.util import rmdir class ZopeInterfaceRecipe(PythonRecipe): @@ -25,11 +26,8 @@ def build_arch(self, arch): def prebuild_arch(self, arch): super().prebuild_arch(arch) with current_directory(self.get_build_dir(arch.arch)): - sh.rm( - '-rf', - 'src/zope/interface/tests', - 'src/zope/interface/common/tests', - ) + rmdir('src/zope/interface/tests') + rmdir('src/zope/interface/common/tests') recipe = ZopeInterfaceRecipe() diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index 38c700740c..7a5461f30d 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -16,7 +16,6 @@ from os.path import (join, dirname, realpath, exists, expanduser, basename) import re import shlex -import shutil import sys from sys import platform @@ -42,7 +41,7 @@ from pythonforandroid.recommendations import ( RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API, print_recommendations) from pythonforandroid.util import ( - current_directory, BuildInterruptingException, load_source) + current_directory, BuildInterruptingException, load_source, rmdir) user_dir = dirname(realpath(os.path.curdir)) toolchain_dir = dirname(__file__) @@ -831,18 +830,16 @@ def clean_dists(self, _args): """Delete all compiled distributions in the internal distribution directory.""" ctx = self.ctx - if exists(ctx.dist_dir): - shutil.rmtree(ctx.dist_dir) + rmdir(ctx.dist_dir) def clean_bootstrap_builds(self, _args): """Delete all the bootstrap builds.""" - if exists(join(self.ctx.build_dir, 'bootstrap_builds')): - shutil.rmtree(join(self.ctx.build_dir, 'bootstrap_builds')) + rmdir(join(self.ctx.build_dir, 'bootstrap_builds')) # for bs in Bootstrap.all_bootstraps(): # bs = Bootstrap.get_bootstrap(bs, self.ctx) # if bs.build_dir and exists(bs.build_dir): # info('Cleaning build for {} bootstrap.'.format(bs.name)) - # shutil.rmtree(bs.build_dir) + # rmdir(bs.build_dir) def clean_builds(self, _args): """Delete all build caches for each recipe, python-install, java code @@ -853,13 +850,10 @@ def clean_builds(self, _args): of a specific recipe. """ ctx = self.ctx - if exists(ctx.build_dir): - shutil.rmtree(ctx.build_dir) - if exists(ctx.python_installs_dir): - shutil.rmtree(ctx.python_installs_dir) + rmdir(ctx.build_dir) + rmdir(ctx.python_installs_dir) libs_dir = join(self.ctx.build_dir, 'libs_collections') - if exists(libs_dir): - shutil.rmtree(libs_dir) + rmdir(libs_dir) def clean_recipe_build(self, args): """Deletes the build files of the given recipe. @@ -889,14 +883,14 @@ def clean_download_cache(self, args): for package in args.recipes: remove_path = join(ctx.packages_path, package) if exists(remove_path): - shutil.rmtree(remove_path) + rmdir(remove_path) info('Download cache removed for: "{}"'.format(package)) else: warning('No download cache found for "{}", skipping'.format( package)) else: if exists(ctx.packages_path): - shutil.rmtree(ctx.packages_path) + rmdir(ctx.packages_path) info('Download cache removed.') else: print('No cache found at "{}"'.format(ctx.packages_path)) diff --git a/pythonforandroid/util.py b/pythonforandroid/util.py index 713c2b46c0..f9d5302ecc 100644 --- a/pythonforandroid/util.py +++ b/pythonforandroid/util.py @@ -1,12 +1,15 @@ import contextlib +from fnmatch import fnmatch +import logging from os.path import exists, join from os import getcwd, chdir, makedirs, walk from platform import uname -import shutil -from fnmatch import fnmatch +from shutil import rmtree from tempfile import mkdtemp + from pythonforandroid.logger import (logger, Err_Fore, error, info) +LOGGER = logging.getLogger("p4a.util") build_platform = "{system}-{machine}".format( system=uname().system, machine=uname().machine @@ -36,16 +39,11 @@ def temp_directory(): temp_dir, Err_Fore.RESET))) yield temp_dir finally: - shutil.rmtree(temp_dir) + rmtree(temp_dir) logger.debug(''.join((Err_Fore.CYAN, ' - temp directory deleted ', temp_dir, Err_Fore.RESET))) -def ensure_dir(filename): - if not exists(filename): - makedirs(filename) - - def walk_valid_filens(base_dir, invalid_dir_names, invalid_file_patterns): """Recursively walks all the files and directories in ``dirn``, ignoring directories that match any pattern in ``invalid_dirns`` @@ -106,3 +104,17 @@ def handle_build_exception(exception): if exception.instructions is not None: info('Instructions: {}'.format(exception.instructions)) exit(1) + + +def rmdir(dn, ignore_errors=False): + if not exists(dn): + return + LOGGER.debug("Remove directory and subdirectory {}".format(dn)) + rmtree(dn, ignore_errors) + + +def ensure_dir(dn): + if exists(dn): + return + LOGGER.debug("Create directory {0}".format(dn)) + makedirs(dn) diff --git a/tests/recipes/test_libmysqlclient.py b/tests/recipes/test_libmysqlclient.py index 1be4b71e50..e484393398 100644 --- a/tests/recipes/test_libmysqlclient.py +++ b/tests/recipes/test_libmysqlclient.py @@ -23,7 +23,7 @@ def test_build_arch( mock_sh_rm, ): # We overwrite the base test method because we need - # to mock a little more (`sh.cp` and `sh.rm`) + # to mock a little more (`sh.cp` and rmdir) super().test_build_arch() # make sure that the mocked methods are actually called mock_sh_cp.assert_called() diff --git a/tests/recipes/test_openal.py b/tests/recipes/test_openal.py index 27634d9013..21f3196798 100644 --- a/tests/recipes/test_openal.py +++ b/tests/recipes/test_openal.py @@ -50,7 +50,7 @@ def test_build_arch( mock_sh_cp, ): # We overwrite the base test method because we need to mock a little - # more with this recipe (`sh.cp` and `sh.rm`) + # more with this recipe. super().test_build_arch() # make sure that the mocked methods are actually called mock_sh_cp.assert_called() diff --git a/tests/recipes/test_openssl.py b/tests/recipes/test_openssl.py index 509c1cc1e8..861e73bd39 100644 --- a/tests/recipes/test_openssl.py +++ b/tests/recipes/test_openssl.py @@ -23,7 +23,7 @@ def test_build_arch( mock_sh_patch, ): # We overwrite the base test method because we need to mock a little - # more with this recipe (`sh.cp` and `sh.rm`) + # more with this recipe. super().test_build_arch() # make sure that the mocked methods are actually called mock_sh_patch.assert_called() diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index e997eba8c2..ee5c0123f9 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -352,12 +352,16 @@ def bootstrap_name(self): @mock.patch("pythonforandroid.util.exists") @mock.patch("pythonforandroid.util.chdir") @mock.patch("pythonforandroid.bootstrap.listdir") - @mock.patch("pythonforandroid.bootstrap.sh.rm") + @mock.patch("pythonforandroid.bootstraps.sdl2.rmdir") + @mock.patch("pythonforandroid.bootstraps.service_only.rmdir") + @mock.patch("pythonforandroid.bootstraps.webview.rmdir") @mock.patch("pythonforandroid.bootstrap.sh.cp") def test_assemble_distribution( self, mock_sh_cp, - mock_sh_rm, + mock_rmdir1, + mock_rmdir2, + mock_rmdir3, mock_listdir, mock_chdir, mock_ensure_dir, @@ -433,7 +437,6 @@ def test_assemble_distribution( ) # check that the other mocks we made are actually called - mock_sh_rm.assert_called() mock_sh_cp.assert_called() mock_chdir.assert_called() mock_listdir.assert_called() @@ -558,11 +561,11 @@ def test_bootstrap_strip( mock_sh_print.assert_called() @mock.patch("pythonforandroid.bootstrap.listdir") - @mock.patch("pythonforandroid.bootstrap.sh.rm") + @mock.patch("pythonforandroid.bootstrap.rmdir") @mock.patch("pythonforandroid.bootstrap.sh.mv") @mock.patch("pythonforandroid.bootstrap.isdir") def test_bootstrap_fry_eggs( - self, mock_isdir, mock_sh_mv, mock_sh_rm, mock_listdir + self, mock_isdir, mock_sh_mv, mock_rmdir, mock_listdir ): mock_listdir.return_value = [ "jnius", @@ -590,7 +593,7 @@ def test_bootstrap_fry_eggs( ] ) self.assertEqual( - mock_sh_rm.call_args[0][1], "pyjnius-1.2.1.dev0-py3.7.egg" + mock_rmdir.call_args[0][0], "pyjnius-1.2.1.dev0-py3.7.egg" ) # check that the other mocks we made are actually called mock_isdir.assert_called() diff --git a/tests/test_distribution.py b/tests/test_distribution.py index 423d572252..404091ca75 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -91,8 +91,8 @@ def test_folder_exist(self, mock_exists): self.ctx.bootstrap.distribution.dist_dir ) - @mock.patch("pythonforandroid.distribution.rmtree") - def test_delete(self, mock_rmtree): + @mock.patch("pythonforandroid.distribution.rmdir") + def test_delete(self, mock_rmdir): """Test that method :meth:`~pythonforandroid.distribution.Distribution.delete` is called once with the proper arguments.""" @@ -100,7 +100,7 @@ def test_delete(self, mock_rmtree): Bootstrap().get_bootstrap("sdl2", self.ctx) ) self.ctx.bootstrap.distribution.delete() - mock_rmtree.assert_called_once_with( + mock_rmdir.assert_called_once_with( self.ctx.bootstrap.distribution.dist_dir ) diff --git a/tests/test_util.py b/tests/test_util.py index ff57dc7a47..495d3d7c1b 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -21,20 +21,20 @@ def test_ensure_dir(self, mock_makedirs): util.ensure_dir("fake_directory") mock_makedirs.assert_called_once_with("fake_directory") - @mock.patch("shutil.rmtree") + @mock.patch("pythonforandroid.util.rmtree") @mock.patch("pythonforandroid.util.mkdtemp") - def test_temp_directory(self, mock_mkdtemp, mock_shutil_rmtree): + def test_temp_directory(self, mock_mkdtemp, mock_rmtree): """ Basic test for method :meth:`~pythonforandroid.util.temp_directory`. We perform this test by `mocking` the command `mkdtemp` and - `shutil.rmtree` and we make sure that those functions are called in the + `rmdir` and we make sure that those functions are called in the proper place. """ mock_mkdtemp.return_value = "/temp/any_directory" with util.temp_directory(): mock_mkdtemp.assert_called_once() - mock_shutil_rmtree.assert_not_called() - mock_shutil_rmtree.assert_called_once_with("/temp/any_directory") + mock_rmtree.assert_not_called() + mock_rmtree.assert_called_once_with("/temp/any_directory") @mock.patch("pythonforandroid.util.chdir") def test_current_directory(self, moch_chdir): From 3107e4e788e152fc3c69e0ff67702d670a23d970 Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 30 Aug 2023 02:25:39 +1000 Subject: [PATCH 28/30] Standardise on move (files, directories) (#2884) * Add new util function (and tests) called `move`. * Change references to sh.mv to use move (as it is faster and cross-platform). * Change conditional "mv -t" to a for loop. --- pythonforandroid/bootstrap.py | 6 ++-- pythonforandroid/recipe.py | 8 +++--- pythonforandroid/util.py | 11 ++++++-- tests/test_bootstrap.py | 6 ++-- tests/test_util.py | 52 +++++++++++++++++++++++++++++++---- 5 files changed, 65 insertions(+), 18 deletions(-) diff --git a/pythonforandroid/bootstrap.py b/pythonforandroid/bootstrap.py index 965ee4e7bf..8bbbcf0eb6 100755 --- a/pythonforandroid/bootstrap.py +++ b/pythonforandroid/bootstrap.py @@ -11,7 +11,7 @@ from pythonforandroid.logger import (shprint, info, logger, debug) from pythonforandroid.util import ( current_directory, ensure_dir, temp_directory, BuildInterruptingException, - rmdir) + rmdir, move) from pythonforandroid.recipe import Recipe @@ -395,8 +395,8 @@ def fry_eggs(self, sitepackages): if isdir(rd) and d.endswith('.egg'): info(' ' + d) files = [join(rd, f) for f in listdir(rd) if f != 'EGG-INFO'] - if files: - shprint(sh.mv, '-t', sitepackages, *files) + for f in files: + move(f, sitepackages) rmdir(d) diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index a2578e6b32..2cb914c3e4 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -18,7 +18,7 @@ from urllib.parse import urlparse from pythonforandroid.logger import (logger, info, warning, debug, shprint, info_main) from pythonforandroid.util import (current_directory, ensure_dir, - BuildInterruptingException, rmdir) + BuildInterruptingException, rmdir, move) from pythonforandroid.util import load_source as import_recipe @@ -457,14 +457,14 @@ def unpack(self, arch): fileh = zipfile.ZipFile(extraction_filename, 'r') root_directory = fileh.filelist[0].filename.split('/')[0] if root_directory != basename(directory_name): - shprint(sh.mv, root_directory, directory_name) + move(root_directory, directory_name) elif extraction_filename.endswith( ('.tar.gz', '.tgz', '.tar.bz2', '.tbz2', '.tar.xz', '.txz')): sh.tar('xf', extraction_filename) root_directory = sh.tar('tf', extraction_filename).stdout.decode( 'utf-8').split('\n')[0].split('/')[0] if root_directory != basename(directory_name): - shprint(sh.mv, root_directory, directory_name) + move(root_directory, directory_name) else: raise Exception( 'Could not extract {} download, it must be .zip, ' @@ -1166,7 +1166,7 @@ def reduce_object_file_names(self, dirn): parts = file_basename.split('.') if len(parts) <= 2: continue - shprint(sh.mv, filen, join(file_dirname, parts[0] + '.so')) + move(filen, join(file_dirname, parts[0] + '.so')) def algsum(alg, filen): diff --git a/pythonforandroid/util.py b/pythonforandroid/util.py index f9d5302ecc..ff5727955c 100644 --- a/pythonforandroid/util.py +++ b/pythonforandroid/util.py @@ -4,7 +4,7 @@ from os.path import exists, join from os import getcwd, chdir, makedirs, walk from platform import uname -from shutil import rmtree +import shutil from tempfile import mkdtemp from pythonforandroid.logger import (logger, Err_Fore, error, info) @@ -39,7 +39,7 @@ def temp_directory(): temp_dir, Err_Fore.RESET))) yield temp_dir finally: - rmtree(temp_dir) + shutil.rmtree(temp_dir) logger.debug(''.join((Err_Fore.CYAN, ' - temp directory deleted ', temp_dir, Err_Fore.RESET))) @@ -110,7 +110,7 @@ def rmdir(dn, ignore_errors=False): if not exists(dn): return LOGGER.debug("Remove directory and subdirectory {}".format(dn)) - rmtree(dn, ignore_errors) + shutil.rmtree(dn, ignore_errors) def ensure_dir(dn): @@ -118,3 +118,8 @@ def ensure_dir(dn): return LOGGER.debug("Create directory {0}".format(dn)) makedirs(dn) + + +def move(source, destination): + LOGGER.debug("Moving {} to {}".format(source, destination)) + shutil.move(source, destination) diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index ee5c0123f9..99620fee75 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -562,10 +562,10 @@ def test_bootstrap_strip( @mock.patch("pythonforandroid.bootstrap.listdir") @mock.patch("pythonforandroid.bootstrap.rmdir") - @mock.patch("pythonforandroid.bootstrap.sh.mv") + @mock.patch("pythonforandroid.bootstrap.move") @mock.patch("pythonforandroid.bootstrap.isdir") def test_bootstrap_fry_eggs( - self, mock_isdir, mock_sh_mv, mock_rmdir, mock_listdir + self, mock_isdir, mock_move, mock_rmdir, mock_listdir ): mock_listdir.return_value = [ "jnius", @@ -597,7 +597,7 @@ def test_bootstrap_fry_eggs( ) # check that the other mocks we made are actually called mock_isdir.assert_called() - mock_sh_mv.assert_called() + mock_move.assert_called() class TestBootstrapSdl2(GenericBootstrapTest, unittest.TestCase): diff --git a/tests/test_util.py b/tests/test_util.py index 495d3d7c1b..cd5caa4802 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,4 +1,6 @@ import os +from pathlib import Path +from tempfile import TemporaryDirectory import types import unittest from unittest import mock @@ -21,20 +23,21 @@ def test_ensure_dir(self, mock_makedirs): util.ensure_dir("fake_directory") mock_makedirs.assert_called_once_with("fake_directory") - @mock.patch("pythonforandroid.util.rmtree") + @mock.patch("shutil.rmtree") @mock.patch("pythonforandroid.util.mkdtemp") - def test_temp_directory(self, mock_mkdtemp, mock_rmtree): + def test_temp_directory(self, mock_mkdtemp, mock_shutil_rmtree): + """ Basic test for method :meth:`~pythonforandroid.util.temp_directory`. We perform this test by `mocking` the command `mkdtemp` and - `rmdir` and we make sure that those functions are called in the + `shutil.rmtree` and we make sure that those functions are called in the proper place. """ mock_mkdtemp.return_value = "/temp/any_directory" with util.temp_directory(): mock_mkdtemp.assert_called_once() - mock_rmtree.assert_not_called() - mock_rmtree.assert_called_once_with("/temp/any_directory") + mock_shutil_rmtree.assert_not_called() + mock_shutil_rmtree.assert_called_once_with("/temp/any_directory") @mock.patch("pythonforandroid.util.chdir") def test_current_directory(self, moch_chdir): @@ -136,3 +139,42 @@ def test_util_exceptions(self): ) with self.assertRaises(SystemExit): util.handle_build_exception(exc) + + def test_move(self): + with mock.patch( + "pythonforandroid.util.LOGGER" + ) as m_logger, TemporaryDirectory() as base_dir: + new_path = Path(base_dir) / "new" + + # Set up source + old_path = Path(base_dir) / "old" + with open(old_path, "w") as outfile: + outfile.write("Temporary content") + + # Non existent source + with self.assertRaises(FileNotFoundError): + util.move(new_path, new_path) + m_logger.debug.assert_called() + m_logger.error.assert_not_called() + m_logger.reset_mock() + assert old_path.exists() + assert not new_path.exists() + + # Successful move + util.move(old_path, new_path) + assert not old_path.exists() + assert new_path.exists() + m_logger.debug.assert_called() + m_logger.error.assert_not_called() + m_logger.reset_mock() + + # Move over existing: + existing_path = Path(base_dir) / "existing" + existing_path.touch() + + util.move(new_path, existing_path) + with open(existing_path, "r") as infile: + assert infile.read() == "Temporary content" + m_logger.debug.assert_called() + m_logger.error.assert_not_called() + m_logger.reset_mock() From 4d50d7d883bb6313f37c22402ecfaefa689bbbed Mon Sep 17 00:00:00 2001 From: Julian Date: Sun, 3 Sep 2023 17:56:15 +1000 Subject: [PATCH 29/30] Standardise on touch files (#2886) Existing code shells out to run the `touch` Unix command. This can be done faster (and cross-platform) with `pathlib`'s touch method. * Created a utility to accept string paths or pathlib paths and applies touch. * Created test for it. * Update all instances to use it, and updated their tests. --- pythonforandroid/recipe.py | 12 +++++++----- pythonforandroid/recipes/protobuf_cpp/__init__.py | 15 ++++++++------- pythonforandroid/recipes/reportlab/__init__.py | 7 ++++--- pythonforandroid/util.py | 5 +++++ tests/recipes/test_reportlab.py | 2 +- tests/test_recipe.py | 2 +- tests/test_util.py | 9 +++++++++ 7 files changed, 35 insertions(+), 17 deletions(-) diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 2cb914c3e4..fa0bb4e790 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -16,9 +16,11 @@ from urlparse import urlparse except ImportError: from urllib.parse import urlparse -from pythonforandroid.logger import (logger, info, warning, debug, shprint, info_main) -from pythonforandroid.util import (current_directory, ensure_dir, - BuildInterruptingException, rmdir, move) +from pythonforandroid.logger import ( + logger, info, warning, debug, shprint, info_main) +from pythonforandroid.util import ( + current_directory, ensure_dir, BuildInterruptingException, rmdir, move, + touch) from pythonforandroid.util import load_source as import_recipe @@ -395,7 +397,7 @@ def download(self): shprint(sh.rm, '-f', marker_filename) self.download_file(self.versioned_url, filename) - shprint(sh.touch, marker_filename) + touch(marker_filename) if exists(filename) and isfile(filename): for alg, expected_digest in expected_digests.items(): @@ -530,7 +532,7 @@ def apply_patches(self, arch, build_dir=None): patch.format(version=self.version, arch=arch.arch), arch.arch, build_dir=build_dir) - shprint(sh.touch, join(build_dir, '.patched')) + touch(join(build_dir, '.patched')) def should_build(self, arch): '''Should perform any necessary test and return True only if it needs diff --git a/pythonforandroid/recipes/protobuf_cpp/__init__.py b/pythonforandroid/recipes/protobuf_cpp/__init__.py index c1149f2714..7209e0909b 100644 --- a/pythonforandroid/recipes/protobuf_cpp/__init__.py +++ b/pythonforandroid/recipes/protobuf_cpp/__init__.py @@ -1,12 +1,13 @@ -from pythonforandroid.recipe import CppCompiledComponentsPythonRecipe -from pythonforandroid.logger import shprint, info_notify -from pythonforandroid.util import current_directory -from os.path import exists, join -import sh from multiprocessing import cpu_count +import os +from os.path import exists, join from pythonforandroid.toolchain import info +import sh import sys -import os + +from pythonforandroid.recipe import CppCompiledComponentsPythonRecipe +from pythonforandroid.logger import shprint, info_notify +from pythonforandroid.util import current_directory, touch class ProtobufCppRecipe(CppCompiledComponentsPythonRecipe): @@ -30,7 +31,7 @@ def prebuild_arch(self, arch): patch_mark = join(self.get_build_dir(arch.arch), '.protobuf-patched') if self.ctx.python_recipe.name == 'python3' and not exists(patch_mark): self.apply_patch('fix-python3-compatibility.patch', arch.arch) - shprint(sh.touch, patch_mark) + touch(patch_mark) # During building, host needs to transpile .proto files to .py # ideally with the same version as protobuf runtime, or with an older one. diff --git a/pythonforandroid/recipes/reportlab/__init__.py b/pythonforandroid/recipes/reportlab/__init__.py index 60f1a078ea..ee5de38214 100644 --- a/pythonforandroid/recipes/reportlab/__init__.py +++ b/pythonforandroid/recipes/reportlab/__init__.py @@ -1,8 +1,9 @@ import os import sh + +from pythonforandroid.logger import info from pythonforandroid.recipe import CompiledComponentsPythonRecipe -from pythonforandroid.util import (current_directory, ensure_dir) -from pythonforandroid.logger import (info, shprint) +from pythonforandroid.util import current_directory, ensure_dir, touch class ReportLabRecipe(CompiledComponentsPythonRecipe): @@ -28,7 +29,7 @@ def prebuild_arch(self, arch): # Apply patches: self.apply_patch('patches/fix-setup.patch', arch.arch) - shprint(sh.touch, os.path.join(recipe_dir, '.patched')) + touch(os.path.join(recipe_dir, '.patched')) ft = self.get_recipe('freetype', self.ctx) ft_dir = ft.get_build_dir(arch.arch) ft_lib_dir = os.environ.get('_FT_LIB_', os.path.join(ft_dir, 'objs', '.libs')) diff --git a/pythonforandroid/util.py b/pythonforandroid/util.py index ff5727955c..af363b2e3f 100644 --- a/pythonforandroid/util.py +++ b/pythonforandroid/util.py @@ -3,6 +3,7 @@ import logging from os.path import exists, join from os import getcwd, chdir, makedirs, walk +from pathlib import Path from platform import uname import shutil from tempfile import mkdtemp @@ -123,3 +124,7 @@ def ensure_dir(dn): def move(source, destination): LOGGER.debug("Moving {} to {}".format(source, destination)) shutil.move(source, destination) + + +def touch(filename): + Path(filename).touch() diff --git a/tests/recipes/test_reportlab.py b/tests/recipes/test_reportlab.py index 83191e5286..6129a6a963 100644 --- a/tests/recipes/test_reportlab.py +++ b/tests/recipes/test_reportlab.py @@ -30,7 +30,7 @@ def test_prebuild_arch(self): # these sh commands are not relevant for the test and need to be mocked with \ patch('sh.patch'), \ - patch('sh.touch'), \ + patch('pythonforandroid.recipe.touch'), \ patch('sh.unzip'), \ patch('os.path.isfile'): self.recipe.prebuild_arch(self.arch) diff --git a/tests/test_recipe.py b/tests/test_recipe.py index 666d089caa..b6e4c99225 100644 --- a/tests/test_recipe.py +++ b/tests/test_recipe.py @@ -134,7 +134,7 @@ def test_download_url_is_set(self): with ( patch_logger_debug()) as m_debug, ( mock.patch.object(Recipe, 'download_file')) as m_download_file, ( - mock.patch('pythonforandroid.recipe.sh.touch')) as m_touch, ( + mock.patch('pythonforandroid.recipe.touch')) as m_touch, ( tempfile.TemporaryDirectory()) as temp_dir: recipe.ctx.setup_dirs(temp_dir) recipe.download() diff --git a/tests/test_util.py b/tests/test_util.py index cd5caa4802..a4d7ea816e 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -178,3 +178,12 @@ def test_move(self): m_logger.debug.assert_called() m_logger.error.assert_not_called() m_logger.reset_mock() + + def test_touch(self): + # Just checking the new file case. + # Assume the existing file timestamp case will work if this does. + with TemporaryDirectory() as base_dir: + new_file_path = Path(base_dir) / "new_file" + assert not new_file_path.exists() + util.touch(new_file_path) + assert new_file_path.exists() From 12ef2a23be5e10aea1a5de1dd8657e4145e569cf Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sun, 17 Sep 2023 16:15:27 +0200 Subject: [PATCH 30/30] Update CHANGELOG.md and bump version to 2023.09.16 --- CHANGELOG.md | 60 ++++++++++++++++++++++++++++++++++++ pythonforandroid/__init__.py | 2 +- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98288f265d..2ce4711cbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,65 @@ # Changelog +## [v2023.09.16](https://github.com/kivy/python-for-android/tree/v2023.09.16) + +[Full Changelog](https://github.com/kivy/python-for-android/compare/v2023.05.21...v2023.09.16) + +**Closed issues:** + +- Microphone And Audio permissions doesn't work [\#2889](https://github.com/kivy/python-for-android/issues/2889) +- Error with Scipy [\#2883](https://github.com/kivy/python-for-android/issues/2883) +- Download failed \( Downloading sqlite3 from https://www.sqlite.org/2021/sqlite-amalgamation-3350500.zip \) during buildozer -v andriod debug [\#2881](https://github.com/kivy/python-for-android/issues/2881) +- ONNXruntime lib open failed due to 64-bit [\#2880](https://github.com/kivy/python-for-android/issues/2880) +- Question: how to write a recipe to download and install\(coz build fail\) a wheel package directly? [\#2878](https://github.com/kivy/python-for-android/issues/2878) +- Impossible to install python for android [\#2867](https://github.com/kivy/python-for-android/issues/2867) +- scipy with kivy [\#2861](https://github.com/kivy/python-for-android/issues/2861) +- False positve parser warning. [\#2856](https://github.com/kivy/python-for-android/issues/2856) +- After successfully building app, its crashes with this error, using firebase-admin [\#2854](https://github.com/kivy/python-for-android/issues/2854) +- Kivy [\#2837](https://github.com/kivy/python-for-android/issues/2837) +- not installing on windows 10 [\#2836](https://github.com/kivy/python-for-android/issues/2836) +- Could not find com.android.tools.build:gradle:7.2.0. in android studio [\#2834](https://github.com/kivy/python-for-android/issues/2834) +- vlc recipe build fail [\#2822](https://github.com/kivy/python-for-android/issues/2822) +- mysqldb recipe build fail [\#2813](https://github.com/kivy/python-for-android/issues/2813) +- babel recipe build fail [\#2803](https://github.com/kivy/python-for-android/issues/2803) +- Python 3.10 cffi build fails [\#2799](https://github.com/kivy/python-for-android/issues/2799) +- \[Recipe request\] OpenColorIO & rawpy \(libraw\) [\#2789](https://github.com/kivy/python-for-android/issues/2789) +- Longer app load time, bigger apk size and unnecessary logs [\#2704](https://github.com/kivy/python-for-android/issues/2704) +- -fp-model argument not found, directory strict not found [\#2329](https://github.com/kivy/python-for-android/issues/2329) +- Kivy crashes on Android: ImportError: dlopen failed: library "libpython3.7m.so" not found [\#2237](https://github.com/kivy/python-for-android/issues/2237) +- pyconfig.h issue [\#2074](https://github.com/kivy/python-for-android/issues/2074) + +**Merged pull requests:** + +- Use Python's touch\(\) rather than shelling out. [\#2886](https://github.com/kivy/python-for-android/pull/2886) ([Julian-O](https://github.com/Julian-O)) +- Standardise on move [\#2884](https://github.com/kivy/python-for-android/pull/2884) ([Julian-O](https://github.com/Julian-O)) +- Remove deprecated FlatDir in Gradle template [\#2876](https://github.com/kivy/python-for-android/pull/2876) ([Julian-O](https://github.com/Julian-O)) +- :rotating\_light: linter fixes [\#2874](https://github.com/kivy/python-for-android/pull/2874) ([AndreMiras](https://github.com/AndreMiras)) +- Standardise ensure\_dir and rmdir [\#2871](https://github.com/kivy/python-for-android/pull/2871) ([Julian-O](https://github.com/Julian-O)) +- Correct check for --sdk option [\#2870](https://github.com/kivy/python-for-android/pull/2870) ([Julian-O](https://github.com/Julian-O)) +- Python versions: Update documentation & CI testing [\#2869](https://github.com/kivy/python-for-android/pull/2869) ([Julian-O](https://github.com/Julian-O)) +- Patching cleanup [\#2868](https://github.com/kivy/python-for-android/pull/2868) ([Julian-O](https://github.com/Julian-O)) +- Factor out dependency checking. Use modern version handling [\#2866](https://github.com/kivy/python-for-android/pull/2866) ([Julian-O](https://github.com/Julian-O)) +- `build_platform` should be all-lowercase [\#2864](https://github.com/kivy/python-for-android/pull/2864) ([misl6](https://github.com/misl6)) +- Fix simple typos in comments [\#2863](https://github.com/kivy/python-for-android/pull/2863) ([Julian-O](https://github.com/Julian-O)) +- Use a pinned version of `Cython` for now, as most of the recipes are incompatible with `Cython==3.x.x` [\#2862](https://github.com/kivy/python-for-android/pull/2862) ([misl6](https://github.com/misl6)) +- Docs: Fix typos and updated command to build apk - README [\#2860](https://github.com/kivy/python-for-android/pull/2860) ([kulothunganug](https://github.com/kulothunganug)) +- Docs: Fix code string - quickstart.rst [\#2859](https://github.com/kivy/python-for-android/pull/2859) ([kulothunganug](https://github.com/kulothunganug)) +- Automatically generate required pre-requisites [\#2858](https://github.com/kivy/python-for-android/pull/2858) ([Julian-O](https://github.com/Julian-O)) +- Use platform.uname instead of os.uname [\#2857](https://github.com/kivy/python-for-android/pull/2857) ([Julian-O](https://github.com/Julian-O)) +- Bump `kivy` version to `2.2.1` [\#2855](https://github.com/kivy/python-for-android/pull/2855) ([misl6](https://github.com/misl6)) +- Correct sys\_platform [\#2852](https://github.com/kivy/python-for-android/pull/2852) ([Julian-O](https://github.com/Julian-O)) +- Changed the url to use https as http fails [\#2846](https://github.com/kivy/python-for-android/pull/2846) ([kuzeyron](https://github.com/kuzeyron)) +- vlc: fix build [\#2841](https://github.com/kivy/python-for-android/pull/2841) ([T-Dynamos](https://github.com/T-Dynamos)) +- Optimize CI runs, by avoiding unnecessary rebuilds [\#2833](https://github.com/kivy/python-for-android/pull/2833) ([misl6](https://github.com/misl6)) +- Remove `pytz` recipe, as it's not needed anymore [\#2830](https://github.com/kivy/python-for-android/pull/2830) ([misl6](https://github.com/misl6)) +- Remove `dateutil` recipe, as it's not needed anymore [\#2829](https://github.com/kivy/python-for-android/pull/2829) ([misl6](https://github.com/misl6)) +- Removes `mysqldb` recipe as does not support Python 3 [\#2828](https://github.com/kivy/python-for-android/pull/2828) ([misl6](https://github.com/misl6)) +- Bump `actions/setup-python` and `actions/checkout` versions, as old ones are deprecated [\#2827](https://github.com/kivy/python-for-android/pull/2827) ([misl6](https://github.com/misl6)) +- Removes `Babel` recipe as it's not needed anymore. [\#2826](https://github.com/kivy/python-for-android/pull/2826) ([misl6](https://github.com/misl6)) +- Cffi update [\#2800](https://github.com/kivy/python-for-android/pull/2800) ([HyTurtle](https://github.com/HyTurtle)) +- Merge master into develop [\#2797](https://github.com/kivy/python-for-android/pull/2797) ([misl6](https://github.com/misl6)) +- Use build rather than pep517 for building [\#2784](https://github.com/kivy/python-for-android/pull/2784) ([s-t-e-v-e-n-k](https://github.com/s-t-e-v-e-n-k)) + ## [v2023.05.21](https://github.com/kivy/python-for-android/tree/v2023.05.21) [Full Changelog](https://github.com/kivy/python-for-android/compare/v2023.02.10...v2023.05.21) diff --git a/pythonforandroid/__init__.py b/pythonforandroid/__init__.py index ec30da5902..bcabd44859 100644 --- a/pythonforandroid/__init__.py +++ b/pythonforandroid/__init__.py @@ -1 +1 @@ -__version__ = '2023.05.21' +__version__ = '2023.09.16'