diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 1faaa18d70..c6c5f55c2f 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -15,9 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout python-for-android - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python 3.x - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.x - name: Run flake8 @@ -32,13 +32,13 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] + python-version: ['3.8', '3.9', '3.10', '3.11'] os: [ubuntu-latest, macOs-latest] steps: - name: Checkout python-for-android - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Tox tests @@ -53,8 +53,8 @@ jobs: parallel: true flag-name: run-${{ matrix.os }}-${{ matrix.python-version }} - ubuntu_build_apk: - name: Unit test apk [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ] + ubuntu_build: + name: Build test APP [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ] needs: [flake8] runs-on: ${{ matrix.runs_on }} continue-on-error: true @@ -68,36 +68,41 @@ jobs: target: testapps-with-scipy - name: webview target: testapps-webview + - name: service_library + target: testapps-service_library-aar steps: - name: Checkout python-for-android - uses: actions/checkout@v2 - # helps with GitHub runner getting out of space - - name: Free disk space - run: | - df -h - sudo swapoff -a - sudo rm -f /swapfile - sudo apt -y clean - docker rmi $(docker image ls -aq) - df -h - - name: Pull docker image - run: | - make docker/pull - - name: Build multi-arch apk Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86) - run: | - mkdir -p apks - make docker/run/make/with-artifact/apk/${{ matrix.bootstrap.target }} - - name: Rename apk artifact to include the build platform name - run: | - mv apks/${{ env.APK_ARTIFACT_FILENAME }} apks/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} - - name: Upload apk artifact - uses: actions/upload-artifact@v1 + uses: actions/checkout@v3 + - name: Build python-for-android docker image + run: | + docker build --tag=kivy/python-for-android . + - name: Build multi-arch ${{ matrix.bootstrap.target }} artifact with docker + run: | + docker run --name p4a-latest kivy/python-for-android make ${{ matrix.bootstrap.target }} + - name: Copy produced artifacts from docker container (*.apk, *.aab) + if: matrix.bootstrap.name != 'service_library' + run: | + mkdir -p dist + docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/${{ env.APK_ARTIFACT_FILENAME }} dist/ + docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/${{ env.AAB_ARTIFACT_FILENAME }} dist/ + - name: Copy produced artifacts from docker container (*.aar) + if: matrix.bootstrap.name == 'service_library' + run: | + mkdir -p dist + docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/${{ env.AAR_ARTIFACT_FILENAME }} dist/ + - name: Rename artifacts to include the build platform name (*.apk, *.aab, *.aar) + run: | + if [ -f dist/${{ env.APK_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.APK_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}; fi + if [ -f dist/${{ env.AAB_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.AAB_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }}; fi + if [ -f dist/${{ env.AAR_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.AAR_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAR_ARTIFACT_FILENAME }}; fi + - name: Upload artifacts + uses: actions/upload-artifact@v3 with: - name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} - path: apks/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} + name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-artifacts + path: dist - macos_build_apk: - name: Unit test apk [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ] + macos_build: + name: Build test APP [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ] needs: [flake8] runs-on: ${{ matrix.runs_on }} continue-on-error: true @@ -116,7 +121,7 @@ jobs: ANDROID_NDK_HOME: ${HOME}/.android/android-ndk steps: - name: Checkout python-for-android - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install python-for-android run: | source ci/osx_ci.sh @@ -137,146 +142,21 @@ jobs: source ci/osx_ci.sh arm64_set_path_and_python_version 3.9.7 make ${{ matrix.bootstrap.target }} - - name: Rename apk artifact to include the build platform name + - name: Copy produced artifacts into dist/ (*.apk, *.aab) run: | - mv testapps/on_device_unit_tests/${{ env.APK_ARTIFACT_FILENAME }} ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} - - name: Upload apk artifact - uses: actions/upload-artifact@v1 - with: - name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} - path: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }} - - ubuntu_build_aab: - name: Unit test aab [ ${{ matrix.runs_on }} ] - needs: [flake8] - runs-on: ${{ matrix.runs_on }} - continue-on-error: true - strategy: - matrix: - runs_on: [ubuntu-latest] - bootstrap: - - name: sdl2 - target: testapps-with-numpy-aab - - name: webview - target: testapps-webview-aab - steps: - - name: Checkout python-for-android - uses: actions/checkout@v2 - # helps with GitHub runner getting out of space - - name: Free disk space - run: | - df -h - sudo swapoff -a - sudo rm -f /swapfile - sudo apt -y clean - docker rmi $(docker image ls -aq) - df -h - - name: Pull docker image - run: | - make docker/pull - - name: Build Android App Bundle Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86) - run: | - mkdir -p aabs - make docker/run/make/with-artifact/aab/${{ matrix.bootstrap.target }} - - name: Rename artifact to include the build platform name - run: | - mv aabs/${{ env.AAB_ARTIFACT_FILENAME }} aabs/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAB_ARTIFACT_FILENAME }} - - name: Upload apk artifact - uses: actions/upload-artifact@v1 - with: - name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAB_ARTIFACT_FILENAME }} - path: aabs/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAB_ARTIFACT_FILENAME }} - - - ubuntu_build_aar: - name: Unit test aar [ ${{ matrix.runs_on }} ] - needs: [flake8] - runs-on: ${{ matrix.runs_on }} - continue-on-error: true - strategy: - matrix: - runs_on: [ubuntu-latest] - bootstrap: - - name: service_library - target: testapps-service_library-aar - steps: - - name: Checkout python-for-android - uses: actions/checkout@v2 - # helps with GitHub runner getting out of space - - name: Free disk space - run: | - df -h - sudo swapoff -a - sudo rm -f /swapfile - sudo apt -y clean - docker rmi $(docker image ls -aq) - df -h - - name: Pull docker image - run: | - make docker/pull - - name: Build Android AAR Python 3 (arm64-v8a) - run: | - mkdir -p aars - make docker/run/make/with-artifact/aar/${{ matrix.bootstrap.target }} - - name: Rename artifact to include the build platform name - run: | - mv aars/${{ env.AAR_ARTIFACT_FILENAME }} aars/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAR_ARTIFACT_FILENAME }} - - name: Upload aar artifact - uses: actions/upload-artifact@v1 - with: - name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAR_ARTIFACT_FILENAME }} - path: aars/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAR_ARTIFACT_FILENAME }} - - - macos_build_aab: - name: Unit test aab [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ] - needs: [flake8] - runs-on: ${{ matrix.runs_on }} - continue-on-error: true - strategy: - matrix: - runs_on: [macos-latest, apple-silicon-m1] - bootstrap: - - name: sdl2 - target: testapps-with-numpy-aab - - name: webview - target: testapps-webview-aab - env: - ANDROID_HOME: ${HOME}/.android - ANDROID_SDK_ROOT: ${HOME}/.android/android-sdk - ANDROID_SDK_HOME: ${HOME}/.android/android-sdk - ANDROID_NDK_HOME: ${HOME}/.android/android-ndk - steps: - - name: Checkout python-for-android - uses: actions/checkout@v2 - - name: Install python-for-android + mkdir -p dist + cp testapps/on_device_unit_tests/*.apk dist/ + cp testapps/on_device_unit_tests/*.aab dist/ + ls -l dist/ + - name: Rename artifacts to include the build platform name (*.apk, *.aab) run: | - source ci/osx_ci.sh - arm64_set_path_and_python_version 3.9.7 - python3 -m pip install -e . - - name: Install prerequisites via pythonforandroid/prerequisites.py (Experimental) - run: | - source ci/osx_ci.sh - arm64_set_path_and_python_version 3.9.7 - python3 pythonforandroid/prerequisites.py - - name: Install dependencies (Legacy) - run: | - source ci/osx_ci.sh - arm64_set_path_and_python_version 3.9.7 - make --file ci/makefiles/osx.mk - - name: Build multi-arch sdl2 aab Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86) - run: | - source ci/osx_ci.sh - arm64_set_path_and_python_version 3.9.7 - make ${{ matrix.bootstrap.target }} - - name: Rename sdl2 artifact to include the build platform name - run: | - mv testapps/on_device_unit_tests/${{ env.AAB_ARTIFACT_FILENAME }} ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }} - - name: Upload sdl2 apk artifact - uses: actions/upload-artifact@v1 + if [ -f dist/${{ env.APK_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.APK_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}; fi + if [ -f dist/${{ env.AAB_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.AAB_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }}; fi + - name: Upload artifacts + uses: actions/upload-artifact@v3 with: - name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }} - path: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }} + name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-artifacts + path: dist ubuntu_rebuild_updated_recipes: name: Test updated recipes for arch ${{ matrix.android_arch }} [ ubuntu-latest ] @@ -289,8 +169,8 @@ jobs: env: REBUILD_UPDATED_RECIPES_EXTRA_ARGS: --arch=${{ matrix.android_arch }} steps: - - name: Checkout python-for-android - uses: actions/checkout@v2 + - name: Checkout python-for-android (all-history) + uses: actions/checkout@v3 with: fetch-depth: 0 # helps with GitHub runner getting out of space @@ -325,8 +205,8 @@ jobs: ANDROID_NDK_HOME: ${HOME}/.android/android-ndk REBUILD_UPDATED_RECIPES_EXTRA_ARGS: --arch=${{ matrix.android_arch }} steps: - - name: Checkout python-for-android - uses: actions/checkout@v2 + - name: Checkout python-for-android (all-history) + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Install python-for-android diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index b5bb5fbb6c..a66a30567e 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -5,11 +5,11 @@ jobs: pypi_release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python 3.x - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: 3.x + python-version: '3.x' - name: Install dependencies run: | python -m pip install --upgrade setuptools wheel twine diff --git a/CHANGELOG.md b/CHANGELOG.md index 98288f265d..2ce4711cbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,65 @@ # Changelog +## [v2023.09.16](https://github.com/kivy/python-for-android/tree/v2023.09.16) + +[Full Changelog](https://github.com/kivy/python-for-android/compare/v2023.05.21...v2023.09.16) + +**Closed issues:** + +- Microphone And Audio permissions doesn't work [\#2889](https://github.com/kivy/python-for-android/issues/2889) +- Error with Scipy [\#2883](https://github.com/kivy/python-for-android/issues/2883) +- Download failed \( Downloading sqlite3 from https://www.sqlite.org/2021/sqlite-amalgamation-3350500.zip \) during buildozer -v andriod debug [\#2881](https://github.com/kivy/python-for-android/issues/2881) +- ONNXruntime lib open failed due to 64-bit [\#2880](https://github.com/kivy/python-for-android/issues/2880) +- Question: how to write a recipe to download and install\(coz build fail\) a wheel package directly? [\#2878](https://github.com/kivy/python-for-android/issues/2878) +- Impossible to install python for android [\#2867](https://github.com/kivy/python-for-android/issues/2867) +- scipy with kivy [\#2861](https://github.com/kivy/python-for-android/issues/2861) +- False positve parser warning. [\#2856](https://github.com/kivy/python-for-android/issues/2856) +- After successfully building app, its crashes with this error, using firebase-admin [\#2854](https://github.com/kivy/python-for-android/issues/2854) +- Kivy [\#2837](https://github.com/kivy/python-for-android/issues/2837) +- not installing on windows 10 [\#2836](https://github.com/kivy/python-for-android/issues/2836) +- Could not find com.android.tools.build:gradle:7.2.0. in android studio [\#2834](https://github.com/kivy/python-for-android/issues/2834) +- vlc recipe build fail [\#2822](https://github.com/kivy/python-for-android/issues/2822) +- mysqldb recipe build fail [\#2813](https://github.com/kivy/python-for-android/issues/2813) +- babel recipe build fail [\#2803](https://github.com/kivy/python-for-android/issues/2803) +- Python 3.10 cffi build fails [\#2799](https://github.com/kivy/python-for-android/issues/2799) +- \[Recipe request\] OpenColorIO & rawpy \(libraw\) [\#2789](https://github.com/kivy/python-for-android/issues/2789) +- Longer app load time, bigger apk size and unnecessary logs [\#2704](https://github.com/kivy/python-for-android/issues/2704) +- -fp-model argument not found, directory strict not found [\#2329](https://github.com/kivy/python-for-android/issues/2329) +- Kivy crashes on Android: ImportError: dlopen failed: library "libpython3.7m.so" not found [\#2237](https://github.com/kivy/python-for-android/issues/2237) +- pyconfig.h issue [\#2074](https://github.com/kivy/python-for-android/issues/2074) + +**Merged pull requests:** + +- Use Python's touch\(\) rather than shelling out. [\#2886](https://github.com/kivy/python-for-android/pull/2886) ([Julian-O](https://github.com/Julian-O)) +- Standardise on move [\#2884](https://github.com/kivy/python-for-android/pull/2884) ([Julian-O](https://github.com/Julian-O)) +- Remove deprecated FlatDir in Gradle template [\#2876](https://github.com/kivy/python-for-android/pull/2876) ([Julian-O](https://github.com/Julian-O)) +- :rotating\_light: linter fixes [\#2874](https://github.com/kivy/python-for-android/pull/2874) ([AndreMiras](https://github.com/AndreMiras)) +- Standardise ensure\_dir and rmdir [\#2871](https://github.com/kivy/python-for-android/pull/2871) ([Julian-O](https://github.com/Julian-O)) +- Correct check for --sdk option [\#2870](https://github.com/kivy/python-for-android/pull/2870) ([Julian-O](https://github.com/Julian-O)) +- Python versions: Update documentation & CI testing [\#2869](https://github.com/kivy/python-for-android/pull/2869) ([Julian-O](https://github.com/Julian-O)) +- Patching cleanup [\#2868](https://github.com/kivy/python-for-android/pull/2868) ([Julian-O](https://github.com/Julian-O)) +- Factor out dependency checking. Use modern version handling [\#2866](https://github.com/kivy/python-for-android/pull/2866) ([Julian-O](https://github.com/Julian-O)) +- `build_platform` should be all-lowercase [\#2864](https://github.com/kivy/python-for-android/pull/2864) ([misl6](https://github.com/misl6)) +- Fix simple typos in comments [\#2863](https://github.com/kivy/python-for-android/pull/2863) ([Julian-O](https://github.com/Julian-O)) +- Use a pinned version of `Cython` for now, as most of the recipes are incompatible with `Cython==3.x.x` [\#2862](https://github.com/kivy/python-for-android/pull/2862) ([misl6](https://github.com/misl6)) +- Docs: Fix typos and updated command to build apk - README [\#2860](https://github.com/kivy/python-for-android/pull/2860) ([kulothunganug](https://github.com/kulothunganug)) +- Docs: Fix code string - quickstart.rst [\#2859](https://github.com/kivy/python-for-android/pull/2859) ([kulothunganug](https://github.com/kulothunganug)) +- Automatically generate required pre-requisites [\#2858](https://github.com/kivy/python-for-android/pull/2858) ([Julian-O](https://github.com/Julian-O)) +- Use platform.uname instead of os.uname [\#2857](https://github.com/kivy/python-for-android/pull/2857) ([Julian-O](https://github.com/Julian-O)) +- Bump `kivy` version to `2.2.1` [\#2855](https://github.com/kivy/python-for-android/pull/2855) ([misl6](https://github.com/misl6)) +- Correct sys\_platform [\#2852](https://github.com/kivy/python-for-android/pull/2852) ([Julian-O](https://github.com/Julian-O)) +- Changed the url to use https as http fails [\#2846](https://github.com/kivy/python-for-android/pull/2846) ([kuzeyron](https://github.com/kuzeyron)) +- vlc: fix build [\#2841](https://github.com/kivy/python-for-android/pull/2841) ([T-Dynamos](https://github.com/T-Dynamos)) +- Optimize CI runs, by avoiding unnecessary rebuilds [\#2833](https://github.com/kivy/python-for-android/pull/2833) ([misl6](https://github.com/misl6)) +- Remove `pytz` recipe, as it's not needed anymore [\#2830](https://github.com/kivy/python-for-android/pull/2830) ([misl6](https://github.com/misl6)) +- Remove `dateutil` recipe, as it's not needed anymore [\#2829](https://github.com/kivy/python-for-android/pull/2829) ([misl6](https://github.com/misl6)) +- Removes `mysqldb` recipe as does not support Python 3 [\#2828](https://github.com/kivy/python-for-android/pull/2828) ([misl6](https://github.com/misl6)) +- Bump `actions/setup-python` and `actions/checkout` versions, as old ones are deprecated [\#2827](https://github.com/kivy/python-for-android/pull/2827) ([misl6](https://github.com/misl6)) +- Removes `Babel` recipe as it's not needed anymore. [\#2826](https://github.com/kivy/python-for-android/pull/2826) ([misl6](https://github.com/misl6)) +- Cffi update [\#2800](https://github.com/kivy/python-for-android/pull/2800) ([HyTurtle](https://github.com/HyTurtle)) +- Merge master into develop [\#2797](https://github.com/kivy/python-for-android/pull/2797) ([misl6](https://github.com/misl6)) +- Use build rather than pep517 for building [\#2784](https://github.com/kivy/python-for-android/pull/2784) ([s-t-e-v-e-n-k](https://github.com/s-t-e-v-e-n-k)) + ## [v2023.05.21](https://github.com/kivy/python-for-android/tree/v2023.05.21) [Full Changelog](https://github.com/kivy/python-for-android/compare/v2023.02.10...v2023.05.21) diff --git a/Makefile b/Makefile index ea5adadcd8..9747c6c43d 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ all: virtualenv $(VIRTUAL_ENV): python3 -m venv $(VIRTUAL_ENV) - $(PIP) install Cython + $(PIP) install Cython==0.29.36 $(PIP) install -e . virtualenv: $(VIRTUAL_ENV) @@ -36,47 +36,51 @@ rebuild_updated_recipes: virtualenv ANDROID_SDK_HOME=$(ANDROID_SDK_HOME) ANDROID_NDK_HOME=$(ANDROID_NDK_HOME) \ $(PYTHON) ci/rebuild_updated_recipes.py $(REBUILD_UPDATED_RECIPES_EXTRA_ARGS) -testapps-with-numpy: virtualenv +testapps-with-numpy: testapps-with-numpy/debug/apk testapps-with-numpy/release/aab + +# testapps-with-numpy/MODE/ARTIFACT +testapps-with-numpy/%: virtualenv + $(eval MODE := $(word 2, $(subst /, ,$@))) + $(eval ARTIFACT := $(word 3, $(subst /, ,$@))) + @echo Building testapps-with-numpy for $(MODE) mode and $(ARTIFACT) artifact . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ - python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + python setup.py $(ARTIFACT) --$(MODE) --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,urllib3,chardet,idna,sqlite3,setuptools,numpy \ --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 \ --permission "(name=android.permission.WRITE_EXTERNAL_STORAGE;maxSdkVersion=18)" --permission "(name=android.permission.INTERNET)" -testapps-with-scipy: virtualenv +testapps-with-scipy: testapps-with-scipy/debug/apk testapps-with-scipy/release/aab + +# testapps-with-scipy/MODE/ARTIFACT +testapps-with-scipy/%: virtualenv + $(eval MODE := $(word 2, $(subst /, ,$@))) + $(eval ARTIFACT := $(word 3, $(subst /, ,$@))) + @echo Building testapps-with-scipy for $(MODE) mode and $(ARTIFACT) artifact . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ export LEGACY_NDK=$(ANDROID_NDK_HOME_LEGACY) && \ - python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + python setup.py $(ARTIFACT) --$(MODE) --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ --requirements python3,scipy,kivy \ --arch=armeabi-v7a --arch=arm64-v8a -testapps-with-numpy-aab: virtualenv - . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ - python setup.py aab --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ - --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,urllib3,chardet,idna,sqlite3,setuptools,numpy \ - --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 --release \ - --permission "(name=android.permission.WRITE_EXTERNAL_STORAGE;maxSdkVersion=18)" --permission "(name=android.permission.INTERNET)" +testapps-webview: testapps-webview/debug/apk testapps-webview/release/aab -testapps-service_library-aar: virtualenv +# testapps-webview/MODE/ARTIFACT +testapps-webview/%: virtualenv + $(eval MODE := $(word 2, $(subst /, ,$@))) + $(eval ARTIFACT := $(word 3, $(subst /, ,$@))) + @echo Building testapps-webview for $(MODE) mode and $(ARTIFACT) artifact . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ - python setup.py aar --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ - --bootstrap service_library \ - --requirements python3 \ - --arch=arm64-v8a --arch=x86 --release - -testapps-webview: virtualenv - . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ - python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + python setup.py $(ARTIFACT) --$(MODE) --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ --bootstrap webview \ --requirements sqlite3,libffi,openssl,pyjnius,flask,python3,genericndkbuild \ --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 -testapps-webview-aab: virtualenv +testapps-service_library-aar: virtualenv . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ - python setup.py aab --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ - --bootstrap webview \ - --requirements sqlite3,libffi,openssl,pyjnius,flask,python3,genericndkbuild \ - --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 --release + python setup.py aar --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + --bootstrap service_library \ + --requirements python3 \ + --arch=arm64-v8a --arch=x86 --release testapps/%: virtualenv $(eval $@_APP_ARCH := $(shell basename $*)) @@ -106,21 +110,6 @@ docker/run/test: docker/build docker/run/command: docker/build docker run --rm --env-file=.env $(DOCKER_IMAGE) /bin/sh -c "$(COMMAND)" -docker/run/make/with-artifact/apk/%: docker/build - docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $* - docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-debug-1.1.apk ./apks - docker rm -fv p4a-latest - -docker/run/make/with-artifact/aar/%: docker/build - docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $* - docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-release-1.1.aar ./aars - docker rm -fv p4a-latest - -docker/run/make/with-artifact/aab/%: docker/build - docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $* - docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-release-1.1.aab ./aabs - docker rm -fv p4a-latest - docker/run/make/rebuild_updated_recipes: docker/build docker run --name p4a-latest -e REBUILD_UPDATED_RECIPES_EXTRA_ARGS --env-file=.env $(DOCKER_IMAGE) make rebuild_updated_recipes diff --git a/README.md b/README.md index 269673a0b6..764ea7b7a7 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ other API levels may not work. With everything installed, build an APK with SDL2 with e.g.: - p4a apk --requirements=kivy --private /home/username/devel/planewave_frozen/ --package=net.inclem.planewavessdl2 --name="planewavessdl2" --version=0.5 --bootstrap=sdl2 + p4a apk --private PATH_TO_YOUR_APP_CODE --package=org.example.myapp --name "My application" --version 0.1 --bootstrap=sdl2 --requirements=python3,kivy **If you need to deploy your app on Google Play, Android App Bundle (aab) is required since 1 August 2021:** @@ -96,7 +96,7 @@ Please refer to the LICENSE file. In 2015 these tools were rewritten to provide a new, easier-to-use and easier-to-extend interface. If you'd like to browse the old toolchain, its -status is recorded for posterity at at +status is recorded for posterity at https://github.com/kivy/python-for-android/tree/old_toolchain. In the last quarter of 2018 the python recipes were changed. The diff --git a/ci/constants.py b/ci/constants.py index 021e64a74a..6a81a23405 100644 --- a/ci/constants.py +++ b/ci/constants.py @@ -33,6 +33,13 @@ class TargetPython(Enum): 'twisted', # genericndkbuild is incompatible with sdl2 (which is build by default when targeting sdl2 bootstrap) 'genericndkbuild', + # libmysqlclient gives a linker failure (See issue #2808) + 'libmysqlclient', + # boost gives errors (requires numpy? syntax error in .jam?) + 'boost', + # libtorrent gives errors (requires boost. Also, see issue #2809, to start with) + 'libtorrent', + ]) BROKEN_RECIPES = { diff --git a/doc/source/buildoptions.rst b/doc/source/buildoptions.rst index 4acefa1885..8ad067b205 100644 --- a/doc/source/buildoptions.rst +++ b/doc/source/buildoptions.rst @@ -8,8 +8,8 @@ This page contains instructions for using different build options. Python versions --------------- -python-for-android supports using Python 3.7 or higher. To explicitly select a Python -version in your requirements, use e.g. ``--requirements=python3==3.7.1,hostpython3==3.7.1``. +python-for-android supports using Python 3.8 or higher. To explicitly select a Python +version in your requirements, use e.g. ``--requirements=python3==3.10.11,hostpython3==3.10.11``. The last python-for-android version supporting Python2 was `v2019.10.06 `__ diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 6797bcadc6..2e3fe21108 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -232,7 +232,7 @@ Exporting the Android App Bundle (aab) for distributing it on Google Play Starting from August 2021 for new apps and from November 2021 for updates to existings apps, Google Play Console will require the Android App Bundle instead of the long lived apk. -python-for-android handles by itself the needed work to accomplish the new requirements: +python-for-android handles by itself the needed work to accomplish the new requirements:: p4a aab --private $HOME/code/myapp --package=org.example.myapp --name="My App" --version 0.1 --bootstrap=sdl2 --requirements=python3,kivy --arch=arm64-v8a --arch=armeabi-v7a --release diff --git a/pythonforandroid/__init__.py b/pythonforandroid/__init__.py index ec30da5902..bcabd44859 100644 --- a/pythonforandroid/__init__.py +++ b/pythonforandroid/__init__.py @@ -1 +1 @@ -__version__ = '2023.05.21' +__version__ = '2023.09.16' diff --git a/pythonforandroid/bdistapk.py b/pythonforandroid/bdistapk.py index 575e0e17e5..b4467b9f91 100644 --- a/pythonforandroid/bdistapk.py +++ b/pythonforandroid/bdistapk.py @@ -1,10 +1,10 @@ +from glob import glob +from os.path import realpath, join, dirname, curdir, basename, split from setuptools import Command - +from shutil import copyfile import sys -from os.path import realpath, join, exists, dirname, curdir, basename, split -from os import makedirs -from glob import glob -from shutil import rmtree, copyfile + +from pythonforandroid.util import rmdir, ensure_dir def argv_contains(t): @@ -90,9 +90,8 @@ def prepare_build_dir(self): 'that.') bdist_dir = 'build/bdist.android-{}'.format(self.arch) - if exists(bdist_dir): - rmtree(bdist_dir) - makedirs(bdist_dir) + rmdir(bdist_dir) + ensure_dir(bdist_dir) globs = [] for directory, patterns in self.distribution.package_data.items(): @@ -107,8 +106,7 @@ def prepare_build_dir(self): if not argv_contains('--launcher'): for filen in filens: new_dir = join(bdist_dir, dirname(filen)) - if not exists(new_dir): - makedirs(new_dir) + ensure_dir(new_dir) print('Including {}'.format(filen)) copyfile(filen, join(bdist_dir, filen)) if basename(filen) in ('main.py', 'main.pyc'): diff --git a/pythonforandroid/bootstrap.py b/pythonforandroid/bootstrap.py index 0a5225e526..8bbbcf0eb6 100755 --- a/pythonforandroid/bootstrap.py +++ b/pythonforandroid/bootstrap.py @@ -10,7 +10,8 @@ from pythonforandroid.logger import (shprint, info, logger, debug) from pythonforandroid.util import ( - current_directory, ensure_dir, temp_directory, BuildInterruptingException) + current_directory, ensure_dir, temp_directory, BuildInterruptingException, + rmdir, move) from pythonforandroid.recipe import Recipe @@ -70,7 +71,6 @@ class Bootstrap: '''An Android project template, containing recipe stuff for compilation and templated fields for APK info. ''' - name = '' jni_subdir = '/jni' ctx = None @@ -395,9 +395,9 @@ def fry_eggs(self, sitepackages): if isdir(rd) and d.endswith('.egg'): info(' ' + d) files = [join(rd, f) for f in listdir(rd) if f != 'EGG-INFO'] - if files: - shprint(sh.mv, '-t', sitepackages, *files) - shprint(sh.rm, '-rf', d) + for f in files: + move(f, sitepackages) + rmdir(d) def expand_dependencies(recipes, ctx): diff --git a/pythonforandroid/bootstraps/common/build/build.py b/pythonforandroid/bootstraps/common/build/build.py index 3aaf51d009..0b6b9832f0 100644 --- a/pythonforandroid/bootstraps/common/build/build.py +++ b/pythonforandroid/bootstraps/common/build/build.py @@ -21,6 +21,8 @@ from fnmatch import fnmatch import jinja2 +from pythonforandroid.util import rmdir, ensure_dir + def get_dist_info_for(key, error_if_missing=True): try: @@ -93,11 +95,6 @@ def get_bootstrap_name(): DEFAULT_PYTHON_SERVICE_JAVA_CLASS = 'org.kivy.android.PythonService' -def ensure_dir(path): - if not exists(path): - makedirs(path) - - def render(template, dest, **kwargs): '''Using jinja2, render `template` to the filename `dest`, supplying the @@ -241,7 +238,7 @@ def make_package(args): assets_dir = "src/main/assets" # Delete the old assets. - shutil.rmtree(assets_dir, ignore_errors=True) + rmdir(assets_dir, ignore_errors=True) ensure_dir(assets_dir) # Add extra environment variable file into tar-able directory: @@ -290,7 +287,7 @@ def make_package(args): not exists( join(main_py_only_dir, dir_path) )): - os.mkdir(join(main_py_only_dir, dir_path)) + ensure_dir(join(main_py_only_dir, dir_path)) # Copy actual file: shutil.copyfile( join(args.private, variant), @@ -328,17 +325,17 @@ def make_package(args): ) finally: for directory in _temp_dirs_to_clean: - shutil.rmtree(directory) + rmdir(directory) # Remove extra env vars tar-able directory: - shutil.rmtree(env_vars_tarpath) + rmdir(env_vars_tarpath) # Prepare some variables for templating process res_dir = "src/main/res" res_dir_initial = "src/res_initial" # make res_dir stateless if exists(res_dir_initial): - shutil.rmtree(res_dir, ignore_errors=True) + rmdir(res_dir, ignore_errors=True) shutil.copytree(res_dir_initial, res_dir) else: shutil.copytree(res_dir, res_dir_initial) @@ -1006,7 +1003,7 @@ def _read_configuration(): print('Billing not yet supported!') sys.exit(1) - if args.sdk_version == -1: + if args.sdk_version != -1: print('WARNING: Received a --sdk argument, but this argument is ' 'deprecated and does nothing.') args.sdk_version = -1 # ensure it is not used diff --git a/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle b/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle index bb000393a4..ce105736d3 100644 --- a/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle +++ b/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle @@ -16,9 +16,6 @@ allprojects { {%- for repo in args.gradle_repositories %} {{repo}} {%- endfor %} - flatDir { - dirs 'libs' - } } } diff --git a/pythonforandroid/bootstraps/sdl2/__init__.py b/pythonforandroid/bootstraps/sdl2/__init__.py index 662d43c0ef..9334724a33 100644 --- a/pythonforandroid/bootstraps/sdl2/__init__.py +++ b/pythonforandroid/bootstraps/sdl2/__init__.py @@ -1,9 +1,11 @@ -from pythonforandroid.toolchain import ( - Bootstrap, shprint, current_directory, info, info_main) -from pythonforandroid.util import ensure_dir from os.path import join + import sh +from pythonforandroid.toolchain import ( + Bootstrap, shprint, current_directory, info, info_main) +from pythonforandroid.util import ensure_dir, rmdir + class SDL2GradleBootstrap(Bootstrap): name = 'sdl2' @@ -15,8 +17,8 @@ class SDL2GradleBootstrap(Bootstrap): def assemble_distribution(self): info_main("# Creating Android project ({})".format(self.name)) + rmdir(self.dist_dir) info("Copying SDL2/gradle build") - shprint(sh.rm, "-rf", self.dist_dir) shprint(sh.cp, "-r", self.build_dir, self.dist_dir) # either the build use environment variable (ANDROID_HOME) diff --git a/pythonforandroid/bootstraps/service_only/__init__.py b/pythonforandroid/bootstraps/service_only/__init__.py index b9e000c012..4f0d6cf20b 100644 --- a/pythonforandroid/bootstraps/service_only/__init__.py +++ b/pythonforandroid/bootstraps/service_only/__init__.py @@ -2,7 +2,7 @@ from os.path import join from pythonforandroid.toolchain import ( Bootstrap, current_directory, info, info_main, shprint) -from pythonforandroid.util import ensure_dir +from pythonforandroid.util import ensure_dir, rmdir class ServiceOnlyBootstrap(Bootstrap): @@ -18,7 +18,7 @@ def assemble_distribution(self): self.name)) info('This currently just copies the build stuff straight from the build dir.') - shprint(sh.rm, '-rf', self.dist_dir) + rmdir(self.dist_dir) shprint(sh.cp, '-r', self.build_dir, self.dist_dir) with current_directory(self.dist_dir): with open('local.properties', 'w') as fileh: diff --git a/pythonforandroid/bootstraps/webview/__init__.py b/pythonforandroid/bootstraps/webview/__init__.py index da33ac72d5..7604ed3b84 100644 --- a/pythonforandroid/bootstraps/webview/__init__.py +++ b/pythonforandroid/bootstraps/webview/__init__.py @@ -1,8 +1,10 @@ -from pythonforandroid.toolchain import Bootstrap, current_directory, info, info_main, shprint -from pythonforandroid.util import ensure_dir from os.path import join + import sh +from pythonforandroid.toolchain import Bootstrap, current_directory, info, info_main, shprint +from pythonforandroid.util import ensure_dir, rmdir + class WebViewBootstrap(Bootstrap): name = 'webview' @@ -15,7 +17,7 @@ def assemble_distribution(self): info_main('# Creating Android project from build and {} bootstrap'.format( self.name)) - shprint(sh.rm, '-rf', self.dist_dir) + rmdir(self.dist_dir) shprint(sh.cp, '-r', self.build_dir, self.dist_dir) with current_directory(self.dist_dir): with open('local.properties', 'w') as fileh: diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index 645b368d00..4777e2f934 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -1,28 +1,29 @@ +from contextlib import suppress +import copy +import glob +import os +from os import environ from os.path import ( abspath, join, realpath, dirname, expanduser, exists ) -from os import environ -import copy -import os -import glob import re -import sh import shutil import subprocess -from contextlib import suppress -from pythonforandroid.util import ( - current_directory, ensure_dir, - BuildInterruptingException, -) -from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint) +import sh + +from pythonforandroid.androidndk import AndroidNDK from pythonforandroid.archs import ArchARM, ArchARMv7_a, ArchAarch_64, Archx86, Archx86_64 +from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint) from pythonforandroid.pythonpackage import get_package_name from pythonforandroid.recipe import CythonRecipe, Recipe from pythonforandroid.recommendations import ( check_ndk_version, check_target_api, check_ndk_api, RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API) -from pythonforandroid.androidndk import AndroidNDK +from pythonforandroid.util import ( + current_directory, ensure_dir, + BuildInterruptingException, rmdir +) def get_targets(sdk_dir): @@ -77,11 +78,6 @@ class Context: # the Android project folder where everything ends up dist_dir = None - # where Android libs are cached after build - # but before being placed in dists - libs_dir = None - aars_dir = None - # Whether setup.py or similar should be used if present: use_setup_py = False @@ -109,6 +105,10 @@ def templates_dir(self): @property def libs_dir(self): + """ + where Android libs are cached after build + but before being placed in dists + """ # Was previously hardcoded as self.build_dir/libs directory = join(self.build_dir, 'libs_collections', self.bootstrap.distribution.name) @@ -642,7 +642,7 @@ def run_setuppy_install(ctx, project_dir, env=None, arch=None): for f in set(copied_over_contents + new_venv_additions): full_path = os.path.join(venv_site_packages_dir, f) if os.path.isdir(full_path): - shutil.rmtree(full_path) + rmdir(full_path) else: os.remove(full_path) finally: diff --git a/pythonforandroid/checkdependencies.py b/pythonforandroid/checkdependencies.py new file mode 100644 index 0000000000..c53115de7a --- /dev/null +++ b/pythonforandroid/checkdependencies.py @@ -0,0 +1,70 @@ +from importlib import import_module +from os import environ +import sys + +from packaging.version import Version + +from pythonforandroid.prerequisites import ( + check_and_install_default_prerequisites, +) + + +def check_python_dependencies(): + """ + Check if the Python requirements are installed. This must appears + before other imports because otherwise they're imported elsewhere. + + Using the ok check instead of failing immediately so that all + errors are printed at once. + """ + + ok = True + + modules = [("colorama", "0.3.3"), "appdirs", ("sh", "1.10"), "jinja2"] + + for module in modules: + if isinstance(module, tuple): + module, version = module + else: + version = None + + try: + import_module(module) + except ImportError: + if version is None: + print( + "ERROR: The {} Python module could not be found, please " + "install it.".format(module) + ) + ok = False + else: + print( + "ERROR: The {} Python module could not be found, " + "please install version {} or higher".format( + module, version + ) + ) + ok = False + else: + if version is None: + continue + try: + cur_ver = sys.modules[module].__version__ + except AttributeError: # this is sometimes not available + continue + if Version(cur_ver) < Version(version): + print( + "ERROR: {} version is {}, but python-for-android needs " + "at least {}.".format(module, cur_ver, version) + ) + ok = False + + if not ok: + print("python-for-android is exiting due to the errors logged above") + exit(1) + + +def check(): + if not environ.get("SKIP_PREREQUISITES_CHECK", "0") == "1": + check_and_install_default_prerequisites() + check_python_dependencies() diff --git a/pythonforandroid/distribution.py b/pythonforandroid/distribution.py index 2b1f1a3f5e..c878e0ea87 100644 --- a/pythonforandroid/distribution.py +++ b/pythonforandroid/distribution.py @@ -1,10 +1,11 @@ -from os.path import exists, join -import glob import json +import glob +from os.path import exists, join -from pythonforandroid.logger import (debug, info, info_notify, warning, Err_Style, Err_Fore) -from pythonforandroid.util import current_directory, BuildInterruptingException -from shutil import rmtree +from pythonforandroid.logger import ( + debug, info, info_notify, warning, Err_Style, Err_Fore) +from pythonforandroid.util import ( + current_directory, BuildInterruptingException, rmdir) class Distribution: @@ -201,7 +202,7 @@ def folder_exists(self): return exists(self.dist_dir) def delete(self): - rmtree(self.dist_dir) + rmdir(self.dist_dir) @classmethod def get_distributions(cls, ctx, extra_dist_dirs=[]): diff --git a/pythonforandroid/graph.py b/pythonforandroid/graph.py index bdaca4349c..4edb8f4c90 100644 --- a/pythonforandroid/graph.py +++ b/pythonforandroid/graph.py @@ -45,7 +45,7 @@ def get_dependency_tuple_list_for_recipe(recipe, blacklist=None): """ if blacklist is None: blacklist = set() - assert type(blacklist) == set + assert type(blacklist) is set if recipe.depends is None: dependencies = [] else: @@ -160,7 +160,7 @@ def obvious_conflict_checker(ctx, name_tuples, blacklist=None): current_to_be_added = list(to_be_added) to_be_added = [] for (added_tuple, adding_recipe) in current_to_be_added: - assert type(added_tuple) == tuple + assert type(added_tuple) is tuple if len(added_tuple) > 1: # No obvious commitment in what to add, don't check it itself # but throw it into deps for later comparing against diff --git a/pythonforandroid/patching.py b/pythonforandroid/patching.py index 96a3b273bb..1e143cef90 100644 --- a/pythonforandroid/patching.py +++ b/pythonforandroid/patching.py @@ -1,89 +1,178 @@ -from os import uname -from distutils.version import LooseVersion +""" + Helper functions for recipes. + Recipes must supply a list of patches. -def check_all(*callables): - def check(**kwargs): - return all(c(**kwargs) for c in callables) - return check + Patches consist of a filename and an optional conditional, which is + any function of the form: + def patch_check(arch: string, recipe : Recipe) -> bool + This library provides some helpful conditionals and mechanisms to + join multiple conditionals. -def check_any(*callables): - def check(**kwargs): - return any(c(**kwargs) for c in callables) - return check + Example: + patches = [ + ("linux_or_darwin_only.patch", + check_any(is_linux, is_darwin), + ("recent_android_API.patch", + is_apt_gte(27)), + ] +""" +from platform import uname +from packaging.version import Version + + +# Platform checks def is_platform(platform): - def is_x(**kwargs): - return uname()[0] == platform - return is_x + """ + Returns true if the host platform matches the parameter given. + """ + def check(arch, recipe): + return uname().system.lower() == platform.lower() + + return check -is_linux = is_platform('Linux') -is_darwin = is_platform('Darwin') + +is_linux = is_platform("Linux") +is_darwin = is_platform("Darwin") +is_windows = is_platform("Windows") def is_arch(xarch): - def is_x(arch, **kwargs): + """ + Returns true if the target architecture platform matches the parameter + given. + """ + + def check(arch): return arch.arch == xarch - return is_x + return check + + +# Android API comparisons: +# Return true if the Android API level being targeted +# is equal (or >, >=, <, <= as appropriate) the given parameter + + +def is_api(apiver: int): + def check(arch, recipe): + return recipe.ctx.android_api == apiver + + return check -def is_api_gt(apiver): - def is_x(recipe, **kwargs): + +def is_api_gt(apiver: int): + def check(arch, recipe): return recipe.ctx.android_api > apiver - return is_x + + return check -def is_api_gte(apiver): - def is_x(recipe, **kwargs): +def is_api_gte(apiver: int): + def check(arch, recipe): return recipe.ctx.android_api >= apiver - return is_x + + return check -def is_api_lt(apiver): - def is_x(recipe, **kwargs): +def is_api_lt(apiver: int): + def check(arch, recipe): return recipe.ctx.android_api < apiver - return is_x + + return check -def is_api_lte(apiver): - def is_x(recipe, **kwargs): +def is_api_lte(apiver: int): + def check(arch, recipe): return recipe.ctx.android_api <= apiver - return is_x - -def is_api(apiver): - def is_x(recipe, **kwargs): - return recipe.ctx.android_api == apiver - return is_x + return check -def will_build(recipe_name): - def will(recipe, **kwargs): - return recipe_name in recipe.ctx.recipe_build_order - return will +# Android API comparisons: def is_ndk(ndk): - def is_x(recipe, **kwargs): + """ + Return true if the Minimum Supported Android NDK level being targeted + is equal the given parameter (which should be an AndroidNDK instance) + """ + + def check(arch, recipe): return recipe.ctx.ndk == ndk - return is_x + + return check + + +# Recipe Version comparisons: +# These compare the Recipe's version with the provided string (or +# Packaging.Version). +# +# Warning: Both strings must conform to PEP 440 - e.g. "3.2.1" or "1.0rc1" def is_version_gt(version): - def is_x(recipe, **kwargs): - return LooseVersion(recipe.version) > version + """Return true if the Recipe's version is greater""" + + def check(arch, recipe): + return Version(recipe.version) > Version(version) + + return check def is_version_lt(version): - def is_x(recipe, **kwargs): - return LooseVersion(recipe.version) < version - return is_x + """Return true if the Recipe's version is less than""" + + def check(arch, recipe): + return Version(recipe.version) < Version(version) + + return check -def version_starts_with(version): - def is_x(recipe, **kwargs): - return recipe.version.startswith(version) - return is_x +def version_starts_with(version_prefix): + def check(arch, recipe): + return recipe.version.startswith(version_prefix) + + return check + + +# Will Build + + +def will_build(recipe_name): + """Return true if the recipe with this name is planned to be included in + the distribution.""" + + def check(arch, recipe): + return recipe_name in recipe.ctx.recipe_build_order + + return check + + +# Conjunctions + + +def check_all(*patch_checks): + """ + Given a collection of patch_checks as params, return if all returned true. + """ + + def check(arch, recipe): + return all(patch_check(arch, recipe) for patch_check in patch_checks) + + return check + + +def check_any(*patch_checks): + """ + Given a collection of patch_checks as params, return if any returned true. + """ + + def check(arch, recipe): + return any(patch_check(arch, recipe) for patch_check in patch_checks) + + return check diff --git a/pythonforandroid/prerequisites.py b/pythonforandroid/prerequisites.py index d85eb0b76d..d5e013f11d 100644 --- a/pythonforandroid/prerequisites.py +++ b/pythonforandroid/prerequisites.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import sys -import platform import os -import subprocess +import platform import shutil +import subprocess +import sys + from pythonforandroid.logger import info, warning, error +from pythonforandroid.util import ensure_dir class Prerequisite(object): @@ -247,13 +249,7 @@ def darwin_installer(self): "~/Library/Java/JavaVirtualMachines" ) info(f"Extracting {filename} to {user_library_java_path}") - subprocess.check_output( - [ - "mkdir", - "-p", - user_library_java_path, - ], - ) + ensure_dir(user_library_java_path) subprocess.check_output( ["tar", "xzf", f"/tmp/{filename}", "-C", user_library_java_path], ) @@ -370,22 +366,19 @@ def darwin_installer(self): def get_required_prerequisites(platform="linux"): - DEFAULT_PREREQUISITES = dict( - darwin=[ - HomebrewPrerequisite(), - AutoconfPrerequisite(), - AutomakePrerequisite(), - LibtoolPrerequisite(), - PkgConfigPrerequisite(), - CmakePrerequisite(), - OpenSSLPrerequisite(), - JDKPrerequisite(), - ], - linux=[], - all_platforms=[], - ) - - return DEFAULT_PREREQUISITES["all_platforms"] + DEFAULT_PREREQUISITES[platform] + return [ + prerequisite_cls() + for prerequisite_cls in [ + HomebrewPrerequisite, + AutoconfPrerequisite, + AutomakePrerequisite, + LibtoolPrerequisite, + PkgConfigPrerequisite, + CmakePrerequisite, + OpenSSLPrerequisite, + JDKPrerequisite, + ] if prerequisite_cls.mandatory.get(platform, False) + ] def check_and_install_default_prerequisites(): diff --git a/pythonforandroid/pythonpackage.py b/pythonforandroid/pythonpackage.py index 3b03a513f5..9e4c29bd81 100644 --- a/pythonforandroid/pythonpackage.py +++ b/pythonforandroid/pythonpackage.py @@ -4,17 +4,17 @@ Usage examples: # Getting package name from pip reference: - from pytonforandroid.pythonpackage import get_package_name + from pythonforandroid.pythonpackage import get_package_name print(get_package_name("pillow")) # Outputs: "Pillow" (note the spelling!) # Getting package dependencies: - from pytonforandroid.pythonpackage import get_package_dependencies + from pythonforandroid.pythonpackage import get_package_dependencies print(get_package_dependencies("pep517")) # Outputs: "['pytoml']" # Get package name from arbitrary package source: - from pytonforandroid.pythonpackage import get_package_name + from pythonforandroid.pythonpackage import get_package_name print(get_package_name("/some/local/project/folder/")) # Outputs package name @@ -34,22 +34,22 @@ import functools +from io import open # needed for python 2 import os import shutil import subprocess import sys import tarfile import tempfile -import textwrap import time -import zipfile -from io import open # needed for python 2 from urllib.parse import unquote as urlunquote from urllib.parse import urlparse +import zipfile import toml -from pep517.envbuild import BuildEnvironment -from pep517.wrappers import Pep517HookCaller +import build.util + +from pythonforandroid.util import rmdir, ensure_dir def transform_dep_for_pip(dependency): @@ -113,42 +113,9 @@ def extract_metainfo_files_from_package( ) package = os.path.join(temp_folder, "package") - # Because PEP517 can be noisy and contextlib.redirect_* fails to - # contain it, we will run the actual analysis in a separate process: - try: - subprocess.check_output([ - sys.executable, - "-c", - "import importlib\n" - "import json\n" - "import os\n" - "import sys\n" - "sys.path = [os.path.dirname(sys.argv[3])] + sys.path\n" - "m = importlib.import_module(\n" - " os.path.basename(sys.argv[3]).partition('.')[0]\n" - ")\n" - "m._extract_metainfo_files_from_package_unsafe(" - " sys.argv[1]," - " sys.argv[2]," - ")", - package, output_folder, os.path.abspath(__file__)], - stderr=subprocess.STDOUT, # make sure stderr is muted. - cwd=os.path.join(os.path.dirname(__file__), "..") - ) - except subprocess.CalledProcessError as e: - output = e.output.decode("utf-8", "replace") - if debug: - print("Got error obtaining meta info.") - print("Detail output:") - print(output) - print("End of Detail output.") - raise ValueError( - "failed to obtain meta info - " - "is '{}' a valid package? " - "Detailed output:\n{}".format(package, output) - ) + _extract_metainfo_files_from_package_unsafe(package, output_folder) finally: - shutil.rmtree(temp_folder) + rmdir(temp_folder) def _get_system_python_executable(): @@ -349,7 +316,7 @@ def get_package_as_folder(dependency): ) # Create download subfolder: - os.mkdir(os.path.join(venv_path, "download")) + ensure_dir(os.path.join(venv_path, "download")) # Write a requirements.txt with our package and download: with open(os.path.join(venv_path, "requirements.txt"), @@ -429,11 +396,11 @@ def to_unicode(s): # Needed for Python 2. # Copy result to new dedicated folder so we can throw away # our entire virtualenv nonsense after returning: result_path = tempfile.mkdtemp() - shutil.rmtree(result_path) + rmdir(result_path) shutil.copytree(result_folder_or_file, result_path) return (dl_type, result_path) finally: - shutil.rmtree(venv_parent) + rmdir(venv_parent) def _extract_metainfo_files_from_package_unsafe( @@ -461,51 +428,17 @@ def _extract_metainfo_files_from_package_unsafe( clean_up_path = True try: - build_requires = [] metadata_path = None if path_type != "wheel": - # We need to process this first to get the metadata. - - # Ensure pyproject.toml is available (pep517 expects it) - if not os.path.exists(os.path.join(path, "pyproject.toml")): - with open(os.path.join(path, "pyproject.toml"), "w") as f: - f.write(textwrap.dedent(u"""\ - [build-system] - requires = ["setuptools", "wheel"] - build-backend = "setuptools.build_meta" - """)) - - # Copy the pyproject.toml: - shutil.copyfile( - os.path.join(path, 'pyproject.toml'), - os.path.join(output_path, 'pyproject.toml') - ) - - # Get build backend and requirements from pyproject.toml: - with open(os.path.join(path, 'pyproject.toml')) as f: - build_sys = toml.load(f)['build-system'] - backend = build_sys["build-backend"] - build_requires.extend(build_sys["requires"]) - - # Get a virtualenv with build requirements and get all metadata: - env = BuildEnvironment() - metadata = None - with env: - hooks = Pep517HookCaller(path, backend) - env.pip_install( - [transform_dep_for_pip(req) for req in build_requires] - ) - reqs = hooks.get_requires_for_build_wheel({}) - env.pip_install([transform_dep_for_pip(req) for req in reqs]) - try: - metadata = hooks.prepare_metadata_for_build_wheel(path) - except Exception: # sadly, pep517 has no good error here - pass - if metadata is not None: - metadata_path = os.path.join( - path, metadata, "METADATA" - ) + # Use a build helper function to fetch the metadata directly + metadata = build.util.project_wheel_metadata(path) + # And write it to a file + metadata_path = os.path.join(output_path, "built_metadata") + with open(metadata_path, 'w') as f: + for key in metadata.keys(): + for value in metadata.get_all(key): + f.write("{}: {}\n".format(key, value)) else: # This is a wheel, so metadata should be in *.dist-info folder: metadata_path = os.path.join( @@ -527,7 +460,7 @@ def _extract_metainfo_files_from_package_unsafe( shutil.copyfile(metadata_path, os.path.join(output_path, "METADATA")) finally: if clean_up_path: - shutil.rmtree(path) + rmdir(path) def is_filesystem_path(dep): @@ -645,7 +578,7 @@ def _extract_info_from_package(dependency, return list(set(requirements)) # remove duplicates finally: - shutil.rmtree(output_folder) + rmdir(output_folder) package_name_cache = dict() diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 67c309e197..fa0bb4e790 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -1,6 +1,5 @@ from os.path import basename, dirname, exists, isdir, isfile, join, realpath, split import glob -from shutil import rmtree import hashlib from re import match @@ -10,16 +9,18 @@ import fnmatch import urllib.request from urllib.request import urlretrieve -from os import listdir, unlink, environ, mkdir, curdir, walk +from os import listdir, unlink, environ, curdir, walk from sys import stdout import time try: from urlparse import urlparse except ImportError: from urllib.parse import urlparse -from pythonforandroid.logger import (logger, info, warning, debug, shprint, info_main) -from pythonforandroid.util import (current_directory, ensure_dir, - BuildInterruptingException) +from pythonforandroid.logger import ( + logger, info, warning, debug, shprint, info_main) +from pythonforandroid.util import ( + current_directory, ensure_dir, BuildInterruptingException, rmdir, move, + touch) from pythonforandroid.util import load_source as import_recipe @@ -218,7 +219,7 @@ def report_hook(index, blksize, size): url = url[4:] # if 'version' is specified, do a shallow clone if self.version: - shprint(sh.mkdir, '-p', target) + ensure_dir(target) with current_directory(target): shprint(sh.git, 'init') shprint(sh.git, 'remote', 'add', 'origin', url) @@ -367,7 +368,7 @@ def download(self): if expected_digest: expected_digests[alg] = expected_digest - shprint(sh.mkdir, '-p', join(self.ctx.packages_path, self.name)) + ensure_dir(join(self.ctx.packages_path, self.name)) with current_directory(join(self.ctx.packages_path, self.name)): filename = shprint(sh.basename, url).stdout[:-1].decode('utf-8') @@ -396,7 +397,7 @@ def download(self): shprint(sh.rm, '-f', marker_filename) self.download_file(self.versioned_url, filename) - shprint(sh.touch, marker_filename) + touch(marker_filename) if exists(filename) and isfile(filename): for alg, expected_digest in expected_digests.items(): @@ -423,9 +424,7 @@ def unpack(self, arch): self.name.lower())) if exists(self.get_build_dir(arch)): return - shprint(sh.rm, '-rf', build_dir) - shprint(sh.mkdir, '-p', build_dir) - shprint(sh.rmdir, build_dir) + rmdir(build_dir) ensure_dir(build_dir) shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch)) return @@ -460,20 +459,20 @@ def unpack(self, arch): fileh = zipfile.ZipFile(extraction_filename, 'r') root_directory = fileh.filelist[0].filename.split('/')[0] if root_directory != basename(directory_name): - shprint(sh.mv, root_directory, directory_name) + move(root_directory, directory_name) elif extraction_filename.endswith( ('.tar.gz', '.tgz', '.tar.bz2', '.tbz2', '.tar.xz', '.txz')): sh.tar('xf', extraction_filename) root_directory = sh.tar('tf', extraction_filename).stdout.decode( 'utf-8').split('\n')[0].split('/')[0] if root_directory != basename(directory_name): - shprint(sh.mv, root_directory, directory_name) + move(root_directory, directory_name) else: raise Exception( 'Could not extract {} download, it must be .zip, ' '.tar.gz or .tar.bz2 or .tar.xz'.format(extraction_filename)) elif isdir(extraction_filename): - mkdir(directory_name) + ensure_dir(directory_name) for entry in listdir(extraction_filename): if entry not in ('.git',): shprint(sh.cp, '-Rv', @@ -533,7 +532,7 @@ def apply_patches(self, arch, build_dir=None): patch.format(version=self.version, arch=arch.arch), arch.arch, build_dir=build_dir) - shprint(sh.touch, join(build_dir, '.patched')) + touch(join(build_dir, '.patched')) def should_build(self, arch): '''Should perform any necessary test and return True only if it needs @@ -614,13 +613,11 @@ def clean_build(self, arch=None): 'build dirs'.format(self.name)) for directory in dirs: - if exists(directory): - info('Deleting {}'.format(directory)) - shutil.rmtree(directory) + rmdir(directory) # Delete any Python distributions to ensure the recipe build # doesn't persist in site-packages - shutil.rmtree(self.ctx.python_installs_dir) + rmdir(self.ctx.python_installs_dir) def install_libs(self, arch, *libs): libs_dir = self.ctx.get_libs_dir(arch.arch) @@ -721,7 +718,7 @@ def prepare_build_dir(self, arch): if self.src_filename is None: raise BuildInterruptingException( 'IncludedFilesBehaviour failed: no src_filename specified') - shprint(sh.rm, '-rf', self.get_build_dir(arch)) + rmdir(self.get_build_dir(arch)) shprint(sh.cp, '-a', join(self.get_recipe_dir(), self.src_filename), self.get_build_dir(arch)) @@ -861,7 +858,7 @@ def clean_build(self, arch=None): build_dir = join(site_packages_dir[0], name) if exists(build_dir): info('Deleted {}'.format(build_dir)) - rmtree(build_dir) + rmdir(build_dir) @property def real_hostpython_location(self): @@ -1171,7 +1168,7 @@ def reduce_object_file_names(self, dirn): parts = file_basename.split('.') if len(parts) <= 2: continue - shprint(sh.mv, filen, join(file_dirname, parts[0] + '.so')) + move(filen, join(file_dirname, parts[0] + '.so')) def algsum(alg, filen): diff --git a/pythonforandroid/recipes/babel/__init__.py b/pythonforandroid/recipes/babel/__init__.py deleted file mode 100644 index fc17f8e4b0..0000000000 --- a/pythonforandroid/recipes/babel/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from pythonforandroid.recipe import PythonRecipe - - -class BabelRecipe(PythonRecipe): - name = 'babel' - version = '2.2.0' - url = 'https://pypi.python.org/packages/source/B/Babel/Babel-{version}.tar.gz' - - depends = ['setuptools', 'pytz'] - - call_hostpython_via_targetpython = False - install_in_hostpython = True - - -recipe = BabelRecipe() diff --git a/pythonforandroid/recipes/cffi/__init__.py b/pythonforandroid/recipes/cffi/__init__.py index a198a3db0d..f0c25a92c9 100644 --- a/pythonforandroid/recipes/cffi/__init__.py +++ b/pythonforandroid/recipes/cffi/__init__.py @@ -7,7 +7,7 @@ class CffiRecipe(CompiledComponentsPythonRecipe): Extra system dependencies: autoconf, automake and libtool. """ name = 'cffi' - version = '1.13.2' + version = '1.15.1' url = 'https://pypi.python.org/packages/source/c/cffi/cffi-{version}.tar.gz' depends = ['setuptools', 'pycparser', 'libffi'] diff --git a/pythonforandroid/recipes/cffi/disable-pkg-config.patch b/pythonforandroid/recipes/cffi/disable-pkg-config.patch index cf2abd5b86..b1a5ff9b4c 100644 --- a/pythonforandroid/recipes/cffi/disable-pkg-config.patch +++ b/pythonforandroid/recipes/cffi/disable-pkg-config.patch @@ -1,19 +1,19 @@ -diff --git a/setup.py b/setup.py -index c1db368..57311c3 100644 +diff --git a/setup.py b/setup copy.py +index 4ce0007..9be4a6d 100644 --- a/setup.py -+++ b/setup.py -@@ -5,8 +5,7 @@ import errno ++++ b/setup +@@ -9,8 +9,7 @@ if sys.platform == "win32": sources = ['c/_cffi_backend.c'] libraries = ['ffi'] -include_dirs = ['/usr/include/ffi', - '/usr/include/libffi'] # may be changed by pkg-config -+include_dirs = os.environ['FFI_INC'].split(",") if 'FFI_INC' in os.environ else [] - define_macros = [] ++include_dirs = os.environ['FFI_INC'].split(',') if 'FFI_INC' in os.environ else [] + define_macros = [('FFI_BUILDING', '1')] # for linking with libffi static library library_dirs = [] extra_compile_args = [] -@@ -67,14 +66,7 @@ def ask_supports_thread(): - sys.stderr.write("The above error message can be safely ignored\n") +@@ -105,14 +104,7 @@ def uses_msvc(): + return config.try_compile('#ifndef _MSC_VER\n#error "not MSVC"\n#endif') def use_pkg_config(): - if sys.platform == 'darwin' and os.path.exists('/usr/local/bin/brew'): @@ -25,6 +25,4 @@ index c1db368..57311c3 100644 - _ask_pkg_config(extra_link_args, '--libs-only-other') - _ask_pkg_config(libraries, '--libs-only-l', '-l') + pass - - def use_homebrew_for_libffi(): - # We can build by setting: + diff --git a/pythonforandroid/recipes/dateutil/__init__.py b/pythonforandroid/recipes/dateutil/__init__.py deleted file mode 100644 index 3367f8d145..0000000000 --- a/pythonforandroid/recipes/dateutil/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -from pythonforandroid.recipe import PythonRecipe - - -class DateutilRecipe(PythonRecipe): - name = 'dateutil' - version = '2.6.0' - url = 'https://pypi.python.org/packages/51/fc/39a3fbde6864942e8bb24c93663734b74e281b984d1b8c4f95d64b0c21f6/python-dateutil-2.6.0.tar.gz' - - depends = ["setuptools"] - call_hostpython_via_targetpython = False - install_in_hostpython = True - - -recipe = DateutilRecipe() diff --git a/pythonforandroid/recipes/freetype/__init__.py b/pythonforandroid/recipes/freetype/__init__.py index 0b04c95da6..e5ddfe1424 100644 --- a/pythonforandroid/recipes/freetype/__init__.py +++ b/pythonforandroid/recipes/freetype/__init__.py @@ -25,7 +25,7 @@ class FreetypeRecipe(Recipe): """ version = '2.10.1' - url = 'http://download.savannah.gnu.org/releases/freetype/freetype-{version}.tar.gz' # noqa + url = 'https://download.savannah.gnu.org/releases/freetype/freetype-{version}.tar.gz' # noqa built_libraries = {'libfreetype.so': 'objs/.libs'} def get_recipe_env(self, arch=None, with_harfbuzz=False): diff --git a/pythonforandroid/recipes/ifaddrs/__init__.py b/pythonforandroid/recipes/ifaddrs/__init__.py index 7d44f9cd72..1317dc2556 100644 --- a/pythonforandroid/recipes/ifaddrs/__init__.py +++ b/pythonforandroid/recipes/ifaddrs/__init__.py @@ -1,10 +1,13 @@ """ ifaddrs for Android """ -from os.path import join, exists +from os.path import join + import sh -from pythonforandroid.logger import info, shprint + +from pythonforandroid.logger import shprint from pythonforandroid.recipe import CompiledComponentsPythonRecipe from pythonforandroid.toolchain import current_directory +from pythonforandroid.util import ensure_dir class IFAddrRecipe(CompiledComponentsPythonRecipe): @@ -19,9 +22,7 @@ class IFAddrRecipe(CompiledComponentsPythonRecipe): def prebuild_arch(self, arch): """Make the build and target directories""" path = self.get_build_dir(arch.arch) - if not exists(path): - info("creating {}".format(path)) - shprint(sh.mkdir, '-p', path) + ensure_dir(path) def build_arch(self, arch): """simple shared compile""" @@ -30,9 +31,7 @@ def build_arch(self, arch): self.get_build_dir(arch.arch), join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Lib'), join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Include')): - if not exists(path): - info("creating {}".format(path)) - shprint(sh.mkdir, '-p', path) + ensure_dir(path) cli = env['CC'].split()[0] # makes sure first CC command is the compiler rather than ccache, refs: # https://github.com/kivy/python-for-android/issues/1398 diff --git a/pythonforandroid/recipes/kivy/__init__.py b/pythonforandroid/recipes/kivy/__init__.py index c213879b64..ebf7b29e82 100644 --- a/pythonforandroid/recipes/kivy/__init__.py +++ b/pythonforandroid/recipes/kivy/__init__.py @@ -22,7 +22,7 @@ def is_kivy_affected_by_deadlock_issue(recipe=None, arch=None): class KivyRecipe(CythonRecipe): - version = '2.2.0' + version = '2.2.1' url = 'https://github.com/kivy/kivy/archive/{version}.zip' name = 'kivy' diff --git a/pythonforandroid/recipes/lapack/__init__.py b/pythonforandroid/recipes/lapack/__init__.py index ae20e69538..b6124dc285 100644 --- a/pythonforandroid/recipes/lapack/__init__.py +++ b/pythonforandroid/recipes/lapack/__init__.py @@ -11,7 +11,7 @@ import sh import shutil from os import environ -from pythonforandroid.util import build_platform +from pythonforandroid.util import build_platform, rmdir arch_to_sysroot = {'armeabi': 'arm', 'armeabi-v7a': 'arm', 'arm64-v8a': 'arm64'} @@ -57,7 +57,8 @@ def build_arch(self, arch): with current_directory(build_target): env = self.get_recipe_env(arch) ndk_dir = environ["LEGACY_NDK"] - shprint(sh.rm, '-rf', 'CMakeFiles/', 'CMakeCache.txt', _env=env) + rmdir('CMakeFiles') + shprint(sh.rm, '-f', 'CMakeCache.txt', _env=env) opts = [ '-DCMAKE_SYSTEM_NAME=Android', '-DCMAKE_POSITION_INDEPENDENT_CODE=1', diff --git a/pythonforandroid/recipes/libglob/__init__.py b/pythonforandroid/recipes/libglob/__init__.py index f63db42628..39a68b7ee2 100644 --- a/pythonforandroid/recipes/libglob/__init__.py +++ b/pythonforandroid/recipes/libglob/__init__.py @@ -2,11 +2,14 @@ android libglob available via '-lglob' LDFLAG """ -from os.path import exists, join +from os.path import join + +import sh + +from pythonforandroid.logger import shprint from pythonforandroid.recipe import Recipe from pythonforandroid.toolchain import current_directory -from pythonforandroid.logger import info, shprint -import sh +from pythonforandroid.util import ensure_dir class LibGlobRecipe(Recipe): @@ -32,9 +35,7 @@ def should_build(self, arch): def prebuild_arch(self, arch): """Make the build and target directories""" path = self.get_build_dir(arch.arch) - if not exists(path): - info("creating {}".format(path)) - shprint(sh.mkdir, '-p', path) + ensure_dir(path) def build_arch(self, arch): """simple shared compile""" @@ -43,9 +44,7 @@ def build_arch(self, arch): self.get_build_dir(arch.arch), join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Lib'), join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Include')): - if not exists(path): - info("creating {}".format(path)) - shprint(sh.mkdir, '-p', path) + ensure_dir(path) cli = env['CC'].split()[0] # makes sure first CC command is the compiler rather than ccache, refs: # https://github.com/kivy/python-for-android/issues/1399 diff --git a/pythonforandroid/recipes/libmysqlclient/__init__.py b/pythonforandroid/recipes/libmysqlclient/__init__.py index 31ebd3c540..84fd8d30ac 100644 --- a/pythonforandroid/recipes/libmysqlclient/__init__.py +++ b/pythonforandroid/recipes/libmysqlclient/__init__.py @@ -26,7 +26,7 @@ def build_arch(self, arch): env = self.get_recipe_env(arch) with current_directory(join(self.get_build_dir(arch.arch), 'libmysqlclient')): shprint(sh.cp, '-t', '.', join(self.get_recipe_dir(), 'p4a.cmake')) - # shprint(sh.mkdir, 'Platform') + # ensure_dir('Platform') # shprint(sh.cp, '-t', 'Platform', join(self.get_recipe_dir(), 'Linux.cmake')) shprint(sh.rm, '-f', 'CMakeCache.txt') shprint(sh.cmake, '-G', 'Unix Makefiles', diff --git a/pythonforandroid/recipes/libtorrent/__init__.py b/pythonforandroid/recipes/libtorrent/__init__.py index 24f94081c6..1086e00fcc 100644 --- a/pythonforandroid/recipes/libtorrent/__init__.py +++ b/pythonforandroid/recipes/libtorrent/__init__.py @@ -1,9 +1,12 @@ -from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory from multiprocessing import cpu_count -from os.path import join, basename from os import listdir, walk +from os.path import join, basename +import shutil + import sh +from pythonforandroid.toolchain import Recipe, shprint, current_directory + # This recipe builds libtorrent with Python bindings # It depends on Boost.Build and the source of several Boost libraries present # in BOOST_ROOT, which is all provided by the boost recipe diff --git a/pythonforandroid/recipes/mysqldb/__init__.py b/pythonforandroid/recipes/mysqldb/__init__.py deleted file mode 100644 index 768cb72af2..0000000000 --- a/pythonforandroid/recipes/mysqldb/__init__.py +++ /dev/null @@ -1,52 +0,0 @@ -from pythonforandroid.recipe import CompiledComponentsPythonRecipe -from os.path import join - - -class MysqldbRecipe(CompiledComponentsPythonRecipe): - name = 'mysqldb' - version = '1.2.5' - url = 'https://pypi.python.org/packages/source/M/MySQL-python/MySQL-python-{version}.zip' - site_packages_name = 'MySQLdb' - - depends = ['setuptools', 'libmysqlclient'] - - patches = ['override-mysql-config.patch', - 'disable-zip.patch'] - - # call_hostpython_via_targetpython = False - - def convert_newlines(self, filename): - print('converting newlines in {}'.format(filename)) - with open(filename, 'rb') as f: - data = f.read() - with open(filename, 'wb') as f: - f.write(data.replace(b'\r\n', b'\n').replace(b'\r', b'\n')) - - def prebuild_arch(self, arch): - super().prebuild_arch(arch) - setupbase = join(self.get_build_dir(arch.arch), 'setup') - self.convert_newlines(setupbase + '.py') - self.convert_newlines(setupbase + '_posix.py') - - def get_recipe_env(self, arch=None): - env = super().get_recipe_env(arch) - - hostpython = self.get_recipe('hostpython3', self.ctx) - # TODO: fix hardcoded path - env['PYTHONPATH'] = (join(hostpython.get_build_dir(arch.arch), - 'build', 'lib.linux-x86_64-2.7') + - ':' + env.get('PYTHONPATH', '')) - - libmysql = self.get_recipe('libmysqlclient', self.ctx) - mydir = join(libmysql.get_build_dir(arch.arch), 'libmysqlclient') - # env['CFLAGS'] += ' -I' + join(mydir, 'include') - # env['LDFLAGS'] += ' -L' + join(mydir) - libdir = self.ctx.get_libs_dir(arch.arch) - env['MYSQL_libs'] = env['MYSQL_libs_r'] = '-L' + libdir + ' -lmysql' - env['MYSQL_cflags'] = env['MYSQL_include'] = '-I' + join(mydir, - 'include') - - return env - - -recipe = MysqldbRecipe() diff --git a/pythonforandroid/recipes/mysqldb/disable-zip.patch b/pythonforandroid/recipes/mysqldb/disable-zip.patch deleted file mode 100644 index 51f804ea2c..0000000000 --- a/pythonforandroid/recipes/mysqldb/disable-zip.patch +++ /dev/null @@ -1,8 +0,0 @@ ---- mysqldb/setup.py 2014-01-02 13:52:50.000000000 -0600 -+++ b/setup.py 2016-01-13 15:48:36.781216443 -0600 -@@ -18,4 +18,5 @@ - metadata['ext_modules'] = [ - setuptools.Extension(sources=['_mysql.c'], **options)] - metadata['long_description'] = metadata['long_description'].replace(r'\n', '') -+metadata['zip_safe'] = False - setuptools.setup(**metadata) diff --git a/pythonforandroid/recipes/mysqldb/override-mysql-config.patch b/pythonforandroid/recipes/mysqldb/override-mysql-config.patch deleted file mode 100644 index 195ebdacbc..0000000000 --- a/pythonforandroid/recipes/mysqldb/override-mysql-config.patch +++ /dev/null @@ -1,21 +0,0 @@ ---- mysqldb/setup_posix.py 2014-01-02 13:52:50.000000000 -0600 -+++ b/setup_posix.py 2016-01-13 15:48:18.732883429 -0600 -@@ -13,17 +13,7 @@ - return "-%s" % f - - def mysql_config(what): -- from os import popen -- -- f = popen("%s --%s" % (mysql_config.path, what)) -- data = f.read().strip().split() -- ret = f.close() -- if ret: -- if ret/256: -- data = [] -- if ret/256 > 1: -- raise EnvironmentError("%s not found" % (mysql_config.path,)) -- return data -+ return os.environ['MYSQL_' + what.replace('-', '_')].strip().split() - mysql_config.path = "mysql_config" - - def get_config(): diff --git a/pythonforandroid/recipes/opencv/__init__.py b/pythonforandroid/recipes/opencv/__init__.py index c760cbdda7..650c77e508 100644 --- a/pythonforandroid/recipes/opencv/__init__.py +++ b/pythonforandroid/recipes/opencv/__init__.py @@ -1,9 +1,11 @@ +from multiprocessing import cpu_count from os.path import join + import sh -from pythonforandroid.recipe import NDKRecipe -from pythonforandroid.util import current_directory + from pythonforandroid.logger import shprint -from multiprocessing import cpu_count +from pythonforandroid.recipe import NDKRecipe +from pythonforandroid.util import current_directory, ensure_dir class OpenCVRecipe(NDKRecipe): @@ -45,7 +47,7 @@ def get_recipe_env(self, arch): def build_arch(self, arch): build_dir = join(self.get_build_dir(arch.arch), 'build') - shprint(sh.mkdir, '-p', build_dir) + ensure_dir(build_dir) opencv_extras = [] if 'opencv_extras' in self.ctx.recipe_build_order: diff --git a/pythonforandroid/recipes/pandas/__init__.py b/pythonforandroid/recipes/pandas/__init__.py index a217ab635a..a43209a339 100644 --- a/pythonforandroid/recipes/pandas/__init__.py +++ b/pythonforandroid/recipes/pandas/__init__.py @@ -7,9 +7,9 @@ class PandasRecipe(CppCompiledComponentsPythonRecipe): version = '1.0.3' url = 'https://github.com/pandas-dev/pandas/releases/download/v{version}/pandas-{version}.tar.gz' # noqa - depends = ['cython', 'numpy', 'pytz', 'libbz2', 'liblzma'] + depends = ['cython', 'numpy', 'libbz2', 'liblzma'] - python_depends = ['python-dateutil'] + python_depends = ['python-dateutil', 'pytz'] patches = ['fix_numpy_includes.patch'] call_hostpython_via_targetpython = False diff --git a/pythonforandroid/recipes/protobuf_cpp/__init__.py b/pythonforandroid/recipes/protobuf_cpp/__init__.py index c1149f2714..7209e0909b 100644 --- a/pythonforandroid/recipes/protobuf_cpp/__init__.py +++ b/pythonforandroid/recipes/protobuf_cpp/__init__.py @@ -1,12 +1,13 @@ -from pythonforandroid.recipe import CppCompiledComponentsPythonRecipe -from pythonforandroid.logger import shprint, info_notify -from pythonforandroid.util import current_directory -from os.path import exists, join -import sh from multiprocessing import cpu_count +import os +from os.path import exists, join from pythonforandroid.toolchain import info +import sh import sys -import os + +from pythonforandroid.recipe import CppCompiledComponentsPythonRecipe +from pythonforandroid.logger import shprint, info_notify +from pythonforandroid.util import current_directory, touch class ProtobufCppRecipe(CppCompiledComponentsPythonRecipe): @@ -30,7 +31,7 @@ def prebuild_arch(self, arch): patch_mark = join(self.get_build_dir(arch.arch), '.protobuf-patched') if self.ctx.python_recipe.name == 'python3' and not exists(patch_mark): self.apply_patch('fix-python3-compatibility.patch', arch.arch) - shprint(sh.touch, patch_mark) + touch(patch_mark) # During building, host needs to transpile .proto files to .py # ideally with the same version as protobuf runtime, or with an older one. diff --git a/pythonforandroid/recipes/pytz/__init__.py b/pythonforandroid/recipes/pytz/__init__.py deleted file mode 100644 index ff9bc37573..0000000000 --- a/pythonforandroid/recipes/pytz/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from pythonforandroid.recipe import PythonRecipe - - -class PytzRecipe(PythonRecipe): - name = 'pytz' - version = '2019.3' - url = 'https://pypi.python.org/packages/source/p/pytz/pytz-{version}.tar.gz' - - depends = [] - - call_hostpython_via_targetpython = False - install_in_hostpython = True - - -recipe = PytzRecipe() diff --git a/pythonforandroid/recipes/reportlab/__init__.py b/pythonforandroid/recipes/reportlab/__init__.py index 60f1a078ea..ee5de38214 100644 --- a/pythonforandroid/recipes/reportlab/__init__.py +++ b/pythonforandroid/recipes/reportlab/__init__.py @@ -1,8 +1,9 @@ import os import sh + +from pythonforandroid.logger import info from pythonforandroid.recipe import CompiledComponentsPythonRecipe -from pythonforandroid.util import (current_directory, ensure_dir) -from pythonforandroid.logger import (info, shprint) +from pythonforandroid.util import current_directory, ensure_dir, touch class ReportLabRecipe(CompiledComponentsPythonRecipe): @@ -28,7 +29,7 @@ def prebuild_arch(self, arch): # Apply patches: self.apply_patch('patches/fix-setup.patch', arch.arch) - shprint(sh.touch, os.path.join(recipe_dir, '.patched')) + touch(os.path.join(recipe_dir, '.patched')) ft = self.get_recipe('freetype', self.ctx) ft_dir = ft.get_build_dir(arch.arch) ft_lib_dir = os.environ.get('_FT_LIB_', os.path.join(ft_dir, 'objs', '.libs')) diff --git a/pythonforandroid/recipes/sqlite3/__init__.py b/pythonforandroid/recipes/sqlite3/__init__.py index 955d808141..1f4292c1eb 100644 --- a/pythonforandroid/recipes/sqlite3/__init__.py +++ b/pythonforandroid/recipes/sqlite3/__init__.py @@ -1,7 +1,8 @@ -from pythonforandroid.recipe import NDKRecipe -from pythonforandroid.toolchain import shutil from os.path import join -import sh +import shutil + +from pythonforandroid.recipe import NDKRecipe +from pythonforandroid.util import ensure_dir class Sqlite3Recipe(NDKRecipe): @@ -16,7 +17,7 @@ def should_build(self, arch): def prebuild_arch(self, arch): super().prebuild_arch(arch) # Copy the Android make file - sh.mkdir('-p', join(self.get_build_dir(arch.arch), 'jni')) + ensure_dir(join(self.get_build_dir(arch.arch), 'jni')) shutil.copyfile(join(self.get_recipe_dir(), 'Android.mk'), join(self.get_build_dir(arch.arch), 'jni/Android.mk')) diff --git a/pythonforandroid/recipes/twisted/__init__.py b/pythonforandroid/recipes/twisted/__init__.py index 0c390a5b14..30a7af4bb9 100644 --- a/pythonforandroid/recipes/twisted/__init__.py +++ b/pythonforandroid/recipes/twisted/__init__.py @@ -1,7 +1,7 @@ import os -import shutil from pythonforandroid.recipe import CythonRecipe +from pythonforandroid.util import rmdir class TwistedRecipe(CythonRecipe): @@ -23,7 +23,7 @@ def prebuild_arch(self, arch): for item in os.walk(source_dir): if os.path.basename(item[0]) == 'test': full_path = os.path.join(source_dir, item[0]) - shutil.rmtree(full_path, ignore_errors=True) + rmdir(full_path, ignore_errors=True) def get_recipe_env(self, arch): env = super().get_recipe_env(arch) diff --git a/pythonforandroid/recipes/vlc/__init__.py b/pythonforandroid/recipes/vlc/__init__.py index 490c4f2a18..0995576f5f 100644 --- a/pythonforandroid/recipes/vlc/__init__.py +++ b/pythonforandroid/recipes/vlc/__init__.py @@ -6,7 +6,7 @@ class VlcRecipe(Recipe): - version = '3.0.0' + version = '3.0.18' url = None name = 'vlc' @@ -52,7 +52,7 @@ def prebuild_arch(self, arch): def build_arch(self, arch): super().build_arch(arch) build_dir = self.get_build_dir(arch.arch) - port_dir = join(build_dir, 'vlc-port-android') + port_dir = join(build_dir, 'vlc-port-android', 'buildsystem') aar = self.aars[arch] if not isfile(aar): with current_directory(port_dir): @@ -67,7 +67,7 @@ def build_arch(self, arch): if not isfile(join('bin', 'VLC-debug.apk')): shprint(sh.Command('./compile.sh'), _env=env, _tail=50, _critical=True) - shprint(sh.Command('./compile-libvlc.sh'), _env=env, + shprint(sh.Command('./compile-medialibrary.sh'), _env=env, _tail=50, _critical=True) shprint(sh.cp, '-a', aar, self.ctx.aars_dir) diff --git a/pythonforandroid/recipes/zope_interface/__init__.py b/pythonforandroid/recipes/zope_interface/__init__.py index 46a1820c2b..7e7fecaea5 100644 --- a/pythonforandroid/recipes/zope_interface/__init__.py +++ b/pythonforandroid/recipes/zope_interface/__init__.py @@ -1,7 +1,8 @@ +from os.path import join + from pythonforandroid.recipe import PythonRecipe from pythonforandroid.toolchain import current_directory -from os.path import join -import sh +from pythonforandroid.util import rmdir class ZopeInterfaceRecipe(PythonRecipe): @@ -25,11 +26,8 @@ def build_arch(self, arch): def prebuild_arch(self, arch): super().prebuild_arch(arch) with current_directory(self.get_build_dir(arch.arch)): - sh.rm( - '-rf', - 'src/zope/interface/tests', - 'src/zope/interface/common/tests', - ) + rmdir('src/zope/interface/tests') + rmdir('src/zope/interface/common/tests') recipe = ZopeInterfaceRecipe() diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index 85404a2359..7a5461f30d 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -6,96 +6,42 @@ This module defines the entry point for command line and programmatic use. """ +from appdirs import user_data_dir +import argparse +from functools import wraps +import glob +import logging +import os from os import environ -from pythonforandroid import __version__ -from pythonforandroid.pythonpackage import get_dep_names_of_package -from pythonforandroid.recommendations import ( - RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API, print_recommendations) -from pythonforandroid.util import BuildInterruptingException, load_source -from pythonforandroid.entrypoints import main -from pythonforandroid.prerequisites import check_and_install_default_prerequisites - - -def check_python_dependencies(): - # Check if the Python requirements are installed. This appears - # before the imports because otherwise they're imported elsewhere. - - # Using the ok check instead of failing immediately so that all - # errors are printed at once - - from distutils.version import LooseVersion - from importlib import import_module - import sys - - ok = True - - modules = [('colorama', '0.3.3'), 'appdirs', ('sh', '1.10'), 'jinja2'] - - for module in modules: - if isinstance(module, tuple): - module, version = module - else: - version = None - - try: - import_module(module) - except ImportError: - if version is None: - print('ERROR: The {} Python module could not be found, please ' - 'install it.'.format(module)) - ok = False - else: - print('ERROR: The {} Python module could not be found, ' - 'please install version {} or higher'.format( - module, version)) - ok = False - else: - if version is None: - continue - try: - cur_ver = sys.modules[module].__version__ - except AttributeError: # this is sometimes not available - continue - if LooseVersion(cur_ver) < LooseVersion(version): - print('ERROR: {} version is {}, but python-for-android needs ' - 'at least {}.'.format(module, cur_ver, version)) - ok = False - - if not ok: - print('python-for-android is exiting due to the errors logged above') - exit(1) - - -if not environ.get('SKIP_PREREQUISITES_CHECK', '0') == '1': - check_and_install_default_prerequisites() -check_python_dependencies() - - -import sys -from sys import platform from os.path import (join, dirname, realpath, exists, expanduser, basename) -import os -import glob -import shutil import re import shlex -from functools import wraps +import sys +from sys import platform -import argparse +# This must be imported and run before other third-party or p4a +# packages. +from pythonforandroid.checkdependencies import check +check() + +from packaging.version import Version, InvalidVersion import sh -from appdirs import user_data_dir -import logging -from distutils.version import LooseVersion -from pythonforandroid.recipe import Recipe -from pythonforandroid.logger import (logger, info, warning, setup_color, - Out_Style, Out_Fore, - info_notify, info_main, shprint) -from pythonforandroid.util import current_directory +from pythonforandroid import __version__ from pythonforandroid.bootstrap import Bootstrap +from pythonforandroid.build import Context, build_recipes from pythonforandroid.distribution import Distribution, pretty_log_dists +from pythonforandroid.entrypoints import main from pythonforandroid.graph import get_recipe_order_and_bootstrap -from pythonforandroid.build import Context, build_recipes +from pythonforandroid.logger import (logger, info, warning, setup_color, + Out_Style, Out_Fore, + info_notify, info_main, shprint) +from pythonforandroid.pythonpackage import get_dep_names_of_package +from pythonforandroid.recipe import Recipe +from pythonforandroid.recommendations import ( + RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API, print_recommendations) +from pythonforandroid.util import ( + current_directory, BuildInterruptingException, load_source, rmdir) user_dir = dirname(realpath(os.path.curdir)) toolchain_dir = dirname(__file__) @@ -884,18 +830,16 @@ def clean_dists(self, _args): """Delete all compiled distributions in the internal distribution directory.""" ctx = self.ctx - if exists(ctx.dist_dir): - shutil.rmtree(ctx.dist_dir) + rmdir(ctx.dist_dir) def clean_bootstrap_builds(self, _args): """Delete all the bootstrap builds.""" - if exists(join(self.ctx.build_dir, 'bootstrap_builds')): - shutil.rmtree(join(self.ctx.build_dir, 'bootstrap_builds')) + rmdir(join(self.ctx.build_dir, 'bootstrap_builds')) # for bs in Bootstrap.all_bootstraps(): # bs = Bootstrap.get_bootstrap(bs, self.ctx) # if bs.build_dir and exists(bs.build_dir): # info('Cleaning build for {} bootstrap.'.format(bs.name)) - # shutil.rmtree(bs.build_dir) + # rmdir(bs.build_dir) def clean_builds(self, _args): """Delete all build caches for each recipe, python-install, java code @@ -906,13 +850,10 @@ def clean_builds(self, _args): of a specific recipe. """ ctx = self.ctx - if exists(ctx.build_dir): - shutil.rmtree(ctx.build_dir) - if exists(ctx.python_installs_dir): - shutil.rmtree(ctx.python_installs_dir) + rmdir(ctx.build_dir) + rmdir(ctx.python_installs_dir) libs_dir = join(self.ctx.build_dir, 'libs_collections') - if exists(libs_dir): - shutil.rmtree(libs_dir) + rmdir(libs_dir) def clean_recipe_build(self, args): """Deletes the build files of the given recipe. @@ -942,14 +883,14 @@ def clean_download_cache(self, args): for package in args.recipes: remove_path = join(ctx.packages_path, package) if exists(remove_path): - shutil.rmtree(remove_path) + rmdir(remove_path) info('Download cache removed for: "{}"'.format(package)) else: warning('No download cache found for "{}", skipping'.format( package)) else: if exists(ctx.packages_path): - shutil.rmtree(ctx.packages_path) + rmdir(ctx.packages_path) info('Download cache removed.') else: print('No cache found at "{}"'.format(ctx.packages_path)) @@ -1068,13 +1009,22 @@ def _build_package(self, args, package_type): self.hook("before_apk_assemble") build_tools_versions = os.listdir(join(ctx.sdk_dir, 'build-tools')) - build_tools_versions = sorted(build_tools_versions, - key=LooseVersion) + + def sort_key(version_text): + try: + # Historically, Android build release candidates have had + # spaces in the version number. + return Version(version_text.replace(" ", "")) + except InvalidVersion: + # Put badly named versions at worst position. + return Version("0") + + build_tools_versions.sort(key=sort_key) build_tools_version = build_tools_versions[-1] info(('Detected highest available build tools ' 'version to be {}').format(build_tools_version)) - if build_tools_version < '25.0': + if Version(build_tools_version.replace(" ", "")) < Version('25.0'): raise BuildInterruptingException( 'build_tools >= 25 is required, but %s is installed' % build_tools_version) if not exists("gradlew"): diff --git a/pythonforandroid/util.py b/pythonforandroid/util.py index f290cdcb25..af363b2e3f 100644 --- a/pythonforandroid/util.py +++ b/pythonforandroid/util.py @@ -1,14 +1,20 @@ import contextlib +from fnmatch import fnmatch +import logging from os.path import exists, join -from os import getcwd, chdir, makedirs, walk, uname +from os import getcwd, chdir, makedirs, walk +from pathlib import Path +from platform import uname import shutil -from fnmatch import fnmatch from tempfile import mkdtemp + from pythonforandroid.logger import (logger, Err_Fore, error, info) +LOGGER = logging.getLogger("p4a.util") -build_platform = '{system}-{machine}'.format( - system=uname()[0], machine=uname()[-1]).lower() +build_platform = "{system}-{machine}".format( + system=uname().system, machine=uname().machine +).lower() """the build platform in the format `system-machine`. We use this string to define the right build system when compiling some recipes or to get the right path for clang compiler""" @@ -39,11 +45,6 @@ def temp_directory(): temp_dir, Err_Fore.RESET))) -def ensure_dir(filename): - if not exists(filename): - makedirs(filename) - - def walk_valid_filens(base_dir, invalid_dir_names, invalid_file_patterns): """Recursively walks all the files and directories in ``dirn``, ignoring directories that match any pattern in ``invalid_dirns`` @@ -104,3 +105,26 @@ def handle_build_exception(exception): if exception.instructions is not None: info('Instructions: {}'.format(exception.instructions)) exit(1) + + +def rmdir(dn, ignore_errors=False): + if not exists(dn): + return + LOGGER.debug("Remove directory and subdirectory {}".format(dn)) + shutil.rmtree(dn, ignore_errors) + + +def ensure_dir(dn): + if exists(dn): + return + LOGGER.debug("Create directory {0}".format(dn)) + makedirs(dn) + + +def move(source, destination): + LOGGER.debug("Moving {} to {}".format(source, destination)) + shutil.move(source, destination) + + +def touch(filename): + Path(filename).touch() diff --git a/setup.py b/setup.py index 57bddc2593..9ced788ea8 100644 --- a/setup.py +++ b/setup.py @@ -21,10 +21,10 @@ # https://github.com/kivy/buildozer/issues/722 install_reqs = [ 'appdirs', 'colorama>=0.3.3', 'jinja2', - 'sh>=1.10, <2.0; sys_platform!="nt"', - 'pep517', 'toml', 'packaging', + 'sh>=1.10, <2.0; sys_platform!="win32"', + 'build', 'toml', 'packaging', ] -# (pep517 and toml are used by pythonpackage.py) +# (build and toml are used by pythonpackage.py) # By specifying every file manually, package_data will be able to diff --git a/tests/recipes/test_libmysqlclient.py b/tests/recipes/test_libmysqlclient.py index 1be4b71e50..e484393398 100644 --- a/tests/recipes/test_libmysqlclient.py +++ b/tests/recipes/test_libmysqlclient.py @@ -23,7 +23,7 @@ def test_build_arch( mock_sh_rm, ): # We overwrite the base test method because we need - # to mock a little more (`sh.cp` and `sh.rm`) + # to mock a little more (`sh.cp` and rmdir) super().test_build_arch() # make sure that the mocked methods are actually called mock_sh_cp.assert_called() diff --git a/tests/recipes/test_openal.py b/tests/recipes/test_openal.py index 27634d9013..21f3196798 100644 --- a/tests/recipes/test_openal.py +++ b/tests/recipes/test_openal.py @@ -50,7 +50,7 @@ def test_build_arch( mock_sh_cp, ): # We overwrite the base test method because we need to mock a little - # more with this recipe (`sh.cp` and `sh.rm`) + # more with this recipe. super().test_build_arch() # make sure that the mocked methods are actually called mock_sh_cp.assert_called() diff --git a/tests/recipes/test_openssl.py b/tests/recipes/test_openssl.py index 509c1cc1e8..861e73bd39 100644 --- a/tests/recipes/test_openssl.py +++ b/tests/recipes/test_openssl.py @@ -23,7 +23,7 @@ def test_build_arch( mock_sh_patch, ): # We overwrite the base test method because we need to mock a little - # more with this recipe (`sh.cp` and `sh.rm`) + # more with this recipe. super().test_build_arch() # make sure that the mocked methods are actually called mock_sh_patch.assert_called() diff --git a/tests/recipes/test_reportlab.py b/tests/recipes/test_reportlab.py index 83191e5286..6129a6a963 100644 --- a/tests/recipes/test_reportlab.py +++ b/tests/recipes/test_reportlab.py @@ -30,7 +30,7 @@ def test_prebuild_arch(self): # these sh commands are not relevant for the test and need to be mocked with \ patch('sh.patch'), \ - patch('sh.touch'), \ + patch('pythonforandroid.recipe.touch'), \ patch('sh.unzip'), \ patch('os.path.isfile'): self.recipe.prebuild_arch(self.arch) diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index e997eba8c2..99620fee75 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -352,12 +352,16 @@ def bootstrap_name(self): @mock.patch("pythonforandroid.util.exists") @mock.patch("pythonforandroid.util.chdir") @mock.patch("pythonforandroid.bootstrap.listdir") - @mock.patch("pythonforandroid.bootstrap.sh.rm") + @mock.patch("pythonforandroid.bootstraps.sdl2.rmdir") + @mock.patch("pythonforandroid.bootstraps.service_only.rmdir") + @mock.patch("pythonforandroid.bootstraps.webview.rmdir") @mock.patch("pythonforandroid.bootstrap.sh.cp") def test_assemble_distribution( self, mock_sh_cp, - mock_sh_rm, + mock_rmdir1, + mock_rmdir2, + mock_rmdir3, mock_listdir, mock_chdir, mock_ensure_dir, @@ -433,7 +437,6 @@ def test_assemble_distribution( ) # check that the other mocks we made are actually called - mock_sh_rm.assert_called() mock_sh_cp.assert_called() mock_chdir.assert_called() mock_listdir.assert_called() @@ -558,11 +561,11 @@ def test_bootstrap_strip( mock_sh_print.assert_called() @mock.patch("pythonforandroid.bootstrap.listdir") - @mock.patch("pythonforandroid.bootstrap.sh.rm") - @mock.patch("pythonforandroid.bootstrap.sh.mv") + @mock.patch("pythonforandroid.bootstrap.rmdir") + @mock.patch("pythonforandroid.bootstrap.move") @mock.patch("pythonforandroid.bootstrap.isdir") def test_bootstrap_fry_eggs( - self, mock_isdir, mock_sh_mv, mock_sh_rm, mock_listdir + self, mock_isdir, mock_move, mock_rmdir, mock_listdir ): mock_listdir.return_value = [ "jnius", @@ -590,11 +593,11 @@ def test_bootstrap_fry_eggs( ] ) self.assertEqual( - mock_sh_rm.call_args[0][1], "pyjnius-1.2.1.dev0-py3.7.egg" + mock_rmdir.call_args[0][0], "pyjnius-1.2.1.dev0-py3.7.egg" ) # check that the other mocks we made are actually called mock_isdir.assert_called() - mock_sh_mv.assert_called() + mock_move.assert_called() class TestBootstrapSdl2(GenericBootstrapTest, unittest.TestCase): diff --git a/tests/test_build.py b/tests/test_build.py index f386b8410f..cf9fa7801d 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -36,10 +36,10 @@ def test_strip_if_with_debug_symbols(self): modules = ["mymodule"] project_dir = None with mock.patch('pythonforandroid.build.info'), \ - mock.patch('sh.Command'),\ - mock.patch('pythonforandroid.build.open'),\ - mock.patch('pythonforandroid.build.shprint'),\ - mock.patch('pythonforandroid.build.current_directory'),\ + mock.patch('sh.Command'), \ + mock.patch('pythonforandroid.build.open'), \ + mock.patch('pythonforandroid.build.shprint'), \ + mock.patch('pythonforandroid.build.current_directory'), \ mock.patch('pythonforandroid.build.CythonRecipe') as m_CythonRecipe, \ mock.patch('pythonforandroid.build.project_has_setup_py') as m_project_has_setup_py, \ mock.patch('pythonforandroid.build.run_setuppy_install'): diff --git a/tests/test_distribution.py b/tests/test_distribution.py index 423d572252..404091ca75 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -91,8 +91,8 @@ def test_folder_exist(self, mock_exists): self.ctx.bootstrap.distribution.dist_dir ) - @mock.patch("pythonforandroid.distribution.rmtree") - def test_delete(self, mock_rmtree): + @mock.patch("pythonforandroid.distribution.rmdir") + def test_delete(self, mock_rmdir): """Test that method :meth:`~pythonforandroid.distribution.Distribution.delete` is called once with the proper arguments.""" @@ -100,7 +100,7 @@ def test_delete(self, mock_rmtree): Bootstrap().get_bootstrap("sdl2", self.ctx) ) self.ctx.bootstrap.distribution.delete() - mock_rmtree.assert_called_once_with( + mock_rmdir.assert_called_once_with( self.ctx.bootstrap.distribution.dist_dir ) diff --git a/tests/test_pythonpackage.py b/tests/test_pythonpackage.py index 2f88cf2aa4..21412e9258 100644 --- a/tests/test_pythonpackage.py +++ b/tests/test_pythonpackage.py @@ -42,7 +42,7 @@ def test_get_package_dependencies(): if "MarkupSafe" in dep ] # Check setuptools not being in non-recursive deps: - # (It will be in recursive ones due to p4a's pep517 dependency) + # (It will be in recursive ones due to p4a's build dependency) assert "setuptools" not in deps_nonrecursive # Check setuptools is present in non-recursive deps, # if we also add build requirements: diff --git a/tests/test_pythonpackage_basic.py b/tests/test_pythonpackage_basic.py index b05344b56b..e98a5f99b0 100644 --- a/tests/test_pythonpackage_basic.py +++ b/tests/test_pythonpackage_basic.py @@ -236,7 +236,7 @@ def run__get_system_python_executable(self, pybin): pybin, "-c", "import importlib\n" - "import json\n" + "import build.util\n" "import os\n" "import sys\n" "sys.path = [os.path.dirname(sys.argv[1])] + sys.path\n" @@ -273,8 +273,8 @@ def test_systemwide_python(self): # Some deps may not be installed, so we just avoid to raise # an exception here, as a missing dep should not make the test # fail. - if "pep517" in str(e.args): - # System python probably doesn't have pep517 available! + if "build" in str(e.args): + # System python probably doesn't have build available! pass elif "toml" in str(e.args): # System python probably doesn't have toml available! @@ -304,11 +304,8 @@ def test_venv(self): ]) subprocess.check_output([ os.path.join(test_dir, "venv", "bin", "pip"), - "install", "-U", "pep517" - ]) - subprocess.check_output([ - os.path.join(test_dir, "venv", "bin", "pip"), - "install", "-U", "toml" + "install", "-U", "build", "toml", "sh<2.0", "colorama", + "appdirs", "jinja2", "packaging" ]) sys_python_path = self.run__get_system_python_executable( os.path.join(test_dir, "venv", "bin", "python") diff --git a/tests/test_recipe.py b/tests/test_recipe.py index 666d089caa..b6e4c99225 100644 --- a/tests/test_recipe.py +++ b/tests/test_recipe.py @@ -134,7 +134,7 @@ def test_download_url_is_set(self): with ( patch_logger_debug()) as m_debug, ( mock.patch.object(Recipe, 'download_file')) as m_download_file, ( - mock.patch('pythonforandroid.recipe.sh.touch')) as m_touch, ( + mock.patch('pythonforandroid.recipe.touch')) as m_touch, ( tempfile.TemporaryDirectory()) as temp_dir: recipe.ctx.setup_dirs(temp_dir) recipe.download() diff --git a/tests/test_util.py b/tests/test_util.py index ff57dc7a47..a4d7ea816e 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,4 +1,6 @@ import os +from pathlib import Path +from tempfile import TemporaryDirectory import types import unittest from unittest import mock @@ -24,6 +26,7 @@ def test_ensure_dir(self, mock_makedirs): @mock.patch("shutil.rmtree") @mock.patch("pythonforandroid.util.mkdtemp") def test_temp_directory(self, mock_mkdtemp, mock_shutil_rmtree): + """ Basic test for method :meth:`~pythonforandroid.util.temp_directory`. We perform this test by `mocking` the command `mkdtemp` and @@ -136,3 +139,51 @@ def test_util_exceptions(self): ) with self.assertRaises(SystemExit): util.handle_build_exception(exc) + + def test_move(self): + with mock.patch( + "pythonforandroid.util.LOGGER" + ) as m_logger, TemporaryDirectory() as base_dir: + new_path = Path(base_dir) / "new" + + # Set up source + old_path = Path(base_dir) / "old" + with open(old_path, "w") as outfile: + outfile.write("Temporary content") + + # Non existent source + with self.assertRaises(FileNotFoundError): + util.move(new_path, new_path) + m_logger.debug.assert_called() + m_logger.error.assert_not_called() + m_logger.reset_mock() + assert old_path.exists() + assert not new_path.exists() + + # Successful move + util.move(old_path, new_path) + assert not old_path.exists() + assert new_path.exists() + m_logger.debug.assert_called() + m_logger.error.assert_not_called() + m_logger.reset_mock() + + # Move over existing: + existing_path = Path(base_dir) / "existing" + existing_path.touch() + + util.move(new_path, existing_path) + with open(existing_path, "r") as infile: + assert infile.read() == "Temporary content" + m_logger.debug.assert_called() + m_logger.error.assert_not_called() + m_logger.reset_mock() + + def test_touch(self): + # Just checking the new file case. + # Assume the existing file timestamp case will work if this does. + with TemporaryDirectory() as base_dir: + new_file_path = Path(base_dir) / "new_file" + assert not new_file_path.exists() + util.touch(new_file_path) + assert new_file_path.exists()