From 7a1d7b8bdadf764065e86715b244d285695cb96a Mon Sep 17 00:00:00 2001 From: Julian-o Date: Tue, 1 Aug 2023 01:49:12 +1000 Subject: [PATCH] Standardise ensure_dir and rmdir --- pythonforandroid/bdistapk.py | 18 +++++------ pythonforandroid/bootstrap.py | 6 ++-- .../bootstraps/common/build/build.py | 17 ++++------ pythonforandroid/bootstraps/sdl2/__init__.py | 10 +++--- .../bootstraps/service_only/__init__.py | 4 +-- .../bootstraps/webview/__init__.py | 8 +++-- pythonforandroid/build.py | 32 ++++++++----------- pythonforandroid/distribution.py | 13 ++++---- pythonforandroid/graph.py | 4 +-- pythonforandroid/prerequisites.py | 16 ++++------ pythonforandroid/pythonpackage.py | 18 ++++++----- pythonforandroid/recipe.py | 25 ++++++--------- pythonforandroid/recipes/ifaddrs/__init__.py | 15 ++++----- pythonforandroid/recipes/lapack/__init__.py | 5 +-- pythonforandroid/recipes/libglob/__init__.py | 17 +++++----- .../recipes/libmysqlclient/__init__.py | 2 +- .../recipes/libtorrent/__init__.py | 7 ++-- pythonforandroid/recipes/opencv/__init__.py | 10 +++--- pythonforandroid/recipes/sqlite3/__init__.py | 9 +++--- pythonforandroid/recipes/twisted/__init__.py | 4 +-- .../recipes/zope_interface/__init__.py | 12 +++---- pythonforandroid/toolchain.py | 24 ++++++-------- pythonforandroid/util.py | 28 +++++++++++----- tests/recipes/test_libmysqlclient.py | 2 +- tests/recipes/test_openal.py | 2 +- tests/recipes/test_openssl.py | 2 +- tests/test_bootstrap.py | 15 +++++---- tests/test_build.py | 8 ++--- tests/test_distribution.py | 6 ++-- tests/test_util.py | 10 +++--- 30 files changed, 175 insertions(+), 174 deletions(-) diff --git a/pythonforandroid/bdistapk.py b/pythonforandroid/bdistapk.py index 575e0e17e5..b4467b9f91 100644 --- a/pythonforandroid/bdistapk.py +++ b/pythonforandroid/bdistapk.py @@ -1,10 +1,10 @@ +from glob import glob +from os.path import realpath, join, dirname, curdir, basename, split from setuptools import Command - +from shutil import copyfile import sys -from os.path import realpath, join, exists, dirname, curdir, basename, split -from os import makedirs -from glob import glob -from shutil import rmtree, copyfile + +from pythonforandroid.util import rmdir, ensure_dir def argv_contains(t): @@ -90,9 +90,8 @@ def prepare_build_dir(self): 'that.') bdist_dir = 'build/bdist.android-{}'.format(self.arch) - if exists(bdist_dir): - rmtree(bdist_dir) - makedirs(bdist_dir) + rmdir(bdist_dir) + ensure_dir(bdist_dir) globs = [] for directory, patterns in self.distribution.package_data.items(): @@ -107,8 +106,7 @@ def prepare_build_dir(self): if not argv_contains('--launcher'): for filen in filens: new_dir = join(bdist_dir, dirname(filen)) - if not exists(new_dir): - makedirs(new_dir) + ensure_dir(new_dir) print('Including {}'.format(filen)) copyfile(filen, join(bdist_dir, filen)) if basename(filen) in ('main.py', 'main.pyc'): diff --git a/pythonforandroid/bootstrap.py b/pythonforandroid/bootstrap.py index 0a5225e526..965ee4e7bf 100755 --- a/pythonforandroid/bootstrap.py +++ b/pythonforandroid/bootstrap.py @@ -10,7 +10,8 @@ from pythonforandroid.logger import (shprint, info, logger, debug) from pythonforandroid.util import ( - current_directory, ensure_dir, temp_directory, BuildInterruptingException) + current_directory, ensure_dir, temp_directory, BuildInterruptingException, + rmdir) from pythonforandroid.recipe import Recipe @@ -70,7 +71,6 @@ class Bootstrap: '''An Android project template, containing recipe stuff for compilation and templated fields for APK info. ''' - name = '' jni_subdir = '/jni' ctx = None @@ -397,7 +397,7 @@ def fry_eggs(self, sitepackages): files = [join(rd, f) for f in listdir(rd) if f != 'EGG-INFO'] if files: shprint(sh.mv, '-t', sitepackages, *files) - shprint(sh.rm, '-rf', d) + rmdir(d) def expand_dependencies(recipes, ctx): diff --git a/pythonforandroid/bootstraps/common/build/build.py b/pythonforandroid/bootstraps/common/build/build.py index 0f5ab5d2a2..0b6b9832f0 100644 --- a/pythonforandroid/bootstraps/common/build/build.py +++ b/pythonforandroid/bootstraps/common/build/build.py @@ -21,6 +21,8 @@ from fnmatch import fnmatch import jinja2 +from pythonforandroid.util import rmdir, ensure_dir + def get_dist_info_for(key, error_if_missing=True): try: @@ -93,11 +95,6 @@ def get_bootstrap_name(): DEFAULT_PYTHON_SERVICE_JAVA_CLASS = 'org.kivy.android.PythonService' -def ensure_dir(path): - if not exists(path): - makedirs(path) - - def render(template, dest, **kwargs): '''Using jinja2, render `template` to the filename `dest`, supplying the @@ -241,7 +238,7 @@ def make_package(args): assets_dir = "src/main/assets" # Delete the old assets. - shutil.rmtree(assets_dir, ignore_errors=True) + rmdir(assets_dir, ignore_errors=True) ensure_dir(assets_dir) # Add extra environment variable file into tar-able directory: @@ -290,7 +287,7 @@ def make_package(args): not exists( join(main_py_only_dir, dir_path) )): - os.mkdir(join(main_py_only_dir, dir_path)) + ensure_dir(join(main_py_only_dir, dir_path)) # Copy actual file: shutil.copyfile( join(args.private, variant), @@ -328,17 +325,17 @@ def make_package(args): ) finally: for directory in _temp_dirs_to_clean: - shutil.rmtree(directory) + rmdir(directory) # Remove extra env vars tar-able directory: - shutil.rmtree(env_vars_tarpath) + rmdir(env_vars_tarpath) # Prepare some variables for templating process res_dir = "src/main/res" res_dir_initial = "src/res_initial" # make res_dir stateless if exists(res_dir_initial): - shutil.rmtree(res_dir, ignore_errors=True) + rmdir(res_dir, ignore_errors=True) shutil.copytree(res_dir_initial, res_dir) else: shutil.copytree(res_dir, res_dir_initial) diff --git a/pythonforandroid/bootstraps/sdl2/__init__.py b/pythonforandroid/bootstraps/sdl2/__init__.py index 662d43c0ef..9334724a33 100644 --- a/pythonforandroid/bootstraps/sdl2/__init__.py +++ b/pythonforandroid/bootstraps/sdl2/__init__.py @@ -1,9 +1,11 @@ -from pythonforandroid.toolchain import ( - Bootstrap, shprint, current_directory, info, info_main) -from pythonforandroid.util import ensure_dir from os.path import join + import sh +from pythonforandroid.toolchain import ( + Bootstrap, shprint, current_directory, info, info_main) +from pythonforandroid.util import ensure_dir, rmdir + class SDL2GradleBootstrap(Bootstrap): name = 'sdl2' @@ -15,8 +17,8 @@ class SDL2GradleBootstrap(Bootstrap): def assemble_distribution(self): info_main("# Creating Android project ({})".format(self.name)) + rmdir(self.dist_dir) info("Copying SDL2/gradle build") - shprint(sh.rm, "-rf", self.dist_dir) shprint(sh.cp, "-r", self.build_dir, self.dist_dir) # either the build use environment variable (ANDROID_HOME) diff --git a/pythonforandroid/bootstraps/service_only/__init__.py b/pythonforandroid/bootstraps/service_only/__init__.py index b9e000c012..4f0d6cf20b 100644 --- a/pythonforandroid/bootstraps/service_only/__init__.py +++ b/pythonforandroid/bootstraps/service_only/__init__.py @@ -2,7 +2,7 @@ from os.path import join from pythonforandroid.toolchain import ( Bootstrap, current_directory, info, info_main, shprint) -from pythonforandroid.util import ensure_dir +from pythonforandroid.util import ensure_dir, rmdir class ServiceOnlyBootstrap(Bootstrap): @@ -18,7 +18,7 @@ def assemble_distribution(self): self.name)) info('This currently just copies the build stuff straight from the build dir.') - shprint(sh.rm, '-rf', self.dist_dir) + rmdir(self.dist_dir) shprint(sh.cp, '-r', self.build_dir, self.dist_dir) with current_directory(self.dist_dir): with open('local.properties', 'w') as fileh: diff --git a/pythonforandroid/bootstraps/webview/__init__.py b/pythonforandroid/bootstraps/webview/__init__.py index da33ac72d5..7604ed3b84 100644 --- a/pythonforandroid/bootstraps/webview/__init__.py +++ b/pythonforandroid/bootstraps/webview/__init__.py @@ -1,8 +1,10 @@ -from pythonforandroid.toolchain import Bootstrap, current_directory, info, info_main, shprint -from pythonforandroid.util import ensure_dir from os.path import join + import sh +from pythonforandroid.toolchain import Bootstrap, current_directory, info, info_main, shprint +from pythonforandroid.util import ensure_dir, rmdir + class WebViewBootstrap(Bootstrap): name = 'webview' @@ -15,7 +17,7 @@ def assemble_distribution(self): info_main('# Creating Android project from build and {} bootstrap'.format( self.name)) - shprint(sh.rm, '-rf', self.dist_dir) + rmdir(self.dist_dir) shprint(sh.cp, '-r', self.build_dir, self.dist_dir) with current_directory(self.dist_dir): with open('local.properties', 'w') as fileh: diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index 645b368d00..9fe4ba6a6a 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -1,28 +1,29 @@ +from contextlib import suppress +import copy +import glob +import os +from os import environ from os.path import ( abspath, join, realpath, dirname, expanduser, exists ) -from os import environ -import copy -import os -import glob import re -import sh import shutil import subprocess -from contextlib import suppress -from pythonforandroid.util import ( - current_directory, ensure_dir, - BuildInterruptingException, -) -from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint) +import sh + +from pythonforandroid.androidndk import AndroidNDK from pythonforandroid.archs import ArchARM, ArchARMv7_a, ArchAarch_64, Archx86, Archx86_64 +from pythonforandroid.logger import (info, warning, info_notify, info_main, shprint) from pythonforandroid.pythonpackage import get_package_name from pythonforandroid.recipe import CythonRecipe, Recipe from pythonforandroid.recommendations import ( check_ndk_version, check_target_api, check_ndk_api, RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API) -from pythonforandroid.androidndk import AndroidNDK +from pythonforandroid.util import ( + current_directory, ensure_dir, + BuildInterruptingException, rmdir +) def get_targets(sdk_dir): @@ -77,11 +78,6 @@ class Context: # the Android project folder where everything ends up dist_dir = None - # where Android libs are cached after build - # but before being placed in dists - libs_dir = None - aars_dir = None - # Whether setup.py or similar should be used if present: use_setup_py = False @@ -642,7 +638,7 @@ def run_setuppy_install(ctx, project_dir, env=None, arch=None): for f in set(copied_over_contents + new_venv_additions): full_path = os.path.join(venv_site_packages_dir, f) if os.path.isdir(full_path): - shutil.rmtree(full_path) + rmdir(full_path) else: os.remove(full_path) finally: diff --git a/pythonforandroid/distribution.py b/pythonforandroid/distribution.py index 2b1f1a3f5e..c878e0ea87 100644 --- a/pythonforandroid/distribution.py +++ b/pythonforandroid/distribution.py @@ -1,10 +1,11 @@ -from os.path import exists, join -import glob import json +import glob +from os.path import exists, join -from pythonforandroid.logger import (debug, info, info_notify, warning, Err_Style, Err_Fore) -from pythonforandroid.util import current_directory, BuildInterruptingException -from shutil import rmtree +from pythonforandroid.logger import ( + debug, info, info_notify, warning, Err_Style, Err_Fore) +from pythonforandroid.util import ( + current_directory, BuildInterruptingException, rmdir) class Distribution: @@ -201,7 +202,7 @@ def folder_exists(self): return exists(self.dist_dir) def delete(self): - rmtree(self.dist_dir) + rmdir(self.dist_dir) @classmethod def get_distributions(cls, ctx, extra_dist_dirs=[]): diff --git a/pythonforandroid/graph.py b/pythonforandroid/graph.py index bdaca4349c..4edb8f4c90 100644 --- a/pythonforandroid/graph.py +++ b/pythonforandroid/graph.py @@ -45,7 +45,7 @@ def get_dependency_tuple_list_for_recipe(recipe, blacklist=None): """ if blacklist is None: blacklist = set() - assert type(blacklist) == set + assert type(blacklist) is set if recipe.depends is None: dependencies = [] else: @@ -160,7 +160,7 @@ def obvious_conflict_checker(ctx, name_tuples, blacklist=None): current_to_be_added = list(to_be_added) to_be_added = [] for (added_tuple, adding_recipe) in current_to_be_added: - assert type(added_tuple) == tuple + assert type(added_tuple) is tuple if len(added_tuple) > 1: # No obvious commitment in what to add, don't check it itself # but throw it into deps for later comparing against diff --git a/pythonforandroid/prerequisites.py b/pythonforandroid/prerequisites.py index 38b1cea49d..d5e013f11d 100644 --- a/pythonforandroid/prerequisites.py +++ b/pythonforandroid/prerequisites.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import sys -import platform import os -import subprocess +import platform import shutil +import subprocess +import sys + from pythonforandroid.logger import info, warning, error +from pythonforandroid.util import ensure_dir class Prerequisite(object): @@ -247,13 +249,7 @@ def darwin_installer(self): "~/Library/Java/JavaVirtualMachines" ) info(f"Extracting {filename} to {user_library_java_path}") - subprocess.check_output( - [ - "mkdir", - "-p", - user_library_java_path, - ], - ) + ensure_dir(user_library_java_path) subprocess.check_output( ["tar", "xzf", f"/tmp/{filename}", "-C", user_library_java_path], ) diff --git a/pythonforandroid/pythonpackage.py b/pythonforandroid/pythonpackage.py index 53b1226c55..9e4c29bd81 100644 --- a/pythonforandroid/pythonpackage.py +++ b/pythonforandroid/pythonpackage.py @@ -34,6 +34,7 @@ import functools +from io import open # needed for python 2 import os import shutil import subprocess @@ -41,14 +42,15 @@ import tarfile import tempfile import time -import zipfile -from io import open # needed for python 2 from urllib.parse import unquote as urlunquote from urllib.parse import urlparse +import zipfile import toml import build.util +from pythonforandroid.util import rmdir, ensure_dir + def transform_dep_for_pip(dependency): if dependency.find("@") > 0 and ( @@ -113,7 +115,7 @@ def extract_metainfo_files_from_package( _extract_metainfo_files_from_package_unsafe(package, output_folder) finally: - shutil.rmtree(temp_folder) + rmdir(temp_folder) def _get_system_python_executable(): @@ -314,7 +316,7 @@ def get_package_as_folder(dependency): ) # Create download subfolder: - os.mkdir(os.path.join(venv_path, "download")) + ensure_dir(os.path.join(venv_path, "download")) # Write a requirements.txt with our package and download: with open(os.path.join(venv_path, "requirements.txt"), @@ -394,11 +396,11 @@ def to_unicode(s): # Needed for Python 2. # Copy result to new dedicated folder so we can throw away # our entire virtualenv nonsense after returning: result_path = tempfile.mkdtemp() - shutil.rmtree(result_path) + rmdir(result_path) shutil.copytree(result_folder_or_file, result_path) return (dl_type, result_path) finally: - shutil.rmtree(venv_parent) + rmdir(venv_parent) def _extract_metainfo_files_from_package_unsafe( @@ -458,7 +460,7 @@ def _extract_metainfo_files_from_package_unsafe( shutil.copyfile(metadata_path, os.path.join(output_path, "METADATA")) finally: if clean_up_path: - shutil.rmtree(path) + rmdir(path) def is_filesystem_path(dep): @@ -576,7 +578,7 @@ def _extract_info_from_package(dependency, return list(set(requirements)) # remove duplicates finally: - shutil.rmtree(output_folder) + rmdir(output_folder) package_name_cache = dict() diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 67c309e197..a2578e6b32 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -1,6 +1,5 @@ from os.path import basename, dirname, exists, isdir, isfile, join, realpath, split import glob -from shutil import rmtree import hashlib from re import match @@ -10,7 +9,7 @@ import fnmatch import urllib.request from urllib.request import urlretrieve -from os import listdir, unlink, environ, mkdir, curdir, walk +from os import listdir, unlink, environ, curdir, walk from sys import stdout import time try: @@ -19,7 +18,7 @@ from urllib.parse import urlparse from pythonforandroid.logger import (logger, info, warning, debug, shprint, info_main) from pythonforandroid.util import (current_directory, ensure_dir, - BuildInterruptingException) + BuildInterruptingException, rmdir) from pythonforandroid.util import load_source as import_recipe @@ -218,7 +217,7 @@ def report_hook(index, blksize, size): url = url[4:] # if 'version' is specified, do a shallow clone if self.version: - shprint(sh.mkdir, '-p', target) + ensure_dir(target) with current_directory(target): shprint(sh.git, 'init') shprint(sh.git, 'remote', 'add', 'origin', url) @@ -367,7 +366,7 @@ def download(self): if expected_digest: expected_digests[alg] = expected_digest - shprint(sh.mkdir, '-p', join(self.ctx.packages_path, self.name)) + ensure_dir(join(self.ctx.packages_path, self.name)) with current_directory(join(self.ctx.packages_path, self.name)): filename = shprint(sh.basename, url).stdout[:-1].decode('utf-8') @@ -423,9 +422,7 @@ def unpack(self, arch): self.name.lower())) if exists(self.get_build_dir(arch)): return - shprint(sh.rm, '-rf', build_dir) - shprint(sh.mkdir, '-p', build_dir) - shprint(sh.rmdir, build_dir) + rmdir(build_dir) ensure_dir(build_dir) shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch)) return @@ -473,7 +470,7 @@ def unpack(self, arch): 'Could not extract {} download, it must be .zip, ' '.tar.gz or .tar.bz2 or .tar.xz'.format(extraction_filename)) elif isdir(extraction_filename): - mkdir(directory_name) + ensure_dir(directory_name) for entry in listdir(extraction_filename): if entry not in ('.git',): shprint(sh.cp, '-Rv', @@ -614,13 +611,11 @@ def clean_build(self, arch=None): 'build dirs'.format(self.name)) for directory in dirs: - if exists(directory): - info('Deleting {}'.format(directory)) - shutil.rmtree(directory) + rmdir(directory) # Delete any Python distributions to ensure the recipe build # doesn't persist in site-packages - shutil.rmtree(self.ctx.python_installs_dir) + rmdir(self.ctx.python_installs_dir) def install_libs(self, arch, *libs): libs_dir = self.ctx.get_libs_dir(arch.arch) @@ -721,7 +716,7 @@ def prepare_build_dir(self, arch): if self.src_filename is None: raise BuildInterruptingException( 'IncludedFilesBehaviour failed: no src_filename specified') - shprint(sh.rm, '-rf', self.get_build_dir(arch)) + rmdir(self.get_build_dir(arch)) shprint(sh.cp, '-a', join(self.get_recipe_dir(), self.src_filename), self.get_build_dir(arch)) @@ -861,7 +856,7 @@ def clean_build(self, arch=None): build_dir = join(site_packages_dir[0], name) if exists(build_dir): info('Deleted {}'.format(build_dir)) - rmtree(build_dir) + rmdir(build_dir) @property def real_hostpython_location(self): diff --git a/pythonforandroid/recipes/ifaddrs/__init__.py b/pythonforandroid/recipes/ifaddrs/__init__.py index 7d44f9cd72..1317dc2556 100644 --- a/pythonforandroid/recipes/ifaddrs/__init__.py +++ b/pythonforandroid/recipes/ifaddrs/__init__.py @@ -1,10 +1,13 @@ """ ifaddrs for Android """ -from os.path import join, exists +from os.path import join + import sh -from pythonforandroid.logger import info, shprint + +from pythonforandroid.logger import shprint from pythonforandroid.recipe import CompiledComponentsPythonRecipe from pythonforandroid.toolchain import current_directory +from pythonforandroid.util import ensure_dir class IFAddrRecipe(CompiledComponentsPythonRecipe): @@ -19,9 +22,7 @@ class IFAddrRecipe(CompiledComponentsPythonRecipe): def prebuild_arch(self, arch): """Make the build and target directories""" path = self.get_build_dir(arch.arch) - if not exists(path): - info("creating {}".format(path)) - shprint(sh.mkdir, '-p', path) + ensure_dir(path) def build_arch(self, arch): """simple shared compile""" @@ -30,9 +31,7 @@ def build_arch(self, arch): self.get_build_dir(arch.arch), join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Lib'), join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Include')): - if not exists(path): - info("creating {}".format(path)) - shprint(sh.mkdir, '-p', path) + ensure_dir(path) cli = env['CC'].split()[0] # makes sure first CC command is the compiler rather than ccache, refs: # https://github.com/kivy/python-for-android/issues/1398 diff --git a/pythonforandroid/recipes/lapack/__init__.py b/pythonforandroid/recipes/lapack/__init__.py index ae20e69538..b6124dc285 100644 --- a/pythonforandroid/recipes/lapack/__init__.py +++ b/pythonforandroid/recipes/lapack/__init__.py @@ -11,7 +11,7 @@ import sh import shutil from os import environ -from pythonforandroid.util import build_platform +from pythonforandroid.util import build_platform, rmdir arch_to_sysroot = {'armeabi': 'arm', 'armeabi-v7a': 'arm', 'arm64-v8a': 'arm64'} @@ -57,7 +57,8 @@ def build_arch(self, arch): with current_directory(build_target): env = self.get_recipe_env(arch) ndk_dir = environ["LEGACY_NDK"] - shprint(sh.rm, '-rf', 'CMakeFiles/', 'CMakeCache.txt', _env=env) + rmdir('CMakeFiles') + shprint(sh.rm, '-f', 'CMakeCache.txt', _env=env) opts = [ '-DCMAKE_SYSTEM_NAME=Android', '-DCMAKE_POSITION_INDEPENDENT_CODE=1', diff --git a/pythonforandroid/recipes/libglob/__init__.py b/pythonforandroid/recipes/libglob/__init__.py index f63db42628..39a68b7ee2 100644 --- a/pythonforandroid/recipes/libglob/__init__.py +++ b/pythonforandroid/recipes/libglob/__init__.py @@ -2,11 +2,14 @@ android libglob available via '-lglob' LDFLAG """ -from os.path import exists, join +from os.path import join + +import sh + +from pythonforandroid.logger import shprint from pythonforandroid.recipe import Recipe from pythonforandroid.toolchain import current_directory -from pythonforandroid.logger import info, shprint -import sh +from pythonforandroid.util import ensure_dir class LibGlobRecipe(Recipe): @@ -32,9 +35,7 @@ def should_build(self, arch): def prebuild_arch(self, arch): """Make the build and target directories""" path = self.get_build_dir(arch.arch) - if not exists(path): - info("creating {}".format(path)) - shprint(sh.mkdir, '-p', path) + ensure_dir(path) def build_arch(self, arch): """simple shared compile""" @@ -43,9 +44,7 @@ def build_arch(self, arch): self.get_build_dir(arch.arch), join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Lib'), join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Include')): - if not exists(path): - info("creating {}".format(path)) - shprint(sh.mkdir, '-p', path) + ensure_dir(path) cli = env['CC'].split()[0] # makes sure first CC command is the compiler rather than ccache, refs: # https://github.com/kivy/python-for-android/issues/1399 diff --git a/pythonforandroid/recipes/libmysqlclient/__init__.py b/pythonforandroid/recipes/libmysqlclient/__init__.py index 31ebd3c540..84fd8d30ac 100644 --- a/pythonforandroid/recipes/libmysqlclient/__init__.py +++ b/pythonforandroid/recipes/libmysqlclient/__init__.py @@ -26,7 +26,7 @@ def build_arch(self, arch): env = self.get_recipe_env(arch) with current_directory(join(self.get_build_dir(arch.arch), 'libmysqlclient')): shprint(sh.cp, '-t', '.', join(self.get_recipe_dir(), 'p4a.cmake')) - # shprint(sh.mkdir, 'Platform') + # ensure_dir('Platform') # shprint(sh.cp, '-t', 'Platform', join(self.get_recipe_dir(), 'Linux.cmake')) shprint(sh.rm, '-f', 'CMakeCache.txt') shprint(sh.cmake, '-G', 'Unix Makefiles', diff --git a/pythonforandroid/recipes/libtorrent/__init__.py b/pythonforandroid/recipes/libtorrent/__init__.py index 24f94081c6..1086e00fcc 100644 --- a/pythonforandroid/recipes/libtorrent/__init__.py +++ b/pythonforandroid/recipes/libtorrent/__init__.py @@ -1,9 +1,12 @@ -from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory from multiprocessing import cpu_count -from os.path import join, basename from os import listdir, walk +from os.path import join, basename +import shutil + import sh +from pythonforandroid.toolchain import Recipe, shprint, current_directory + # This recipe builds libtorrent with Python bindings # It depends on Boost.Build and the source of several Boost libraries present # in BOOST_ROOT, which is all provided by the boost recipe diff --git a/pythonforandroid/recipes/opencv/__init__.py b/pythonforandroid/recipes/opencv/__init__.py index c760cbdda7..650c77e508 100644 --- a/pythonforandroid/recipes/opencv/__init__.py +++ b/pythonforandroid/recipes/opencv/__init__.py @@ -1,9 +1,11 @@ +from multiprocessing import cpu_count from os.path import join + import sh -from pythonforandroid.recipe import NDKRecipe -from pythonforandroid.util import current_directory + from pythonforandroid.logger import shprint -from multiprocessing import cpu_count +from pythonforandroid.recipe import NDKRecipe +from pythonforandroid.util import current_directory, ensure_dir class OpenCVRecipe(NDKRecipe): @@ -45,7 +47,7 @@ def get_recipe_env(self, arch): def build_arch(self, arch): build_dir = join(self.get_build_dir(arch.arch), 'build') - shprint(sh.mkdir, '-p', build_dir) + ensure_dir(build_dir) opencv_extras = [] if 'opencv_extras' in self.ctx.recipe_build_order: diff --git a/pythonforandroid/recipes/sqlite3/__init__.py b/pythonforandroid/recipes/sqlite3/__init__.py index 955d808141..1f4292c1eb 100644 --- a/pythonforandroid/recipes/sqlite3/__init__.py +++ b/pythonforandroid/recipes/sqlite3/__init__.py @@ -1,7 +1,8 @@ -from pythonforandroid.recipe import NDKRecipe -from pythonforandroid.toolchain import shutil from os.path import join -import sh +import shutil + +from pythonforandroid.recipe import NDKRecipe +from pythonforandroid.util import ensure_dir class Sqlite3Recipe(NDKRecipe): @@ -16,7 +17,7 @@ def should_build(self, arch): def prebuild_arch(self, arch): super().prebuild_arch(arch) # Copy the Android make file - sh.mkdir('-p', join(self.get_build_dir(arch.arch), 'jni')) + ensure_dir(join(self.get_build_dir(arch.arch), 'jni')) shutil.copyfile(join(self.get_recipe_dir(), 'Android.mk'), join(self.get_build_dir(arch.arch), 'jni/Android.mk')) diff --git a/pythonforandroid/recipes/twisted/__init__.py b/pythonforandroid/recipes/twisted/__init__.py index 0c390a5b14..30a7af4bb9 100644 --- a/pythonforandroid/recipes/twisted/__init__.py +++ b/pythonforandroid/recipes/twisted/__init__.py @@ -1,7 +1,7 @@ import os -import shutil from pythonforandroid.recipe import CythonRecipe +from pythonforandroid.util import rmdir class TwistedRecipe(CythonRecipe): @@ -23,7 +23,7 @@ def prebuild_arch(self, arch): for item in os.walk(source_dir): if os.path.basename(item[0]) == 'test': full_path = os.path.join(source_dir, item[0]) - shutil.rmtree(full_path, ignore_errors=True) + rmdir(full_path, ignore_errors=True) def get_recipe_env(self, arch): env = super().get_recipe_env(arch) diff --git a/pythonforandroid/recipes/zope_interface/__init__.py b/pythonforandroid/recipes/zope_interface/__init__.py index 46a1820c2b..7e7fecaea5 100644 --- a/pythonforandroid/recipes/zope_interface/__init__.py +++ b/pythonforandroid/recipes/zope_interface/__init__.py @@ -1,7 +1,8 @@ +from os.path import join + from pythonforandroid.recipe import PythonRecipe from pythonforandroid.toolchain import current_directory -from os.path import join -import sh +from pythonforandroid.util import rmdir class ZopeInterfaceRecipe(PythonRecipe): @@ -25,11 +26,8 @@ def build_arch(self, arch): def prebuild_arch(self, arch): super().prebuild_arch(arch) with current_directory(self.get_build_dir(arch.arch)): - sh.rm( - '-rf', - 'src/zope/interface/tests', - 'src/zope/interface/common/tests', - ) + rmdir('src/zope/interface/tests') + rmdir('src/zope/interface/common/tests') recipe = ZopeInterfaceRecipe() diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index 38c700740c..7a5461f30d 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -16,7 +16,6 @@ from os.path import (join, dirname, realpath, exists, expanduser, basename) import re import shlex -import shutil import sys from sys import platform @@ -42,7 +41,7 @@ from pythonforandroid.recommendations import ( RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API, print_recommendations) from pythonforandroid.util import ( - current_directory, BuildInterruptingException, load_source) + current_directory, BuildInterruptingException, load_source, rmdir) user_dir = dirname(realpath(os.path.curdir)) toolchain_dir = dirname(__file__) @@ -831,18 +830,16 @@ def clean_dists(self, _args): """Delete all compiled distributions in the internal distribution directory.""" ctx = self.ctx - if exists(ctx.dist_dir): - shutil.rmtree(ctx.dist_dir) + rmdir(ctx.dist_dir) def clean_bootstrap_builds(self, _args): """Delete all the bootstrap builds.""" - if exists(join(self.ctx.build_dir, 'bootstrap_builds')): - shutil.rmtree(join(self.ctx.build_dir, 'bootstrap_builds')) + rmdir(join(self.ctx.build_dir, 'bootstrap_builds')) # for bs in Bootstrap.all_bootstraps(): # bs = Bootstrap.get_bootstrap(bs, self.ctx) # if bs.build_dir and exists(bs.build_dir): # info('Cleaning build for {} bootstrap.'.format(bs.name)) - # shutil.rmtree(bs.build_dir) + # rmdir(bs.build_dir) def clean_builds(self, _args): """Delete all build caches for each recipe, python-install, java code @@ -853,13 +850,10 @@ def clean_builds(self, _args): of a specific recipe. """ ctx = self.ctx - if exists(ctx.build_dir): - shutil.rmtree(ctx.build_dir) - if exists(ctx.python_installs_dir): - shutil.rmtree(ctx.python_installs_dir) + rmdir(ctx.build_dir) + rmdir(ctx.python_installs_dir) libs_dir = join(self.ctx.build_dir, 'libs_collections') - if exists(libs_dir): - shutil.rmtree(libs_dir) + rmdir(libs_dir) def clean_recipe_build(self, args): """Deletes the build files of the given recipe. @@ -889,14 +883,14 @@ def clean_download_cache(self, args): for package in args.recipes: remove_path = join(ctx.packages_path, package) if exists(remove_path): - shutil.rmtree(remove_path) + rmdir(remove_path) info('Download cache removed for: "{}"'.format(package)) else: warning('No download cache found for "{}", skipping'.format( package)) else: if exists(ctx.packages_path): - shutil.rmtree(ctx.packages_path) + rmdir(ctx.packages_path) info('Download cache removed.') else: print('No cache found at "{}"'.format(ctx.packages_path)) diff --git a/pythonforandroid/util.py b/pythonforandroid/util.py index 713c2b46c0..f9d5302ecc 100644 --- a/pythonforandroid/util.py +++ b/pythonforandroid/util.py @@ -1,12 +1,15 @@ import contextlib +from fnmatch import fnmatch +import logging from os.path import exists, join from os import getcwd, chdir, makedirs, walk from platform import uname -import shutil -from fnmatch import fnmatch +from shutil import rmtree from tempfile import mkdtemp + from pythonforandroid.logger import (logger, Err_Fore, error, info) +LOGGER = logging.getLogger("p4a.util") build_platform = "{system}-{machine}".format( system=uname().system, machine=uname().machine @@ -36,16 +39,11 @@ def temp_directory(): temp_dir, Err_Fore.RESET))) yield temp_dir finally: - shutil.rmtree(temp_dir) + rmtree(temp_dir) logger.debug(''.join((Err_Fore.CYAN, ' - temp directory deleted ', temp_dir, Err_Fore.RESET))) -def ensure_dir(filename): - if not exists(filename): - makedirs(filename) - - def walk_valid_filens(base_dir, invalid_dir_names, invalid_file_patterns): """Recursively walks all the files and directories in ``dirn``, ignoring directories that match any pattern in ``invalid_dirns`` @@ -106,3 +104,17 @@ def handle_build_exception(exception): if exception.instructions is not None: info('Instructions: {}'.format(exception.instructions)) exit(1) + + +def rmdir(dn, ignore_errors=False): + if not exists(dn): + return + LOGGER.debug("Remove directory and subdirectory {}".format(dn)) + rmtree(dn, ignore_errors) + + +def ensure_dir(dn): + if exists(dn): + return + LOGGER.debug("Create directory {0}".format(dn)) + makedirs(dn) diff --git a/tests/recipes/test_libmysqlclient.py b/tests/recipes/test_libmysqlclient.py index 1be4b71e50..e484393398 100644 --- a/tests/recipes/test_libmysqlclient.py +++ b/tests/recipes/test_libmysqlclient.py @@ -23,7 +23,7 @@ def test_build_arch( mock_sh_rm, ): # We overwrite the base test method because we need - # to mock a little more (`sh.cp` and `sh.rm`) + # to mock a little more (`sh.cp` and rmdir) super().test_build_arch() # make sure that the mocked methods are actually called mock_sh_cp.assert_called() diff --git a/tests/recipes/test_openal.py b/tests/recipes/test_openal.py index 27634d9013..21f3196798 100644 --- a/tests/recipes/test_openal.py +++ b/tests/recipes/test_openal.py @@ -50,7 +50,7 @@ def test_build_arch( mock_sh_cp, ): # We overwrite the base test method because we need to mock a little - # more with this recipe (`sh.cp` and `sh.rm`) + # more with this recipe. super().test_build_arch() # make sure that the mocked methods are actually called mock_sh_cp.assert_called() diff --git a/tests/recipes/test_openssl.py b/tests/recipes/test_openssl.py index 509c1cc1e8..861e73bd39 100644 --- a/tests/recipes/test_openssl.py +++ b/tests/recipes/test_openssl.py @@ -23,7 +23,7 @@ def test_build_arch( mock_sh_patch, ): # We overwrite the base test method because we need to mock a little - # more with this recipe (`sh.cp` and `sh.rm`) + # more with this recipe. super().test_build_arch() # make sure that the mocked methods are actually called mock_sh_patch.assert_called() diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index e997eba8c2..ee5c0123f9 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -352,12 +352,16 @@ def bootstrap_name(self): @mock.patch("pythonforandroid.util.exists") @mock.patch("pythonforandroid.util.chdir") @mock.patch("pythonforandroid.bootstrap.listdir") - @mock.patch("pythonforandroid.bootstrap.sh.rm") + @mock.patch("pythonforandroid.bootstraps.sdl2.rmdir") + @mock.patch("pythonforandroid.bootstraps.service_only.rmdir") + @mock.patch("pythonforandroid.bootstraps.webview.rmdir") @mock.patch("pythonforandroid.bootstrap.sh.cp") def test_assemble_distribution( self, mock_sh_cp, - mock_sh_rm, + mock_rmdir1, + mock_rmdir2, + mock_rmdir3, mock_listdir, mock_chdir, mock_ensure_dir, @@ -433,7 +437,6 @@ def test_assemble_distribution( ) # check that the other mocks we made are actually called - mock_sh_rm.assert_called() mock_sh_cp.assert_called() mock_chdir.assert_called() mock_listdir.assert_called() @@ -558,11 +561,11 @@ def test_bootstrap_strip( mock_sh_print.assert_called() @mock.patch("pythonforandroid.bootstrap.listdir") - @mock.patch("pythonforandroid.bootstrap.sh.rm") + @mock.patch("pythonforandroid.bootstrap.rmdir") @mock.patch("pythonforandroid.bootstrap.sh.mv") @mock.patch("pythonforandroid.bootstrap.isdir") def test_bootstrap_fry_eggs( - self, mock_isdir, mock_sh_mv, mock_sh_rm, mock_listdir + self, mock_isdir, mock_sh_mv, mock_rmdir, mock_listdir ): mock_listdir.return_value = [ "jnius", @@ -590,7 +593,7 @@ def test_bootstrap_fry_eggs( ] ) self.assertEqual( - mock_sh_rm.call_args[0][1], "pyjnius-1.2.1.dev0-py3.7.egg" + mock_rmdir.call_args[0][0], "pyjnius-1.2.1.dev0-py3.7.egg" ) # check that the other mocks we made are actually called mock_isdir.assert_called() diff --git a/tests/test_build.py b/tests/test_build.py index f386b8410f..cf9fa7801d 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -36,10 +36,10 @@ def test_strip_if_with_debug_symbols(self): modules = ["mymodule"] project_dir = None with mock.patch('pythonforandroid.build.info'), \ - mock.patch('sh.Command'),\ - mock.patch('pythonforandroid.build.open'),\ - mock.patch('pythonforandroid.build.shprint'),\ - mock.patch('pythonforandroid.build.current_directory'),\ + mock.patch('sh.Command'), \ + mock.patch('pythonforandroid.build.open'), \ + mock.patch('pythonforandroid.build.shprint'), \ + mock.patch('pythonforandroid.build.current_directory'), \ mock.patch('pythonforandroid.build.CythonRecipe') as m_CythonRecipe, \ mock.patch('pythonforandroid.build.project_has_setup_py') as m_project_has_setup_py, \ mock.patch('pythonforandroid.build.run_setuppy_install'): diff --git a/tests/test_distribution.py b/tests/test_distribution.py index 423d572252..404091ca75 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -91,8 +91,8 @@ def test_folder_exist(self, mock_exists): self.ctx.bootstrap.distribution.dist_dir ) - @mock.patch("pythonforandroid.distribution.rmtree") - def test_delete(self, mock_rmtree): + @mock.patch("pythonforandroid.distribution.rmdir") + def test_delete(self, mock_rmdir): """Test that method :meth:`~pythonforandroid.distribution.Distribution.delete` is called once with the proper arguments.""" @@ -100,7 +100,7 @@ def test_delete(self, mock_rmtree): Bootstrap().get_bootstrap("sdl2", self.ctx) ) self.ctx.bootstrap.distribution.delete() - mock_rmtree.assert_called_once_with( + mock_rmdir.assert_called_once_with( self.ctx.bootstrap.distribution.dist_dir ) diff --git a/tests/test_util.py b/tests/test_util.py index ff57dc7a47..495d3d7c1b 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -21,20 +21,20 @@ def test_ensure_dir(self, mock_makedirs): util.ensure_dir("fake_directory") mock_makedirs.assert_called_once_with("fake_directory") - @mock.patch("shutil.rmtree") + @mock.patch("pythonforandroid.util.rmtree") @mock.patch("pythonforandroid.util.mkdtemp") - def test_temp_directory(self, mock_mkdtemp, mock_shutil_rmtree): + def test_temp_directory(self, mock_mkdtemp, mock_rmtree): """ Basic test for method :meth:`~pythonforandroid.util.temp_directory`. We perform this test by `mocking` the command `mkdtemp` and - `shutil.rmtree` and we make sure that those functions are called in the + `rmdir` and we make sure that those functions are called in the proper place. """ mock_mkdtemp.return_value = "/temp/any_directory" with util.temp_directory(): mock_mkdtemp.assert_called_once() - mock_shutil_rmtree.assert_not_called() - mock_shutil_rmtree.assert_called_once_with("/temp/any_directory") + mock_rmtree.assert_not_called() + mock_rmtree.assert_called_once_with("/temp/any_directory") @mock.patch("pythonforandroid.util.chdir") def test_current_directory(self, moch_chdir):