From 7938754e65e9301216bf3a156bc9ae4a46e18c53 Mon Sep 17 00:00:00 2001 From: Julian-o Date: Fri, 25 Aug 2023 23:17:15 +1000 Subject: [PATCH] Migrate to use Buildop's rmtree, file_copy, file_copytree and rename This is part of a bigger refactor to reduce the size and complexity of the Buildozer class. It adds no new functionality (apart from making logging more consistent) * Remove file_rename, file_copy and file_copytree from Buildozer. * Update all references to those Buildozer methods to refer to buildops equivalents. * Note: `file_rename` is now called rename as, in practice, it was also used to rename directories. * Updated all references to shutil.copyfile and shutil.rmtree to use the Buildops equivalents * Note this adds implicit debug logging, making them more consistent. * Updated all references to check_call() and check_output() that called the `rm` and `cp` shell commands. * Note this adds implicit debug logging, making them more consistent, and improves performance. * Updated mock in test. Cheekily, also removed some obsolete references to Python 2 (only) standard libraries to make my IDE stop warning me about them while I edited the file. --- buildozer/__init__.py | 56 +++++++---------------------------- buildozer/target.py | 4 +-- buildozer/targets/android.py | 18 ++++++----- buildozer/targets/osx.py | 19 ++++++------ tests/targets/test_android.py | 2 +- 5 files changed, 34 insertions(+), 65 deletions(-) diff --git a/buildozer/__init__.py b/buildozer/__init__.py index 60c5630d2..dab75b4d0 100644 --- a/buildozer/__init__.py +++ b/buildozer/__init__.py @@ -17,7 +17,7 @@ import re from re import search import select -from shutil import copyfile, rmtree, copytree, move, which +from shutil import which from subprocess import Popen, PIPE, TimeoutExpired import sys from sys import stdout, stderr, exit @@ -476,23 +476,6 @@ def file_matches(self, patterns): def file_exists(self, *args): return exists(join(*args)) - def file_rename(self, source, target, cwd=None): - if cwd: - source = join(cwd, source) - target = join(cwd, target) - self.logger.debug('Rename {0} to {1}'.format(source, target)) - if not os.path.isdir(os.path.dirname(target)): - self.logger.error(('Rename {0} to {1} fails because {2} is not a ' - 'directory').format(source, target, target)) - move(source, target) - - def file_copy(self, source, target, cwd=None): - if cwd: - source = join(cwd, source) - target = join(cwd, target) - self.logger.debug('Copy {0} to {1}'.format(source, target)) - copyfile(source, target) - def file_extract(self, archive, cwd=None): if archive.endswith('.tgz') or archive.endswith('.tar.gz'): self.cmd(["tar", "xzf", archive], cwd=cwd) @@ -515,24 +498,11 @@ def file_extract(self, archive, cwd=None): raise Exception('Unhandled extraction for type {0}'.format(archive)) - def file_copytree(self, src, dest): - print('copy {} to {}'.format(src, dest)) - if os.path.isdir(src): - if not os.path.isdir(dest): - os.makedirs(dest) - files = os.listdir(src) - for f in files: - self.file_copytree( - os.path.join(src, f), - os.path.join(dest, f)) - else: - copyfile(src, dest) - def clean_platform(self): self.logger.info('Clean the platform build directory') if not exists(self.platform_dir): return - rmtree(self.platform_dir) + buildops.rmdir(self.platform_dir) def download(self, url, filename, cwd=None): def report_hook(index, blksize, size): @@ -615,7 +585,7 @@ def _copy_application_sources(self): self.logger.debug('Copy application source from {}'.format(source_dir)) - rmtree(self.app_dir) + buildops.rmdir(self.app_dir) for root, dirs, files in walk(source_dir, followlinks=True): # avoid hidden directory @@ -693,14 +663,14 @@ def _copy_application_sources(self): # copy! self.logger.debug('Copy {0}'.format(sfn)) - copyfile(sfn, rfn) + buildops.file_copy(sfn, rfn) def _copy_application_libs(self): # copy also the libs - copytree(self.applibs_dir, join(self.app_dir, '_applibs')) + buildops.file_copytree(self.applibs_dir, join(self.app_dir, '_applibs')) def _add_sitecustomize(self): - copyfile(join(dirname(__file__), 'sitecustomize.py'), + buildops.file_copy(join(dirname(__file__), 'sitecustomize.py'), join(self.app_dir, 'sitecustomize.py')) main_py = join(self.app_dir, 'service', 'main.py') @@ -946,7 +916,7 @@ def cmd_init(self, *args): if exists('buildozer.spec'): print('ERROR: You already have a buildozer.spec file.') exit(1) - copyfile(join(dirname(__file__), 'default.spec'), 'buildozer.spec') + buildops.file_copy(join(dirname(__file__), 'default.spec'), 'buildozer.spec') print('File buildozer.spec created, ready to customize!') def cmd_distclean(self, *args): @@ -958,7 +928,7 @@ def cmd_distclean(self, *args): self.logger.info('Clean the global build directory') if not exists(self.global_buildozer_dir): return - rmtree(self.global_buildozer_dir) + buildops.rmdir(self.global_buildozer_dir) def cmd_appclean(self, *args): '''Clean the .buildozer folder in the app directory. @@ -973,7 +943,7 @@ def cmd_appclean(self, *args): 'not attempt to delete files in a user-specified build directory.').format(self.user_build_dir)) elif exists(self.buildozer_dir): self.logger.info('Deleting {}'.format(self.buildozer_dir)) - rmtree(self.buildozer_dir) + buildops.rmdir(self.buildozer_dir) else: self.logger.error('{} already deleted, skipping.'.format(self.buildozer_dir)) @@ -996,12 +966,8 @@ def cmd_version(self, *args): def cmd_serve(self, *args): '''Serve the bin directory via SimpleHTTPServer ''' - try: - from http.server import SimpleHTTPRequestHandler - from socketserver import TCPServer - except ImportError: - from SimpleHTTPServer import SimpleHTTPRequestHandler - from SocketServer import TCPServer + from http.server import SimpleHTTPRequestHandler + from socketserver import TCPServer os.chdir(self.bin_dir) handler = SimpleHTTPRequestHandler diff --git a/buildozer/target.py b/buildozer/target.py index 75251a765..d401300c2 100644 --- a/buildozer/target.py +++ b/buildozer/target.py @@ -255,12 +255,12 @@ def install_or_update_repo(self, repo, **kwargs): if not self.buildozer.file_exists(install_dir): if custom_dir: buildops.mkdir(install_dir) - cmd(["cp", "-a", f"{custom_dir}/*", f"{install_dir}/"]) + buildops.file_copytree(custom_dir, install_dir) else: cmd(["git", "clone", "--branch", clone_branch, clone_url], cwd=self.buildozer.platform_dir) elif self.platform_update: if custom_dir: - cmd(["cp", "-a", f"{custom_dir}/*", f"{install_dir}/"]) + buildops.file_copytree(custom_dir, install_dir) else: cmd(["git", "clean", "-dxf"], cwd=install_dir) cmd(["git", "pull", "origin", clone_branch], cwd=install_dir) diff --git a/buildozer/targets/android.py b/buildozer/targets/android.py index cd81c8c60..53d53e6c0 100644 --- a/buildozer/targets/android.py +++ b/buildozer/targets/android.py @@ -28,7 +28,7 @@ from platform import architecture import re import shlex -from shutil import copyfile, rmtree, which +from shutil import which from sys import platform, executable from time import sleep import traceback @@ -36,6 +36,7 @@ from distutils.version import LooseVersion import pexpect +import buildozer.buildops as buildops from buildozer.exceptions import BuildozerException from buildozer.logger import USE_COLOR from buildozer.target import Target @@ -450,9 +451,10 @@ def _install_android_ndk(self): self.logger.info('Unpacking Android NDK') self.buildozer.file_extract(archive, cwd=self.buildozer.global_platform_dir) - self.buildozer.file_rename(unpacked, - ndk_dir, - cwd=self.buildozer.global_platform_dir) + buildops.rename( + unpacked, + ndk_dir, + cwd=self.buildozer.global_platform_dir) self.logger.info('Android NDK installation done.') return ndk_dir @@ -681,7 +683,7 @@ def _install_p4a(self): self.logger.info( f"Detected old url/branch ({cur_url}/{cur_branch}), deleting..." ) - rmtree(p4a_dir) + buildops.rmdir(p4a_dir) if not self.buildozer.file_exists(p4a_dir): cmd( @@ -1052,7 +1054,7 @@ def build_package(self): self.logger.debug('Search and copy libs for {}'.format(lib_dir)) for fn in self.buildozer.file_matches(patterns): - self.buildozer.file_copy( + buildops.file_copy( join(self.buildozer.root_dir, fn), join(dist_dir, 'libs', lib_dir, basename(fn))) @@ -1289,7 +1291,9 @@ def build_package(self): arch=self.archs_snake, artifact_format=self.artifact_format) # copy to our place - copyfile(join(artifact_dir, artifact), join(self.buildozer.bin_dir, artifact_dest)) + buildops.file_copy( + join(artifact_dir, artifact), + join(self.buildozer.bin_dir, artifact_dest)) self.logger.info('Android packaging done!') self.logger.info( diff --git a/buildozer/targets/osx.py b/buildozer/targets/osx.py index 92623a0f4..6c7dd01ab 100644 --- a/buildozer/targets/osx.py +++ b/buildozer/targets/osx.py @@ -9,6 +9,7 @@ from os.path import exists, join, abspath, dirname from subprocess import check_call, check_output +import buildozer.buildops as buildops from buildozer.target import Target @@ -32,17 +33,14 @@ def ensure_sdk(self): 'https://github.com/kivy/kivy-sdk-packager/archive/master.zip'), cwd=platdir) check_call(('unzip', 'master.zip'), cwd=platdir) - check_call(('rm', 'master.zip'), cwd=platdir) + buildops.file_remove(join(platdir, 'master.zip')) def download_kivy(self, cwd): current_kivy_vers = self.buildozer.config.get('app', 'osx.kivy_version') if exists('/Applications/Kivy.app'): self.logger.info('Kivy found in Applications dir...') - check_call( - ('cp', '-a', '/Applications/Kivy.app', - 'Kivy.app'), cwd=cwd) - + buildops.file_copy('/Applications/Kivy.app', 'Kivy.app', cwd=cwd) else: if not exists(join(cwd, 'Kivy.dmg')): self.logger.info('Downloading kivy...') @@ -56,12 +54,13 @@ def download_kivy(self, cwd): self.logger.error( "Unable to download the Kivy App. Check osx.kivy_version in your buildozer.spec, and verify " "Kivy servers are accessible. https://kivy.org/downloads/") - check_call(("rm", "Kivy.dmg"), cwd=cwd) + buildops.file_remove(join(cwd, "Kivy.dmg")) sys.exit(1) self.logger.info('Extracting and installing Kivy...') check_call(('hdiutil', 'attach', cwd + '/Kivy.dmg')) - check_call(('cp', '-a', '/Volumes/Kivy/Kivy.app', './Kivy.app'), cwd=cwd) + buildops.file_copy( + '/Volumes/Kivy/Kivy.app', './Kivy.app', cwd=cwd) def ensure_kivyapp(self): self.logger.info('check if Kivy.app exists in local dir') @@ -139,9 +138,9 @@ def build_package(self): binpath = join( self.buildozer.user_build_dir or dirname(abspath(self.buildozer.specfilename)), 'bin') - check_output( - ('cp', '-a', package_name + '.dmg', binpath), - cwd=cwd) + buildops.file_copytree( + join(cwd, package_name + '.dmg'), + binpath) self.logger.info('All Done!') def compile_platform(self): diff --git a/tests/targets/test_android.py b/tests/targets/test_android.py index 8c9668fa4..2dd5fba35 100644 --- a/tests/targets/test_android.py +++ b/tests/targets/test_android.py @@ -65,7 +65,7 @@ def call_build_package(target_android): with patch_target_android('_update_libraries_references') as m_update_libraries_references, \ patch_target_android('_generate_whitelist') as m_generate_whitelist, \ mock.patch('buildozer.targets.android.TargetAndroid.execute_build_package') as m_execute_build_package, \ - mock.patch('buildozer.targets.android.copyfile') as m_copyfile, \ + mock.patch('buildozer.targets.android.buildops.file_copy') as m_copyfile, \ mock.patch('buildozer.targets.android.os.listdir') as m_listdir: m_listdir.return_value = ['30.0.0-rc2'] target_android.build_package()