Migrate to use Buildop's rmtree, file_copy, file_copytree and rename (k…

…ivy#1669)

* Migrate to use Buildop's rmtree, file_copy, file_copytree and rename

This is part of a bigger refactor to reduce the size and complexity of the Buildozer class. It adds no new functionality (apart from making logging more consistent)

* Remove file_rename, file_copy and file_copytree from Buildozer.
* Update all references to those Buildozer methods to refer to buildops equivalents.
   * Note: `file_rename` is now called rename as, in practice, it was also used to rename directories.
* Updated all references to shutil.copyfile and shutil.rmtree to use the Buildops equivalents
  * Note this adds implicit debug logging, making them more consistent.
* Updated all references to check_call() and check_output() that called the `rm` and `cp` shell commands.
  * Note this adds implicit debug logging, making them more consistent, and improves performance.
* Updated mock in test.

Cheekily, also removed some obsolete references to Python 2 (only) standard libraries to make my IDE stop warning me about them while I edited the file.

* Fixed missed file_rename reference in IOS

buildozer/__init__.py

@@ -17,7 +17,7 @@
 import re
 from re import search
 import select
-from shutil import copyfile, rmtree, copytree, move, which
+from shutil import which
 from subprocess import Popen, PIPE, TimeoutExpired
 import sys
 from sys import stdout, stderr, exit
@@ -476,23 +476,6 @@ def file_matches(self, patterns):
     def file_exists(self, *args):
         return exists(join(*args))
 
-    def file_rename(self, source, target, cwd=None):
-        if cwd:
-            source = join(cwd, source)
-            target = join(cwd, target)
-        self.logger.debug('Rename {0} to {1}'.format(source, target))
-        if not os.path.isdir(os.path.dirname(target)):
-            self.logger.error(('Rename {0} to {1} fails because {2} is not a '
-                        'directory').format(source, target, target))
-        move(source, target)
-
-    def file_copy(self, source, target, cwd=None):
-        if cwd:
-            source = join(cwd, source)
-            target = join(cwd, target)
-        self.logger.debug('Copy {0} to {1}'.format(source, target))
-        copyfile(source, target)
-
     def file_extract(self, archive, cwd=None):
         if archive.endswith('.tgz') or archive.endswith('.tar.gz'):
             self.cmd(["tar", "xzf", archive], cwd=cwd)
@@ -515,24 +498,11 @@ def file_extract(self, archive, cwd=None):
 
         raise Exception('Unhandled extraction for type {0}'.format(archive))
 
-    def file_copytree(self, src, dest):
-        print('copy {} to {}'.format(src, dest))
-        if os.path.isdir(src):
-            if not os.path.isdir(dest):
-                os.makedirs(dest)
-            files = os.listdir(src)
-            for f in files:
-                self.file_copytree(
-                    os.path.join(src, f),
-                    os.path.join(dest, f))
-        else:
-            copyfile(src, dest)
-
     def clean_platform(self):
         self.logger.info('Clean the platform build directory')
         if not exists(self.platform_dir):
             return
-        rmtree(self.platform_dir)
+        buildops.rmdir(self.platform_dir)
 
     def download(self, url, filename, cwd=None):
         def report_hook(index, blksize, size):
@@ -615,7 +585,7 @@ def _copy_application_sources(self):
 
         self.logger.debug('Copy application source from {}'.format(source_dir))
 
-        rmtree(self.app_dir)
+        buildops.rmdir(self.app_dir)
 
         for root, dirs, files in walk(source_dir, followlinks=True):
             # avoid hidden directory
@@ -693,14 +663,14 @@ def _copy_application_sources(self):
 
                 # copy!
                 self.logger.debug('Copy {0}'.format(sfn))
-                copyfile(sfn, rfn)
+                buildops.file_copy(sfn, rfn)
 
     def _copy_application_libs(self):
         # copy also the libs
-        copytree(self.applibs_dir, join(self.app_dir, '_applibs'))
+        buildops.file_copytree(self.applibs_dir, join(self.app_dir, '_applibs'))
 
     def _add_sitecustomize(self):
-        copyfile(join(dirname(__file__), 'sitecustomize.py'),
+        buildops.file_copy(join(dirname(__file__), 'sitecustomize.py'),
                 join(self.app_dir, 'sitecustomize.py'))
 
         main_py = join(self.app_dir, 'service', 'main.py')
@@ -946,7 +916,7 @@ def cmd_init(self, *args):
         if exists('buildozer.spec'):
             print('ERROR: You already have a buildozer.spec file.')
             exit(1)
-        copyfile(join(dirname(__file__), 'default.spec'), 'buildozer.spec')
+        buildops.file_copy(join(dirname(__file__), 'default.spec'), 'buildozer.spec')
         print('File buildozer.spec created, ready to customize!')
 
     def cmd_distclean(self, *args):
@@ -958,7 +928,7 @@ def cmd_distclean(self, *args):
             self.logger.info('Clean the global build directory')
             if not exists(self.global_buildozer_dir):
                 return
-            rmtree(self.global_buildozer_dir)
+            buildops.rmdir(self.global_buildozer_dir)
 
     def cmd_appclean(self, *args):
         '''Clean the .buildozer folder in the app directory.
@@ -973,7 +943,7 @@ def cmd_appclean(self, *args):
                  'not attempt to delete files in a user-specified build directory.').format(self.user_build_dir))
         elif exists(self.buildozer_dir):
             self.logger.info('Deleting {}'.format(self.buildozer_dir))
-            rmtree(self.buildozer_dir)
+            buildops.rmdir(self.buildozer_dir)
         else:
             self.logger.error('{} already deleted, skipping.'.format(self.buildozer_dir))
 
@@ -996,12 +966,8 @@ def cmd_version(self, *args):
     def cmd_serve(self, *args):
         '''Serve the bin directory via SimpleHTTPServer
         '''
-        try:
-            from http.server import SimpleHTTPRequestHandler
-            from socketserver import TCPServer
-        except ImportError:
-            from SimpleHTTPServer import SimpleHTTPRequestHandler
-            from SocketServer import TCPServer
+        from http.server import SimpleHTTPRequestHandler
+        from socketserver import TCPServer
 
         os.chdir(self.bin_dir)
         handler = SimpleHTTPRequestHandler

buildozer/target.py

@@ -255,12 +255,12 @@ def install_or_update_repo(self, repo, **kwargs):
         if not self.buildozer.file_exists(install_dir):
             if custom_dir:
                 buildops.mkdir(install_dir)
-                cmd(["cp", "-a", f"{custom_dir}/*", f"{install_dir}/"])
+                buildops.file_copytree(custom_dir, install_dir)
             else:
                 cmd(["git", "clone", "--branch", clone_branch, clone_url], cwd=self.buildozer.platform_dir)
         elif self.platform_update:
             if custom_dir:
-                cmd(["cp", "-a", f"{custom_dir}/*", f"{install_dir}/"])
+                buildops.file_copytree(custom_dir, install_dir)
             else:
                 cmd(["git", "clean", "-dxf"], cwd=install_dir)
                 cmd(["git", "pull", "origin", clone_branch], cwd=install_dir)

buildozer/targets/android.py

@@ -28,14 +28,15 @@
 from platform import architecture
 import re
 import shlex
-from shutil import copyfile, rmtree, which
+from shutil import which
 from sys import platform, executable
 from time import sleep
 import traceback
 
 from distutils.version import LooseVersion
 import pexpect
 
+import buildozer.buildops as buildops
 from buildozer.exceptions import BuildozerException
 from buildozer.logger import USE_COLOR
 from buildozer.target import Target
@@ -450,9 +451,10 @@ def _install_android_ndk(self):
         self.logger.info('Unpacking Android NDK')
         self.buildozer.file_extract(archive,
                                     cwd=self.buildozer.global_platform_dir)
-        self.buildozer.file_rename(unpacked,
-                                   ndk_dir,
-                                   cwd=self.buildozer.global_platform_dir)
+        buildops.rename(
+            unpacked,
+            ndk_dir,
+            cwd=self.buildozer.global_platform_dir)
         self.logger.info('Android NDK installation done.')
         return ndk_dir
 
@@ -681,7 +683,7 @@ def _install_p4a(self):
                     self.logger.info(
                         f"Detected old url/branch ({cur_url}/{cur_branch}), deleting..."
                     )
-                    rmtree(p4a_dir)
+                    buildops.rmdir(p4a_dir)
 
             if not self.buildozer.file_exists(p4a_dir):
                 cmd(
@@ -1052,7 +1054,7 @@ def build_package(self):
 
             self.logger.debug('Search and copy libs for {}'.format(lib_dir))
             for fn in self.buildozer.file_matches(patterns):
-                self.buildozer.file_copy(
+                buildops.file_copy(
                     join(self.buildozer.root_dir, fn),
                     join(dist_dir, 'libs', lib_dir, basename(fn)))
 
@@ -1289,7 +1291,9 @@ def build_package(self):
             arch=self.archs_snake, artifact_format=self.artifact_format)
 
         # copy to our place
-        copyfile(join(artifact_dir, artifact), join(self.buildozer.bin_dir, artifact_dest))
+        buildops.file_copy(
+            join(artifact_dir, artifact),
+            join(self.buildozer.bin_dir, artifact_dest))
 
         self.logger.info('Android packaging done!')
         self.logger.info(

buildozer/targets/ios.py

@@ -310,7 +310,7 @@ def build_package(self):
             cwd=build_dir)
 
         self.logger.info('Moving IPA to bin...')
-        self.buildozer.file_rename(ipa_tmp, ipa)
+        buildops.file_rename(ipa_tmp, ipa)
 
         self.logger.info('iOS packaging done!')
         self.logger.info('IPA {0} available in the bin directory'.format(

buildozer/targets/osx.py

@@ -9,6 +9,7 @@
 from os.path import exists, join, abspath, dirname
 from subprocess import check_call, check_output
 
+import buildozer.buildops as buildops
 from buildozer.target import Target
 
 
@@ -32,17 +33,14 @@ def ensure_sdk(self):
                 'https://github.com/kivy/kivy-sdk-packager/archive/master.zip'),
             cwd=platdir)
         check_call(('unzip', 'master.zip'), cwd=platdir)
-        check_call(('rm', 'master.zip'), cwd=platdir)
+        buildops.file_remove(join(platdir, 'master.zip'))
 
     def download_kivy(self, cwd):
         current_kivy_vers = self.buildozer.config.get('app', 'osx.kivy_version')
 
         if exists('/Applications/Kivy.app'):
             self.logger.info('Kivy found in Applications dir...')
-            check_call(
-                ('cp', '-a', '/Applications/Kivy.app',
-                    'Kivy.app'), cwd=cwd)
-
+            buildops.file_copy('/Applications/Kivy.app', 'Kivy.app', cwd=cwd)
         else:
             if not exists(join(cwd, 'Kivy.dmg')):
                 self.logger.info('Downloading kivy...')
@@ -56,12 +54,13 @@ def download_kivy(self, cwd):
                     self.logger.error(
                         "Unable to download the Kivy App. Check osx.kivy_version in your buildozer.spec, and verify "
                         "Kivy servers are accessible. https://kivy.org/downloads/")
-                    check_call(("rm", "Kivy.dmg"), cwd=cwd)
+                    buildops.file_remove(join(cwd, "Kivy.dmg"))
                     sys.exit(1)
 
             self.logger.info('Extracting and installing Kivy...')
             check_call(('hdiutil', 'attach', cwd + '/Kivy.dmg'))
-            check_call(('cp', '-a', '/Volumes/Kivy/Kivy.app', './Kivy.app'), cwd=cwd)
+            buildops.file_copy(
+                '/Volumes/Kivy/Kivy.app', './Kivy.app', cwd=cwd)
 
     def ensure_kivyapp(self):
         self.logger.info('check if Kivy.app exists in local dir')
@@ -139,9 +138,9 @@ def build_package(self):
         binpath = join(
             self.buildozer.user_build_dir or
             dirname(abspath(self.buildozer.specfilename)), 'bin')
-        check_output(
-            ('cp', '-a', package_name + '.dmg', binpath),
-            cwd=cwd)
+        buildops.file_copytree(
+            join(cwd, package_name + '.dmg'),
+            binpath)
         self.logger.info('All Done!')
 
     def compile_platform(self):

tests/targets/test_android.py

@@ -65,7 +65,7 @@ def call_build_package(target_android):
     with patch_target_android('_update_libraries_references') as m_update_libraries_references, \
          patch_target_android('_generate_whitelist') as m_generate_whitelist, \
          mock.patch('buildozer.targets.android.TargetAndroid.execute_build_package') as m_execute_build_package, \
-         mock.patch('buildozer.targets.android.copyfile') as m_copyfile, \
+         mock.patch('buildozer.targets.android.buildops.file_copy') as m_copyfile, \
          mock.patch('buildozer.targets.android.os.listdir') as m_listdir:
         m_listdir.return_value = ['30.0.0-rc2']
         target_android.build_package()