diff --git a/CHANGELOG.md b/CHANGELOG.md index d21cc60e..42fdb4ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,27 @@ # Changelog -## udocker (1.3.13) +## udocker (1.3.14) - 2024-04-04 + +* Support for runsc as engine for execution mode R1: closes #414 +* New option `login --password-stdin`: closes #168 +* New option `run --pull=reuse` to be used with --name= and with + and image name as argument. Instead of always pulling and creating + a new container --pull=reuse allows to execute an existing container + and only pull+create if the container does not exist +* New option `run --httpproxy=`: closes #418 +* Improve handling of registry names in login: closes #168 +* Improve handling of image names in pull: closes #168 +* Improve handling of mount point removal: closes #406, #399 +* Support for AWS ECR registries: closes #168 +* Remove pycurl dependency on unit tests +* Documentation fixes + +## udocker (1.3.13) - 2024-02-05 -* udocker improve binary executables identification +* udocker improve binary executables selection * udocker fix fakechroot parsing of so, exec_path and add cmd subst * udocker implement minor pylint compliance improvements -* udocker mode Pn make links2symlinks feature disabled by default in config: solves #412 +* udocker mode Pn make links2symlinks disabled by default in config: closes #412 * New udockertools 1.2.11 tarball * udockertools mode Fn glibc fix dladdr(), dlopen(), dlmopen(), dl_iterate_phdr() * udockertools mode Fn glibc add dladdr1() @@ -22,105 +38,105 @@ * udockertools mode Fn added support for Fedora 39 (x86_64, aarch64, ppc64le) * udockertools mode Rn include runc 1.1.12 -## udocker (1.3.12) +## udocker (1.3.12) - 2023-11-02 -* fix units tests, no modifications w.r.t. 1.3.11 +* Fix unit tests, no modifications w.r.t. 1.3.11 -## udocker (1.3.11) +## udocker (1.3.11) - 2023-10-23 -* add support for hard link to symbolic link conversion in Pn modes - as hard links cannot be created by unprivileged users - partially - addresses: #388 -* check of availability of network extensions for port mapping and +* Add support for hard link to symbolic link conversion in Pn modes + as hard links cannot be created by unprivileged users: partially + addresses #388 +* Check of availability of network extensions for port mapping and netcoop in Pn modes and only use them if supported by the proot engine being invoked. -* improve image metadata generated by udocker on import - closes: #398 +* Improve image metadata generated by udocker on import: closes #398 -## udocker (1.3.10) +## udocker (1.3.10) - 2023-07-03 -* improved handling of container platform information -* added support for QEMU on Pn modes enabling execution of containers +* Improved handling of container platform information +* Added support for QEMU on Pn modes enabling execution of containers with architectures different than the host -* selection of executable for Sn mode now defaults to apptainer and +* Selection of executable for Sn mode now defaults to apptainer and in second place to singularity -* the new command `manifest inspect` allows display of image manifests +* The new command `manifest inspect` allows display of image manifests therefore enabling access to the catalogue of platforms supported by a given image -* the new command `tag` enables changing the name of an existing image -* new option `pull --platform=os/architecture` enables pulling of images +* The new command `tag` enables changing the name of an existing image +* New option `pull --platform=os/architecture` enables pulling of images of a given architecture possibly different from the host -* new option `run --platform=os/architecture` enables pull and run of +* New option `run --platform=os/architecture` enables pull and run of images of a given architecture possibly different from the host -* new option `import --platform=os/architecture` enables to specify +* New option `import --platform=os/architecture` enables to specify an architecture for the image -* new option `ps -p` enables list of the architectures of containers -* new option `images -p` enables list of the architectures of containers -* build udockertools 1.2.10 and set it as default -* the udockertools support for Fn now includes Ubuntu 23:04, Fedora 38, +* New option `ps -p` enables list of the architectures of containers +* New option `images -p` enables list of the architectures of containers +* Build udockertools 1.2.10 and set it as default +* The udockertools support for Fn now includes Ubuntu 23:04, Fedora 38, Alpine 3.17 and 3.18. -* experimental support for native Fn execution on arm64 for Fedora 36, +* Experimental support for native Fn execution on arm64 for Fedora 36, Fedora 37, Fedora 38, CentOS 7, AlmaLinux 8, AlmaLinux 9 and Ubuntu 22, Ubuntu 20, Ubuntu 18 and similar. -* experimental support for native Fn execution on ppc64le for CentOS 7, +* Experimental support for native Fn execution on ppc64le for CentOS 7, AlmaLinux 8, AlmaLinux 9, Ubuntu 22, Ubuntu 20, Ubuntu 18 and similar. -* experimental support for runc in arm64 and ppc64le -* updated version of Pn engines for x86, x86_64, arm64. +* Experimental support for runc in arm64 and ppc64le +* Updated version of Pn engines for x86, x86_64, arm64: addresses #393 -## udocker (1.3.9) +## udocker (1.3.9) - 2023-06-07 -* add support to access non-config metadata from containers -* added support for multiplatform manifests and indices solves #392 and #355 +* Add support to access non-config metadata from containers +* Added support for multiplatform manifests and indices: closes #392, #355 -## udocker (1.3.8) +## udocker (1.3.8) - 2023-03-24 -* build udockertools 1.2.9 and set it as default -* add Fn support for Ubuntu:22 -* remove files to be installed -* set Fn preference to use runc +* Build udockertools 1.2.9 and set it as default +* Add Fn support for Ubuntu:22 +* Remove files to be installed +* Set Fn preference to use runc -## udocker (1.3.7) +## udocker (1.3.7) - 2023-01-24 * Remove deprecated unit tests. udocker is the same as version 1.3.6 -## udocker (1.3.6) +## udocker (1.3.6) - 2023-01-19 -* re-implement udocker namespace solves #380 -* login fails all the time solves #379 -* Ignore image loading if already exists solves #378 +* Re-implement udocker namespace: closes #380 +* Login fails all the time: closes #379 +* Ignore image loading if already exists: closes #378 -## udocker (1.3.5) +## udocker (1.3.5) - 2022-10-21 -* fix python backwards compatibility issues - closes: #374 -* fix incorrectly reported errors by image verification -* fix image search returning empty results -* fix issue with logical links in the udocker executable path -* add check to verify if container name exists before creation +* Fix python backwards compatibility issues: closes #374 +* Fix incorrectly reported errors by image verification +* Fix image search returning empty results +* Fix issue with logical links in the udocker executable path +* Add check to verify if container name exists before creation or cloning -* add --force option to create and clone to allow creation +* Add --force option to create and clone to allow creation of container even if the intended name given by --name exists -* prevent closing of file descriptors upon engine invocation +* Prevent closing of file descriptors upon engine invocation improves PMI process management interface interoperability -* fix issues in import and export while using pipes. -* fix image name parsing where "library" component is missing - closes: #359 +* Fix issues in import and export while using pipes. +* Fix image name parsing where "library" component is missing: closes #359 -## udocker (1.3.4) +## udocker (1.3.4) - 2022-08-26 -* fix 2 unit tests +* Fix 2 unit tests -## udocker (1.3.3) +## udocker (1.3.3) - 2022-08-23 -* image list does not truncate long names - solve #349 -* fix conditional warning in verify image -* fix and improve udocker high level tests +* Image list does not truncate long names: closes #349 +* Fix conditional warning in verify image +* Fix and improve udocker high level tests -## udocker (1.3.2) +## udocker (1.3.2) - 2022-08-17 -* fix missing f (format) for string -* fix bugs with dict .items() -* solving several pylint issues -* remove use2to3, fix issue #358 +* Fix missing f (format) for string +* Fix bugs with dict .items() +* Solving several pylint issues +* Remove use2to3: closes #358 -## udocker (1.3.1) +## udocker (1.3.1) - 2021-06-24 * Add --entrypoint to run --help * Set docker hub registry registry-1.docker.io @@ -136,7 +152,7 @@ * Update documentation: README, user and install manuals * Fix sqa and config -## udocker (1.3.0) +## udocker (1.3.0) - 2021-06-05 * Prepare to move the stable code for Python 3 and Python 2 >= 2.6 to master * Installation procedure changed since 1.1.x series see the `installation_manual` @@ -147,18 +163,18 @@ * Fix support for `newfstatat()` in Pn execution modes * Add Fn libraries for Fedora 34 and Ubuntu 21.04 * Remove broken links in FileUtil.remove() -* update minimum udocker tools tarball to 1.2.8 +* Update minimum udocker tools tarball to 1.2.8 * Cmd and entrypoint metadata and arguments processing changed to mimic docker * Improve removal of files and links in install and filebind restore * Add follow location option to GetURL() -* Implement use of `--entrypoint=` to force execution of command - closes: #306 -* Implement use of `--entrypoint=""` to bypass entrypoint in metadata - closes: #306 +* Implement use of `--entrypoint=` to force execution of command: closes #306 +* Implement use of `--entrypoint=""` to bypass entrypoint in metadata: closes #306 -## udocker (1.2.9) +## udocker (1.2.9) - 2021-05-24 -* method Unshare.unshare os.strerror() takes one argument - closes: #254 +* Method Unshare.unshare os.strerror() takes one argument: closes #254 * Add unit test for #254 -* Method chown udocker.utils.fileutil FileUtil - closes: #276 +* Method chown udocker.utils.fileutil FileUtil: closes #276 * Several fixes of unit tests and pylint * Fix confusion between exit code 0 and inferred False * Dereference on `safe_prefixes` @@ -173,40 +189,46 @@ * Improve keystore logic * Fix pull /v2 -## udocker (1.2.8b2) +## udocker (1.2.8b2) - 2021-05-04 * Fix Rn modes to enable containers execution from readonly dirs * Documentation centralized installation and readonly setups * Fix handling of dockerhub repository names in /v2 -* Improve documentation and algn with 1.1.8b2 +* Improve documentation and align with 1.1.8b2 * Add credits -* Fix delete of paths with symlinks - closes: #267, #265 -* Fix issues with login credentials - closes: #310 -* Fix pull images from docker hub in Termux - closes: #307 -* Fix issues on running udocker in googlecolab - closes: #286 -* Fix execution with Pn modes in alternate /tmp - closes: #284 +* Fix delete of paths with symlinks: closes #267, #265 +* Fix issues with login credentials: closes #310 +* Fix pull images from docker hub in Termux: closes #307 +* Fix issues on running udocker in googlecolab: closes #286 +* Fix execution with Pn modes in alternate /tmp: closes #284 * Add conditional delay-directory-restore to untar layers * Add exclude of whiteouts on layer untar * Add --nobanner to udocker run -## udocker (1.2.7) +## udocker (1.2.7) - 2021-01-26 * Major restructuring of the code * Major restructuring of the unit tests * Porting to Python 3, still supports python 2.7 -* all fixes up to previous 1.1.7 version have been applied -* added scripts tests udocker: `utils/udocker_test.sh utils/udocker_test-run.sh` +* All fixes up to previous 1.1.7 version have been applied +* Added scripts tests udocker: `utils/udocker_test.sh utils/udocker_test-run.sh` + +## udocker (1.1.8) - 2021-06-16 + +* Last 1.1.x release +* Fix Rn modes to enable containers execution from readonly dirs +* Documentation centralized installation and readonly setups -## udocker (1.1.7) +## udocker (1.1.7) - 2021-02-21 -* Fix P1 when Linux 4.8.0 SECCOMP is backported, affects newer CentOS 7 - closes: #282 -* Check for file ownership on remove wrongly follows symlinks - closes: #266, #267 -* udocker unexpectedly uses P1 exec mode instead of P2 - closes: #274 -* Allow passing of `PROOT_TMP_DIR` environment variable - closes: #284 +* Fix P1 when Linux 4.8.0 SECCOMP is backported, affects newer CentOS 7: closes #282 +* Check for file ownership on remove wrongly follows symlinks: closes #266, #267 +* udocker unexpectedly uses P1 exec mode instead of P2: closes #274 +* Allow passing of `PROOT_TMP_DIR` environment variable: closes #284 ## udocker (1.1.6) -* Complete fix for of ELF paths in modes Fn for $ORIGIN:$ORIGIN - closes: #255 +* Complete fix for of ELF paths in modes Fn for "$ORIGIN:$ORIGIN": closes #255 ## udocker (1.1.5) @@ -214,7 +236,7 @@ * Add Fn libraries for Ubuntu20, Fedora32, Fedora33 * Add Fn libraries for Alpine 3.12, 3.13 -## udocker (1.1.4-1) +## udocker (1.1.4-1) - 2020-01-10 * Fix run --location * Fix udocker integrated help @@ -224,7 +246,7 @@ * `os._exit` from Unshare.unshare() * Disable `FAKECHROOT_DISALLOW_ENV_CHANGES` in F4 mode -## udocker (1.1.4) +## udocker (1.1.4) - 2020-01-07 * Use hub.docker.com as default registry * Search using v1 and v2 APIs @@ -260,66 +282,66 @@ * Improved fix of SECCOMP accelerated mode for P1 mode * Added loading and handling of container images in OCI format * Fixes for udocker in ARM aarch64 -* Fix processing of --dri in Sn mode - closes: #241 -* Improve handling of container and host authentication - partially addresses: #239 -* Fixes to address authentication and redirects in pull - closes: #225, #230 -* Added minimal support to load OCI images - closes: #111 -* Added Pn support for newer distributions - closes: #192 -* Improve the installation of udockertools - closes: #220, #228 -* Read environment variables from file with --env-file= - closes: #212 -* Prepare for pypy - closes: #211 -* Fixes for verification of container images - closes: #209 -* Fix command line processing for "-" in argument - closes: #202 -* Fix file protections on extraction making files u+r - closes: #202, #206 -* Fix comparison of kernel versions having non-integers - closes: #183 -* Support for both manifest V2 schema 1 and schema 2 - closes: #218, #225 -* Further improved pathname translation in Fn modes - closes: #160 -* Implement save images in docker format - closes: #74 -* useradd and groupadd not working in containers - closes: #141 -* fix return code when exporting to stdin - closes: #202 - -## udocker (1.1.3) - -* Support for nvidia drivers on ubuntu - closes: #162 -* Installation improvements - closes: #166 -* Fix issue on Fn mode symlink conversion - partially addresses: #160 - -## udocker (1.1.2) - -* Improve parsing of quotes in the command line - closes: #98 -* Fix version command to exit with 0 - closes: #107 +* Fix processing of --dri in Sn mode: closes #241 +* Improve handling of container and host authentication: addresses #239 +* Fixes to address authentication and redirects in pull: closes #225, #230 +* Added minimal support to load OCI images: closes #111 +* Added Pn support for newer distributions: closes #192 +* Improve the installation of udockertools: closes #220, #228 +* Add --env-file= - to read environment variables from file: closes #212 +* Prepare for pypy: closes #211 +* Fixes for verification of container images: closes #209 +* Fix command line processing for "-" in argument: closes #202 +* Fix file protections on extraction making files u+r : closes #202, #206 +* Fix comparison of kernel versions having non-integers: closes #183 +* Support for both manifest V2 schema 1 and schema 2: closes #218, #225 +* Further improved pathname translation in Fn modes: closes #160 +* Implement save images in docker format: closes #74 +* useradd and groupadd not working in containers: closes #141 +* fix return code when exporting to stdin: closes #202 + +## udocker (1.1.3) - 2018-11-01 + +* Support for nvidia drivers on ubuntu: closes #162 +* Installation improvements: closes #166 +* Fix issue on Fn mode symlink conversion: addresses #160 + +## udocker (1.1.2) - 2018-10-29 + +* Improve parsing of quotes in the command line: closes #98 +* Fix version command to exit with 0: closes #107 * Add kill-on-exit to proot on Pn modes * Improve download of udocker utils -* Handle authentication headers when pulling - closes: #110 +* Handle authentication headers when pulling: closes #110 * Handle of redirects when pulling * Fix registries table * Support search quay.io * Fix auth header when no standard Docker registry is used * Add registry detection on image name * Add --version option -* Force python2 as interpreter - closes: #131 +* Force python2 as interpreter: closes #131 * Fix handling of volumes in metadata * Handle empty metadata -* Fix http proxy functionality - closes: #115 -* Ignore --no-trunc and --all in the images command - closes: #108 +* Fix http proxy functionality: closes #115 +* Ignore --no-trunc and --all in the images command: closes #108 * Implement verification of layers in manifest * Add --nvidia to support GPUs and related drivers * Send download messages to stderr * Enable override of curl executable -* Fix building on CentOS 6 - closes: #157 -* Mitigation for upstream limitation in runC without tty - closes: #132 -* Fix detection of executable with symlinks in container - closes: #118 +* Fix building on CentOS 6: closes #157 +* Mitigation for upstream limitation in runC without tty: closes #132 +* Fix detection of executable with symlinks in container: closes #118 * Updated runC to v1.0.0-rc5 * Experimental support for Alpine in Fn modes -* Improve pathname translation in Fn modes for mounted dirs - partially addresses: #160 +* Improve pathname translation in Fn modes for mounted dirs: addresses #160 -## udocker (1.1.1) +## udocker (1.1.1) - 2017-11-24 * New execution engine using singularity * Updated documentation with OpenMPI information and examples * Additional unit tests * Redirect messages to stderr -* Improved parsing of quotes in the command line - closes: #87 +* Improved parsing of quotes in the command line: closes #87 * Allow override of the HOME environment variable * Allow override of libfakechroot.so at the container level * Automatic selection of libfakechroot.so from container info @@ -331,10 +353,10 @@ * Load, import and export to/from stdin/stdout * Clone existing containers * Support for TCP/IP port remap in execution modes Pn -* Fix run with basenames failing - closes: #89 -* Allow run as root flag - closes: #91 +* Fix run with basenames failing: closes #89 +* Allow run as root flag: closes #91 -## udocker (1.1.0) +## udocker (1.1.0) - 2017-09-30 * Support image names prefixed by registry similarly to docker * Add execution engine selection logic @@ -343,34 +365,34 @@ * Improve proot tmp files cleanup on non ext filesystems * Improve search returning empty on Docker repositories * Improve runC execution portability -* Add environment variable `UDOCKER_KEYSTORE` - closes: #75 -* Prevent creation of .udocker when `UDOCKER_KEYSTORE` is used - closes: #75 +* Add environment variable `UDOCKER_KEYSTORE`: closes #75 +* Prevent creation of .udocker when `UDOCKER_KEYSTORE` is used: closes #75 -## udocker (1.0.4) +## udocker (1.0.4) - 2017-09-26 * Documentation fixes -## udocker (1.0.3) +## udocker (1.0.3) - 2017-03-30 * Support for import Docker containers in newer metadata structure * Improve the command line parsing * Improve temporary file handling and removal * Support for additional execution engines to be provided in the future -* Improved parsing of entrypoint and cmd metadata - closes: #53 -* Increase name alias length - closes: #52 -* Add support for change dir into volume directories - closes: #51 -* Fix deletion of files upon container import - closes: #50 -* Fix exporting of host environment variables to the containers - closes: #48 -* Change misleading behavior of import tarball from move to copy - closes: #44 -* Fix validation of volumes specification - closes: #43 +* Improved parsing of entrypoint and cmd metadata: closes #53 +* Increase name alias length: closes #52 +* Add support for change dir into volume directories: closes #51 +* Fix deletion of files upon container import: closes #50 +* Fix exporting of host environment variables to the containers: closes #48 +* Change misleading behavior of import tarball from move to copy: closes #44 +* Fix validation of volumes specification: closes #43 -## udocker (1.0.2) +## udocker (1.0.2) - 2017-02-13 * Improve download on repositories that fail authentication on /v2 * Improve run verification of binaries with recursive symbolic links -* Improve accelerated seccomp on kernels >= 4.8.0 - closes: #40 +* Improve accelerated seccomp on kernels >= 4.8.0 : closes #40 -## udocker (1.0.1) +## udocker (1.0.1) - 2017-01-31 * Minor bugfixes * Executable name changed from udocker.py to udocker @@ -383,11 +405,11 @@ * Insecure flag fixed * Address seccomp change introduced on kernels >= 4.8.0 * Utilities for packaging -* Improved verbose levels, messaging and output - closes: #24, #23 -* Fully implement support for registry selection --registry parameter - closes: #29 -* Provide support for private repositories e.g. gitlab registries - closes: #30 -* Provide --insecure command line parameter for SSL requests - closes: #31 +* Improved verbose levels, messaging and output: closes #24, #23 +* Fully implement support for registry selection --registry parameter: closes #29 +* Provide support for private repositories e.g. gitlab registries: closes #30 +* Provide --insecure command line parameter for SSL requests: closes #31 -## udocker (1.0.0) +## udocker (1.0.0) - 2016-06-06 * Initial version diff --git a/README.md b/README.md index 27652247..d44720a5 100644 --- a/README.md +++ b/README.md @@ -294,7 +294,6 @@ the nvidia drivers are installed in the host system. udocker setup --nvidia mytensorflow ``` - ## Security By default udocker via PRoot offers the emulation of the root user. This @@ -354,7 +353,7 @@ containers execution in user space. udocker is particularly suited to run user applications encapsulated in docker containers. Debugging or using strace with the PRoot engine will not work as both -the debuggers and PRoot use the same tracing mechanism. +the debuggers and PRoot use the same tracing mechanism. ## Execution mode specific limitations diff --git a/codemeta.json b/codemeta.json index 6bc24d8c..9315e3f1 100644 --- a/codemeta.json +++ b/codemeta.json @@ -6,7 +6,7 @@ "@type": "SoftwareSourceCode", "identifier": "udocker", "name": "udocker", - "version": "1.3.13", + "version": "1.3.14", "description": "A basic user tool to execute simple docker containers in batch or interactive systems without root privileges", "license": "Apache Software License 2.0, OSI Approved :: Apache Software License", "author": [ diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000..bc8db6e5 --- /dev/null +++ b/conftest.py @@ -0,0 +1,3 @@ +""" +This file is required for pytest to find and load udocker as module. +""" diff --git a/docs/installation_manual.md b/docs/installation_manual.md index 9bf4acf5..9ac38773 100644 --- a/docs/installation_manual.md +++ b/docs/installation_manual.md @@ -32,18 +32,18 @@ udocker requires: Download a release tarball from : ```bash -wget https://github.com/indigo-dc/udocker/releases/download/1.3.13/udocker-1.3.13.tar.gz -tar zxvf udocker-1.3.13.tar.gz -export PATH=`pwd`/udocker-1.3.13/udocker:$PATH +wget https://github.com/indigo-dc/udocker/releases/download/1.3.14/udocker-1.3.14.tar.gz +tar zxvf udocker-1.3.14.tar.gz +export PATH=`pwd`/udocker-1.3.14/udocker:$PATH ``` Alternatively use `curl` instead of `wget` as follows: ```bash -curl -L https://github.com/indigo-dc/udocker/releases/download/1.3.13/udocker-1.3.13.tar.gz \ - > udocker-1.3.13.tar.gz -tar zxvf udocker-1.3.13.tar.gz -export PATH=`pwd`/udocker-1.3.13/udocker:$PATH +curl -L https://github.com/indigo-dc/udocker/releases/download/1.3.14/udocker-1.3.14.tar.gz \ + > udocker-1.3.14.tar.gz +tar zxvf udocker-1.3.14.tar.gz +export PATH=`pwd`/udocker-1.3.14/udocker:$PATH ``` udocker executes containers using external tools and libraries that @@ -169,7 +169,7 @@ The configuration of udocker has the following hierarchy: it will override 1. 3. If environment variables are set ([section 5. Environment variables](#5-environment-variables)), they will override 2. -4. The presence of general udocker command line options, will override 3. . +4. The presence of general udocker command line options, will override 3. ### 3.1. Directories @@ -257,8 +257,8 @@ the environment variables described below together with the default behavior. A value of `UDOCKER` will force the usage of the executables provided by the udocker installation. -A full pathname can be used to select a specific executable (or library) from the -host or from the udocker installation. +A full pathname can be used to force selection of a specific executable (or library) +from the host or from the udocker installation. * `UDOCKER_USE_PROOT_EXECUTABLE`: path to proot, default is proot from udocker. * `UDOCKER_USE_RUNC_EXECUTABLE`: path to runc, default is search the host and @@ -266,10 +266,23 @@ host or from the udocker installation. * `UDOCKER_USE_SINGULARITY_EXECUTABLE`: path to singularity, default is search the host. * `UDOCKER_FAKECHROOT_SO`: path to a fakechroot library, default is search - in udocker. + in udocker under `$HOME/.udocker/lib`. * `UDOCKER_DEFAULT_EXECUTION_MODE`: default execution mode can be P1, P2, F1, S1, R1, R2 or R3. +Several executables and libraries are shipped with udocker. For instance +the executable for the Rn modes can be selected to be either `runc` or +`crun`. This can be accomplished by setting `UDOCKER_USE_RUNC_EXECUTABLE` +to the path of the desired executable. If `runsc` is available in the +host it can also be selected in this manner. + +``` +# Forcing the use of crun instead of runc +export UDOCKER_USE_RUNC_EXECUTABLE=$HOME/.udocker/bin/crun-x86_64 +export UDOCKER_DEFAULT_EXECUTION_MODE=R1 +udocker run +``` + ## 6. External tools and libraries ### 6.1. Source code repositories @@ -323,10 +336,33 @@ versions. The tools are also delivered for several architectures. | Mode | Supported architecture | |-------|:---------------------------------------------| | **P** | x86_64, i386, aarch64 and arm | -| **F** | x86_64 | -| **R** | x86_64 | +| **F** | x86_64, aarch64, ppc64le | +| **R** | x86_64 aarch64 | | **S** | uses the binaries present in the host system | +### 6.3. Compiling + +udocker already provides executables and libraries for the engines. These +are statically compiled to be used across different Linux distributions. +In some cases these executables may not work and may require recompilation. +Use the repositories in section 6.2 if you which to compile the executables +or libraries. Notice that the git repositories that are specific of udocker +have branches or tags like `UDOCKER-x` where `x` is a number. Use the branch +or tag with the highest number. + +A notable case are the fakechroot libraries used in the Fn modes that need +to match the libc in the container. This means that a libfakechroot.so must +be produced for each different distribution release and intended architecture. +Two implementations of the `libc` are supported `glibc` and `musl`, choose +the one that matches the distribution inside the container. Once compiled the +selection of the library can be forced by setting the environment variable +`UDOCKER_FAKECHROOT_SO`. + +``` +udocker setup --execmode=F3 +UDOCKER_FAKECHROOT_SO=$HOME/mylibfakechroot.so udocker run +``` + The latest binary tarball can be produced from the source code using: ```bash @@ -353,8 +389,8 @@ The udocker tool should be installed as shown in section 2.1: ```bash cd /sw -wget https://github.com/indigo-dc/udocker/releases/download/1.3.13/udocker-1.3.13.tar.gz -tar zxvf udocker-1.3.13.tar.gz +wget https://github.com/indigo-dc/udocker/releases/download/1.3.14/udocker-1.3.14.tar.gz +tar zxvf udocker-1.3.14.tar.gz ``` Directing users to the central udocker installation can be done using the diff --git a/docs/udocker.1 b/docs/udocker.1 index 20de9e8a..6dddf673 100644 --- a/docs/udocker.1 +++ b/docs/udocker.1 @@ -1,7 +1,7 @@ .\" Manpage for udocker .\" Contact udocker@lip.pt to correct errors or typos. .\" To read this man page use: man -l udocker.1 -.TH udocker 1 "5 Feb 2024" "version 1.3.13" "udocker man page" +.TH udocker 1 "4 Apr 2024" "version 1.3.14" "udocker man page" .SH NAME udocker \- execute Docker containers in user space without privileges .SH SYNOPSIS @@ -112,7 +112,7 @@ Obtain and print information about an IMAGE manifest from a remote registry. .BR mkrepo " " DIRECTORY Setup a local repository in the host DIRECTORY. The required directory structure is created. .TP -.BR login " " \--username=USERNAME " " | " " \--password=PASSWORD " " | " " \--registry=REGISTRY +.BR login " " \--username=USERNAME " " | " " \--password=PASSWORD " " | " " \--password-stdin " " | " " \--registry=REGISTRY Setup of authentication information for access to remote Docker registries. Enables "pull" of IMAGES from private registries. The option --registry can be used to access registries other than the default dockerhub. If USERNAME or PASSWORD are not provided in the command line, the user will be prompted to provide them. .TP .BR logout " " [ " " \-a " " ] " " | " " \--registry=REGISTRY @@ -196,7 +196,10 @@ Override the container metadata entrypoint. Specify the operating system and/or architecture of the image to be pulled and executed. .TP --pull=WHEN -Specify when to pull the image. The argument WHEN can take the values of "missing", "never" or "always". +Specify when to pull the image. The argument WHEN can take the values of "missing", "never", "always" or "reuse". +.TP +--httpproxy=PROXY +Specify an http, socks4 or socks5 proxy, see the "pull" command for the syntax. .RE .SH ENVIRONMENT diff --git a/docs/user_manual.md b/docs/user_manual.md index 3024f6fe..b5c20410 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -275,6 +275,7 @@ udocker pull --httpproxy=socks5h://host:port busybox udocker pull --httpproxy=socks4a://user:pass@host:port busybox udocker pull --httpproxy=socks5h://user:pass@host:port busybox udocker pull --platform=linux/arm64 fedora:latest +udocker pull --platform=linux/ppc64le centos:7 ``` ### 3.6. images @@ -638,7 +639,8 @@ Options: * `--kernel=KERNELID` use a specific kernel id to emulate useful when the host kernel is too old * `--location=DIR` execute a container in a given directory * `--platform=os/architecture` specify a different platform to be pulled -* `--pull=missing|never|always` specify when to pull the image +* `--pull=missing|never|always|reuse` specify when to pull the image +* `--httpproxy=PROXY` uses an http or socks proxy, see `pull` Options valid only in Pn execution modes: @@ -661,11 +663,20 @@ udocker create --name=myfed fedora:29 # execute a cat inside of the container udocker run myfed cat /etc/redhat-release -# The above three operations could have done with a single command -# However each time udocker is invoked this way a new container -# directory tree is created consuming additional space and time +# The above three operations can be done with a single command +# However each time udocker is invoked in this way a new container +# directory tree is created. This will consume additional space +# and may considerably increase the time for the container to start. udocker run fedora:29 cat /etc/redhat-release +# For repeated invocations of the same container image the issue +# described above can be prevented by using --pull=reuse with --name. +# With the option --pull=reuse udocker will first try to execute +# a container with the same name specified by --name and only if +# it doesn't exist will it pull and create. In this way repeated +# calls to run only create a single container that is then reused. +udocker run --name=F29 --pull=reuse fedora:29 cat /etc/redhat-release + # In this example the host /tmp is mapped to the container /tmp udocker run --volume=/tmp myfed /bin/bash @@ -736,7 +747,7 @@ UDOCKER_LOGLEVEL=2 udocker run busybox:latest /bin/ls ### 3.23. login ```bash -udocker login [--username=USERNAME] [--password=PASSWORD] [--registry=REGISTRY] +udocker login [--username=USERNAME] [--password=PASSWORD | --password-stdin ] [--registry=REGISTRY] ``` Login into a Docker registry using v2 API. Only basic authentication @@ -748,15 +759,22 @@ Options: * `--username=USERNAME` provide the username in the command line * `--password=PASSWORD` provide the password in the command line +* `--password-stdin` provide the password via stdin * `--registry=REGISTRY` credentials are for this registry Examples: ```bash +# To use dockerhub private repositories udocker login --username=xxxx --password=yyyy -udocker login --registry="https://hostname:5000" + +# To use a different container registry (the https:// is optional) +udocker login --registry=https://hostname username: xxxx password: **** + +# To use a private repository at AWS ECR +aws ecr get-login-password --region eu-north-1 | udocker login --username=AWS --password-stdin --registry=000000000000.dkr.ecr.eu-north-1.amazonaws.com ``` ### 3.24. logout @@ -929,7 +947,7 @@ Newer versions of Singularity may run without requiring privileges but need a recent kernel in the host system with support for rootless user mode namespaces similar to runc in mode R1. Singularity cannot be compiled statically due to dependencies on -dynamic libraries and therefore is not provided with udocker. +dynamic libraries and therefore is not shipped with udocker. In CentOS 6 and CentOS 7 Singularity must be installed with privileges by a system administrator as it requires suid or capabilities. The S1 mode also offers root emulation to facilitate software installation @@ -966,7 +984,7 @@ changed through the configuration files by changing the attribute **UDOCKER_DEFAULT_EXECUTION_MODE**. Only the following modes can be used as default modes: **P1**, **P2**, **F1**, **S1**, and **R1**. Changing the default execution -mode can be useful in case the default does not work as expected. +mode can be useful if the default does not work as expected. Example: @@ -1000,11 +1018,20 @@ udocker manifest inspect REPO/IMAGE:TAG ``` Obtain and print information about an IMAGE manifest from a remote registry. +Can be used to obtain the platform architectures supported by the IMAGE. + +Options: + +* `--index=url` specify an index other than index.docker.io +* `--registry=url` specify a registry other than registry-1.docker.io +* `--httpproxy=proxy` specify a socks proxy for downloading, see `pull` +* `--platform=os/architecture` specify a platform to be inspected Example: ```bash udocker manifest inspect centos:centos7 +udocker manifest --platform=linux/ppc64le inspect centos:7 ``` ## 4. Running MPI jobs @@ -1252,7 +1279,7 @@ as root. In other modes execution as root is achieved by invoking run with the `--user=root` option: ```bash -udocker run --user=root ` +udocker run --user=root ``` ### 7.1. Running as root in Pn modes @@ -1381,18 +1408,48 @@ considerations may hold: mode may exhibit a large performance penalty. This also applies to P1 in older kernels without **SECCOMP filtering** * Fn modes are generally faster than Pn modes and do not have - multi threading or I/O limitations. + the multi threading or I/O limitations. * Singularity and runc should provide similar performances. * Depending on application the Fn modes are often faster than all other modes. ## 10. Hardware architectures -The udocker Python code was the built-in logic to support several hardware +The udocker Python code has the built-in logic to support several hardware architectures namely i386, x86_64, arm (32 bit) and aarch64 (arm 64 bit). However the required engine binaries and/or libraries must also be provided -for each of the architectures. Currently only the Pn modes are provided with -compiled executables to support execution on x86, x86_64, ARM and ARM64. +for each of the architectures. Currently only some modes have compiled +binaries to support execution on x86, x86_64, ARM, ARM64 and +ppc64le. The executables and libraries for the execution engines shipped +with udocker have a suffix that identifies the architecture, check the +relevant udocker installation directories usually `$HOME/.udocker/bin` +and `$HOME/.udocker/lib`. + +Users may compile the same executables shipped in the udockertools in +their linux hosts to support different or newer distributions, and/or +architectures. See the [installation manual](installation_manual.md) +for further information. + +Checking which architectures are supported by a given container can +be verified using `udocker manifest inspect IMAGE`. If the intended architecture +is available it can be pulled using `udocker pull --platform=OS/ARCH`. + +```bash +udocker manifest inspect centos:7 +udocker pull --platform=linux/arm64 centos:7 +udocker create --name=C7 centos:7 +udocker run C7 +``` + +In general, if the binaries in the container have been compiled for +an architecture that is different from the host then the execution +will not be possible. However, execution may still be possible provided +that `qemu-user` is locally installed. In many distributions `qemu-user` +is provided by the package `qemu-user-static`. In such case the default +engine of udocker Pn will automatically use the qemu emulation to support +the execution. Since the architecture is emulated the execution will be +much slower. Emulation for the Fn modes may also work if the `qemu-user` +binaries are both installed and also appear in `/proc/sys/fs/binfmt_misc/`. ## 11. Host environment specific notes @@ -1412,9 +1469,11 @@ udocker run arm64v8/fedora:35 udocker can run on Google Colab using the **P** or **F** modes. ```bash -!PATH=`pwd`/udocker:$PATH udocker --allow-root pull centos:centos7 -!PATH=`pwd`/udocker:$PATH udocker --allow-root create --name=c7 centos:centos7 -!PATH=`pwd`/udocker:$PATH udocker --allow-root run c7 +! pip install udocker +! udocker install +! udocker --allow-root pull centos:centos7 +! udocker --allow-root create --name=c7 centos:centos7 +! udocker --allow-root run c7 ``` ### 11.3. Docker @@ -1431,15 +1490,13 @@ udocker --allow-root run ub18 ## 12. Issues -To avoid corruption the execution of data backups and container copies should -only be performed when the container is not being executed (not locally nor -in any other host if the filesystem is shared). - Containers should only be copied for transfer when they are in the execution modes Pn or Rn. The modes Fn perform changes to the containers that will make them fail if they are execute in a different host where the absolute pathname to the container location is different. In this later case convert back to P1 -(using: `udocker setup --execmode=P1`) before performing the backup. +(using: `udocker setup --execmode=P1`) before performing the backup. Sharing +of containers can be done across hosts in an homogeneous cluster or between +hosts with the very same directory structure. When experiencing issues in the default execution mode (P1) you may try to setup the container to execute using mode P2 or one of the Fn or @@ -1448,7 +1505,8 @@ Rn modes. See section 3.27 for information on changing execution modes. Some execution modes require the creation of auxiliary files, directories and mount points. These can be purged from a given container using `setup --purge`, however this operation must be performed when the -container is not being executed. +container is not being executed (nor locally nor in another host of the +cluster). ## Acknowledgments diff --git a/setup.cfg b/setup.cfg index 17f44a03..10dbfa16 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,9 +8,8 @@ exclude = docs # Define setup.py command aliases here test = pytest -[tool:pytest] -collect_ignore = ['setup.py'] - +# [tool:pytest] +# collect_ignore = ['setup.py'] [codespell] # Ref: https://github.com/codespell-project/codespell#using-a-config-file diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 56398e1d..6f49094e 100755 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -46,12 +46,12 @@ def tearDown(self): @patch('udocker.cli.LocalFileAPI') @patch('udocker.cli.KeyStore') @patch('udocker.cli.DockerIoAPI') - def test_01_init(self, mock_dioapi, mock_ks, mock_lfapi): + def test_01_init(self, mock_dockerio, mock_ks, mock_lfapi): """Test01 UdockerCLI() constructor.""" # Test Config().conf['keystore'] starts with / Config().conf['keystore'] = "/xxx" UdockerCLI(self.local) - self.assertTrue(mock_dioapi.called) + self.assertTrue(mock_dockerio.called) self.assertTrue(mock_lfapi.called) self.assertTrue(mock_ks.called_with(Config().conf['keystore'])) @@ -60,8 +60,9 @@ def test_01_init(self, mock_dioapi, mock_ks, mock_lfapi): UdockerCLI(self.local) self.assertTrue(mock_ks.called_with(Config().conf['keystore'])) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.FileUtil.isdir') - def test_02__cdrepo(self, mock_isdir): + def test_02__cdrepo(self, mock_isdir, mock_dockerio): """Test02 UdockerCLI()._cdrepo().""" argv = ["udocker", "-h"] cmdp = CmdParser() @@ -90,9 +91,11 @@ def test_02__cdrepo(self, mock_isdir): self.assertTrue(status) self.assertTrue(self.local.setup.called) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.DockerIoAPI.is_repo_name') @patch('udocker.cli.Msg') - def test_03__check_imagespec(self, mock_msg, mock_reponame): + def test_03__check_imagespec(self, mock_msg, mock_reponame, + mock_dockerio): """Test03 UdockerCLI()._check_imagespec().""" mock_msg.level = 0 mock_reponame.return_value = False @@ -110,9 +113,11 @@ def test_03__check_imagespec(self, mock_msg, mock_reponame): status = udoc._check_imagespec("AAA:45") self.assertEqual(status, ("AAA", "45")) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.DockerIoAPI.is_repo_name') @patch('udocker.cli.Msg') - def test_04__check_imagerepo(self, mock_msg, mock_reponame): + def test_04__check_imagerepo(self, mock_msg, mock_reponame, + mock_dockerio): """Test04 UdockerCLI()._check_imagerepo().""" mock_msg.level = 0 mock_reponame.return_value = False @@ -125,38 +130,38 @@ def test_04__check_imagerepo(self, mock_msg, mock_reponame): status = udoc._check_imagerepo("AAA") self.assertEqual(status, "AAA") - @patch('udocker.cli.DockerIoAPI.set_index') - @patch('udocker.cli.DockerIoAPI.set_registry') - @patch('udocker.cli.DockerIoAPI.set_proxy') + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.Msg') - def test_05__set_repository(self, mock_msg, mock_proxy, - mock_reg, mock_idx): + def test_05__set_repository(self, mock_msg, mock_dockerio): """Test05 UdockerCLI()._set_repository().""" mock_msg.level = 0 regist = "registry.io" idxurl = "dockerhub.io" imgrepo = "dockerhub.io/myimg:1.2" - mock_proxy.return_value = None - mock_reg.side_effect = [None, None, None, None] - mock_idx.side_effect = [None, None, None, None] + mock_dockerio.set_proxy.return_value = None + mock_dockerio.set_registry.side_effect = [None, None, None, None] + mock_dockerio.set_index.side_effect = [None, None, None, None] udoc = UdockerCLI(self.local) + udoc.dockerioapi = mock_dockerio status = udoc._set_repository(regist, idxurl, imgrepo, True) self.assertTrue(status) - self.assertTrue(mock_proxy.called) - self.assertTrue(mock_reg.called) - self.assertTrue(mock_idx.called) + self.assertTrue(mock_dockerio.set_proxy.called) + self.assertTrue(mock_dockerio.set_registry.called) + self.assertTrue(mock_dockerio.set_index.called) regist = "" idxurl = "" imgrepo = "https://dockerhub.io/myimg:1.2" - mock_proxy.return_value = None - mock_reg.side_effect = [None, None, None, None] - mock_idx.side_effect = [None, None, None, None] + mock_dockerio.set_proxy.return_value = None + mock_dockerio.set_registry.side_effect = [None, None, None, None] + mock_dockerio.set_index.side_effect = [None, None, None, None] udoc = UdockerCLI(self.local) + udoc.dockerioapi = mock_dockerio status = udoc._set_repository(regist, idxurl, imgrepo, False) self.assertTrue(status) - def test_06__split_imagespec(self): + @patch('udocker.cli.DockerIoAPI') + def test_06__split_imagespec(self, mock_dockerio): """Test06 UdockerCLI()._split_imagespec().""" imgrepo = "" res = ("", "", "", "") @@ -176,9 +181,10 @@ def test_06__split_imagespec(self): status = udoc._split_imagespec(imgrepo) self.assertEqual(status, res) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.os.path.exists') @patch('udocker.cli.Msg') - def test_07_do_mkrepo(self, mock_msg, mock_exists): + def test_07_do_mkrepo(self, mock_msg, mock_exists, mock_dockerio): """Test07 UdockerCLI().do_mkrepo().""" mock_msg.level = 0 @@ -238,29 +244,29 @@ def test_07_do_mkrepo(self, mock_msg, mock_exists): # status = udoc._search_repositories("ipyrad") # self.assertEqual(status, 0) - @patch('udocker.cli.DockerIoAPI.get_tags') - def test_10__list_tags(self, mock_gettags): + @patch('udocker.cli.DockerIoAPI') + def test_10__list_tags(self, mock_dockerio): """Test10 UdockerCLI()._list_tags().""" - mock_gettags.return_value = ["t1"] + mock_dockerio.get_tags.return_value = ["t1"] udoc = UdockerCLI(self.local) + udoc.dockerioapi = mock_dockerio status = udoc._list_tags("t1") self.assertEqual(status, 0) - mock_gettags.return_value = None + mock_dockerio.get_tags.return_value = None udoc = UdockerCLI(self.local) + udoc.dockerioapi = mock_dockerio status = udoc._list_tags("t1") self.assertEqual(status, 1) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.KeyStore.get') - @patch('udocker.cli.DockerIoAPI.set_v2_login_token') - @patch('udocker.cli.DockerIoAPI.search_init') @patch.object(UdockerCLI, '_search_repositories') @patch.object(UdockerCLI, '_list_tags') @patch.object(UdockerCLI, '_split_imagespec') @patch.object(UdockerCLI, '_set_repository') def test_11_do_search(self, mock_setrepo, mock_split, mock_listtags, - mock_searchrepo, mock_doiasearch, mock_doiasetv2, - mock_ksget): + mock_searchrepo, mock_ksget, mock_dockerio): """Test11 UdockerCLI().do_search().""" argv = ["udocker", "-h"] cmdp = CmdParser() @@ -274,17 +280,18 @@ def test_11_do_search(self, mock_setrepo, mock_split, mock_listtags, cmdp.parse(argv) mock_setrepo.return_value = None mock_split.return_value = ("d1", "d2", "ipyrad", "d3") - mock_doiasearch.return_value = None + mock_dockerio.search_init.return_value = None mock_ksget.return_value = "v2token1" - mock_doiasetv2.return_value = None + mock_dockerio.set_v2_login_token.return_value = None mock_listtags.return_value = ["t1", "t2"] udoc = UdockerCLI(self.local) + udoc.dockerioapi = mock_dockerio status = udoc.do_search(cmdp) self.assertEqual(status, ["t1", "t2"]) self.assertTrue(mock_setrepo.called) - self.assertTrue(mock_doiasearch.called) + self.assertTrue(mock_dockerio.search_init.called) self.assertTrue(mock_ksget.called) - self.assertTrue(mock_doiasetv2.called) + self.assertTrue(mock_dockerio.set_v2_login_token.called) self.assertTrue(mock_listtags.called) argv = ["udocker", "search", "ipyrad"] @@ -292,19 +299,22 @@ def test_11_do_search(self, mock_setrepo, mock_split, mock_listtags, cmdp.parse(argv) mock_setrepo.return_value = None mock_split.return_value = ("d1", "d2", "ipyrad", "d3") - mock_doiasearch.return_value = None + mock_dockerio.search_init.return_value = None mock_ksget.return_value = "v2token1" - mock_doiasetv2.return_value = None + mock_dockerio.set_v2_login_token.return_value = None mock_searchrepo.return_value = 0 udoc = UdockerCLI(self.local) + udoc.dockerioapi = mock_dockerio status = udoc.do_search(cmdp) self.assertEqual(status, 0) self.assertTrue(mock_searchrepo.called) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.Msg') @patch('udocker.cli.LocalFileAPI.load') @patch.object(UdockerCLI, '_check_imagerepo') - def test_12_do_load(self, mock_chkimg, mock_load, mock_msg): + def test_12_do_load(self, mock_chkimg, mock_load, mock_msg, + mock_dockerio): """Test12 UdockerCLI().do_load().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -342,11 +352,13 @@ def test_12_do_load(self, mock_chkimg, mock_load, mock_msg): status = udoc.do_load(cmdp) self.assertEqual(status, 0) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.Msg') @patch('udocker.cli.os.path.exists') @patch('udocker.cli.LocalFileAPI.save') @patch.object(UdockerCLI, '_check_imagespec') - def test_13_do_save(self, mock_chkimg, mock_save, mock_exists, mock_msg): + def test_13_do_save(self, mock_chkimg, mock_save, mock_exists, + mock_msg, mock_dockerio): """Test13 UdockerCLI().do_save().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -391,13 +403,14 @@ def test_13_do_save(self, mock_chkimg, mock_save, mock_exists, mock_msg): self.assertTrue(mock_save.called) self.assertEqual(status, 0) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.LocalFileAPI.import_toimage') @patch('udocker.cli.LocalFileAPI.import_tocontainer') @patch('udocker.cli.LocalFileAPI.import_clone') @patch('udocker.cli.Msg') @patch.object(UdockerCLI, '_check_imagespec') def test_14_do_import(self, mock_chkimg, mock_msg, mock_impclone, - mock_impcont, mock_impimg): + mock_impcont, mock_impimg, mock_dockerio): """Test14 UdockerCLI().do_import().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -453,9 +466,10 @@ def test_14_do_import(self, mock_chkimg, mock_msg, mock_impclone, self.assertEqual(status, 0) self.assertTrue(mock_impcont.called) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.Msg') @patch('udocker.cli.ContainerStructure') - def test_15_do_export(self, mock_cs, mock_msg): + def test_15_do_export(self, mock_cs, mock_msg, mock_dockerio): """Test15 UdockerCLI().do_export().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -503,9 +517,10 @@ def test_15_do_export(self, mock_cs, mock_msg): status = udoc.do_export(cmdp) self.assertEqual(status, 0) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.LocalFileAPI.clone_container') @patch('udocker.cli.Msg') - def test_16_do_clone(self, mock_msg, mock_clone): + def test_16_do_clone(self, mock_msg, mock_clone, mock_dockerio): """Test16 UdockerCLI().do_clone().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -535,12 +550,12 @@ def test_16_do_clone(self, mock_msg, mock_clone): self.assertEqual(status, 0) self.assertTrue(mock_clone.called) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.Msg') @patch('udocker.cli.KeyStore.put') - @patch('udocker.cli.DockerIoAPI.get_v2_login_token') @patch.object(UdockerCLI, '_set_repository') - def test_17_do_login(self, mock_setrepo, mock_dioalog, - mock_ksput, mock_msg): + def test_17_do_login(self, mock_setrepo, mock_ksput, mock_msg, + mock_dockerio): """Test17 UdockerCLI().do_login().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -555,13 +570,14 @@ def test_17_do_login(self, mock_setrepo, mock_dioalog, cmdp = CmdParser() cmdp.parse(argv) mock_setrepo.return_value = True - mock_dioalog.return_value = "zx1" + mock_dockerio.get_v2_login_token.return_value = "zx1" mock_ksput.return_value = 1 udoc = UdockerCLI(self.local) + udoc.dockerioapi = mock_dockerio status = udoc.do_login(cmdp) self.assertEqual(status, 1) self.assertTrue(mock_setrepo.called) - self.assertTrue(mock_dioalog.called) + self.assertTrue(mock_dockerio.get_v2_login_token.called) self.assertTrue(mock_ksput.called) argv = ["udocker", "login", "--username", "u1", @@ -569,19 +585,22 @@ def test_17_do_login(self, mock_setrepo, mock_dioalog, cmdp = CmdParser() cmdp.parse(argv) mock_setrepo.return_value = None - mock_dioalog.return_value = "zx1" + mock_dockerio.get_v2_login_token.return_value = "zx1" mock_ksput.return_value = 0 udoc = UdockerCLI(self.local) + udoc.dockerioapi = mock_dockerio status = udoc.do_login(cmdp) self.assertEqual(status, 0) self.assertTrue(mock_setrepo.called) - self.assertTrue(mock_dioalog.called) + self.assertTrue(mock_dockerio.get_v2_login_token.called) self.assertTrue(mock_ksput.called) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.Msg') @patch('udocker.cli.KeyStore') @patch.object(UdockerCLI, '_set_repository') - def test_18_do_logout(self, mock_setrepo, mock_ks, mock_msg): + def test_18_do_logout(self, mock_setrepo, mock_ks, mock_msg, + mock_dockerio): """Test18 UdockerCLI().do_logout().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -618,7 +637,7 @@ def test_18_do_logout(self, mock_setrepo, mock_ks, mock_msg): @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.KeyStore.get') @patch('udocker.cli.Msg') - def test_19_do_pull(self, mock_msg, mock_ksget, mock_dioa, + def test_19_do_pull(self, mock_msg, mock_ksget, mock_dockerio, mock_chkimg, mock_setrepo): """Test19 UdockerCLI().do_pull().""" mock_msg.level = 0 @@ -627,6 +646,7 @@ def test_19_do_pull(self, mock_msg, mock_ksget, mock_dioa, cmdp.parse(argv) mock_chkimg.return_value = ("ipyrad", "latest") udoc = UdockerCLI(self.local) + udoc.dockerioapi = mock_dockerio status = udoc.do_pull(cmdp) self.assertEqual(status, 1) @@ -636,16 +656,17 @@ def test_19_do_pull(self, mock_msg, mock_ksget, mock_dioa, mock_chkimg.return_value = ("ipyrad", "latest") mock_setrepo.return_value = None mock_ksget.return_value = "zx1" - mock_dioa.return_value.set_v2_login_token.return_value = None - mock_dioa.return_value.get.return_value = False + mock_dockerio.set_v2_login_token.return_value = None + mock_dockerio.get.return_value = False udoc = UdockerCLI(self.local) + udoc.dockerioapi = mock_dockerio status = udoc.do_pull(cmdp) self.assertEqual(status, 1) self.assertTrue(mock_chkimg.called) self.assertTrue(mock_setrepo.called) self.assertTrue(mock_ksget.called) - self.assertTrue(mock_dioa.return_value.set_v2_login_token.called) - self.assertTrue(mock_dioa.return_value.get.called) + self.assertTrue(mock_dockerio.set_v2_login_token.called) + self.assertTrue(mock_dockerio.get.called) argv = ["udocker", "pull", "ipyrad:latest"] cmdp = CmdParser() @@ -653,9 +674,10 @@ def test_19_do_pull(self, mock_msg, mock_ksget, mock_dioa, mock_chkimg.return_value = ("ipyrad", "latest") mock_setrepo.return_value = None mock_ksget.return_value = "zx1" - mock_dioa.return_value.set_v2_login_token.return_value = None - mock_dioa.return_value.get.return_value = True + mock_dockerio.set_v2_login_token.return_value = None + mock_dockerio.get.return_value = True udoc = UdockerCLI(self.local) + udoc.dockerioapi = mock_dockerio status = udoc.do_pull(cmdp) self.assertEqual(status, 0) @@ -663,33 +685,37 @@ def test_19_do_pull(self, mock_msg, mock_ksget, mock_dioa, @patch('udocker.cli.ContainerStructure') @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.Msg') - def test_20__create(self, mock_msg, mock_dioapi, + def test_20__create(self, mock_msg, mock_dockerio, mock_cstruct, mock_chkimg): """Test20 UdockerCLI()._create().""" mock_msg.level = 0 - mock_dioapi.return_value.is_repo_name.return_value = False + mock_dockerio.is_repo_name.return_value = False udoc = UdockerCLI(self.local) + udoc.dockerioapi = mock_dockerio status = udoc._create("IMAGE:TAG") self.assertFalse(status) self.assertTrue(mock_msg.return_value.err.called) - mock_dioapi.return_value.is_repo_name.return_value = True + mock_dockerio.is_repo_name.return_value = True mock_chkimg.return_value = ("", "TAG") mock_cstruct.return_value.create.return_value = True udoc = UdockerCLI(self.local) + udoc.dockerioapi = mock_dockerio status = udoc._create("IMAGE:TAG") self.assertFalse(status) - mock_dioapi.return_value.is_repo_name.return_value = True + mock_dockerio.is_repo_name.return_value = True mock_chkimg.return_value = ("IMAGE", "TAG") mock_cstruct.return_value.create.return_value = True udoc = UdockerCLI(self.local) + udoc.dockerioapi = mock_dockerio status = udoc._create("IMAGE:TAG") self.assertTrue(status) + @patch('udocker.cli.DockerIoAPI') @patch.object(UdockerCLI, '_create') @patch('udocker.cli.Msg') - def test_21_do_create(self, mock_msg, mock_create): + def test_21_do_create(self, mock_msg, mock_create, mock_dockerio): """Test21 UdockerCLI().do_create().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -731,13 +757,14 @@ def test_21_do_create(self, mock_msg, mock_create): # def test_22__get_run_options(self): # """Test22 UdockerCLI()._get_run_options()""" + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.ExecutionMode') @patch('udocker.cli.Msg') @patch.object(UdockerCLI, 'do_pull') @patch.object(UdockerCLI, '_create') @patch.object(UdockerCLI, '_check_imagespec') def test_23_do_run(self, mock_chkimg, mock_create, mock_pull, - mock_msg, mock_exec): + mock_msg, mock_exec, mock_dockerio): """Test23 UdockerCLI().do_run().""" mock_msg.level = 0 mock_pull.return_value = None @@ -804,7 +831,8 @@ def test_23_do_run(self, mock_chkimg, mock_create, mock_pull, exeng_patch.stop() - def test_24_do_images(self): + @patch('udocker.cli.DockerIoAPI') + def test_24_do_images(self, mock_dockerio): """Test24 UdockerCLI().do_images().""" argv = ["udocker", "-h"] cmdp = CmdParser() @@ -828,8 +856,9 @@ def test_24_do_images(self): self.assertTrue(self.local.cd_imagerepo.called) self.assertTrue(self.local.get_layers.called) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.ExecutionMode') - def test_25_do_ps(self, mock_exec): + def test_25_do_ps(self, mock_exec, mock_dockerio): """Test25 UdockerCLI().do_ps().""" argv = ["udocker", "-h"] cmdp = CmdParser() @@ -856,8 +885,9 @@ def test_25_do_ps(self, mock_exec): self.assertEqual(status, 0) exeng_patch.stop() + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.Msg') - def test_26_do_rm(self, mock_msg): + def test_26_do_rm(self, mock_msg, mock_dockerio): """Test26 UdockerCLI().do_rm().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -917,9 +947,10 @@ def test_26_do_rm(self, mock_msg): status = udoc.do_rm(cmdp) self.assertEqual(status, 0) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.Msg') @patch.object(UdockerCLI, '_check_imagespec') - def test_27_do_rmi(self, mock_chkimg, mock_msg): + def test_27_do_rmi(self, mock_chkimg, mock_msg, mock_dockerio): """Test27 UdockerCLI().do_rmi().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -972,9 +1003,10 @@ def test_27_do_rmi(self, mock_chkimg, mock_msg): self.assertEqual(status, 0) self.assertTrue(self.local.del_imagerepo.called) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.Msg') @patch.object(UdockerCLI, '_check_imagespec') - def test_28_do_protect(self, mock_chkimg, mock_msg): + def test_28_do_protect(self, mock_chkimg, mock_msg, mock_dockerio): """Test28 UdockerCLI().do_protect().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -1038,9 +1070,10 @@ def test_28_do_protect(self, mock_chkimg, mock_msg): status = udoc.do_protect(cmdp) self.assertEqual(status, 1) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.Msg') @patch.object(UdockerCLI, '_check_imagespec') - def test_29_do_unprotect(self, mock_chkimg, mock_msg): + def test_29_do_unprotect(self, mock_chkimg, mock_msg, mock_dockerio): """Test29 UdockerCLI().do_unprotect().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -1104,8 +1137,9 @@ def test_29_do_unprotect(self, mock_chkimg, mock_msg): status = udoc.do_unprotect(cmdp) self.assertEqual(status, 1) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.Msg') - def test_30_do_name(self, mock_msg): + def test_30_do_name(self, mock_msg, mock_dockerio): """Test30 UdockerCLI().do_name().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -1144,8 +1178,9 @@ def test_30_do_name(self, mock_msg): status = udoc.do_name(cmdp) self.assertEqual(status, 0) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.Msg') - def test_31_do_rename(self, mock_msg): + def test_31_do_rename(self, mock_msg, mock_dockerio): """Test31 UdockerCLI().do_rename().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -1206,8 +1241,9 @@ def test_31_do_rename(self, mock_msg): self.assertEqual(status, 0) self.assertTrue(self.local.set_container_name.call_count, 1) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.Msg') - def test_32_do_rmname(self, mock_msg): + def test_32_do_rmname(self, mock_msg, mock_dockerio): """Test32 UdockerCLI().do_rmname().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -1243,12 +1279,13 @@ def test_32_do_rmname(self, mock_msg): self.assertEqual(status, 0) self.assertTrue(self.local.del_container_name.called) + @patch('udocker.cli.DockerIoAPI') @patch.object(UdockerCLI, '_check_imagespec') @patch('udocker.cli.json.dumps') @patch('udocker.cli.ContainerStructure.get_container_attr') @patch('udocker.cli.Msg') def test_33_do_inspect(self, mock_msg, mock_csattr, mock_jdump, - mock_chkimg): + mock_chkimg, mock_dockerio): """Test33 UdockerCLI().do_inspect().""" cont_insp = { "architecture": "amd64", @@ -1339,9 +1376,10 @@ def test_33_do_inspect(self, mock_msg, mock_csattr, mock_jdump, status = udoc.do_inspect(cmdp) self.assertEqual(status, 0) + @patch('udocker.cli.DockerIoAPI') @patch.object(UdockerCLI, '_check_imagespec') @patch('udocker.cli.Msg') - def test_34_do_verify(self, mock_msg, mock_chkimg): + def test_34_do_verify(self, mock_msg, mock_chkimg, mock_dockerio): """Test34 UdockerCLI().do_verify().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -1371,6 +1409,7 @@ def test_34_do_verify(self, mock_msg, mock_chkimg): status = udoc.do_verify(cmdp) self.assertEqual(status, 0) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.ExecutionMode') @patch('udocker.cli.NvidiaMode') @patch('udocker.cli.FileUtil.rchmod') @@ -1379,7 +1418,8 @@ def test_34_do_verify(self, mock_msg, mock_chkimg): @patch('udocker.cli.FileBind') @patch('udocker.cli.Msg') def test_35_do_setup(self, mock_msg, mock_fb, mock_mp, - mock_unshr, mock_furchmod, mock_nv, mock_execm): + mock_unshr, mock_furchmod, mock_nv, mock_execm, + mock_dockerio): """Test35 UdockerCLI().do_setup().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -1457,9 +1497,10 @@ def test_35_do_setup(self, mock_msg, mock_fb, mock_mp, status = udoc.do_setup(cmdp) self.assertEqual(status, 0) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.UdockerTools') @patch('udocker.cli.Msg') - def test_36_do_install(self, mock_msg, mock_utools): + def test_36_do_install(self, mock_msg, mock_utools, mock_dockerio): """Test36 UdockerCLI().do_install().""" mock_msg.level = 0 argv = ["udocker", "-h"] @@ -1487,8 +1528,9 @@ def test_36_do_install(self, mock_msg, mock_utools): status = udoc.do_install(cmdp) self.assertEqual(status, 0) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.Msg') - def test_37_do_showconf(self, mock_msg): + def test_37_do_showconf(self, mock_msg, mock_dockerio): """Test37 UdockerCLI().do_showconf().""" mock_msg.level = 0 argv = ["udocker", "showconf"] @@ -1499,8 +1541,9 @@ def test_37_do_showconf(self, mock_msg): self.assertEqual(status, 0) self.assertTrue(mock_msg.return_value.out.called) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.Msg') - def test_38_do_version(self, mock_msg): + def test_38_do_version(self, mock_msg, mock_dockerio): """Test38 UdockerCLI().do_version().""" mock_msg.level = 0 argv = ["udocker", "version"] @@ -1511,8 +1554,9 @@ def test_38_do_version(self, mock_msg): self.assertEqual(status, 0) self.assertTrue(mock_msg.return_value.out.called) + @patch('udocker.cli.DockerIoAPI') @patch('udocker.cli.Msg') - def test_39_do_help(self, mock_msg): + def test_39_do_help(self, mock_msg, mock_dockerio): """Test39 UdockerCLI().do_help().""" mock_msg.level = 0 argv = ["udocker", "-h"] diff --git a/tests/unit/test_commonlocalfile.py b/tests/unit/test_commonlocalfile.py index 2f6ab12d..d486b690 100755 --- a/tests/unit/test_commonlocalfile.py +++ b/tests/unit/test_commonlocalfile.py @@ -136,7 +136,7 @@ def test_06_create_container_meta(self, mock_arch, mock_version, status = clfapi.create_container_meta(layer_id, comment) self.assertEqual(status["id"], layer_id) self.assertEqual(status["comment"], comment) - self.assertEqual(status["rootfs"]["diff_ids"], ["sha256:" + 'abc123',]) + self.assertEqual(status["rootfs"]["diff_ids"], ["sha256:" + 'abc123', ]) self.assertTrue(mock_arch.called) self.assertTrue(mock_version.called) self.assertTrue(mock_size.called) diff --git a/tests/unit/test_curl.py b/tests/unit/test_curl.py index c2c2365f..4eb870b0 100755 --- a/tests/unit/test_curl.py +++ b/tests/unit/test_curl.py @@ -90,7 +90,7 @@ def setUp(self): Config().conf['http_agent'] = "" Config().conf['http_proxy'] = "" Config().conf['http_insecure'] = 0 - Config().conf['use_curl_exec'] = "" + Config().conf['use_curl_executable'] = "" def tearDown(self): pass @@ -136,7 +136,8 @@ def test_02__select_implementation(self, mock_gupycurl, with self.assertRaises(NameError): GetURL() - def test_03_get_content_length(self): + @patch('udocker.utils.curl.GetURL._select_implementation') + def test_03_get_content_length(self, mock_sel): """Test03 GetURL().get_content_length().""" hdr = type('test', (object,), {})() hdr.data = {"content-length": 10, } @@ -168,27 +169,27 @@ def test_05_set_proxy(self, mock_gupycurl): geturl.set_proxy("http://host") self.assertEqual(geturl.http_proxy, "http://host") - def test_06_get(self): - """Test06 GetURL().get().""" - geturl = GetURL() - self.assertRaises(TypeError, geturl.get) - - geturl = GetURL() - geturl._geturl = type('test', (object,), {})() - geturl._geturl.get = self._get - self.assertEqual(geturl.get("http://host"), "http://host") - - def test_07_post(self): - """Test07 GetURL().post().""" - geturl = GetURL() - self.assertRaises(TypeError, geturl.post) - self.assertRaises(TypeError, geturl.post, "http://host") - - geturl = GetURL() - geturl._geturl = type('test', (object,), {})() - geturl._geturl.get = self._get - status = geturl.post("http://host", {"DATA": 1, }) - self.assertEqual(status, "http://host") + # def test_06_get(self): + # """Test06 GetURL().get().""" + # geturl = GetURL() + # self.assertRaises(TypeError, geturl.get) + # + # geturl = GetURL() + # geturl._geturl = type('test', (object,), {})() + # geturl._geturl.get = self._get + # self.assertEqual(geturl.get("http://host"), "http://host") + + # def test_07_post(self): + # """Test07 GetURL().post().""" + # geturl = GetURL() + # self.assertRaises(TypeError, geturl.post) + # self.assertRaises(TypeError, geturl.post, "http://host") + # + # geturl = GetURL() + # geturl._geturl = type('test', (object,), {})() + # geturl._geturl.get = self._get + # status = geturl.post("http://host", {"DATA": 1, }) + # self.assertEqual(status, "http://host") # def test_08_get_status_code(self): # """Test08 GetURL().get_status_code().""" @@ -205,6 +206,7 @@ def setUp(self): Config().conf['http_agent'] = "" Config().conf['http_proxy'] = "" Config().conf['http_insecure'] = 0 + Config().conf['use_curl_executable'] = "" def tearDown(self): pass @@ -233,34 +235,34 @@ def test_02_is_available(self, mock_gupycurl, mock_msg): # def test_03__select_implementation(self): # """Test03 GetURLpyCurl()._select_implementation().""" - @patch.object(GetURLpyCurl, 'is_available') - @patch('udocker.utils.curl.Msg') - @patch('udocker.utils.curl.pycurl') - @patch('udocker.utils.curl.CurlHeader') - def test_04__set_defaults(self, mock_hdr, mock_pyc, - mock_msg, mock_selinsec): - """Test04 GetURLpyCurl()._set_defaults().""" - mock_selinsec.return_value = True - mock_msg.level = 0 - mock_msg.VER = 4 - geturl = GetURLpyCurl() - geturl._set_defaults(mock_pyc, mock_hdr) - self.assertTrue(mock_pyc.setopt.called) + # @patch.object(GetURLpyCurl, 'is_available') + # @patch('udocker.utils.curl.Msg') + # @patch('udocker.utils.curl.pycurl') + # @patch('udocker.utils.curl.CurlHeader') + # def test_04__set_defaults(self, mock_hdr, mock_pyc, + # mock_msg, mock_selinsec): + # """Test04 GetURLpyCurl()._set_defaults().""" + # mock_selinsec.return_value = True + # mock_msg.level = 0 + # mock_msg.VER = 4 + # geturl = GetURLpyCurl() + # geturl._set_defaults(mock_pyc, mock_hdr) + # self.assertTrue(mock_pyc.setopt.called) - # when Msg.level >= Msg.VER = 4: AND insecure - mock_msg.level = 5 - mock_msg.VER = 4 - geturl = GetURLpyCurl() - geturl._set_defaults(mock_pyc, mock_hdr) - self.assertEqual(mock_pyc.setopt.call_count, 18) + # # when Msg.level >= Msg.VER = 4: AND insecure + # mock_msg.level = 5 + # mock_msg.VER = 4 + # geturl = GetURLpyCurl() + # geturl._set_defaults(mock_pyc, mock_hdr) + # self.assertEqual(mock_pyc.setopt.call_count, 18) - mock_selinsec.return_value = True - # when Msg.level < Msg.VER = 4: AND secure - mock_msg.level = 2 - mock_msg.VER = 4 - geturl = GetURLpyCurl() - geturl._set_defaults(mock_pyc, mock_hdr) - self.assertEqual(mock_pyc.setopt.call_count, 27) + # mock_selinsec.return_value = True + # # when Msg.level < Msg.VER = 4: AND secure + # mock_msg.level = 2 + # mock_msg.VER = 4 + # geturl = GetURLpyCurl() + # geturl._set_defaults(mock_pyc, mock_hdr) + # self.assertEqual(mock_pyc.setopt.call_count, 27) # @patch.object(GetURLpyCurl, 'is_available') # @patch('utils.curl.Msg') @@ -293,6 +295,7 @@ def setUp(self): Config().conf['http_agent'] = "" Config().conf['http_proxy'] = "" Config().conf['http_insecure'] = 0 + Config().conf['use_curl_executable'] = "" def tearDown(self): pass diff --git a/tests/unit/test_dockerioapi.py b/tests/unit/test_dockerioapi.py index c473d570..f348bd57 100755 --- a/tests/unit/test_dockerioapi.py +++ b/tests/unit/test_dockerioapi.py @@ -1,13 +1,16 @@ #!/usr/bin/env python + +# -*- coding: utf-8 -*- """ udocker unit tests: DockerIoAPI """ from unittest import TestCase, main -from unittest.mock import patch, Mock +from unittest.mock import Mock, patch from io import BytesIO as strio -from udocker.docker import DockerIoAPI from udocker.config import Config +from udocker.docker import DockerIoAPI +from udocker.utils.curl import GetURLpyCurl import collections collections.Callable = collections.abc.Callable @@ -56,19 +59,22 @@ def test_02_set_proxy(self, mock_geturl): doia.set_proxy(url) self.assertTrue(mock_geturl.return_value.set_proxy.called_with(url)) - def test_03_set_registry(self): + @patch('udocker.docker.GetURL') + def test_03_set_registry(self, mock_geturl): """Test03 DockerIoAPI().set_registry().""" doia = DockerIoAPI(self.local) doia.set_registry("https://registry-1.docker.io") self.assertEqual(doia.registry_url, "https://registry-1.docker.io") - def test_04_set_index(self): + @patch('udocker.docker.GetURL') + def test_04_set_index(self, mock_geturl): """Test04 DockerIoAPI().set_index().""" doia = DockerIoAPI(self.local) doia.set_index("https://index.docker.io/v1") self.assertEqual(doia.index_url, "https://index.docker.io/v1") - def test_05_is_repo_name(self): + @patch('udocker.docker.GetURL') + def test_05_is_repo_name(self, mock_geturl): """Test05 DockerIoAPI().is_repo_name().""" doia = DockerIoAPI(self.local) self.assertFalse(doia.is_repo_name("")) @@ -80,10 +86,13 @@ def test_05_is_repo_name(self): self.assertTrue(doia.is_repo_name("lipcomputing/os-cli-centos7")) self.assertTrue(doia.is_repo_name("lipcomputing/os-cli-centos7:latest")) + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.docker.GetURL.get_status_code') @patch('udocker.docker.GetURL.get') - def test_06__get_url(self, mock_get, mock_getstatus): + def test_06__get_url(self, mock_get, mock_getstatus, mock_gupycurl): """Test06 DockerIoAPI()._get_url().""" + mock_gupycurl.return_value = True + args = ["http://some1.org"] kwargs = list() hdr = type('test', (object,), {})() @@ -123,14 +132,17 @@ def test_06__get_url(self, mock_get, mock_getstatus): status = doia._get_url(args, kwargs) self.assertEqual(status, (hdr, buff)) + @patch.object(GetURLpyCurl, 'is_available') @patch.object(DockerIoAPI, '_get_url') @patch('udocker.docker.GetURL.get_status_code') @patch('udocker.docker.FileUtil.size') @patch('udocker.docker.GetURL.get_content_length') @patch('udocker.docker.ChkSUM.hash') def test_07__get_file(self, mock_hash, mock_getlength, - mock_fusize, mock_status, mock_geturl): + mock_fusize, mock_status, mock_geturl, mock_gupycurl): """Test07 DockerIoAPI()._get_file().""" + mock_gupycurl.return_value = True + cks = "af98ca7807fd3859c5bd876004fa7e960cecebddb342de1bc7f3b0e6f7dab415" url = "http://some1.org/file1" fname = "/sha256:" + cks @@ -177,16 +189,22 @@ def test_07__get_file(self, mock_hash, mock_getlength, status = doia._get_file(url, fname, cache) self.assertTrue(status) - def test_08__split_fields(self): + @patch.object(GetURLpyCurl, 'is_available') + def test_08__split_fields(self, mock_gupycurl): """Test08 DockerIoAPI()._split_fields().""" + mock_gupycurl.return_value = True + buff = 'k1="v1",k2="v2"' doia = DockerIoAPI(self.local) status = doia._split_fields(buff) self.assertEqual(status, {"k1": "v1", "k2": "v2"}) + @patch.object(GetURLpyCurl, 'is_available') @patch.object(DockerIoAPI, '_get_url') - def test_09_is_v1(self, mock_geturl): + def test_09_is_v1(self, mock_geturl, mock_gupycurl): """Test09 DockerIoAPI().is_v1().""" + mock_gupycurl.return_value = True + hdr = type('test', (object,), {})() hdr.data = {"content-length": 10, "X-ND-HTTPSTATUS": "HTTP-Version 200 Reason-Phrase", @@ -207,9 +225,12 @@ def test_09_is_v1(self, mock_geturl): status = doia.is_v1() self.assertFalse(status) + @patch.object(GetURLpyCurl, 'is_available') @patch.object(DockerIoAPI, '_get_url') - def test_10_has_search_v1(self, mock_geturl): + def test_10_has_search_v1(self, mock_geturl, mock_gupycurl): """Test10 DockerIoAPI().has_search_v1().""" + mock_gupycurl.return_value = True + url = "http://some1.org/file1" hdr = type('test', (object,), {})() hdr.data = {"content-length": 10, @@ -231,10 +252,13 @@ def test_10_has_search_v1(self, mock_geturl): status = doia.has_search_v1(url) self.assertFalse(status) + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.docker.json.loads') @patch.object(DockerIoAPI, '_get_url') - def test_11_get_v1_repo(self, mock_geturl, mock_jload): + def test_11_get_v1_repo(self, mock_geturl, mock_jload, mock_gupycurl): """Test11 DockerIoAPI().get_v1_repo""" + mock_gupycurl.return_value = True + imagerepo = "REPO" hdr = type('test', (object,), {})() hdr.data = {"x-docker-token": "12345"} @@ -246,8 +270,11 @@ def test_11_get_v1_repo(self, mock_geturl, mock_jload): status = doia.get_v1_repo(imagerepo) self.assertEqual(status, ({"x-docker-token": "12345"}, {"k1": "v1"})) - def test_12__get_v1_auth(self): + @patch.object(GetURLpyCurl, 'is_available') + def test_12__get_v1_auth(self, mock_gupycurl): """Test12 DockerIoAPI()._get_v1_auth""" + mock_gupycurl.return_value = True + doia = DockerIoAPI(self.local) doia.v1_auth_header = "Not Empty" www_authenticate = ['Other Stuff'] @@ -260,11 +287,15 @@ def test_12__get_v1_auth(self): out = doia._get_v1_auth(www_authenticate) self.assertEqual(out, "Not Empty") + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.docker.Msg') @patch('udocker.utils.curl.CurlHeader') @patch.object(DockerIoAPI, '_get_url') - def test_13_get_v1_image_tags(self, mock_dgu, mock_hdr, mock_msg): + def test_13_get_v1_image_tags(self, mock_dgu, mock_hdr, mock_msg, + mock_gupycurl): """Test13 DockerIoAPI().get_v1_image_tags""" + mock_gupycurl.return_value = True + mock_msg.level = 0 mock_dgu.return_value = (mock_hdr, []) endpoint = "docker.io" @@ -273,11 +304,15 @@ def test_13_get_v1_image_tags(self, mock_dgu, mock_hdr, mock_msg): out = doia.get_v1_image_tags(endpoint, imagerepo) self.assertIsInstance(out, list) + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.docker.Msg') @patch('udocker.utils.curl.CurlHeader') @patch.object(DockerIoAPI, '_get_url') - def test_14_get_v1_image_tag(self, mock_dgu, mock_hdr, mock_msg): + def test_14_get_v1_image_tag(self, mock_dgu, mock_hdr, mock_msg, + mock_gupycurl): """Test14 DockerIoAPI().get_v1_image_tag""" + mock_gupycurl.return_value = True + mock_msg.level = 0 mock_dgu.return_value = (mock_hdr, []) endpoint = "docker.io" @@ -287,11 +322,15 @@ def test_14_get_v1_image_tag(self, mock_dgu, mock_hdr, mock_msg): out = doia.get_v1_image_tag(endpoint, imagerepo, tag) self.assertIsInstance(out, tuple) + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.docker.Msg') @patch('udocker.utils.curl.CurlHeader') @patch.object(DockerIoAPI, '_get_url') - def test_15_get_v1_image_ancestry(self, mock_dgu, mock_hdr, mock_msg): + def test_15_get_v1_image_ancestry(self, mock_dgu, mock_hdr, mock_msg, + mock_gupycurl): """Test15 DockerIoAPI().get_v1_image_ancestry""" + mock_gupycurl.return_value = True + mock_msg.level = 0 mock_dgu.return_value = (mock_hdr, []) endpoint = "docker.io" @@ -300,10 +339,13 @@ def test_15_get_v1_image_ancestry(self, mock_dgu, mock_hdr, mock_msg): out = doia.get_v1_image_ancestry(endpoint, image_id) self.assertIsInstance(out, tuple) + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.docker.Msg') @patch.object(DockerIoAPI, '_get_file') - def test_16_get_v1_image_json(self, mock_dgf, mock_msg): + def test_16_get_v1_image_json(self, mock_dgf, mock_msg, mock_gupycurl): """Test16 DockerIoAPI().get_v1_image_json""" + mock_gupycurl.return_value = True + mock_msg.level = 0 mock_dgf.return_value = True endpoint = "docker.io" @@ -317,10 +359,13 @@ def test_16_get_v1_image_json(self, mock_dgf, mock_msg): status = doia.get_v1_image_json(endpoint, layer_id) self.assertFalse(status) + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.docker.Msg') @patch.object(DockerIoAPI, '_get_file') - def test_17_get_v1_image_layer(self, mock_dgf, mock_msg): + def test_17_get_v1_image_layer(self, mock_dgf, mock_msg, mock_gupycurl): """Test17 DockerIoAPI().get_v1_image_layer""" + mock_gupycurl.return_value = True + mock_msg.level = 0 mock_dgf.return_value = True endpoint = "docker.io" @@ -334,10 +379,13 @@ def test_17_get_v1_image_layer(self, mock_dgf, mock_msg): status = doia.get_v1_image_layer(endpoint, layer_id) self.assertFalse(status) + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.docker.Msg') @patch.object(DockerIoAPI, '_get_file') - def test_18_get_v1_layers_all(self, mock_dgf, mock_msg): + def test_18_get_v1_layers_all(self, mock_dgf, mock_msg, mock_gupycurl): """Test18 DockerIoAPI().get_v1_layers_all""" + mock_gupycurl.return_value = True + mock_msg.level = 0 mock_dgf.return_value = True endpoint = "docker.io" @@ -351,11 +399,15 @@ def test_18_get_v1_layers_all(self, mock_dgf, mock_msg): out = doia.get_v1_layers_all(endpoint, layer_list) self.assertEqual(out, ['b.json', 'b.layer', 'a.json', 'a.layer']) + @patch.object(GetURLpyCurl, 'is_available') @patch.object(DockerIoAPI, '_get_url') @patch('udocker.utils.curl.CurlHeader') @patch('udocker.docker.json.loads') - def test_19__get_v2_auth(self, mock_jloads, mock_hdr, mock_dgu): + def test_19__get_v2_auth(self, mock_jloads, mock_hdr, mock_dgu, + mock_gupycurl): """Test19 DockerIoAPI()._get_v2_auth""" + mock_gupycurl.return_value = True + fakedata = strio('token'.encode('utf-8')) www_authenticate = "Other Stuff" mock_dgu.return_value = (mock_hdr, fakedata) @@ -380,8 +432,11 @@ def test_19__get_v2_auth(self, mock_jloads, mock_hdr, mock_dgu): out = doia._get_v2_auth(www_authenticate, False) self.assertEqual(out, "Authorization: Basic %s" % doia.v2_auth_token) - def test_20_get_v2_login_token(self): + @patch.object(GetURLpyCurl, 'is_available') + def test_20_get_v2_login_token(self, mock_gupycurl): """Test20 DockerIoAPI().get_v2_login_token""" + mock_gupycurl.return_value = True + doia = DockerIoAPI(self.local) out = doia.get_v2_login_token("username", "password") self.assertIsInstance(out, str) @@ -390,16 +445,22 @@ def test_20_get_v2_login_token(self): out = doia.get_v2_login_token("", "") self.assertEqual(out, "") - def test_21_set_v2_login_token(self): + @patch.object(GetURLpyCurl, 'is_available') + def test_21_set_v2_login_token(self, mock_gupycurl): """Test21 DockerIoAPI().set_v2_login_token""" + mock_gupycurl.return_value = True + doia = DockerIoAPI(self.local) doia.set_v2_login_token("BIG-FAT-TOKEN") self.assertEqual(doia.v2_auth_token, "BIG-FAT-TOKEN") + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.utils.curl.CurlHeader') @patch.object(DockerIoAPI, '_get_url') - def test_22_is_v2(self, mock_dgu, mock_hdr): + def test_22_is_v2(self, mock_dgu, mock_hdr, mock_gupycurl): """Test22 DockerIoAPI().is_v2""" + mock_gupycurl.return_value = True + mock_dgu.return_value = (mock_hdr, []) doia = DockerIoAPI(self.local) doia.registry_url = "http://www.docker.io" @@ -417,9 +478,12 @@ def test_22_is_v2(self, mock_dgu, mock_hdr): out = doia.is_v2() self.assertTrue(out) + @patch.object(GetURLpyCurl, 'is_available') @patch.object(DockerIoAPI, '_get_url') - def test_23_has_search_v2(self, mock_dgu): + def test_23_has_search_v2(self, mock_dgu, mock_gupycurl): """Test23 DockerIoAPI().has_search_v2""" + mock_gupycurl.return_value = True + hdr = type('test', (object,), {})() hdr.data = {"content-length": 10, "X-ND-HTTPSTATUS": "HTTP-Version 400 Reason-Phrase", @@ -442,10 +506,13 @@ def test_23_has_search_v2(self, mock_dgu): out = doia.has_search_v2() self.assertTrue(out) + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.docker.json.loads') @patch.object(DockerIoAPI, '_get_url') - def test_24_get_v2_image_tags(self, mock_dgu, mock_jload): + def test_24_get_v2_image_tags(self, mock_dgu, mock_jload, mock_gupycurl): """Test24 DockerIoAPI().get_v2_image_tags""" + mock_gupycurl.return_value = True + imgrepo = "img1" hdr = type('test', (object,), {})() hdr.data = {"content-length": 10, @@ -472,11 +539,15 @@ def test_24_get_v2_image_tags(self, mock_dgu, mock_jload): out = doia.get_v2_image_tags(imgrepo) self.assertEqual(out, ["tag1", "tag2"]) + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.docker.Msg') @patch('udocker.utils.curl.CurlHeader') @patch.object(DockerIoAPI, '_get_url') - def test_25_get_v2_image_manifest(self, mock_dgu, mock_hdr, mock_msg): + def test_25_get_v2_image_manifest(self, mock_dgu, mock_hdr, mock_msg, + mock_gupycurl): """Test25 DockerIoAPI().get_v2_image_manifest""" + mock_gupycurl.return_value = True + mock_msg.level = 0 imagerepo = "REPO" tag = "TAG" @@ -486,10 +557,13 @@ def test_25_get_v2_image_manifest(self, mock_dgu, mock_hdr, mock_msg): out = doia.get_v2_image_manifest(imagerepo, tag) self.assertIsInstance(out, tuple) + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.docker.Msg') @patch.object(DockerIoAPI, '_get_file') - def test_26_get_v2_image_layer(self, mock_dgf, mock_msg): + def test_26_get_v2_image_layer(self, mock_dgf, mock_msg, mock_gupycurl): """Test26 DockerIoAPI().get_v2_image_layer""" + mock_gupycurl.return_value = True + mock_msg.level = 0 imagerepo = "REPO" layer_id = "LAYERID" @@ -506,10 +580,13 @@ def test_26_get_v2_image_layer(self, mock_dgf, mock_msg): out = doia.get_v2_image_layer(imagerepo, layer_id) self.assertFalse(out) + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.docker.Msg') @patch.object(DockerIoAPI, 'get_v2_image_layer') - def test_27_get_v2_layers_all(self, mock_v2il, mock_msg): + def test_27_get_v2_layers_all(self, mock_v2il, mock_msg, mock_gupycurl): """Test27 DockerIoAPI().get_v2_layers_all""" + mock_gupycurl.return_value = True + mock_msg.level = 0 mock_v2il.return_value = False imagerepo = "REPO" @@ -526,14 +603,17 @@ def test_27_get_v2_layers_all(self, mock_v2il, mock_msg): out = doia.get_v2_layers_all(imagerepo, fslayers) self.assertEqual(out, ['foolayername']) + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.docker.Msg') @patch('udocker.docker.GetURL.get_status_code') @patch.object(DockerIoAPI, 'get_v2_image_manifest') @patch.object(DockerIoAPI, 'get_v2_layers_all') @patch.object(DockerIoAPI, '_get_url') def test_28_get_v2(self, mock_dgu, mock_dgv2, mock_manif, - mock_getstatus, mock_msg): + mock_getstatus, mock_msg, mock_gupycurl): """Test28 DockerIoAPI().get_v2""" + mock_gupycurl.return_value = True + imgrepo = "img1" hdr = type('test', (object,), {})() hdr_data = {"content-length": 10, @@ -580,8 +660,11 @@ def test_28_get_v2(self, mock_dgu, mock_dgv2, mock_manif, out = doia.get_v2(imgrepo, tag) self.assertEqual(out, ["foolayername"]) - def test_29__get_v1_id_from_tags(self): + @patch.object(GetURLpyCurl, 'is_available') + def test_29__get_v1_id_from_tags(self, mock_gupycurl): """Test29 DockerIoAPI()._get_v1_id_from_tags""" + mock_gupycurl.return_value = True + tobj = {"tag1": "t1"} tag = "tag1" doia = DockerIoAPI(self.local) @@ -594,8 +677,11 @@ def test_29__get_v1_id_from_tags(self): out = doia._get_v1_id_from_tags(tobj, tag) self.assertEqual(out, "l1") - def test_30__get_v1_id_from_images(self): + @patch.object(GetURLpyCurl, 'is_available') + def test_30__get_v1_id_from_images(self, mock_gupycurl): """Test30 DockerIoAPI()._get_v1_id_from_images""" + mock_gupycurl.return_value = True + imgarr = list() shortid = "" doia = DockerIoAPI(self.local) @@ -608,6 +694,7 @@ def test_30__get_v1_id_from_images(self): out = doia._get_v1_id_from_images(imgarr, shortid) self.assertEqual(out, "1234567890") + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.docker.Msg') @patch('udocker.docker.GetURL.get_status_code') @patch.object(DockerIoAPI, 'get_v1_layers_all') @@ -618,8 +705,10 @@ def test_30__get_v1_id_from_images(self): @patch.object(DockerIoAPI, 'get_v1_repo') def test_31_get_v1(self, mock_dgv1repo, mock_v1imgtag, mock_v1idtag, mock_v1idimg, mock_v1ances, - mock_v1layer, mock_status, mock_msg): + mock_v1layer, mock_status, mock_msg, mock_gupycurl): """Test31 DockerIoAPI().get_v1""" + mock_gupycurl.return_value = True + imgarr = [{"id": "1234567890"}] imagerepo = "REPO" tag = "TAG" @@ -691,19 +780,26 @@ def test_31_get_v1(self, mock_dgv1repo, mock_v1imgtag, out = doia.get_v1(imagerepo, tag) self.assertEqual(out, ["file1"]) - def test_32__parse_imagerepo(self): + @patch.object(GetURLpyCurl, 'is_available') + def test_32__parse_imagerepo(self, mock_gupycurl): """Test32 DockerIoAPI()._parse_imagerepo""" + mock_gupycurl.return_value = True + imagerepo = "https://hub.docker.com/_/mysql" doia = DockerIoAPI(self.local) out = doia._parse_imagerepo(imagerepo) self.assertEqual(out, (imagerepo, "https://hub.docker.com/_/mysql")) + @patch.object(GetURLpyCurl, 'is_available') @patch.object(DockerIoAPI, 'get_v1') @patch.object(DockerIoAPI, 'get_v2') @patch.object(DockerIoAPI, 'is_v2') @patch.object(DockerIoAPI, '_parse_imagerepo') - def test_33_get(self, mock_parse, mock_isv2, mock_getv2, mock_getv1): + def test_33_get(self, mock_parse, mock_isv2, mock_getv2, mock_getv1, + mock_gupycurl): """Test33 DockerIoAPI().get""" + mock_gupycurl.return_value = True + imagerepo = "REPO" tag = "TAG" mock_parse.return_value = ("REPO", "https://registry-1.docker.io") @@ -726,11 +822,15 @@ def test_33_get(self, mock_parse, mock_isv2, mock_getv2, mock_getv1): out = doia.get(imagerepo, tag) self.assertEqual(out, ["a", "b"]) + @patch.object(GetURLpyCurl, 'is_available') @patch.object(DockerIoAPI, 'get_v1_image_tags') @patch.object(DockerIoAPI, 'get_v2_image_tags') @patch.object(DockerIoAPI, 'is_v2') - def test_34_get_tags(self, mock_isv2, mock_getv2, mock_getv1): + def test_34_get_tags(self, mock_isv2, mock_getv2, mock_getv1, + mock_gupycurl): """Test34 DockerIoAPI().get_tags""" + mock_gupycurl.return_value = True + imagerepo = "REPO" mock_isv2.return_value = False mock_getv1.return_value = ["a", "b"] @@ -744,18 +844,25 @@ def test_34_get_tags(self, mock_isv2, mock_getv2, mock_getv1): out = doia.get_tags(imagerepo) self.assertEqual(out, ["a", "b"]) - def test_35_search_init(self): + @patch.object(GetURLpyCurl, 'is_available') + def test_35_search_init(self, mock_gupycurl): """Test35 DockerIoAPI().search_init""" + mock_gupycurl.return_value = True + doia = DockerIoAPI(self.local) doia.search_init("PAUSE") self.assertEqual(doia.search_pause, "PAUSE") self.assertEqual(doia.search_page, 0) self.assertEqual(doia.search_ended, False) + @patch.object(GetURLpyCurl, 'is_available') @patch.object(DockerIoAPI, '_get_url') @patch('udocker.docker.json.loads') - def test_36_search_get_page_v1(self, mock_jload, mock_dgu): + def test_36_search_get_page_v1(self, mock_jload, mock_dgu, + mock_gupycurl): """Test36 DockerIoAPI().set_index""" + mock_gupycurl.return_value = True + hdr = type('test', (object,), {})() hdr_data = {"content-length": 10, "X-ND-HTTPSTATUS": "HTTP-Version 200 Reason-Phrase", @@ -770,10 +877,13 @@ def test_36_search_get_page_v1(self, mock_jload, mock_dgu): out = doia.search_get_page_v1("SOMETHING", url) self.assertEqual(out, {"page": 1, "num_pages": 1}) + @patch.object(GetURLpyCurl, 'is_available') @patch.object(DockerIoAPI, '_get_url') @patch('udocker.docker.json.loads') - def test_37_search_get_page_v2(self, mock_jload, mock_dgu): + def test_37_search_get_page_v2(self, mock_jload, mock_dgu, mock_gupycurl): """Test37 DockerIoAPI().search_get_page_v2""" + mock_gupycurl.return_value = True + hdr = type('test', (object,), {})() hdr_data = {"content-length": 10, "X-ND-HTTPSTATUS": "HTTP-Version 200 Reason-Phrase", @@ -788,13 +898,16 @@ def test_37_search_get_page_v2(self, mock_jload, mock_dgu): out = doia.search_get_page_v2("SOMETHING", url) self.assertEqual(out, {"count": 1, "num_pages": 1}) + @patch.object(GetURLpyCurl, 'is_available') @patch.object(DockerIoAPI, 'search_get_page_v1') @patch.object(DockerIoAPI, 'search_get_page_v2') @patch.object(DockerIoAPI, 'has_search_v1') @patch.object(DockerIoAPI, 'has_search_v2') def test_38_search_get_page(self, mock_searchv2, mock_searchv1, - mock_getv2, mock_getv1): + mock_getv2, mock_getv1, mock_gupycurl): """Test38 DockerIoAPI().search_get_page""" + mock_gupycurl.return_value = True + mock_searchv2.return_value = True mock_searchv1.return_value = False mock_getv2.return_value = {"count": 1, "num_pages": 1} diff --git a/tests/unit/test_geturl.py b/tests/unit/test_geturl.py index 695567ad..4bbcdbbd 100755 --- a/tests/unit/test_geturl.py +++ b/tests/unit/test_geturl.py @@ -28,6 +28,7 @@ def setUp(self): Config().conf['http_proxy'] = "" Config().conf['http_insecure'] = 0 Config().conf['use_curl_exec'] = "" + Config().conf['use_curl_executable'] = "" def tearDown(self): pass @@ -43,9 +44,9 @@ def _get(self, *args, **kwargs): def test_01_init(self, mock_msg, mock_gupycurl, mock_guexecurl, mock_select): """Test01 GetURL() constructor.""" - mock_msg.level = 0 mock_gupycurl.return_value = False mock_guexecurl.return_value = True + mock_msg.level = 0 geturl = GetURL() mock_select.assert_called() self.assertEqual(geturl.ctimeout, Config().conf['ctimeout']) @@ -58,9 +59,8 @@ def test_01_init(self, mock_msg, mock_gupycurl, def test_02__select_implementation(self, mock_gupycurl, mock_guexecurl, mock_msg): """Test02 GetURL()._select_implementation().""" - Config.conf['use_curl_executable'] = "" - mock_msg.level = 0 mock_gupycurl.return_value = True + mock_msg.level = 0 geturl = GetURL() geturl._select_implementation() self.assertTrue(geturl.cache_support) @@ -78,8 +78,11 @@ def test_02__select_implementation(self, mock_gupycurl, geturl._select_implementation() self.assertEqual(nameerr.exception.code, 1) - def test_03_get_content_length(self): + @patch.object(GetURLpyCurl, 'is_available') + def test_03_get_content_length(self, mock_gupycurl): """Test03 GetURL().get_content_length().""" + mock_gupycurl.return_value = True + hdr = type('test', (object,), {})() hdr.data = {"content-length": 10, } geturl = GetURL() @@ -110,8 +113,10 @@ def test_05_set_proxy(self, mock_gupycurl): geturl.set_proxy("http://host") self.assertEqual(geturl.http_proxy, "http://host") - def test_06_get(self): + @patch.object(GetURLpyCurl, 'is_available') + def test_06_get(self, mock_gupycurl): """Test06 GetURL().get().""" + mock_gupycurl.return_value = True geturl = GetURL() self.assertRaises(TypeError, geturl.get) @@ -120,8 +125,10 @@ def test_06_get(self): geturl._geturl.get = self._get self.assertEqual(geturl.get("http://host"), "http://host") - def test_07_post(self): + @patch.object(GetURLpyCurl, 'is_available') + def test_07_post(self, mock_gupycurl): """Test07 GetURL().post().""" + mock_gupycurl.return_value = True geturl = GetURL() self.assertRaises(TypeError, geturl.post) self.assertRaises(TypeError, geturl.post, "http://host") @@ -132,8 +139,10 @@ def test_07_post(self): status = geturl.post("http://host", {"DATA": 1, }) self.assertEqual(status, "http://host") - def test_08_get_status_code(self): + @patch.object(GetURLpyCurl, 'is_available') + def test_08_get_status_code(self, mock_gupycurl): """Test08 GetURL().get_status_code().""" + mock_gupycurl.return_value = True sline = "HTTP-Version 400 Reason-Phrase" geturl = GetURL() status = geturl.get_status_code(sline) diff --git a/tests/unit/test_geturlpycurl.py b/tests/unit/test_geturlpycurl.py index e77d54f2..7556d7f4 100755 --- a/tests/unit/test_geturlpycurl.py +++ b/tests/unit/test_geturlpycurl.py @@ -6,8 +6,9 @@ """ from unittest import TestCase, main -from unittest.mock import patch, Mock -from io import BytesIO as strio +from unittest.mock import patch +# from unittest.mock import patch, Mock +# from io import BytesIO as strio from udocker.utils.curl import GetURLpyCurl from udocker.config import Config import collections @@ -40,92 +41,92 @@ def test_01_init(self): geturl = GetURLpyCurl() self.assertEqual(geturl._url, None) - @patch('udocker.utils.curl.pycurl.Curl') - def test_02_is_available(self, mock_pycurl): - """Test02 GetURLpyCurl().is_available().""" - mock_pycurl.return_value = True - geturl = GetURLpyCurl() - geturl.is_available() - self.assertTrue(geturl.is_available()) - self.assertTrue(mock_pycurl.called) - - # mock_pycurl.side_effect = (NameError, AttributeError) - # geturl = GetURLpyCurl() - # with self.assertRaises(NameError, AttributeError): - # status = geturl.is_available() - # self.assertTrue(mock_pycurl.called) - # self.assertFalse(geturl.is_available()) - - # mock_pycurl.return_value = None - # geturl = GetURLpyCurl() - # self.assertFalse(geturl.is_available()) + # @patch('udocker.utils.curl.pycurl.Curl') + # def test_02_is_available(self, mock_pycurl): + # """Test02 GetURLpyCurl().is_available().""" + # mock_pycurl.return_value = True + # geturl = GetURLpyCurl() + # geturl.is_available() + # self.assertTrue(geturl.is_available()) + # self.assertTrue(mock_pycurl.called) + # + # # mock_pycurl.side_effect = (NameError, AttributeError) + # # geturl = GetURLpyCurl() + # # with self.assertRaises(NameError, AttributeError): + # # status = geturl.is_available() + # # self.assertTrue(mock_pycurl.called) + # # self.assertFalse(geturl.is_available()) + # + # # mock_pycurl.return_value = None + # # geturl = GetURLpyCurl() + # # self.assertFalse(geturl.is_available()) # def test_03__select_implementation(self): # """Test03 GetURLpyCurl()._select_implementation().""" - @patch.object(GetURLpyCurl, 'is_available') - @patch('udocker.utils.curl.Msg') - @patch('udocker.utils.curl.pycurl.Curl') - @patch('udocker.utils.curl.CurlHeader') - def test_04__set_defaults(self, mock_hdr, mock_pyc, - mock_msg, mock_selinsec): - """Test04 GetURLpyCurl()._set_defaults().""" - mock_selinsec.return_value = True - mock_msg.level = 0 - mock_msg.VER = 4 - geturl = GetURLpyCurl() - geturl._set_defaults(mock_pyc, mock_hdr) - self.assertTrue(mock_pyc.setopt.called) - - # when Msg.level >= Msg.VER = 4: AND insecure - mock_msg.level = 5 - mock_msg.VER = 4 - geturl = GetURLpyCurl() - geturl._set_defaults(mock_pyc, mock_hdr) - self.assertEqual(mock_pyc.setopt.call_count, 18) - - mock_selinsec.return_value = True - # when Msg.level < Msg.VER = 4: AND secure - mock_msg.level = 2 - mock_msg.VER = 4 - geturl = GetURLpyCurl() - geturl._set_defaults(mock_pyc, mock_hdr) - self.assertEqual(mock_pyc.setopt.call_count, 27) - - @patch('udocker.utils.curl.json.dumps') - def test_05__mkpycurl(self, mock_jdump): - """Test05 GetURLpyCurl()._mkpycurl().""" - curl_patch = patch("udocker.utils.curl.CurlHeader") - curlhdr = curl_patch.start() - mock_curlhdr = Mock() - curlhdr.return_value = mock_curlhdr - - pyc_patch = patch("udocker.utils.curl.pycurl.Curl") - pycurl = pyc_patch.start() - mock_pycurl = Mock() - pycurl.return_value = mock_pycurl - - buff = strio() - argl = ["http://host"] - - geturl = GetURLpyCurl() - status = geturl._mkpycurl(pycurl, curlhdr, buff, argl) - self.assertTrue(pycurl.setopt.called) - self.assertEqual(status, ("", None)) - - kargl = {"post": "pst1", "sizeonly": True, - "proxy": "http://proxy", "ctimeout": 1000, - "header": "Authorization: Bearer", "v": True, - "nobody": True, "timeout": 50, } - mock_jdump.return_value = {"post": "pst1"} - geturl = GetURLpyCurl() - status = geturl._mkpycurl(pycurl, curlhdr, buff, argl, kargl) - self.assertTrue(pycurl.setopt.called) - self.assertTrue(curlhdr.sizeonly) - self.assertEqual(status, ("", None)) - - curlhdr = curl_patch.stop() - pycurl = pyc_patch.stop() + # @patch.object(GetURLpyCurl, 'is_available') + # @patch('udocker.utils.curl.Msg') + # @patch('udocker.utils.curl.pycurl.Curl') + # @patch('udocker.utils.curl.CurlHeader') + # def test_04__set_defaults(self, mock_hdr, mock_pyc, + # mock_msg, mock_selinsec): + # """Test04 GetURLpyCurl()._set_defaults().""" + # mock_selinsec.return_value = True + # mock_msg.level = 0 + # mock_msg.VER = 4 + # geturl = GetURLpyCurl() + # geturl._set_defaults(mock_pyc, mock_hdr) + # self.assertTrue(mock_pyc.setopt.called) + # + # # when Msg.level >= Msg.VER = 4: AND insecure + # mock_msg.level = 5 + # mock_msg.VER = 4 + # geturl = GetURLpyCurl() + # geturl._set_defaults(mock_pyc, mock_hdr) + # self.assertEqual(mock_pyc.setopt.call_count, 18) + # + # mock_selinsec.return_value = True + # # when Msg.level < Msg.VER = 4: AND secure + # mock_msg.level = 2 + # mock_msg.VER = 4 + # geturl = GetURLpyCurl() + # geturl._set_defaults(mock_pyc, mock_hdr) + # self.assertEqual(mock_pyc.setopt.call_count, 27) + + # @patch('udocker.utils.curl.json.dumps') + # def test_05__mkpycurl(self, mock_jdump): + # """Test05 GetURLpyCurl()._mkpycurl().""" + # curl_patch = patch("udocker.utils.curl.CurlHeader") + # curlhdr = curl_patch.start() + # mock_curlhdr = Mock() + # curlhdr.return_value = mock_curlhdr + # + # pyc_patch = patch("udocker.utils.curl.pycurl.Curl") + # pycurl = pyc_patch.start() + # mock_pycurl = Mock() + # pycurl.return_value = mock_pycurl + # + # buff = strio() + # argl = ["http://host"] + # + # geturl = GetURLpyCurl() + # status = geturl._mkpycurl(pycurl, curlhdr, buff, argl) + # self.assertTrue(pycurl.setopt.called) + # self.assertEqual(status, ("", None)) + # + # kargl = {"post": "pst1", "sizeonly": True, + # "proxy": "http://proxy", "ctimeout": 1000, + # "header": "Authorization: Bearer", "v": True, + # "nobody": True, "timeout": 50, } + # mock_jdump.return_value = {"post": "pst1"} + # geturl = GetURLpyCurl() + # status = geturl._mkpycurl(pycurl, curlhdr, buff, argl, kargl) + # self.assertTrue(pycurl.setopt.called) + # self.assertTrue(curlhdr.sizeonly) + # self.assertEqual(status, ("", None)) + # + # curlhdr = curl_patch.stop() + # pycurl = pyc_patch.stop() @patch.object(GetURLpyCurl, 'is_available') def test_06_get(self, mock_sel): diff --git a/tests/unit/test_hostinfo.py b/tests/unit/test_hostinfo.py index 5399adf7..b2e6c982 100755 --- a/tests/unit/test_hostinfo.py +++ b/tests/unit/test_hostinfo.py @@ -82,13 +82,13 @@ def test_05_oskernel_isgreater(self, mock_kernel): status = HostInfo().oskernel_isgreater([1, 1, 1]) self.assertFalse(status) - def test_06_cmd_has_option(self): - """Test06 HostInfo().cmd_has_option.""" - status = HostInfo().cmd_has_option("ls", "-a") - self.assertTrue(status) - - status = HostInfo().cmd_has_option("ls", "-z") - self.assertFalse(status) + # def test_06_cmd_has_option(self): + # """Test06 HostInfo().cmd_has_option.""" + # status = HostInfo().cmd_has_option("ls", "-a") + # self.assertTrue(status) + # + # status = HostInfo().cmd_has_option("ls", "-z") + # self.assertFalse(status) @patch('udocker.helper.hostinfo.Uprocess.check_output') def test_07_termsize(self, mock_chkout): diff --git a/tests/unit/test_tools.py b/tests/unit/test_tools.py index 42f13d80..cac97cd3 100755 --- a/tests/unit/test_tools.py +++ b/tests/unit/test_tools.py @@ -3,14 +3,15 @@ udocker unit tests: UdockerTools """ -import tarfile -from tarfile import TarInfo +# import tarfile +# from tarfile import TarInfo from unittest import TestCase, main from unittest.mock import Mock, patch from io import StringIO from udocker.config import Config from udocker.utils.curl import CurlHeader from udocker.tools import UdockerTools +from udocker.utils.curl import GetURLpyCurl import collections collections.Callable = collections.abc.Callable @@ -40,21 +41,30 @@ def test_01_init(self, mock_geturl): self.assertTrue(mock_geturl.called) self.assertEqual(utools.localrepo, self.local) + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.tools.Msg') - def test_02__instructions(self, mock_msg): + def test_02__instructions(self, mock_msg, mock_gupycurl): """Test02 UdockerTools()._instructions().""" + mock_gupycurl.return_value = True + utools = UdockerTools(self.local) utools._instructions() self.assertTrue(mock_msg.return_value.out.call_count, 2) - def test_03__version2int(self): + @patch.object(GetURLpyCurl, 'is_available') + def test_03__version2int(self, mock_gupycurl): """Test03 UdockerTools()._version2int().""" + mock_gupycurl.return_value = True + utools = UdockerTools(self.local) status = utools._version2int("2.4") self.assertEqual(status, 2004000) - def test_04__version_isok(self): + @patch.object(GetURLpyCurl, 'is_available') + def test_04__version_isok(self, mock_gupycurl): """Test04 UdockerTools()._version_isok().""" + mock_gupycurl.return_value = True + Config.conf['tarball_release'] = "1.3" utools = UdockerTools(self.local) status = utools._version_isok("2.4") @@ -65,20 +75,27 @@ def test_04__version_isok(self): status = utools._version_isok("1.4") self.assertFalse(status) + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.tools.FileUtil.getdata') - def test_05_is_available(self, mock_fuget): + def test_05_is_available(self, mock_fuget, mock_gupycurl): """Test05 UdockerTools().is_available().""" + mock_gupycurl.return_value = True + Config.conf['tarball_release'] = "2.3" mock_fuget.return_value = "2.3\n" utools = UdockerTools(self.local) status = utools.is_available() self.assertTrue(status) + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.tools.FileUtil.remove') @patch('udocker.tools.FileUtil.register_prefix') @patch('udocker.tools.os.listdir') - def test_06_purge(self, mock_lsdir, mock_fureg, mock_furm): + def test_06_purge(self, mock_lsdir, mock_fureg, mock_furm, + mock_gupycurl): """Test06 UdockerTools().purge().""" + mock_gupycurl.return_value = True + mock_lsdir.side_effect = [["f1", "f2"], ["f3", "f4"], ["f5", "f6"]] @@ -90,11 +107,15 @@ def test_06_purge(self, mock_lsdir, mock_fureg, mock_furm): self.assertTrue(mock_fureg.call_count, 4) self.assertTrue(mock_furm.call_count, 4) + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.tools.GetURL.get') @patch('udocker.tools.FileUtil.remove') @patch('udocker.tools.FileUtil.mktmp') - def test_07__download(self, mock_fumktmp, mock_furm, mock_geturl): + def test_07__download(self, mock_fumktmp, mock_furm, mock_geturl, + mock_gupycurl): """Test07 UdockerTools()._download().""" + mock_gupycurl.return_value = True + url = "https://down/file" hdr = CurlHeader() hdr.data["X-ND-HTTPSTATUS"] = "HTTP/1.1 200 OK" @@ -117,123 +138,130 @@ def test_07__download(self, mock_fumktmp, mock_furm, mock_geturl): self.assertTrue(mock_furm.called) self.assertEqual(status, "") - @patch('udocker.tools.os.path.isfile') - @patch('udocker.tools.os.path.realpath') - @patch('udocker.tools.os.path.exists') - @patch.object(UdockerTools, '_download') - def test_08__get_file(self, mock_downl, mock_exists, mock_rpath, - mock_isfile): - """Test08 UdockerTools()._get_file().""" - url = "" - mock_downl.return_value = "" - mock_exists.return_value = False - mock_isfile.return_value = False - utools = UdockerTools(self.local) - status = utools._get_file(url) - self.assertFalse(mock_downl.called) - self.assertTrue(mock_exists.called) - self.assertEqual(status, "") - - url = "https://down/file" - mock_downl.return_value = "/tmp/file" - mock_exists.return_value = True - mock_isfile.return_value = True - mock_rpath.return_value = "/tmp/file" - utools = UdockerTools(self.local) - status = utools._get_file(url) - self.assertTrue(mock_downl.called) - self.assertTrue(mock_exists.called) - self.assertTrue(mock_isfile.called) - self.assertEqual(status, "/tmp/file") - - @patch.object(UdockerTools, '_version_isok') - @patch('udocker.tools.FileUtil.remove') - @patch('udocker.tools.FileUtil.getdata') - @patch('udocker.tools.os.path.basename') - @patch('udocker.tools.FileUtil.mktmpdir') - @patch('udocker.tools.os.path.isfile') - def test_09__verify_version(self, mock_isfile, mock_fumktmp, - mock_osbase, mock_fugetdata, - mock_furm, mock_versionok): - """Test09 UdockerTools()._verify_version().""" - tball = "/home/udocker.tar" - mock_isfile.return_value = False - utools = UdockerTools(self.local) - status = utools._verify_version(tball) - self.assertTrue(mock_isfile.called) - self.assertEqual(status, (False, "")) - - tball = "/home/udocker.tar" - mock_isfile.return_value = True - mock_fumktmp.return_value = "" - utools = UdockerTools(self.local) - status = utools._verify_version(tball) - self.assertTrue(mock_isfile.called) - self.assertTrue(mock_fumktmp.called) - self.assertEqual(status, (False, "")) - - tball = "/home/udocker.tar" - tinfo1 = TarInfo("udocker_dir/lib/VERSION") - tinfo2 = TarInfo("a") - mock_isfile.return_value = True - mock_fumktmp.return_value = "/home/tmp" - mock_osbase.return_value = "VERSION" - mock_fugetdata.return_value = "1.2.7" - mock_furm.return_value = None - mock_versionok.return_value = True - with patch.object(tarfile, 'open', autospec=True) as open_mock: - open_mock.return_value.getmembers.return_value = [tinfo2, tinfo1] - open_mock.return_value.extract.return_value = None - utools = UdockerTools(self.local) - status = utools._verify_version(tball) - self.assertEqual(status, (True, "1.2.7")) - self.assertTrue(mock_furm.called) - - @patch.object(UdockerTools, '_clean_install') - @patch('udocker.tools.os.path.basename') - @patch('udocker.tools.FileUtil') - @patch('udocker.tools.os.path.isfile') - def test_10__install(self, mock_isfile, mock_futil, mock_osbase, mock_cleaninstall): - """Test10 UdockerTools()._install().""" - tfile = "" - mock_isfile.return_value = False - mock_cleaninstall.return_value = None - utools = UdockerTools(self.local) - status = utools._install(tfile) - self.assertFalse(status) - - tinfo1 = TarInfo("udocker_dir/bin/ls") - tinfo2 = TarInfo("udocker_dir/lib/lib1") - tfile = "udocker.tar" - mock_isfile.return_value = True - mock_futil.return_value.chmod.return_value = None - mock_futil.return_value.rchmod.side_effect = [None, None, None, - None, None, None] - mock_osbase.side_effect = ["ls", "ls", "lib1", "lib1", "doc", "doc1"] - self.local.create_repo.return_value = None - with patch.object(tarfile, 'open', autospec=True) as open_mock: - open_mock.return_value.getmembers.side_effect = [[tinfo1, tinfo2], - [tinfo1, tinfo2], - [tinfo1, tinfo2]] - open_mock.return_value.extract.side_effect = [None, None] - utools = UdockerTools(self.local) - status = utools._install(tfile) - self.assertTrue(status) - self.assertTrue(mock_futil.called) - self.assertTrue(mock_futil.return_value.rchmod.call_count, 4) - - def test_11__get_mirrors(self): + # @patch('udocker.tools.os.path.isfile') + # @patch('udocker.tools.os.path.realpath') + # @patch('udocker.tools.os.path.exists') + # @patch.object(UdockerTools, '_download') + # def test_08__get_file(self, mock_downl, mock_exists, mock_rpath, + # mock_isfile): + # """Test08 UdockerTools()._get_file().""" + # url = "" + # mock_downl.return_value = "" + # mock_exists.return_value = False + # mock_isfile.return_value = False + # utools = UdockerTools(self.local) + # status = utools._get_file(url) + # self.assertFalse(mock_downl.called) + # self.assertTrue(mock_exists.called) + # self.assertEqual(status, "") + # + # url = "https://down/file" + # mock_downl.return_value = "/tmp/file" + # mock_exists.return_value = True + # mock_isfile.return_value = True + # mock_rpath.return_value = "/tmp/file" + # utools = UdockerTools(self.local) + # status = utools._get_file(url) + # self.assertTrue(mock_downl.called) + # self.assertTrue(mock_exists.called) + # self.assertTrue(mock_isfile.called) + # self.assertEqual(status, "/tmp/file") + + # @patch.object(UdockerTools, '_version_isok') + # @patch('udocker.tools.FileUtil.remove') + # @patch('udocker.tools.FileUtil.getdata') + # @patch('udocker.tools.os.path.basename') + # @patch('udocker.tools.FileUtil.mktmpdir') + # @patch('udocker.tools.os.path.isfile') + # def test_09__verify_version(self, mock_isfile, mock_fumktmp, + # mock_osbase, mock_fugetdata, + # mock_furm, mock_versionok): + # """Test09 UdockerTools()._verify_version().""" + # tball = "/home/udocker.tar" + # mock_isfile.return_value = False + # utools = UdockerTools(self.local) + # status = utools._verify_version(tball) + # self.assertTrue(mock_isfile.called) + # self.assertEqual(status, (False, "")) + # + # tball = "/home/udocker.tar" + # mock_isfile.return_value = True + # mock_fumktmp.return_value = "" + # utools = UdockerTools(self.local) + # status = utools._verify_version(tball) + # self.assertTrue(mock_isfile.called) + # self.assertTrue(mock_fumktmp.called) + # self.assertEqual(status, (False, "")) + # + # tball = "/home/udocker.tar" + # tinfo1 = TarInfo("udocker_dir/lib/VERSION") + # tinfo2 = TarInfo("a") + # mock_isfile.return_value = True + # mock_fumktmp.return_value = "/home/tmp" + # mock_osbase.return_value = "VERSION" + # mock_fugetdata.return_value = "1.2.7" + # mock_furm.return_value = None + # mock_versionok.return_value = True + # with patch.object(tarfile, 'open', autospec=True) as open_mock: + # open_mock.return_value.getmembers.return_value = [tinfo2, tinfo1] + # open_mock.return_value.extract.return_value = None + # utools = UdockerTools(self.local) + # status = utools._verify_version(tball) + # self.assertEqual(status, (True, "1.2.7")) + # self.assertTrue(mock_furm.called) + + # @patch.object(UdockerTools, '_clean_install') + # @patch('udocker.tools.os.path.basename') + # @patch('udocker.tools.FileUtil') + # @patch('udocker.tools.os.path.isfile') + # def test_10__install(self, mock_isfile, mock_futil, mock_osbase, mock_cleaninstall): + # """Test10 UdockerTools()._install().""" + # tfile = "" + # mock_isfile.return_value = False + # mock_cleaninstall.return_value = None + # utools = UdockerTools(self.local) + # status = utools._install(tfile) + # self.assertFalse(status) + # + # tinfo1 = TarInfo("udocker_dir/bin/ls") + # tinfo2 = TarInfo("udocker_dir/lib/lib1") + # tfile = "udocker.tar" + # mock_isfile.return_value = True + # mock_futil.return_value.chmod.return_value = None + # mock_futil.return_value.rchmod.side_effect = [None, None, None, + # None, None, None] + # mock_osbase.side_effect = ["ls", "ls", "lib1", "lib1", "doc", "doc1"] + # self.local.create_repo.return_value = None + # with patch.object(tarfile, 'open', autospec=True) as open_mock: + # open_mock.return_value.getmembers.side_effect = [[tinfo1, tinfo2], + # [tinfo1, tinfo2], + # [tinfo1, tinfo2]] + # open_mock.return_value.extract.side_effect = [None, None] + # utools = UdockerTools(self.local) + # status = utools._install(tfile) + # self.assertTrue(status) + # self.assertTrue(mock_futil.called) + # self.assertTrue(mock_futil.return_value.rchmod.call_count, 4) + + @patch.object(GetURLpyCurl, 'is_available') + def test_11__get_mirrors(self, mock_gupycurl): """Test11 UdockerTools()._get_mirrors().""" + mock_gupycurl.return_value = True + mirrors = "https://download.ncg.ingrid.pt/udocker-1.2.7.tar.gz" utools = UdockerTools(self.local) status = utools._get_mirrors(mirrors) self.assertEqual(status, [mirrors]) + @patch.object(GetURLpyCurl, 'is_available') @patch.object(UdockerTools, '_get_file') @patch.object(UdockerTools, '_get_mirrors') @patch('udocker.tools.json.load') - def test_12_get_installinfo(self, mock_jload, mock_mirr, mock_getf): + def test_12_get_installinfo(self, mock_jload, mock_mirr, mock_getf, + mock_gupycurl): """Test12 UdockerTools().get_installinfo().""" + mock_gupycurl.return_value = True + Config.conf['installinfo'] = "/home/info.json" res = {"tarversion": "1.2.7"} mock_jload.return_value = {"tarversion": "1.2.7"} @@ -247,14 +275,17 @@ def test_12_get_installinfo(self, mock_jload, mock_mirr, mock_getf): status = utools.get_installinfo() self.assertEqual(status, res) + @patch.object(GetURLpyCurl, 'is_available') @patch.object(UdockerTools, '_install') @patch.object(UdockerTools, '_verify_version') @patch.object(UdockerTools, '_get_file') @patch.object(UdockerTools, '_get_mirrors') @patch('udocker.tools.FileUtil.remove') def test_13__install_logic(self, mock_furm, mock_getmirr, mock_getfile, - mock_verversion, mock_install): + mock_verversion, mock_install, mock_gupycurl): """Test13 UdockerTools()._install_logic().""" + mock_gupycurl.return_value = True + mock_furm.return_value = None mock_getmirr.return_value = "https://down.pt/udocker-1.2.7.tar.gz" mock_getfile.return_value = "udocker-1.2.7.tar.gz" @@ -286,13 +317,16 @@ def test_13__install_logic(self, mock_furm, mock_getmirr, mock_getfile, status = utools._install_logic(False) self.assertFalse(status) + @patch.object(GetURLpyCurl, 'is_available') @patch('udocker.tools.Msg') @patch.object(UdockerTools, 'get_installinfo') @patch.object(UdockerTools, '_install_logic') @patch.object(UdockerTools, 'is_available') def test_14_install(self, mock_isavail, mock_instlog, - mock_getinfo, mock_msg): + mock_getinfo, mock_msg, mock_gupycurl): """Test14 UdockerTools().install().""" + mock_gupycurl.return_value = True + mock_msg.level = 0 Config.conf['autoinstall'] = True Config.conf['tarball'] = "udocker-1.2.7.tar.gz" diff --git a/tox.ini b/tox.ini index 67706c36..4a02db4c 100644 --- a/tox.ini +++ b/tox.ini @@ -20,11 +20,11 @@ deps = setenv = LC_ALL=C.UTF-8 -changedir = - py37-unit: tests +# changedir = +# py37-unit: tests commands = - py37-unit: discover --pattern='tests_*.py' -v + py37-unit: discover --pattern='tests_*.py' -v tests/unit [testenv:bandit] envdir = {toxworkdir}/shared @@ -33,5 +33,3 @@ commands = bandit udocker -r -f html -o bandit.html [testenv:cover] envdir = {toxworkdir}/shared commands = nosetests -v --with-xcoverage --cover-package=udocker tests/unit - - diff --git a/udocker/__init__.py b/udocker/__init__.py index 57561a24..c28dee0e 100644 --- a/udocker/__init__.py +++ b/udocker/__init__.py @@ -32,5 +32,5 @@ "Singularity http://singularity.lbl.gov" ] __license__ = "Licensed under the Apache License, Version 2.0" -__version__ = "1.3.13" +__version__ = "1.3.14" __date__ = "2024" diff --git a/udocker/cli.py b/udocker/cli.py index 6661f5d2..d54a9cec 100644 --- a/udocker/cli.py +++ b/udocker/cli.py @@ -480,17 +480,21 @@ def do_login(self, cmdp): login: authenticate into docker repository e.g. dockerhub --username=username --password=password - --registry= ex. https://registry-1.docker.io + --password-stdin :read password from stdin + --registry= :optional ex. quay.io, public.ecr.aws """ username = cmdp.get("--username=") password = cmdp.get("--password=") + password_stdin = cmdp.get("--password-stdin") registry_url = cmdp.get("--registry=") if cmdp.missing_options(): # syntax error return self.STATUS_ERROR self._set_repository(registry_url, None, None, None) if not username: username = GET_INPUT("username: ") - if not password: + if password_stdin: + password = input() + elif not password: password = getpass("password: ") if password and password == password.upper(): Msg().out("Warning: password in uppercase", "Caps Lock ?", l=Msg.WAR) @@ -720,9 +724,6 @@ def _get_run_options(self, cmdp, exec_engine=None): if option_value or last_value is None: exec_engine.opt[option] = option_value elif cmdp_args["act"] == "E": # action is extend - # if option == "env": - # print (type(option_value)) - # print (option_value) exec_engine.opt[option].extend(option_value) last_value = option_value @@ -754,7 +755,8 @@ def do_run(self, cmdp): --nobanner :don't print a startup banner --entrypoint :override the container metadata entrypoint --platform=os/arch :pull image for OS and architecture - --pull= :when to pull (missing|never|always) + --pull= :when to pull (missing|never|always|reuse) + --httpproxy= :use http proxy, see udocker pull --help Only available in Rn execution modes: --device=/dev/xxx :pass device to container (R1 mode only) @@ -766,8 +768,10 @@ def do_run(self, cmdp): run executes an existing container, previously created from an image by using: create - run always creates a new container from the image - if needed the image is pulled. This is slow and may waste storage. + run always creates a new container from the image. + If needed the image is pulled. This is slow and may waste storage. + Using run --name= --pull=reuse allows to use existing + container and only pull/create if the does not exist. """ self._get_run_options(cmdp) container_or_image = cmdp.get("P1") @@ -775,18 +779,25 @@ def do_run(self, cmdp): delete = cmdp.get("--rm") name = cmdp.get("--name=") pull = cmdp.get("--pull=") - dummy = cmdp.get("--pull") # if invoked without option + cmdp.get("--pull") # if invoked without option + cmdp.get("--index=") # used in do_pull() + cmdp.get("--registry=") # used in do_pull() + cmdp.get("--httpproxy=") # used in do_pull() if cmdp.missing_options(): # syntax error return self.STATUS_ERROR + container_id = "" if Config.conf['location']: - container_id = "" + pass elif not container_or_image: Msg().err("Error: must specify container_id or image:tag") return self.STATUS_ERROR else: - container_id = self.localrepo.get_container_id(container_or_image) + if pull == "reuse" and name: + container_id = self.localrepo.get_container_id(name) + if not container_id: + container_id = self.localrepo.get_container_id(container_or_image) if not container_id: (imagerepo, tag) = self._check_imagespec(container_or_image) if (imagerepo and @@ -801,8 +812,9 @@ def do_run(self, cmdp): return self.STATUS_ERROR if name and container_id: if not self.localrepo.set_container_name(container_id, name): - Msg().err("Error: invalid container name format") - return self.STATUS_ERROR + if pull != "reuse": + Msg().err("Error: invalid container name") + return self.STATUS_ERROR exec_mode = ExecutionMode(self.localrepo, container_id) exec_engine = exec_mode.get_engine() diff --git a/udocker/commonlocalfile.py b/udocker/commonlocalfile.py index e08d9192..0a015ebc 100644 --- a/udocker/commonlocalfile.py +++ b/udocker/commonlocalfile.py @@ -105,7 +105,7 @@ def create_container_meta(self, layer_id, platform=""): if layer_chksum: container_json["rootfs"] = {} container_json["rootfs"]["type"] = "layers" - container_json["rootfs"]["diff_ids"] = ["sha256:" + layer_chksum,] + container_json["rootfs"]["diff_ids"] = ["sha256:" + layer_chksum, ] container_json["container_config"] = { "Hostname": "", "Domainname": "", diff --git a/udocker/config.py b/udocker/config.py index 4ba11d02..044ef316 100644 --- a/udocker/config.py +++ b/udocker/config.py @@ -50,7 +50,7 @@ class Config(object): # default path for executables conf['root_path'] = "/usr/sbin:/sbin:/usr/bin:/bin" - conf['user_path'] = "/usr/local/bin:/usr/bin:/bin" + conf['user_path'] = "/usr/sbin:/sbin:/usr/local/bin:/usr/bin:/bin" # directories to be mapped in containers with: run --sysdirs conf['sysdirs_list'] = ("/dev", "/proc", "/sys", diff --git a/udocker/docker.py b/udocker/docker.py index 1ba1a315..6b964e0f 100644 --- a/udocker/docker.py +++ b/udocker/docker.py @@ -572,9 +572,13 @@ def _parse_imagerepo(self, imagerepo): if '.' in components[0] and len(components) >= 2: registry = components[0] del components[0] + # if ('.' not in components[0] and + # components[0] != "library" and len(components) == 1): if ('.' not in components[0] and components[0] != "library" and len(components) == 1): - components.insert(0, "library") + if ((not registry) or + "docker.io" in registry or "docker.com" in registry): + components.insert(0, "library") remoterepo = '/'.join(components) if registry: try: diff --git a/udocker/engine/fakechroot.py b/udocker/engine/fakechroot.py index f0c24e1c..fceb1bd9 100644 --- a/udocker/engine/fakechroot.py +++ b/udocker/engine/fakechroot.py @@ -94,7 +94,7 @@ def _get_libc_pathname(self): libc_relative_path = libc_abs_path[len(self.container_root):] (dummy, filetype) = \ OSInfo(self.container_root).get_filetype(libc_relative_path) - if "ELF" in filetype and ( "dynamic" in filetype or "DYN" in filetype): + if "ELF" in filetype and ("dynamic" in filetype or "DYN" in filetype): return libc_relative_path return "" diff --git a/udocker/engine/runc.py b/udocker/engine/runc.py index dca70cf7..2796e112 100644 --- a/udocker/engine/runc.py +++ b/udocker/engine/runc.py @@ -44,13 +44,15 @@ def select_runc(self): self.executable = FileUtil("runc").find_exec() if not self.executable: self.executable = FileUtil("crun").find_exec() + if not self.executable: + self.executable = FileUtil("runsc").find_exec() - arch = HostInfo().arch() + self.arch = HostInfo().arch() if self.executable == "UDOCKER" or not self.executable: self.executable = "" image_list = [] - eng = ["runc", "crun"] - image_list = [eng[0]+"-"+arch, eng[0], eng[1]+"-"+arch, eng[1]] + eng = ["runc", "crun", "runsc"] + image_list = [eng[0]+"-"+self.arch, eng[0], eng[1]+"-"+self.arch, eng[1]] f_util = FileUtil(self.localrepo.bindir) self.executable = f_util.find_file_in_dir(image_list) @@ -58,7 +60,7 @@ def select_runc(self): if not os.path.exists(self.executable): Msg().err("Error: runc or crun executable not found") Msg().out("Info: Host architecture might not be supported by", - "this execution mode:", arch, + "this execution mode:", self.arch, "\n specify path to runc or crun with environment", "UDOCKER_USE_RUNC_EXECUTABLE", "\n or choose other execution mode with: udocker", @@ -68,6 +70,8 @@ def select_runc(self): self.engine_type = "crun" elif "runc" in os.path.basename(self.executable): self.engine_type = "runc" + elif "runsc" in os.path.basename(self.executable): + self.engine_type = "runsc" def _load_spec(self, new=False): """Generate runc spec file""" @@ -76,7 +80,15 @@ def _load_spec(self, new=False): FileUtil(self._container_specfile).remove() if FileUtil(self._container_specfile).size() == -1: - cmd_l = [self.executable, "spec", "--rootless", ] + if self.engine_type == "runsc": + f_util = FileUtil(self.localrepo.bindir) + runc_executable = f_util.find_file_in_dir(["runc-"+self.arch]) + if runc_executable: + cmd_l = [runc_executable, "spec", "--rootless", ] + # cmd_l = [self.executable, "spec", ] + else: + cmd_l = [self.executable, "spec", "--rootless", ] + status = subprocess.call(cmd_l, shell=False, stderr=Msg.chlderr, close_fds=True, cwd=os.path.realpath(self._container_specdir)) @@ -399,6 +411,8 @@ def run(self, container_id): self._add_devices() self._add_capabilities_spec() self._mod_mount_spec("shm", "/dev/shm", {"options": ["size=2g"]}) + if self.engine_type == "runsc": + self._del_namespace_spec("user") self._proot_overlay() self._save_spec() if Msg.level >= Msg.DBG: @@ -413,6 +427,9 @@ def run(self, container_id): cmd_l = self._set_cpu_affinity() cmd_l.append(self.executable) cmd_l.extend(runc_debug) + if self.engine_type == "runsc": + cmd_l.extend(["--network=host", "--ignore-cgroups"]) + cmd_l.extend(["--rootless", ]) cmd_l.extend(["--root", self._container_specdir, "run"]) cmd_l.extend(["--bundle", self._container_specdir, self.execution_id]) Msg().err("CMD =", cmd_l, l=Msg.VER) diff --git a/udocker/helper/keystore.py b/udocker/helper/keystore.py index 98118c90..1466cc55 100644 --- a/udocker/helper/keystore.py +++ b/udocker/helper/keystore.py @@ -3,6 +3,7 @@ import os import json +import re from udocker.helper.hostinfo import HostInfo from udocker.utils.fileutil import FileUtil @@ -75,6 +76,12 @@ def get(self, url): return self.credential["auth"] except KeyError: pass + try: + url = re.sub(r'https?://', '', url) + self.credential = auths[url] + return self.credential["auth"] + except KeyError: + pass return "" def put(self, url, credential, email):