Merge pull request #398 from SpheMakh/udocker

Udocker
ratt-ru · Jun 18, 2019 · 1eda051 · 1eda051
2 parents 48e449c + 049857c
commit 1eda051
Show file tree

Hide file tree

Showing 17 changed files with 473 additions and 74 deletions.
diff --git a/.travis/mypy.docker b/.travis/mypy.docker
@@ -1,7 +1,8 @@
 FROM kernsuite/base:4
-RUN docker-apt-install python3-pip
-RUN pip3 install mypy
+RUN docker-apt-install python3-pip git
+RUN pip3 install -U mypy pip setuptools
 ENV USER root
 ADD . /code
 WORKDIR /code
+RUN pip install /code
 RUN mypy --ignore-missing-import stimela
diff --git a/.travis/py2.docker b/.travis/py2.docker
@@ -1,7 +1,8 @@
 FROM kernsuite/base:4
-RUN docker-apt-install python-pip
+RUN docker-apt-install python-pip git
 ADD . /code
 ENV USER root
 WORKDIR /code
+RUN pip install pip setuptools -U
 RUN pip install /code
 RUN python -c "import stimela"
diff --git a/.travis/py3.docker b/.travis/py3.docker
@@ -1,7 +1,8 @@
 FROM kernsuite/base:4
-RUN docker-apt-install python3-pip
+RUN docker-apt-install python3-pip git
 ENV USER root
 ADD . /code
 WORKDIR /code
+RUN pip3 install pip setuptools -U
 RUN pip3 install .
 RUN python3 -c "import stimela"
diff --git a/examples/simulation_pipeline.py b/examples/simulation_pipeline.py
@@ -1,12 +1,17 @@
 # import stimela package
 import stimela
+import os
 
 # Recipe I/O configuration
 INPUT = "input" # This folder must exist
 OUTPUT = "output"
 MSDIR = "msdir"
 PREFIX = "stimela-example"  # Prefix for output images
-SINGULARTITY_IMAGE_DIR = "/home/sphe/work/github/Stimela/examples/IMAGES/"
+try:
+    SINGULARTITY_IMAGE_DIR = os.environ["STIMELA_SINGULARTITY_IMAGE_DIR"]
+except KeyError:
+    pass
+
 
 # MS name
 MS = "meerkat_simulation_example.ms"
@@ -21,6 +26,7 @@
 #                  singularity_image_dir=SINGULARTITY_IMAGE_DIR,
                   )
 
+pipeline.JOB_TYPE = "udocker"
 # 1: Make empty MS 
 pipeline.add("cab/simms",                   # Executor image to start container from 
              "simms_example",               # Container name

diff --git a/setup.py b/setup.py
@@ -1,11 +1,22 @@
 #!/usr/bin/env python
 
 import os
+import sys
 from setuptools import setup
 
 
+requirements = ["pyyaml", 
+                "nose>=1.3.7",
+                "future",
+               ],
+
+if sys.version_info <= (3, 0):
+    requirements += [
+                "udocker @ git+https://github.com/indigo-dc/udocker@master#egg=udocker=1.1.2",
+               ],
+
 PACKAGE_NAME = "stimela"
-__version__ = "1.0.6"
+__version__ = "1.0.7"
 
 setup(name = PACKAGE_NAME,
     version = __version__,
@@ -25,9 +36,7 @@
                                    "cab/*/src/tdlconf.profiles",
                                    "cab/singularity_run",
                                    ]},
-    install_requires = ["pyyaml", 
-                        "nose>=1.3.7",
-                        "future"],
+    install_requires = requirements,
     scripts = ["bin/" + i for i in os.listdir("bin")],
     classifiers = [],
      )
diff --git a/stimela/__init__.py b/stimela/__init__.py
@@ -13,7 +13,7 @@
 from stimela.cargo import cab
 from stimela.recipe import Recipe as Pipeline
 from stimela.recipe import Recipe, PipelineException
-from stimela import docker, singularity
+from stimela import docker, singularity, udocker
 import pkg_resources
 import warnings
 
@@ -317,10 +317,25 @@ def pull(argv):
             help="Tag")
 
     add("-s", "--singularity", action="store_true",
-            help="Use singularity instead of docker."
+            help="Pull base images using singularity."
                  "Images will be pulled into the directory specified by the enviroment varaible, SINGULARITY_PULLFOLDER. $PWD by default")
 
+    add("-d", "--docker", action="store_true",
+            help="Pull base images using docker.")
+
+    add("-pf", "--pull-folder", help="Images will be placed in this folder. Else, if the environmnental variable 'SINGULARITY_PULLFOLDER' is set, then images will be placed there. "
+                                    "Else, images will be placed in the current directory")
+
     args = parser.parse_args(argv)
+
+    if args.pull_folder:
+        pull_folder = args.pull_folder
+    else:
+        try:
+            pull_folder = os.environ["SINGULARITY_PULLFOLDER"]
+        except KeyError:
+            pull_folder = "."
+
     log = logger.StimelaLogger(LOG_FILE)
     images = log.read()['images']
 
@@ -330,10 +345,13 @@ def pull(argv):
             simage = image.replace("/", "_")
             simage = simage.replace(":", "_") + ".img"
             if args.singularity:
-                singularity.pull(image, simage)
-            else:
+                singularity.pull(image, simage, directory=pull_folder)
+            elif args.docker:
                 docker.pull(image)
                 log.log_image(image, 'pulled')
+            else:
+                udocker.pull(image)
+                log.log_image(image, 'pulled')
     else:
 
         base = []
@@ -344,14 +362,16 @@ def pull(argv):
         base = set(base)
 
         for image in base:
-            if image not in ["stimela/ddfacet", "radioastro/ddfacet"]:
-                if args.singularity:
-                    simage = image.replace("/", "_")
-                    simage = simage.replace(":", "_") + ".img"
-                    singularity.pull(image, simage)
-                else:
-                    docker.pull(image)
-                    log.log_image(image, 'pulled')
+            if args.singularity:
+                simage = image.replace("/", "_")
+                simage = simage.replace(":", "_") + ".img"
+                singularity.pull(image, simage, directory=pull_folder)
+            elif args.docker:
+                docker.pull(image)
+                log.log_image(image, 'pulled')
+            else:
+               udocker.pull(image)
+               log.log_image(image, 'pulled')
 
     log.write()
 

diff --git a/stimela/cargo/cab/aimfast/parameters.json b/stimela/cargo/cab/aimfast/parameters.json
@@ -6,6 +6,7 @@
     "prefix": "--", 
     "binary": "aimfast", 
     "msdir": false, 
+    "use_graphics": true,
     "parameters": [
         {
             "info": "Residual image to extract the four (4) statistical moments of distribution", 

diff --git a/stimela/cargo/cab/casa_plotants/parameters.json b/stimela/cargo/cab/casa_plotants/parameters.json
@@ -6,6 +6,7 @@
     "prefix": "", 
     "binary": "plotants", 
     "msdir": true, 
+    "use_graphics": true,
     "parameters": [
         {
             "info": "Name of measurement set", 
@@ -23,4 +24,4 @@
             "dtype": "file"
         }
     ]
-}
+}
diff --git a/stimela/cargo/cab/casa_plotcal/parameters.json b/stimela/cargo/cab/casa_plotcal/parameters.json
@@ -6,6 +6,7 @@
     "prefix": "-", 
     "binary": "plotcal", 
     "msdir": false, 
+    "use_graphics": true,
     "parameters": [
         {
             "info": "Name of input calibration table", 
@@ -173,4 +174,4 @@
             "name": "showgui"
         }
     ]
-}
+}
diff --git a/stimela/cargo/cab/casa_plotms/parameters.json b/stimela/cargo/cab/casa_plotms/parameters.json
@@ -6,6 +6,7 @@
     "prefix": " ", 
     "binary": "plotms", 
     "msdir": true, 
+    "use_graphics": true,
     "parameters": [
         {
             "info": "MS name", 
@@ -517,4 +518,4 @@
             "name": "showgui"
         }
     ]
-}
+}
diff --git a/stimela/cargo/cab/ragavi/parameters.json b/stimela/cargo/cab/ragavi/parameters.json
@@ -6,12 +6,13 @@
     "prefix": "--", 
     "binary": "ragavi", 
     "msdir": false, 
+    "use_graphics": true,
     "parameters": [
         {
             "info": "Plot only this antenna, or comma-separated list of antennas. Default plots all", 
             "name": "ant",
             "dtype": "str", 
-            "required": false,
+            "required": false
         },
         {
             "info": "Correlation index to plot (usually just 0 or 1)", 
@@ -42,13 +43,13 @@
             "info": "Field ID to plot", 
             "name": "field", 
             "dtype": ["list:str","list:int"],
-            "required": true, 
+            "required": true 
         },
         {
             "info": "The gain type of table(s) to be plotted. Options: ['B', 'F', 'G', 'K']", 
             "name": "gaintype",
             "dtype": "list:str", 
-            "required": true,
+            "required": true
         },
         {
             "info": "Output html file name", 
@@ -69,7 +70,7 @@
             "name": "table",
             "dtype": "list:file", 
             "required": true,              
-            "io": "input",
+            "io": "input"
         }, 
         {
             "info": "Minimum time to plot (default = full range)", 

diff --git a/stimela/cargo/cab/rfinder/parameters.json b/stimela/cargo/cab/rfinder/parameters.json
@@ -6,6 +6,7 @@
     "prefix": "--",
     "binary": "rfinder",
     "msdir": true,
+    "use_graphics" : true,
     "parameters": [
         {
             "info": "Name of telescope",

diff --git a/stimela/cargo/cab/sunblocker/parameters.json b/stimela/cargo/cab/sunblocker/parameters.json
@@ -6,6 +6,7 @@
     "prefix": " ", 
     "binary": "msutils", 
     "msdir": true, 
+    "use_graphics":true,
     "parameters": [
         {
             "info": "MSUtils command to execute",