diff --git a/examples/simulation_pipeline.py b/examples/simulation_pipeline.py index ea1c57f1..7b8b7bef 100644 --- a/examples/simulation_pipeline.py +++ b/examples/simulation_pipeline.py @@ -25,7 +25,7 @@ log_dir=os.path.join(OUTPUT, "logs"), ) -pipeline.JOB_TYPE = "docker" +pipeline.JOB_TYPE = "singularity" # 1: Make empty MS pipeline.add("cab/simms", # Executor image to start container from diff --git a/setup.py b/setup.py index 23d534ba..1a2b7a21 100644 --- a/setup.py +++ b/setup.py @@ -9,11 +9,10 @@ requirements = ["pyyaml", "nose>=1.3.7", "future-fstrings", - "udocker", ], PACKAGE_NAME = "stimela" -__version__ = "1.4.6" +__version__ = "1.5.1" setup(name=PACKAGE_NAME, version=__version__, @@ -25,13 +24,11 @@ "stimela/utils", "stimela/cargo/cab", "stimela/cargo/base"], package_data={"stimela/cargo": [ - "cab/*/Dockerfile", "base/*/Dockerfile", + "base/*.template", "cab/*/src/*.py", - "cab/*/src/*.sh", "cab/*/src/*.json", "base/*/xvfb.init.d", - "cab/*/xvfb.init.d", "cab/*/parameters.json", "cab/*/src/tdlconf.profiles", ]}, diff --git a/stimela/cargo/base/cubical/Dockerfile b/stimela/cargo/base/cubical/Dockerfile index 3b06c7fa..49593d40 100644 --- a/stimela/cargo/base/cubical/Dockerfile +++ b/stimela/cargo/base/cubical/Dockerfile @@ -1,13 +1,13 @@ -FROM stimela/ddfacet:1.3.3 -MAINTAINER -ENV GIT_LFS_SKIP_SMUDGE 1 -RUN pip3 install -U pip setuptools wheel -RUN apt-get update && apt-get install -y wget git-all -RUN git clone https://github.com/ratt-ru/CubiCal.git -WORKDIR CubiCal -RUN git checkout v1.5.1 -RUN pip3 install ".[lsm-support,degridder-support]" -RUN DDF.py --help +FROM stimela/base:1.2.5 +RUN docker-apt-install casacore-dev \ + casacore-data \ + build-essential \ + python3-pip \ + libboost-all-dev \ + wcslib-dev \ + git \ + libcfitsio-dev +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.6 10 +RUN pip3 install --upgrade --force pip +RUN pip install "cubical[lsm-support]@git+https://github.com/ratt-ru/CubiCal.git@v1.5.2" RUN gocubical --help -ENV NUMBA_CACHE_DIR /tmp -ENTRYPOINT [] diff --git a/stimela/cargo/base/cubical_ddf/Dockerfile b/stimela/cargo/base/cubical_ddf/Dockerfile new file mode 100644 index 00000000..6100cc5b --- /dev/null +++ b/stimela/cargo/base/cubical_ddf/Dockerfile @@ -0,0 +1,13 @@ +FROM stimela/ddfacet:1.3.3 +MAINTAINER +ENV GIT_LFS_SKIP_SMUDGE 1 +RUN pip3 install -U pip setuptools wheel +RUN apt-get update && apt-get install -y wget git-all +RUN git clone -b decasacore300fication https://github.com/ratt-ru/CubiCal +WORKDIR CubiCal +#RUN git checkout decasacore300fication +RUN pip3 install -I ".[lsm-support,degridder-support]" +RUN DDF.py --help +RUN gocubical --help +ENV NUMBA_CACHE_DIR /tmp +ENTRYPOINT [] diff --git a/stimela/cargo/base/group.template b/stimela/cargo/base/group.template new file mode 100644 index 00000000..135ac16a --- /dev/null +++ b/stimela/cargo/base/group.template @@ -0,0 +1,42 @@ +root:x:0: +daemon:x:1: +bin:x:2: +sys:x:3: +adm:x:4: +tty:x:5: +disk:x:6: +lp:x:7: +mail:x:8: +news:x:9: +uucp:x:10: +man:x:12: +proxy:x:13: +kmem:x:15: +dialout:x:20: +fax:x:21: +voice:x:22: +cdrom:x:24: +floppy:x:25: +tape:x:26: +sudo:x:27: +audio:x:29: +dip:x:30: +www-data:x:33: +backup:x:34: +operator:x:37: +list:x:38: +irc:x:39: +src:x:40: +gnats:x:41: +shadow:x:42: +utmp:x:43: +video:x:44: +sasl:x:45: +plugdev:x:46: +staff:x:50: +games:x:60: +users:x:100: +nogroup:x:65534: +crontab:x:101: +messagebus:x:102: +ssh:x:103: diff --git a/stimela/cargo/base/passwd.template b/stimela/cargo/base/passwd.template new file mode 100644 index 00000000..c9043069 --- /dev/null +++ b/stimela/cargo/base/passwd.template @@ -0,0 +1,20 @@ +root:x:0:0:root:/root:/bin/bash +daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin +bin:x:2:2:bin:/bin:/usr/sbin/nologin +sys:x:3:3:sys:/dev:/usr/sbin/nologin +sync:x:4:65534:sync:/bin:/bin/sync +games:x:5:60:games:/usr/games:/usr/sbin/nologin +man:x:6:12:man:/var/cache/man:/usr/sbin/nologin +lp:x:7:7:lp:/var/spool/lpd:/usr/sbin/nologin +mail:x:8:8:mail:/var/mail:/usr/sbin/nologin +news:x:9:9:news:/var/spool/news:/usr/sbin/nologin +uucp:x:10:10:uucp:/var/spool/uucp:/usr/sbin/nologin +proxy:x:13:13:proxy:/bin:/usr/sbin/nologin +www-data:x:33:33:www-data:/var/www:/usr/sbin/nologin +backup:x:34:34:backup:/var/backups:/usr/sbin/nologin +list:x:38:38:Mailing List Manager:/var/list:/usr/sbin/nologin +irc:x:39:39:ircd:/var/run/ircd:/usr/sbin/nologin +gnats:x:41:41:Gnats Bug-Reporting System (admin):/var/lib/gnats:/usr/sbin/nologin +nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin +_apt:x:100:65534::/nonexistent:/usr/sbin/nologin +messagebus:x:101:102::/nonexistent:/usr/sbin/nologin diff --git a/stimela/cargo/base/ragavi/Dockerfile b/stimela/cargo/base/ragavi/Dockerfile index c9fe505b..50525cbe 100644 --- a/stimela/cargo/base/ragavi/Dockerfile +++ b/stimela/cargo/base/ragavi/Dockerfile @@ -2,12 +2,8 @@ FROM stimela/base:1.2.5 RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.6 10 RUN pip3 install -U pip setuptools pyyaml -RUN pip3 install -U python-casacore git+https://github.com/ska-sa/dask-ms@remove-table-proxy-weakref-finalize -#dask-ms==0.2.3 -# RUN pip3 install -U ragavi -RUN pip3 install git+https://github.com/ratt-ru/ragavi@dev -# RUN pip install --force-reinstall -U git+https://github.com/o-smirnov/datashader +RUN pip3 install -U python-casacore +RUN pip3 install -U ragavi RUN pip3 install -I numpy RUN ragavi-gains -h RUN ragavi-vis -h -#RUN python3 -c "import yaml" diff --git a/stimela/cargo/cab/cubical/parameters.json b/stimela/cargo/cab/cubical/parameters.json index afc000f3..be411f76 100644 --- a/stimela/cargo/cab/cubical/parameters.json +++ b/stimela/cargo/cab/cubical/parameters.json @@ -1,10 +1,10 @@ { "task": "cubical", "base": "stimela/cubical", - "tag": "1.4.2", + "tag": "1.5.00", "description": "CubiCal is a suite of fast radio interferometric calibration routines exploiting complex optimisation.", "prefix": "--", - "version":"1.5.1", + "version":"1.5.2", "binary": "gocubical", "junk":["cubical.last", "montblanc.log"], "msdir": true, diff --git a/stimela/cargo/cab/cubical_ddf/parameters.json b/stimela/cargo/cab/cubical_ddf/parameters.json new file mode 100644 index 00000000..009df1df --- /dev/null +++ b/stimela/cargo/cab/cubical_ddf/parameters.json @@ -0,0 +1,2099 @@ +{ + "task": "cubical_ddf", + "base": "stimela/cubical", + "tag": "1.4.2", + "description": "CubiCal is a suite of fast radio interferometric calibration routines exploiting complex optimisation.", + "prefix": "--", + "version":"1.5.1", + "binary": "gocubical", + "junk":["cubical.last", "montblanc.log"], + "msdir": true, + "parameters": [ + { + "info": "Name of measurement set (MS)", + "dtype": "file", + "default": null, + "name": "data-ms", + "io": "msfile" + }, + { + "info": "Name of MS column to read for data.", + "dtype": "str", + "default": null, + "name": "data-column" + }, + { + "info": "Chunk data up by this number of timeslots. This limits the amount of data\nprocessed at once. Smaller chunks allow for a smaller RAM footprint and\ngreater parallelism, but this sets an upper limit on the solution intervals\nthat may be employed. 0 means use full time axis.", + "dtype": "int", + "default": null, + "name": "data-time-chunk" + }, + { + "info": "Chunk data by this number of channels. See time-chunk for info.\n0 means full frequency axis.", + "dtype": "int", + "default": null, + "name": "data-freq-chunk" + }, + { + "info": "If set, then time chunks will be broken up whenever the value in the named\ncolumn(s) jumps by >JUMPSIZE. Multiple column names may be given, separated\nby commas. Use None to disable.", + "dtype": "str", + "default": null, + "name": "data-chunk-by" + }, + { + "info": "The jump size used in conjunction with chunk-by. If 0, then any change in\nvalue is a jump. If n, then the change must be >n.", + "dtype": "float", + "default": null, + "name": "data-chunk-by-jump" + }, + { + "info": "If set, processes just one chunk of data matching the chunk ID. Useful for\ndebugging.", + "dtype": "str", + "default": null, + "name": "data-single-chunk" + }, + { + "info": "FIELD_ID to read from the MS.", + "dtype": "int", + "default": null, + "name": "sel-field" + }, + { + "info": "DATA_DESC_IDs to read from the MS. Default reads all. Can be specified as\ne.g. \"5\", \"5,6,7\", \"5~7\" (inclusive range), \"5:8\" (exclusive range),\n\"5:\" (from 5 to last).", + "dtype": "str", + "default": null, + "name": "sel-ddid" + }, + { + "info": "Additional TaQL selection string. Combined with other selection options.", + "dtype": "str", + "default": null, + "name": "sel-taql" + }, + { + "info": "Channels to read (within each DDID). Default reads all. Can be specified as\ne.g. \"5\", \"10~20\" (10 to 20 inclusive), \"10:21\" (same), \"10:\" (from 10 to\nend), \":10\" (0 to 9 inclusive), \"~9\" (same).", + "dtype": "str", + "default": null, + "name": "sel-chan" + }, + { + "info": "Predict model visibilities from given LSM (using Montblanc). Expects pattern of the style source(+-?source)*(:source(+-?source)*)* where source at present can be a column name, lsm or DDFacet DicoModel. Colons split directions as specifiable as 0-indexed indicies through the gain directions and fix directions switches. If an lsm file is spcified a tagged selection of sources can be picked by appending a @[user tag] option. If a DicoModel is used regions can be extracted from the model components through a @[DS9 polygon regions file].", + "default": null, + "check_io": false, + "name": "model-list", + "dtype": "file", + "io": "output" + }, + { + "info": "Enable direction-dependent models. If 'auto', this is determined\nby --sol-jones and --model-list, otherwise, enable/disable\nexplicitly.", + "dtype": [ + "str", + "bool" + ], + "default": "auto", + "name": "model-ddes" + }, + { + "info": "Apply beams if specified eg. 'beam_$(corr)_$(reim).fits' or\n'beam_$(CORR)_$(REIM).fits'", + "check_io": false, + "name": "model-beam-pattern", + "io": "input", + "default": null, + "dtype": "file" + }, + { + "info": "Beam l axis", + "dtype": "str", + "default": null, + "name": "model-beam-l-axis" + }, + { + "info": "Beam m axis", + "dtype": "str", + "default": null, + "name": "model-beam-m-axis" + }, + { + "info": "Use CPU or GPU for simulation.", + "dtype": "str", + "default": null, + "name": "montblanc-device-type" + }, + { + "info": "Precision for simulation.", + "dtype": "str", + "default": null, + "name": "montblanc-dtype" + }, + { + "info": "Simulate using linear or circular feeds.", + "dtype": "str", + "default": null, + "name": "montblanc-feed-type" + }, + { + "info": "Memory budget in MB for simulation.", + "dtype": "int", + "default": null, + "name": "montblanc-mem-budget" + }, + { + "info": "verbosity level of Montblanc's console output", + "dtype": "str", + "default": null, + "name": "montblanc-verbosity" + }, + { + "info": "Column to read weights from. Weights are applied by default. Specify an\nempty string to disable.", + "dtype": "str", + "default": null, + "name": "weight-column" + }, + { + "info": "Which flagsets will be applied prior to calibration. \nUse \"-FLAGSET\" to apply everything except the named flagset (\"-cubical\" is\nuseful, to ignore the flags of a previous CubiCal run).", + "dtype": "str", + "default": null, + "name": "flags-apply" + }, + { + "info": "Insert BITFLAG column if it is missing, and initialize a named flagset\nfrom FLAG/FLAG_ROW.", + "dtype": "str", + "default": null, + "name": "flags-auto-init" + }, + { + "info": "Save flags to named flagset in BITFLAG. If none or 0, will not save.", + "dtype": "str", + "default": null, + "name": "flags-save" + }, + { + "info": "If true, reninitializes BITFLAG column from scratch. Useful if you ended up\nwith a dead one.", + "dtype": "bool", + "default": null, + "name": "flags-reinit-bitflags" + }, + { + "info": "If True, will do an extra round of flagging at the end (post-solution)\nbased on solutions statistics, as per the following options.", + "dtype": "bool", + "default": null, + "name": "flags-post-sol" + }, + { + "info": "Intervals with chi-squared values larger than this value times the median\nwill be flagged.", + "dtype": "str", + "default": null, + "name": "flags-tf-chisq-median" + }, + { + "info": "Minimum percentage of unflagged visibilities per time/frequncy slot\nrequired to prevent flagging.", + "dtype": "str", + "default": null, + "name": "flags-tf-np-median" + }, + { + "info": "Minimum percentage of unflagged visibilities along the time axis required\nto prevent flagging.", + "dtype": "str", + "default": null, + "name": "flags-time-density" + }, + { + "info": "Minimum percentage of unflagged visibilities along the frequency axis\nrequired to prevent flagging.", + "dtype": "str", + "default": null, + "name": "flags-chan-density" + }, + { + "info": "Minimum percentage of unflagged visibilities along the DDID axis\nrequired to prevent flagging.", + "dtype": "str", + "default": null, + "name": "flags-ddid-density" + }, + { + "info": "Enable Mad Max flagging in the solver. This computes the median absolute residual (i.e. median absolute deviation from zero), and flags visibilities exceeding the thresholds", + "dtype": "bool", + "name": "madmax-enable" + }, + { + "info": "MAD estimation mode. Use 'corr' for a separate estimate per each baseline and correlation. Otherwise, a single estimate per baseline is computed using 'all' correlations, or only the 'diag' or 'offdiag' correlations.", + "dtype": "str", + "name": "madmax-estimate", + "choices": [ + "corr", + "all", + "diag", + "offdiag" + ] + }, + { + "info": "Enable plots for Mad Max flagging. Use 'show' to show figures interactively. Plots will show the worst flagged baseline, and a median flagged baseline, provided the fraction of flagged visibilities is above --flags-mad-plot-thr.", + "dtype": "bool", + "name": "madmax-plot" + }, + { + "info": "Threshold (in terms of fraction of visibilities flagged) above which plots will be generated.", + "dtype": "float", + "name": "madmax-plot-frac-above" + }, + { + "info": "Flag on on-diagonal (parallel-hand) residuals", + "dtype": "bool", + "name": "madmax-diag" + }, + { + "info": "Flag on off-diagonal (cross-hand) residuals", + "dtype": "bool", + "name": "madmax-offdiag" + }, + { + "info": "Threshold for global median MAD (MMAD) flagging. MMAD is computed as the median of the per-baseline MADs. Residuals exceeding S*MMAD/1.428 will be flagged. Can be specified", + "dtype": "list:float", + "name": "madmax-global-threshold" + }, + { + "info": "Threshold for MAD flagging per baseline (specified in sigmas). Residuals exceeding mad-thr*MAD/1.428 will be flagged. MAD is computed per baseline. This can be specified as a list e.g. N1,N2,N3,... The first value is used to flag residuals before a solution starts (use 0 to disable), the next value is used when the residuals are first recomputed during the solution several iteratins later (see -chi-int), etc. A final pass may be done at the end of the solution. The last value in the list is reused if necessary. Using a list with gradually decreasing values may be sensible. #metavar:SIGMAS", + "dtype": "list:float", + "name": "madmax-threshold" + }, + { + "info": "Comma-separated list of Jones terms to enable, e.g. \"G,B,dE\"\n(default: G)", + "dtype": "str", + "default": null, + "name": "sol-jones" + }, + { + "info": "If true, then data, model and gains are taken to be diagonal. Off-diagonal terms in data and model are ignored. This option is then enforced on all Jones terms.", + "dtype": "bool", + "name": "sel-diag" + }, + { + "info": "if true, then data, model and gains are taken to be diagonal. off-diagonal terms in data and model are ignored. this option is then enforced on all jones terms.", + "dtype": "bool", + "name": "g-diag-only" + }, + { + "info": "If true, then data, model and gains are taken to be diagonal. Off-diagonal terms in data and model are ignored. This option is then enforced on all Jones terms.", + "dtype": "bool", + "name": "dd-diag-only" + }, + { + "info": "Solve in single or double precision", + "dtype": "str", + "default": null, + "name": "sol-precision" + }, + { + "info": "Theshold for gain accuracy - gains which improve by less than this value\nare considered converged.", + "dtype": "str", + "default": null, + "name": "sol-delta-g" + }, + { + "info": "Theshold for solution stagnancy - if the chi-squared is improving by less\nthan this value, the gain is considered stalled.", + "dtype": "str", + "default": null, + "name": "sol-delta-chi" + }, + { + "info": "Number of iterations to perform between chi-suqared checks. This is done to\navoid computing the expensive chi-squared test evey iteration.", + "dtype": "str", + "default": null, + "name": "sol-chi-int" + }, + { + "info": "Re-estimate chi-squred and noise at the end of a solution cycle. Disabling\nlast rites can save a bit of time, but makes the post-solution stats less\ninformative.", + "dtype": "bool", + "default": null, + "name": "sol-last-rites" + }, + { + "info": "Minimum percentage of solutions which must have stalled before terminating\nthe solver.", + "dtype": "float", + "default": null, + "name": "sol-stall-quorum" + }, + { + "info": "Number of iterations per Jones term. If empty, then each Jones\nterm is solved for once, up to convergence, or up to its -max-iter\nsetting.\nOtherwise, set to a list giving the number of iterations per Jones term.\nFor example, given two Jones terms and --sol-num-iter 10,20,10, it will\ndo 10 iterations on the first term, 20 on the second, and 10 again on the\nfirst.", + "dtype": [ + "list:int", + "list:str" + ], + "default": [50], + "name": "sol-term-iters" + }, + { + "info": "Min baseline length to solve for", + "dtype": "float", + "default": null, + "name": "sol-min-bl" + }, + { + "info": "Max baseline length to solve for. If 0, no maximum is applied.", + "dtype": "float", + "default": null, + "name": "sol-max-bl" + }, + { + "info": "Additional subset of data to actually solve for. Any TaQL string may be\nused.", + "dtype": "str", + "default": null, + "name": "sol-subset" + }, + { + "info": "Load and apply BBCs computed in a previous run. Apply with care! This will\ntend to suppress all unmodelled flux towards the centre of the field.", + "dtype": "file", + "default": null, + "name": "bbc-load-from", + "io": "input" + }, + { + "info": "Compute full 2x2 BBCs (as opposed to diagonal-only). Only useful if you\nreally trust the polarisation information in your sky model.", + "dtype": "bool", + "default": null, + "name": "bbc-compute-2x2" + }, + { + "info": "Apply full 2x2 BBCs (as opposed to diagonal-only). Only enable this if you\nreally trust the polarisation information in your sky model.", + "dtype": "bool", + "default": null, + "name": "bbc-apply-2x2" + }, + { + "info": "Compute suggested BBCs at end of run,\nand save them to the given database. It can be useful to have this always\nenabled, since the BBCs provide useful diagnostics of the solution quality\n(and are not actually applied without a load-from setting).", + "dtype": "file", + "default": "{out[name]}-BBC-field_{sel[field]}-ddid_{sel[ddid]}.parmdb", + "name": "bbc-save-to", + "io": "output" + }, + { + "info": "Compute BBCs per-channel (else across entire band).", + "dtype": "bool", + "default": null, + "name": "bbc-per-chan" + }, + { + "info": "Generate output BBC plots.", + "dtype": "bool", + "default": null, + "name": "bbc-plot" + }, + { + "info": "Number of CPUs (processes) to use (0 or 1 disables parallelism).", + "dtype": "int", + "default": null, + "name": "dist-ncpu" + }, + { + "info": "Maximum number of time/freq data-chunks to load into memory simultaneously. If 0, then as many as possible will be loaded.", + "dtype": "int", + "default": 0, + "name": "dist-max-chunks" + }, + { + "info": "Number of OMP threads to use. 0: determine automatically.", + "dtype": "int", + "name": "dist-nthread" + }, + { + "info": "If empty or None, processes will not be pinned to cores. Otherwise, set to the starting core number, or 'N:K' to start with N and step by K", + "dtype": "int", + "name": "dist-pin" + }, + { + "info": "If not 0, pins the I/O & Montblanc process to a separate core, or cores if --montblanc-threads is specified). Ignored if --dist-pin is not set", + "dtype": "bool", + "name": "dist-pin-io" + }, + { + "info": "If set, pins the main process to a separate core. If set to, pins it to the same core as the I/O process, if I/O process is pinned. Ignored if --dist-pin is not set", + "dtype": [ + "bool", + "str" + ], + "name": "pin-main" + }, + { + "info": "Minimum number of time/freq data-chunks to load into memory\nsimultaneously. This number should be divisible by ncpu-1 for optimal\nperformance.", + "dtype": "str", + "default": null, + "name": "dist-min-chunks" + }, + { + "info": "Base name of output files.", + "dtype": "file", + "default": null, + "name": "out-name", + "io": "output" + }, + { + "info": "Allow overwriting of existing output files. If this is set, and the output parset file exists, will raise an exception", + "dtype": "bool", + "default": true, + "name": "out-overwrite" + }, + { + "info": "Operational mode.\n[so] solve only;\n[sc] solve and generate corrected visibilities;\n[sr] solve and generate corrected residuals;\n[ss] solve and generate uncorrected residuals;\n[ac] apply solutions, generate corrected visibilities;\n[ar] apply solutions, generate corrected residuals;\n[as] apply solutions, generate uncorrected residuals;", + "dtype": "str", + "default": null, + "name": "out-mode", + "choices": [ + "so", + "sc", + "sr", + "ss", + "ac", + "ar", + "as" + ] + }, + { + "info": "Output MS column name (if applicable).", + "dtype": "str", + "default": null, + "name": "out-column" + }, + { + "info": "If set, model visibilities will be written to the specified column.", + "dtype": "str", + "default": null, + "name": "out-model-column" + }, + { + "info": "Reinitialize output MS column. Useful if the column is in a half-filled\nor corrupt state.", + "dtype": "bool", + "default": null, + "name": "out-reinit-column" + }, + { + "info": "Which model to subtract, if generating residuals.", + "dtype": "int", + "default": null, + "name": "out-subtract-model" + }, + { + "info": "Which model directions to subtract, if generating residuals. \":\"\nsubtracts all. Can also be specified as \"N\", \"N:M\", \":N\", \"N:\", \"N,M,K\".", + "dtype": [ + "str", + "int" + ], + "default": null, + "name": "out-subtract-dirs" + }, + { + "info": "Export gaintables to CASA caltable format. Tables are exported to same directory as set for cubical databases", + "dtype": "bool", + "default": false, + "name": "out-casa-gaintables" + }, + { + "info": "Generate summary plots.", + "dtype": "bool", + "default": true, + "name": "out-plots" + }, + { + "info": "Number of processes", + "dtype": "int", + "default": 0, + "name": "dist-nworker" + }, + { + "info": "Log memory usage.", + "dtype": "bool", + "default": null, + "name": "log-memory" + }, + { + "info": "Disable progress bars and some console output.", + "dtype": "bool", + "default": null, + "name": "log-boring" + }, + { + "info": "Append to log file if it exists.", + "dtype": "bool", + "default": null, + "name": "log-append" + }, + { + "info": "Default console output verbosity level. \nCan either be a single number, or a sequence of \"name=level,name=level,...\"\nassignments.", + "dtype": "str", + "default": null, + "name": "log-verbose" + }, + { + "info": "Default logfile output verbosity level. \nCan either be a single number, or a sequence of \"name=level,name=level,...\"\nassignments. If None, then this simply follows the console level.", + "dtype": "str", + "default": null, + "name": "log-file-verbose" + }, + { + "info": "Jumps into pdb on error.", + "dtype": "bool", + "default": null, + "name": "debug-pdb" + }, + { + "info": "Throw an error if a visibility amplitude in the results exceeds the given value.\nUseful for troubleshooting.", + "dtype": "float", + "default": null, + "name": "debug-panic-amplitude" + }, + { + "info": "Invoke pdb before entering the solver.", + "dtype": "bool", + "default": null, + "name": "debug-stop-before-solver" + }, + { + "info": "Seed random number generator with explicit seed. Useful for reproducibility\nof the random-based optimizations (sparsification, etc.).", + "dtype": "str", + "default": null, + "name": "misc-random-seed" + }, + { + "info": "Parset version number, for migration purposes. Can't be specified on command\nline.", + "dtype": "str", + "default": null, + "name": "misc-parset-version" + }, + { + "info": "Determines whether this term is direction dependent. --model-ddes must", + "dtype": "bool", + "default": null, + "name": "g-dd-term" + }, + { + "info": "Time solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "float", + "int" + ], + "default": null, + "name": "g-time-int" + }, + { + "info": "Frequency solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "g-freq-int" + }, + { + "info": "Amplitude clipping - flag solutions with diagonal amplitudes below this\nvalue.", + "dtype": "float", + "default": null, + "name": "g-clip-low" + }, + { + "info": "Amplitude clipping - flag solutions with any amplitudes above this value.", + "dtype": "float", + "default": null, + "name": "g-clip-high" + }, + { + "info": "Number of iterations after which to clip this gain.", + "dtype": "int", + "default": null, + "name": "g-clip-after" + }, + { + "info": "Minimum percentage of converged solutions to accept.", + "dtype": "float", + "default": null, + "name": "g-conv-quorum" + }, + { + "info": "Reference antenna - its phase is guaranteed to be zero.", + "dtype": "str", + "default": null, + "name": "g-ref-ant" + }, + { + "info": "Set to 0 (and specify -load-from or -xfer-from) to load a non-solvable\nterm is loaded from disk. Not to be confused with --sol-jones, which\ndetermines the active Jones terms.", + "dtype": "bool", + "default": null, + "name": "g-solvable" + }, + { + "info": "Type of Jones matrix to solve for. Note that if multiple Jones terms are\nenabled, then only complex-2x2 is supported.", + "dtype": "str", + "default": null, + "name": "g-type" + }, + { + "info": "Load solutions from given database. The DB must define solutions\non the same time/frequency grid (i.e. should normally come from\ncalibrating the same pointing/observation). By default, the Jones\nmatrix label is used to form up parameter names, but his may be\noverridden by adding an explicit \"//LABEL\" to the database filename.", + "dtype": "file", + "default": null, + "name": "g-load-from", + "io": "input" + }, + { + "info": "Transfer solutions from given database. Similar to -load-from, but\nsolutions will be interpolated onto the required time/frequency grid,\nso they can originate from a different field (e.g. from a calibrator).", + "dtype": "file", + "default": null, + "name": "g-xfer-from", + "io": "input" + }, + { + "info": "Save solutions to given database.", + "dtype": "file", + "default": null, + "name": "g-save-to", + "io": "output" + }, + { + "info": "For DD terms, makes the listed directions non-solvable.", + "dtype": [ + "str", + "int" + ], + "default": null, + "name": "g-fix-dirs" + }, + { + "info": "Determines update type. This does not change the Jones solver type, but\nrestricts the update rule to pin the solutions within a certain subspace:\n'full' is the default behaviour;\n'diag' pins the off-diagonal terms to 0;\n'phase-diag' also pins the amplitudes of the diagonal terms to unity;\n'amp-diag' also pins the phases to 0.", + "dtype": "str", + "default": null, + "name": "g-update-type" + }, + { + "info": "Flag solution intervals where the prior error estimate is above this value.", + "dtype": "float", + "default": null, + "name": "g-max-prior-error" + }, + { + "info": "Flag solution intervals where the posterior variance estimate is above this value.", + "dtype": "float", + "default": null, + "name": "g-max-post-error" + }, + { + "info": "Maximum number of iterations spent on this term.", + "dtype": "int", + "default": null, + "name": "g-max-iter" + }, + { + "info": "Flag propagation policy. Determines how flags raised on gains propagate back\ninto the data. Options are 'never' to never propagate, 'always' to always\npropagate, 'default' to only propagate flags from direction-independent gains.", + "dtype": "str", + "default": null, + "name": "g-prop-flags" + }, + { + "info": "Determines whether this term is direction dependent. --model-ddes must", + "dtype": "bool", + "default": null, + "name": "g1-dd-term" + }, + { + "info": "Time solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "g1-time-int" + }, + { + "info": "Frequency solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "g1-freq-int" + }, + { + "info": "Amplitude clipping - flag solutions with diagonal amplitudes below this\nvalue.", + "dtype": "float", + "default": null, + "name": "g1-clip-low" + }, + { + "info": "Amplitude clipping - flag solutions with any amplitudes above this value.", + "dtype": "float", + "default": null, + "name": "g1-clip-high" + }, + { + "info": "Number of iterations after which to clip this gain.", + "dtype": "int", + "default": null, + "name": "g1-clip-after" + }, + { + "info": "Minimum percentage of converged solutions to accept.", + "dtype": "float", + "default": null, + "name": "g1-conv-quorum" + }, + { + "info": "Reference antenna - its phase is guaranteed to be zero.", + "dtype": "str", + "default": null, + "name": "g1-ref-ant" + }, + { + "info": "Set to 0 (and specify -load-from or -xfer-from) to load a non-solvable\nterm is loaded from disk. Not to be confused with --sol-jones, which\ndetermines the active Jones terms.", + "dtype": "bool", + "default": null, + "name": "g1-solvable" + }, + { + "info": "Type of Jones matrix to solve for. Note that if multiple Jones terms are\nenabled, then only complex-2x2 is supported.", + "dtype": "str", + "default": null, + "name": "g1-type" + }, + { + "info": "Load solutions from given database. The DB must define solutions\non the same time/frequency grid (i.e. should normally come from\ncalibrating the same pointing/observation). By default, the Jones\nmatrix label is used to form up parameter names, but his may be\noverridden by adding an explicit \"//LABEL\" to the database filename.", + "dtype": "file", + "default": null, + "name": "g1-load-from", + "io": "input" + }, + { + "info": "Transfer solutions from given database. Similar to -load-from, but\nsolutions will be interpolated onto the required time/frequency grid,\nso they can originate from a different field (e.g. from a calibrator).", + "dtype": "file", + "default": null, + "name": "g1-xfer-from", + "io": "input" + }, + { + "info": "Save solutions to given database.", + "dtype": "file", + "default": null, + "name": "g1-save-to", + "io": "output" + }, + { + "info": "For DD terms, makes the listed directions non-solvable.", + "dtype": [ + "str", + "int" + ], + "default": null, + "name": "g1-fix-dirs" + }, + { + "info": "Determines update type. This does not change the Jones solver type, but\nrestricts the update rule to pin the solutions within a certain subspace:\n'full' is the default behaviour;\n'diag' pins the off-diagonal terms to 0;\n'phase-diag' also pins the amplitudes of the diagonal terms to unity;\n'amp-diag' also pins the phases to 0.", + "dtype": "str", + "default": null, + "name": "g1-update-type" + }, + { + "info": "Flag solution intervals where the prior error estimate is above this value.", + "dtype": "float", + "default": null, + "name": "g1-max-prior-error" + }, + { + "info": "Flag solution intervals where the posterior variance estimate is above this value.", + "dtype": "float", + "default": null, + "name": "g1-max-post-error" + }, + { + "info": "Maximum number of iterations spent on this term.", + "dtype": "int", + "default": null, + "name": "g1-max-iter" + }, + { + "info": "Flag propagation policy. Determines how flags raised on gains propagate back\ninto the data. Options are 'never' to never propagate, 'always' to always\npropagate, 'default' to only propagate flags from direction-independent gains.", + "dtype": "str", + "default": null, + "name": "g1-prop-flags" + }, + { + "info": "Determines whether this term is direction dependent. --model-ddes must", + "dtype": "bool", + "default": null, + "name": "g2-dd-term" + }, + { + "info": "Time solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "g2-time-int" + }, + { + "info": "Frequency solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "g2-freq-int" + }, + { + "info": "Amplitude clipping - flag solutions with diagonal amplitudes below this\nvalue.", + "dtype": "float", + "default": null, + "name": "g2-clip-low" + }, + { + "info": "Amplitude clipping - flag solutions with any amplitudes above this value.", + "dtype": "float", + "default": null, + "name": "g2-clip-high" + }, + { + "info": "Number of iterations after which to clip this gain.", + "dtype": "int", + "default": null, + "name": "g2-clip-after" + }, + { + "info": "Minimum percentage of converged solutions to accept.", + "dtype": "float", + "default": null, + "name": "g2-conv-quorum" + }, + { + "info": "Reference antenna - its phase is guaranteed to be zero.", + "dtype": "str", + "default": null, + "name": "g2-ref-ant" + }, + { + "info": "Set to 0 (and specify -load-from or -xfer-from) to load a non-solvable\nterm is loaded from disk. Not to be confused with --sol-jones, which\ndetermines the active Jones terms.", + "dtype": "bool", + "default": null, + "name": "g2-solvable" + }, + { + "info": "Type of Jones matrix to solve for. Note that if multiple Jones terms are\nenabled, then only complex-2x2 is supported.", + "dtype": "str", + "default": null, + "name": "g2-type" + }, + { + "info": "Load solutions from given database. The DB must define solutions\non the same time/frequency grid (i.e. should normally come from\ncalibrating the same pointing/observation). By default, the Jones\nmatrix label is used to form up parameter names, but his may be\noverridden by adding an explicit \"//LABEL\" to the database filename.", + "dtype": "file", + "default": null, + "name": "g2-load-from", + "io": "input" + }, + { + "info": "Transfer solutions from given database. Similar to -load-from, but\nsolutions will be interpolated onto the required time/frequency grid,\nso they can originate from a different field (e.g. from a calibrator).", + "dtype": "file", + "default": null, + "name": "g2-xfer-from", + "io": "input" + }, + { + "info": "Save solutions to given database.", + "dtype": "file", + "default": null, + "name": "g2-save-to", + "io": "output" + }, + { + "info": "For DD terms, makes the listed directions non-solvable.", + "dtype": [ + "str", + "int" + ], + "default": null, + "name": "g2-fix-dirs" + }, + { + "info": "Determines update type. This does not change the Jones solver type, but\nrestricts the update rule to pin the solutions within a certain subspace:\n'full' is the default behaviour;\n'diag' pins the off-diagonal terms to 0;\n'phase-diag' also pins the amplitudes of the diagonal terms to unity;\n'amp-diag' also pins the phases to 0.", + "dtype": "str", + "default": null, + "name": "g2-update-type" + }, + { + "info": "Flag solution intervals where the prior error estimate is above this value.", + "dtype": "float", + "default": null, + "name": "g2-max-prior-error" + }, + { + "info": "Flag solution intervals where the posterior variance estimate is above this value.", + "dtype": "float", + "default": null, + "name": "g2-max-post-error" + }, + { + "info": "Maximum number of iterations spent on this term.", + "dtype": "int", + "default": null, + "name": "g2-max-iter" + }, + { + "info": "Flag propagation policy. Determines how flags raised on gains propagate back\ninto the data. Options are 'never' to never propagate, 'always' to always\npropagate, 'default' to only propagate flags from direction-independent gains.", + "dtype": "str", + "default": null, + "name": "g2-prop-flags" + }, + { + "info": "Determines whether this term is direction dependent. --model-ddes must", + "dtype": "bool", + "default": null, + "name": "g3-dd-term" + }, + { + "info": "Time solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "g3-time-int" + }, + { + "info": "Frequency solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "g3-freq-int" + }, + { + "info": "Amplitude clipping - flag solutions with diagonal amplitudes below this\nvalue.", + "dtype": "float", + "default": null, + "name": "g3-clip-low" + }, + { + "info": "Amplitude clipping - flag solutions with any amplitudes above this value.", + "dtype": "float", + "default": null, + "name": "g3-clip-high" + }, + { + "info": "Number of iterations after which to clip this gain.", + "dtype": "int", + "default": null, + "name": "g3-clip-after" + }, + { + "info": "Minimum percentage of converged solutions to accept.", + "dtype": "float", + "default": null, + "name": "g3-conv-quorum" + }, + { + "info": "Reference antenna - its phase is guaranteed to be zero.", + "dtype": "str", + "default": null, + "name": "g3-ref-ant" + }, + { + "info": "Set to 0 (and specify -load-from or -xfer-from) to load a non-solvable\nterm is loaded from disk. Not to be confused with --sol-jones, which\ndetermines the active Jones terms.", + "dtype": "bool", + "default": null, + "name": "g3-solvable" + }, + { + "info": "Type of Jones matrix to solve for. Note that if multiple Jones terms are\nenabled, then only complex-2x2 is supported.", + "dtype": "str", + "default": null, + "name": "g3-type" + }, + { + "info": "Load solutions from given database. The DB must define solutions\non the same time/frequency grid (i.e. should normally come from\ncalibrating the same pointing/observation). By default, the Jones\nmatrix label is used to form up parameter names, but his may be\noverridden by adding an explicit \"//LABEL\" to the database filename.", + "dtype": "file", + "default": null, + "name": "g3-load-from", + "io": "input" + }, + { + "info": "Transfer solutions from given database. Similar to -load-from, but\nsolutions will be interpolated onto the required time/frequency grid,\nso they can originate from a different field (e.g. from a calibrator).", + "dtype": "file", + "default": null, + "name": "g3-xfer-from", + "io": "input" + }, + { + "info": "Save solutions to given database.", + "dtype": "file", + "default": null, + "name": "g3-save-to", + "io": "output" + }, + { + "info": "For DD terms, makes the listed directions non-solvable.", + "dtype": [ + "str", + "int" + ], + "default": null, + "name": "g3-fix-dirs" + }, + { + "info": "Determines update type. This does not change the Jones solver type, but\nrestricts the update rule to pin the solutions within a certain subspace:\n'full' is the default behaviour;\n'diag' pins the off-diagonal terms to 0;\n'phase-diag' also pins the amplitudes of the diagonal terms to unity;\n'amp-diag' also pins the phases to 0.", + "dtype": "str", + "default": null, + "name": "g3-update-type" + }, + { + "info": "Flag solution intervals where the prior error estimate is above this value.", + "dtype": "float", + "default": null, + "name": "g3-max-prior-error" + }, + { + "info": "Flag solution intervals where the posterior variance estimate is above this value.", + "dtype": "float", + "default": null, + "name": "g3-max-post-error" + }, + { + "info": "Maximum number of iterations spent on this term.", + "dtype": "int", + "default": null, + "name": "g3-max-iter" + }, + { + "info": "Flag propagation policy. Determines how flags raised on gains propagate back\ninto the data. Options are 'never' to never propagate, 'always' to always\npropagate, 'default' to only propagate flags from direction-independent gains.", + "dtype": "str", + "default": null, + "name": "g3-prop-flags" + }, + { + "info": "Determines whether this term is direction dependent. --model-ddes must", + "dtype": "bool", + "default": null, + "name": "b-dd-term" + }, + { + "info": "Time solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "b-time-int" + }, + { + "info": "Frequency solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "b-freq-int" + }, + { + "info": "Amplitude clipping - flag solutions with diagonal amplitudes below this\nvalue.", + "dtype": "float", + "default": null, + "name": "b-clip-low" + }, + { + "info": "Amplitude clipping - flag solutions with any amplitudes above this value.", + "dtype": "float", + "default": null, + "name": "b-clip-high" + }, + { + "info": "Number of iterations after which to clip this gain.", + "dtype": "int", + "default": null, + "name": "b-clip-after" + }, + { + "info": "Minimum percentage of converged solutions to accept.", + "dtype": "float", + "default": null, + "name": "b-conv-quorum" + }, + { + "info": "Reference antenna - its phase is guaranteed to be zero.", + "dtype": "str", + "default": null, + "name": "b-ref-ant" + }, + { + "info": "Set to 0 (and specify -load-from or -xfer-from) to load a non-solvable\nterm is loaded from disk. Not to be confused with --sol-jones, which\ndetermines the active Jones terms.", + "dtype": "bool", + "default": null, + "name": "b-solvable" + }, + { + "info": "Type of Jones matrix to solve for. Note that if multiple Jones terms are\nenabled, then only complex-2x2 is supported.", + "dtype": "str", + "default": null, + "name": "b-type" + }, + { + "info": "Load solutions from given database. The DB must define solutions\non the same time/frequency grid (i.e. should normally come from\ncalibrating the same pointing/observation). By default, the Jones\nmatrix label is used to form up parameter names, but his may be\noverridden by adding an explicit \"//LABEL\" to the database filename.", + "dtype": "file", + "default": null, + "name": "b-load-from", + "io": "input" + }, + { + "info": "Transfer solutions from given database. Similar to -load-from, but\nsolutions will be interpolated onto the required time/frequency grid,\nso they can originate from a different field (e.g. from a calibrator).", + "dtype": "file", + "default": null, + "name": "b-xfer-from", + "io": "input" + }, + { + "info": "Save solutions to given database.", + "dtype": "file", + "default": null, + "name": "b-save-to", + "io": "output" + }, + { + "info": "For DD terms, makes the listed directions non-solvable.", + "dtype": [ + "str", + "int" + ], + "default": null, + "name": "b-fix-dirs" + }, + { + "info": "Determines update type. This does not change the Jones solver type, but\nrestricts the update rule to pin the solutions within a certain subspace:\n'full' is the default behaviour;\n'diag' pins the off-diagonal terms to 0;\n'phase-diag' also pins the amplitudes of the diagonal terms to unity;\n'amp-diag' also pins the phases to 0.", + "dtype": "str", + "default": null, + "name": "b-update-type" + }, + { + "info": "Flag solution intervals where the prior error estimate is above this value.", + "dtype": "float", + "default": null, + "name": "b-max-prior-error" + }, + { + "info": "Flag solution intervals where the posterior variance estimate is above this value.", + "dtype": "float", + "default": null, + "name": "b-max-post-error" + }, + { + "info": "Maximum number of iterations spent on this term.", + "dtype": "int", + "default": null, + "name": "b-max-iter" + }, + { + "info": "Flag propagation policy. Determines how flags raised on gains propagate back\ninto the data. Options are 'never' to never propagate, 'always' to always\npropagate, 'default' to only propagate flags from direction-independent gains.", + "dtype": "str", + "default": null, + "name": "b-prop-flags" + }, + { + "info": "Determines whether this term is direction dependent. --model-ddes must", + "dtype": "bool", + "default": null, + "name": "b1-dd-term" + }, + { + "info": "Time solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "b1-time-int" + }, + { + "info": "Frequency solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "b1-freq-int" + }, + { + "info": "Amplitude clipping - flag solutions with diagonal amplitudes below this\nvalue.", + "dtype": "float", + "default": null, + "name": "b1-clip-low" + }, + { + "info": "Amplitude clipping - flag solutions with any amplitudes above this value.", + "dtype": "float", + "default": null, + "name": "b1-clip-high" + }, + { + "info": "Number of iterations after which to clip this gain.", + "dtype": "int", + "default": null, + "name": "b1-clip-after" + }, + { + "info": "Minimum percentage of converged solutions to accept.", + "dtype": "float", + "default": null, + "name": "b1-conv-quorum" + }, + { + "info": "Reference antenna - its phase is guaranteed to be zero.", + "dtype": "str", + "default": null, + "name": "b1-ref-ant" + }, + { + "info": "Set to 0 (and specify -load-from or -xfer-from) to load a non-solvable\nterm is loaded from disk. Not to be confused with --sol-jones, which\ndetermines the active Jones terms.", + "dtype": "bool", + "default": null, + "name": "b1-solvable" + }, + { + "info": "Type of Jones matrix to solve for. Note that if multiple Jones terms are\nenabled, then only complex-2x2 is supported.", + "dtype": "str", + "default": null, + "name": "b1-type" + }, + { + "info": "Load solutions from given database. The DB must define solutions\non the same time/frequency grid (i.e. should normally come from\ncalibrating the same pointing/observation). By default, the Jones\nmatrix label is used to form up parameter names, but his may be\noverridden by adding an explicit \"//LABEL\" to the database filename.", + "dtype": "file", + "default": null, + "name": "b1-load-from", + "io": "input" + }, + { + "info": "Transfer solutions from given database. Similar to -load-from, but\nsolutions will be interpolated onto the required time/frequency grid,\nso they can originate from a different field (e.g. from a calibrator).", + "dtype": "file", + "default": null, + "name": "b1-xfer-from", + "io": "input" + }, + { + "info": "Save solutions to given database.", + "dtype": "file", + "default": null, + "name": "b1-save-to", + "io": "output" + }, + { + "info": "For DD terms, makes the listed directions non-solvable.", + "dtype": [ + "str", + "int" + ], + "default": null, + "name": "b1-fix-dirs" + }, + { + "info": "Determines update type. This does not change the Jones solver type, but\nrestricts the update rule to pin the solutions within a certain subspace:\n'full' is the default behaviour;\n'diag' pins the off-diagonal terms to 0;\n'phase-diag' also pins the amplitudes of the diagonal terms to unity;\n'amp-diag' also pins the phases to 0.", + "dtype": "str", + "default": null, + "name": "b1-update-type" + }, + { + "info": "Flag solution intervals where the prior error estimate is above this value.", + "dtype": "float", + "default": null, + "name": "b1-max-prior-error" + }, + { + "info": "Flag solution intervals where the posterior variance estimate is above this value.", + "dtype": "float", + "default": null, + "name": "b1-max-post-error" + }, + { + "info": "Maximum number of iterations spent on this term.", + "dtype": "int", + "default": null, + "name": "b1-max-iter" + }, + { + "info": "Flag propagation policy. Determines how flags raised on gains propagate back\ninto the data. Options are 'never' to never propagate, 'always' to always\npropagate, 'default' to only propagate flags from direction-independent gains.", + "dtype": "str", + "default": null, + "name": "b1-prop-flags" + }, + { + "info": "Determines whether this term is direction dependent. --model-ddes must", + "dtype": "bool", + "default": null, + "name": "b2-dd-term" + }, + { + "info": "Time solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "b2-time-int" + }, + { + "info": "Frequency solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "b2-freq-int" + }, + { + "info": "Amplitude clipping - flag solutions with diagonal amplitudes below this\nvalue.", + "dtype": "float", + "default": null, + "name": "b2-clip-low" + }, + { + "info": "Amplitude clipping - flag solutions with any amplitudes above this value.", + "dtype": "float", + "default": null, + "name": "b2-clip-high" + }, + { + "info": "Number of iterations after which to clip this gain.", + "dtype": "int", + "default": null, + "name": "b2-clip-after" + }, + { + "info": "Minimum percentage of converged solutions to accept.", + "dtype": "float", + "default": null, + "name": "b2-conv-quorum" + }, + { + "info": "Reference antenna - its phase is guaranteed to be zero.", + "dtype": "str", + "default": null, + "name": "b2-ref-ant" + }, + { + "info": "Set to 0 (and specify -load-from or -xfer-from) to load a non-solvable\nterm is loaded from disk. Not to be confused with --sol-jones, which\ndetermines the active Jones terms.", + "dtype": "bool", + "default": null, + "name": "b2-solvable" + }, + { + "info": "Type of Jones matrix to solve for. Note that if multiple Jones terms are\nenabled, then only complex-2x2 is supported.", + "dtype": "str", + "default": null, + "name": "b2-type" + }, + { + "info": "Load solutions from given database. The DB must define solutions\non the same time/frequency grid (i.e. should normally come from\ncalibrating the same pointing/observation). By default, the Jones\nmatrix label is used to form up parameter names, but his may be\noverridden by adding an explicit \"//LABEL\" to the database filename.", + "dtype": "file", + "default": null, + "name": "b2-load-from", + "io": "input" + }, + { + "info": "Transfer solutions from given database. Similar to -load-from, but\nsolutions will be interpolated onto the required time/frequency grid,\nso they can originate from a different field (e.g. from a calibrator).", + "dtype": "file", + "default": null, + "name": "b2-xfer-from", + "io": "input" + }, + { + "info": "Save solutions to given database.", + "dtype": "file", + "default": null, + "name": "b2-save-to", + "io": "output" + }, + { + "info": "For DD terms, makes the listed directions non-solvable.", + "dtype": [ + "str", + "int" + ], + "default": null, + "name": "b2-fix-dirs" + }, + { + "info": "Determines update type. This does not change the Jones solver type, but\nrestricts the update rule to pin the solutions within a certain subspace:\n'full' is the default behaviour;\n'diag' pins the off-diagonal terms to 0;\n'phase-diag' also pins the amplitudes of the diagonal terms to unity;\n'amp-diag' also pins the phases to 0.", + "dtype": "str", + "default": null, + "name": "b2-update-type" + }, + { + "info": "Flag solution intervals where the prior error estimate is above this value.", + "dtype": "float", + "default": null, + "name": "b2-max-prior-error" + }, + { + "info": "Flag solution intervals where the posterior variance estimate is above this value.", + "dtype": "float", + "default": null, + "name": "b2-max-post-error" + }, + { + "info": "Maximum number of iterations spent on this term.", + "dtype": "int", + "default": null, + "name": "b2-max-iter" + }, + { + "info": "Flag propagation policy. Determines how flags raised on gains propagate back\ninto the data. Options are 'never' to never propagate, 'always' to always\npropagate, 'default' to only propagate flags from direction-independent gains.", + "dtype": "str", + "default": null, + "name": "b2-prop-flags" + }, + { + "info": "Determines whether this term is direction dependent. --model-ddes must", + "dtype": "bool", + "default": null, + "name": "b3-dd-term" + }, + { + "info": "Time solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "b3-time-int" + }, + { + "info": "Frequency solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "b3-freq-int" + }, + { + "info": "Amplitude clipping - flag solutions with diagonal amplitudes below this\nvalue.", + "dtype": "float", + "default": null, + "name": "b3-clip-low" + }, + { + "info": "Amplitude clipping - flag solutions with any amplitudes above this value.", + "dtype": "float", + "default": null, + "name": "b3-clip-high" + }, + { + "info": "Number of iterations after which to clip this gain.", + "dtype": "int", + "default": null, + "name": "b3-clip-after" + }, + { + "info": "Minimum percentage of converged solutions to accept.", + "dtype": "float", + "default": null, + "name": "b3-conv-quorum" + }, + { + "info": "Reference antenna - its phase is guaranteed to be zero.", + "dtype": "str", + "default": null, + "name": "b3-ref-ant" + }, + { + "info": "Set to 0 (and specify -load-from or -xfer-from) to load a non-solvable\nterm is loaded from disk. Not to be confused with --sol-jones, which\ndetermines the active Jones terms.", + "dtype": "bool", + "default": null, + "name": "b3-solvable" + }, + { + "info": "Type of Jones matrix to solve for. Note that if multiple Jones terms are\nenabled, then only complex-2x2 is supported.", + "dtype": "str", + "default": null, + "name": "b3-type" + }, + { + "info": "Load solutions from given database. The DB must define solutions\non the same time/frequency grid (i.e. should normally come from\ncalibrating the same pointing/observation). By default, the Jones\nmatrix label is used to form up parameter names, but his may be\noverridden by adding an explicit \"//LABEL\" to the database filename.", + "dtype": "file", + "default": null, + "name": "b3-load-from", + "io": "input" + }, + { + "info": "Transfer solutions from given database. Similar to -load-from, but\nsolutions will be interpolated onto the required time/frequency grid,\nso they can originate from a different field (e.g. from a calibrator).", + "dtype": "file", + "default": null, + "name": "b3-xfer-from", + "io": "input" + }, + { + "info": "Save solutions to given database.", + "dtype": "file", + "default": null, + "name": "b3-save-to", + "io": "output" + }, + { + "info": "For DD terms, makes the listed directions non-solvable.", + "dtype": [ + "str", + "int" + ], + "default": null, + "name": "b3-fix-dirs" + }, + { + "info": "Determines update type. This does not change the Jones solver type, but\nrestricts the update rule to pin the solutions within a certain subspace:\n'full' is the default behaviour;\n'diag' pins the off-diagonal terms to 0;\n'phase-diag' also pins the amplitudes of the diagonal terms to unity;\n'amp-diag' also pins the phases to 0.", + "dtype": "str", + "default": null, + "name": "b3-update-type" + }, + { + "info": "Flag solution intervals where the prior error estimate is above this value.", + "dtype": "float", + "default": null, + "name": "b3-max-prior-error" + }, + { + "info": "Flag solution intervals where the posterior variance estimate is above this value.", + "dtype": "float", + "default": null, + "name": "b3-max-post-error" + }, + { + "info": "Maximum number of iterations spent on this term.", + "dtype": "int", + "default": null, + "name": "b3-max-iter" + }, + { + "info": "Flag propagation policy. Determines how flags raised on gains propagate back\ninto the data. Options are 'never' to never propagate, 'always' to always\npropagate, 'default' to only propagate flags from direction-independent gains.", + "dtype": "str", + "default": null, + "name": "b3-prop-flags" + }, + { + "info": "Determines whether this term is direction dependent. --model-ddes must", + "dtype": "bool", + "default": true, + "name": "dd-dd-term" + }, + { + "info": "Time solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "dd-time-int" + }, + { + "info": "Frequency solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "dd-freq-int" + }, + { + "info": "Amplitude clipping - flag solutions with diagonal amplitudes below this\nvalue.", + "dtype": "float", + "default": null, + "name": "dd-clip-low" + }, + { + "info": "Amplitude clipping - flag solutions with any amplitudes above this value.", + "dtype": "float", + "default": null, + "name": "dd-clip-high" + }, + { + "info": "Number of iterations after which to clip this gain.", + "dtype": "int", + "default": null, + "name": "dd-clip-after" + }, + { + "info": "Minimum percentage of converged solutions to accept.", + "dtype": "float", + "default": null, + "name": "dd-conv-quorum" + }, + { + "info": "Reference antenna - its phase is guaranteed to be zero.", + "dtype": "str", + "default": null, + "name": "dd-ref-ant" + }, + { + "info": "Set to 0 (and specify -load-from or -xfer-from) to load a non-solvable\nterm is loaded from disk. Not to be confused with --sol-jones, which\ndetermines the active Jones terms.", + "dtype": "bool", + "default": null, + "name": "dd-solvable" + }, + { + "info": "Type of Jones matrix to solve for. Note that if multiple Jones terms are\nenabled, then only complex-2x2 is supported.", + "dtype": "str", + "default": null, + "name": "dd-type" + }, + { + "info": "Load solutions from given database. The DB must define solutions\non the same time/frequency grid (i.e. should normally come from\ncalibrating the same pointing/observation). By default, the Jones\nmatrix label is used to form up parameter names, but his may be\noverridden by adding an explicit \"//LABEL\" to the database filename.", + "dtype": "file", + "default": null, + "name": "dd-load-from", + "io": "input" + }, + { + "info": "Transfer solutions from given database. Similar to -load-from, but\nsolutions will be interpolated onto the required time/frequency grid,\nso they can originate from a different field (e.g. from a calibrator).", + "dtype": "file", + "default": null, + "name": "dd-xfer-from", + "io": "input" + }, + { + "info": "Save solutions to given database.", + "dtype": "file", + "default": null, + "name": "dd-save-to", + "io": "output" + }, + { + "info": "For DD terms, makes the listed directions non-solvable.", + "dtype": "str", + "default": null, + "name": "dd-fix-dirs" + }, + { + "info": "Determines update type. This does not change the Jones solver type, but\nrestricts the update rule to pin the solutions within a certain subspace:\n'full' is the default behaviour;\n'diag' pins the off-diagonal terms to 0;\n'phase-diag' also pins the amplitudes of the diagonal terms to unity;\n'amp-diag' also pins the phases to 0.", + "dtype": "str", + "default": null, + "name": "dd-update-type" + }, + { + "info": "Flag solution intervals where the prior error estimate is above this value.", + "dtype": "float", + "default": null, + "name": "dd-max-prior-error" + }, + { + "info": "Flag solution intervals where the posterior variance estimate is above this value.", + "dtype": "float", + "default": null, + "name": "dd-max-post-error" + }, + { + "info": "Maximum number of iterations spent on this term.", + "dtype": "int", + "default": null, + "name": "dd-max-iter" + }, + { + "info": "Flag propagation policy. Determines how flags raised on gains propagate back\ninto the data. Options are 'never' to never propagate, 'always' to always\npropagate, 'default' to only propagate flags from direction-independent gains.", + "dtype": "str", + "default": null, + "name": "dd-prop-flags" + }, + { + "info": "Time solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "dd1-time-int" + }, + { + "info": "Frequency solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "dd1-freq-int" + }, + { + "info": "Amplitude clipping - flag solutions with diagonal amplitudes below this\nvalue.", + "dtype": "float", + "default": null, + "name": "dd1-clip-low" + }, + { + "info": "Amplitude clipping - flag solutions with any amplitudes above this value.", + "dtype": "float", + "default": null, + "name": "dd1-clip-high" + }, + { + "info": "Number of iterations after which to clip this gain.", + "dtype": "int", + "default": null, + "name": "dd1-clip-after" + }, + { + "info": "Minimum percentage of converged solutions to accept.", + "dtype": "float", + "default": null, + "name": "dd1-conv-quorum" + }, + { + "info": "Reference antenna - its phase is guaranteed to be zero.", + "dtype": "str", + "default": null, + "name": "dd1-ref-ant" + }, + { + "info": "Set to 0 (and specify -load-from or -xfer-from) to load a non-solvable\nterm is loaded from disk. Not to be confused with --sol-jones, which\ndetermines the active Jones terms.", + "dtype": "bool", + "default": null, + "name": "dd1-solvable" + }, + { + "info": "Type of Jones matrix to solve for. Note that if multiple Jones terms are\nenabled, then only complex-2x2 is supported.", + "dtype": "str", + "default": null, + "name": "dd1-type" + }, + { + "info": "Load solutions from given database. The DB must define solutions\non the same time/frequency grid (i.e. should normally come from\ncalibrating the same pointing/observation). By default, the Jones\nmatrix label is used to form up parameter names, but his may be\noverridden by adding an explicit \"//LABEL\" to the database filename.", + "dtype": "file", + "default": null, + "name": "dd1-load-from", + "io": "input" + }, + { + "info": "Transfer solutions from given database. Similar to -load-from, but\nsolutions will be interpolated onto the required time/frequency grid,\nso they can originate from a different field (e.g. from a calibrator).", + "dtype": "file", + "default": null, + "name": "dd1-xfer-from", + "io": "input" + }, + { + "info": "Save solutions to given database.", + "dtype": "file", + "default": null, + "name": "dd1-save-to", + "io": "output" + }, + { + "info": "For DD terms, makes the listed directions non-solvable.", + "dtype": "str", + "default": null, + "name": "dd1-fix-dirs" + }, + { + "info": "Determines update type. This does not change the Jones solver type, but\nrestricts the update rule to pin the solutions within a certain subspace:\n'full' is the default behaviour;\n'diag' pins the off-diagonal terms to 0;\n'phase-diag' also pins the amplitudes of the diagonal terms to unity;\n'amp-diag' also pins the phases to 0.", + "dtype": "str", + "default": null, + "name": "dd1-update-type" + }, + { + "info": "Flag solution intervals where the prior error estimate is above this value.", + "dtype": "float", + "default": null, + "name": "dd1-max-prior-error" + }, + { + "info": "Flag solution intervals where the posterior variance estimate is above this value.", + "dtype": "float", + "default": null, + "name": "dd1-max-post-error" + }, + { + "info": "Maximum number of iterations spent on this term.", + "dtype": "int", + "default": null, + "name": "dd1-max-iter" + }, + { + "info": "Flag propagation policy. Determines how flags raised on gains propagate back\ninto the data. Options are 'never' to never propagate, 'always' to always\npropagate, 'default' to only propagate flags from direction-independent gains.", + "dtype": "str", + "default": null, + "name": "dd1-prop-flags" + }, + { + "info": "Time solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "dd2-time-int" + }, + { + "info": "Frequency solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "dd2-freq-int" + }, + { + "info": "Amplitude clipping - flag solutions with diagonal amplitudes below this\nvalue.", + "dtype": "float", + "default": null, + "name": "dd2-clip-low" + }, + { + "info": "Amplitude clipping - flag solutions with any amplitudes above this value.", + "dtype": "float", + "default": null, + "name": "dd2-clip-high" + }, + { + "info": "Number of iterations after which to clip this gain.", + "dtype": "int", + "default": null, + "name": "dd2-clip-after" + }, + { + "info": "Minimum percentage of converged solutions to accept.", + "dtype": "float", + "default": null, + "name": "dd2-conv-quorum" + }, + { + "info": "Reference antenna - its phase is guaranteed to be zero.", + "dtype": "str", + "default": null, + "name": "dd2-ref-ant" + }, + { + "info": "Set to 0 (and specify -load-from or -xfer-from) to load a non-solvable\nterm is loaded from disk. Not to be confused with --sol-jones, which\ndetermines the active Jones terms.", + "dtype": "bool", + "default": null, + "name": "dd2-solvable" + }, + { + "info": "Type of Jones matrix to solve for. Note that if multiple Jones terms are\nenabled, then only complex-2x2 is supported.", + "dtype": "str", + "default": null, + "name": "dd2-type" + }, + { + "info": "Load solutions from given database. The DB must define solutions\non the same time/frequency grid (i.e. should normally come from\ncalibrating the same pointing/observation). By default, the Jones\nmatrix label is used to form up parameter names, but his may be\noverridden by adding an explicit \"//LABEL\" to the database filename.", + "dtype": "file", + "default": null, + "name": "dd2-load-from", + "io": "input" + }, + { + "info": "Transfer solutions from given database. Similar to -load-from, but\nsolutions will be interpolated onto the required time/frequency grid,\nso they can originate from a different field (e.g. from a calibrator).", + "dtype": "file", + "default": null, + "name": "dd2-xfer-from", + "io": "input" + }, + { + "info": "Save solutions to given database.", + "dtype": "file", + "default": null, + "name": "dd2-save-to", + "io": "output" + }, + { + "info": "For DD terms, makes the listed directions non-solvable.", + "dtype": "str", + "default": null, + "name": "dd2-fix-dirs" + }, + { + "info": "Determines update type. This does not change the Jones solver type, but\nrestricts the update rule to pin the solutions within a certain subspace:\n'full' is the default behaviour;\n'diag' pins the off-diagonal terms to 0;\n'phase-diag' also pins the amplitudes of the diagonal terms to unity;\n'amp-diag' also pins the phases to 0.", + "dtype": "str", + "default": null, + "name": "dd2-update-type" + }, + { + "info": "Flag solution intervals where the prior error estimate is above this value.", + "dtype": "float", + "default": null, + "name": "dd2-max-prior-error" + }, + { + "info": "Flag solution intervals where the posterior variance estimate is above this value.", + "dtype": "float", + "default": null, + "name": "dd2-max-post-error" + }, + { + "info": "Maximum number of iterations spent on this term.", + "dtype": "int", + "default": null, + "name": "dd2-max-iter" + }, + { + "info": "Flag propagation policy. Determines how flags raised on gains propagate back\ninto the data. Options are 'never' to never propagate, 'always' to always\npropagate, 'default' to only propagate flags from direction-independent gains.", + "dtype": "str", + "default": null, + "name": "dd2-prop-flags" + }, + { + "info": "Time solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "dd3-time-int" + }, + { + "info": "Frequency solution interval for this term. 0 means use entire chunk.", + "dtype": [ + "int", + "float" + ], + "default": null, + "name": "dd3-freq-int" + }, + { + "info": "Amplitude clipping - flag solutions with diagonal amplitudes below this\nvalue.", + "dtype": "float", + "default": null, + "name": "dd3-clip-low" + }, + { + "info": "Amplitude clipping - flag solutions with any amplitudes above this value.", + "dtype": "float", + "default": null, + "name": "dd3-clip-high" + }, + { + "info": "Number of iterations after which to clip this gain.", + "dtype": "int", + "default": null, + "name": "dd3-clip-after" + }, + { + "info": "Minimum percentage of converged solutions to accept.", + "dtype": "float", + "default": null, + "name": "dd3-conv-quorum" + }, + { + "info": "Reference antenna - its phase is guaranteed to be zero.", + "dtype": "str", + "default": null, + "name": "dd3-ref-ant" + }, + { + "info": "Set to 0 (and specify -load-from or -xfer-from) to load a non-solvable\nterm is loaded from disk. Not to be confused with --sol-jones, which\ndetermines the active Jones terms.", + "dtype": "bool", + "default": null, + "name": "dd3-solvable" + }, + { + "info": "Type of Jones matrix to solve for. Note that if multiple Jones terms are\nenabled, then only complex-2x2 is supported.", + "dtype": "str", + "default": null, + "name": "dd3-type" + }, + { + "info": "Load solutions from given database. The DB must define solutions\non the same time/frequency grid (i.e. should normally come from\ncalibrating the same pointing/observation). By default, the Jones\nmatrix label is used to form up parameter names, but his may be\noverridden by adding an explicit \"//LABEL\" to the database filename.", + "dtype": "file", + "default": null, + "name": "dd3-load-from", + "io": "input" + }, + { + "info": "Transfer solutions from given database. Similar to -load-from, but\nsolutions will be interpolated onto the required time/frequency grid,\nso they can originate from a different field (e.g. from a calibrator).", + "dtype": "file", + "default": null, + "name": "dd3-xfer-from", + "io": "input" + }, + { + "info": "Save solutions to given database.", + "dtype": "file", + "default": null, + "name": "dd3-save-to", + "io": "output" + }, + { + "info": "For DD terms, makes the listed directions non-solvable.", + "dtype": "str", + "default": null, + "name": "dd3-fix-dirs" + }, + { + "info": "Determines update type. This does not change the Jones solver type, but\nrestricts the update rule to pin the solutions within a certain subspace:\n'full' is the default behaviour;\n'diag' pins the off-diagonal terms to 0;\n'phase-diag' also pins the amplitudes of the diagonal terms to unity;\n'amp-diag' also pins the phases to 0.", + "dtype": "str", + "default": null, + "name": "dd3-update-type" + }, + { + "info": "Flag solution intervals where the prior error estimate is above this value.", + "dtype": "float", + "default": null, + "name": "dd3-max-prior-error" + }, + { + "info": "Flag solution intervals where the posterior variance estimate is above this value.", + "dtype": "float", + "default": null, + "name": "dd3-max-post-error" + }, + { + "info": "Maximum number of iterations spent on this term.", + "dtype": "int", + "default": null, + "name": "dd3-max-iter" + }, + { + "info": "Flag propagation policy. Determines how flags raised on gains propagate back\ninto the data. Options are 'never' to never propagate, 'always' to always\npropagate, 'default' to only propagate flags from direction-independent gains.", + "dtype": "str", + "default": null, + "name": "dd3-prop-flags" + }, + { + "info": "Default console output verbosity level", + "dtype": [ + "bool", + "str" + ], + "name": "log-verbose" + }, + { + "info": "Oversampling factor.", + "dtype": "int", + "name": "degridding-OverS", + "default": 11 + }, + { + "info": "CF support size.", + "dtype": "int", + "name": "degridding-Support", + "default": 7 + }, + { + "info": "Number of w-planes.", + "dtype": "int", + "name": "degridding-Nw", + "default": 100 + }, + { + "info": "Maximum w coordinate (meters). Visibilities with larger w will not be gridded. If 0, no maximum is imposed.", + "dtype": "float", + "name": "degridding-wmax", + "default": 0 + }, + { + "info": "Facet padding factor.", + "dtype": "float", + "name": "degridding-Padding", + "default": 1.7 + }, + { + "info": "Number of image bands for degridding. 0 means degrid each channel.", + "dtype": "int", + "name": "degridding-NDegridBand", + "default": 16 + }, + { + "info": "Maximum facet size in degrees", + "dtype": "float", + "name": "degridding-MaxFacetSize", + "default": 0.25 + }, + { + "info": "Minimum number of facets per direction", + "dtype": "int", + "name": "degridding-MinNFacetPerAxis", + "default": 1 + } + ] +} diff --git a/stimela/cargo/cab/cubical_ddf/src/DefaultParset.cfg b/stimela/cargo/cab/cubical_ddf/src/DefaultParset.cfg new file mode 100644 index 00000000..f6723344 --- /dev/null +++ b/stimela/cargo/cab/cubical_ddf/src/DefaultParset.cfg @@ -0,0 +1,506 @@ +[data] +_Help = Visibility data options +ms = # Name of measurement set (MS) #type:str +column = DATA # Name of MS column to read for data. #metavar:COLUMN #type:str +time-chunk = 32 # Chunk data up by this number of timeslots. This limits the amount of data + processed at once. Smaller chunks allow for a smaller RAM footprint and + greater parallelism, but this sets an upper limit on the solution intervals + that may be employed. Specify as an integer number of timeslots, or a value with a unit + (e.g. '300s'). 0 means use full time axis. #metavar:TIME +freq-chunk = 32 # Chunk data by this number of channels. See time-chunk for info. Specify as an + integer number of channels, or a value with a unit (e.g. '128MHz'). + 0 means full frequency axis. #metavar:FREQ +rebin-time = 1 # Rebin data in time on the fly. Specify as a number of timeslots to average + together, or a value with a unit (e.g. '5s'). #metavar:TIME +rebin-freq = 1 # Rebin data in frequency on the fly. Specify as a number of channels to average + together, or a value with a unit (e.g. '4MHz'). #metavar:FREQ +chunk-by = SCAN_NUMBER # If set, then time chunks will be broken up whenever the value in the named + column(s) jumps by >JUMPSIZE. Multiple column names may be given, separated + by commas. Use None to disable. #metavar:COLUMNS +chunk-by-jump = 1 # The jump size used in conjunction with chunk-by. If 0, then any change in + value is a jump. If n, then the change must be >n. #metavar:JUMPSIZE + #type:float +single-chunk = # If set, processes just one chunk of data matching the chunk ID. Useful for + debugging. #metavar:CHUNK_ID + +[sel] +_Help = Data selection options +field = 0 # FIELD_ID to read from the MS. #metavar:FIELD #type:int +ddid = None # DATA_DESC_IDs to read from the MS. Default reads all. Can be specified as + e.g. "5", "5,6,7", "5~7" (inclusive range), "5:8" (exclusive range), + "5:" (from 5 to last). #metavar:DDID +taql = # Additional TaQL selection string. Combined with other selection options. + #metavar:TAQL #type:str +chan = # Channels to read (within each DDID). Default reads all. Can be specified as + e.g. "5", "10~20" (10 to 20 inclusive), "10:21" (same), "10:" (from 10 to + end), ":10:2" (0 to 9 inclusive, stepped by 2), "~9:2" (same).#metavar:CHANNELS + +[model] +_Help = Calibration model options +list = # Predict model visibilities from given LSM (using Montblanc). + #metavar:FILENAME #type:str +ddes = auto # Enable direction-dependent models. If 'auto', this is determined + by --sol-jones and --model-list, otherwise, enable/disable + explicitly. #options:never|auto|always +beam-pattern = None # Apply beams if specified eg. 'beam_$(corr)_$(reim).fits' or + 'beam_$(CORR)_$(REIM).fits' +beam-l-axis = None +beam-m-axis = None + +[montblanc] +_Help = Montblanc simulation options +device-type = CPU # Use CPU or GPU for simulation. #options:CPU|GPU +dtype = float # Precision for simulation. DEPRECATED: forcing float for now since MS data is float + anyway. #options:float|double #type:str +feed-type = linear # Simulate using linear or circular feeds. #options:linear|circular +mem-budget = 1024 # Memory budget in MB for simulation. #type:int +verbosity = WARNING # verbosity level of Montblanc's console output #metavar:LEVEL + #options:DEBUG|INFO|WARNING|ERROR +threads = 0 # Number of OMP threads to run. Note that --dist-pin-io overrides this, if set. + If 0, uses default setting. #metavar:N #type:int + +[weight] +_Help = Weighting options +column = WEIGHT_SPECTRUM # Column to read weights from. Weights are applied by default. Specify an + empty string to disable. + +[flags] +_Help = General flagging options +apply = -cubical # Which flagsets will be applied prior to calibration. #metavar:[-]FLAGSET(s) + Use a comma-separated list of flagset name to use bitflags. Use "-FLAGSET" to apply + all bitflags except the named flagset ("-cubical" is useful, to ignore the flags of + a previous CubiCal run). Use "FLAG" to just apply FLAG/FLAG_ROW. +auto-init = legacy # Insert BITFLAG column if it is missing, and initialize the named flagset + from FLAG/FLAG_ROW. +save = cubical # Save flags to named flagset in BITFLAG. If none or 0, will not save. +save-legacy = auto # Controls whether output flags are written to FLAG/FLAG_ROW. If set to 'auto', then + follows the --flag-save option. If set to 'apply', then fills FLAG/FLAG_ROW based + on the --flags-apply setting. #options:0|1|auto|apply +reinit-bitflags = 0 # If true, reinitializes BITFLAG column from scratch. Useful if you ended up + with a botched one, but be careful what the state of the FLAG/FLAG_ROW column + is when you use this option.#type:bool +warn-thr = 0.3 # If more than this fraction of data is flagged by the solver, issues gentle warnings + #type:float +see-no-evil = 0 # Proceed even if flag columns appear to be botched or damaged. + #type:bool + +[postmortem] +_Help = Options for "postmortem" flagging based on solution statistics +enable = 0 # If True, will do an extra round of flagging at the end based on solution statistics + NB: EXPERIMENTAL. USE AT OWN RISK. #type:bool +tf-chisq-median = 1.2 # Intervals with chi-squared values larger than X times the median + chi-square value will be flagged. #metavar:X #type:float +tf-np-median = 0.5 # Intervals with a number of valid point less than X times the + median number of valid points will be flagged. #metavar:X #type:float +time-density = 0.5 # If more than the given fraction of data in a timeslot is flagged, flag entire timeslot. + #metavar:FRAC #type:float +chan-density = 0.5 # If more than the given fraction of data in a timeslot is flagged, flag entire channel. + #metavar:FRAC #type:float +ddid-density = 0.5 # If more than the given fraction of data in a DDID is flagged, flag entire DDID. + #metavar:FRAC #type:float + +[madmax] +_Help = Options for the "Mad Max" flagger +enable = 0 # Enable Mad Max flagging in the solver. This computes the median absolute residual + (i.e. median absolute deviation from zero), and flags visibilities exceeding the thresholds + set below. Use 1 to enable. For a trial run, if you don't want the flags to be written + out, use either 'pretend' to compute Mad Max flags (and report statistics and make plots), + then throw them away, or 'trial' to also apply the flags during solutions, but not write + them out. + #options:0|1|pretend|trial +estimate = corr # MAD estimation mode. Use 'corr' for a separate estimate per each baseline and + correlation. Otherwise, a single estimate per baseline is computed using 'all' correlations, + or only the 'diag' or 'offdiag' correlations. #options:corr|all|diag|offdiag +diag = 1 # Flag on on-diagonal (parallel-hand) residuals. #type:bool +offdiag = 1 # Flag on off-diagonal (cross-hand) residuals. #type:bool +threshold = 0,10 # Threshold for MAD flagging per baseline (specified in sigmas). Residuals exceeding + S*MAD/1.428 will be flagged. MAD is computed per baseline. + This can be specified as a list e.g. N1,N2,N3,... The first value is used to flag + residuals before a solution starts (use 0 to disable), the next value is used when the residuals + are first recomputed during the solution several iteratins later (see -chi-int), etc. + A final pass may be done at the end of the solution. The last value in the list is reused + if necessary. Using a list with gradually decreasing values may be sensible. #metavar:S +global-threshold = 0,12 # Threshold for global median MAD (MMAD) flagging. MMAD is computed as the median of the + per-baseline MADs. Residuals exceeding S*MMAD/1.428 will be flagged. Can be specified + as a list, with the same semantics as --madmax-threshold. #metavar:S +plot = 1 # Enable plots for Mad Max flagging. Use 'show' to show figures interactively. + Plots will show the worst flagged baseline, and a median flagged baseline, provided the + fraction of flagged visibilities is above --madmax-plot-frac-above. + #options:0|1|show +plot-frac-above = 0.01 # Threshold (in terms of fraction of visibilities flagged) above which Mad Max plots + will be generated. +plot-bl = # Given baseline will always be plotted #type:str +flag-ant = 0 # Flag antennas with excessive residuals, based on MAD criterion. Note that currently + --madmax-plot must be enabled for this to work. +flag-ant-thr = 5 # Threshold (in sigmas) used to flag bad antennas. + + + + +[sol] +_Help = Solution options which apply at the solver level +jones = G # Comma-separated list of Jones terms to enable, e.g. "G,B,dE" + (default: %default) +precision = 32 # Solve in single or double precision #options:32|64 +delta-g = 1e-6 # Theshold for gain accuracy - gains which improve by less than this value + are considered converged. +delta-chi = 1e-6 # Theshold for solution stagnancy - if the chi-squared is improving by less + than this value, the gain is considered stalled. +chi-int = 5 # Number of iterations to perform between chi-suqared checks. This is done to + avoid computing the expensive chi-squared test evey iteration. +last-rites = 1 # Re-estimate chi-squred and noise at the end of a solution cycle. Disabling + last rites can save a bit of time, but makes the post-solution stats less + informative. #type:bool +stall-quorum = 0.99 # Minimum percentage of solutions which must have stalled before terminating + the solver. +diag-diag = 0 # If true, then data, model and gains are taken to be diagonal. Off-diagonal + terms in data and model are ignored. This option is then enforced on + all Jones terms. #type:bool +term-iters = # Number of iterations per Jones term. If empty, then each Jones + term is solved for once, up to convergence, or up to its -max-iter + setting. + Otherwise, set to a list giving the number of iterations per Jones term. + For example, given two Jones terms and --sol-num-iter 10,20,10, it will + do 10 iterations on the first term, 20 on the second, and 10 again on the + first. + #metavar:N1,N2,... +min-bl = 0 # Min baseline length to solve for #metavar:METERS #type:float +max-bl = 0 # Max baseline length to solve for. If 0, no maximum is applied. + #metavar:METERS #type:float +subset = # Additional subset of data to actually solve for. Any TaQL string may be + used. #metavar:TaQL #type:str + + +[bbc] +_Help = Options for baseline-based corrections (a.k.a. BBCs, a.k.a. interferometer gains). +load-from = # Load and apply BBCs computed in a previous run. Apply with care! This will + tend to suppress all unmodelled flux towards the centre of the field. + #metavar:FILENAME +compute-2x2 = 0 # Compute full 2x2 BBCs (as opposed to diagonal-only). Only useful if you + really trust the polarisation information in your sky model. #type:bool +apply-2x2 = 0 # Apply full 2x2 BBCs (as opposed to diagonal-only). Only enable this if you + really trust the polarisation information in your sky model. #type:bool +save-to = {data[ms]}/BBC-field_{sel[field]}-ddid_{sel[ddid]}.parmdb # Compute suggested BBCs at end of run, + and save them to the given database. It can be useful to have this always + enabled, since the BBCs provide useful diagnostics of the solution quality + (and are not actually applied without a load-from setting). #metavar:FILENAME +per-chan = 1 # Compute BBCs per-channel (else across entire band). #type:bool +plot = 1 # Generate output BBC plots. #type:bool + +[dist] +_Help = Parallelization and distribution options +ncpu = 0 # Max number of CPU cores to use. #type:int #metavar:N +nworker = 0 # Number of worker processes to launch (excluding the I/O worker). + 0: determine automatically from the --dist-ncpu setting. #type:int #metavar:N +nthread = 0 # Number of OMP threads to use. 0: determine automatically. #type:int #metavar:N +max-chunks = 0 # Maximum number of time/freq data-chunks to load into memory + simultaneously. If 0, then as many as possible will be loaded. #type:int +min-chunks = 0 # Minimum number of time/freq data-chunks to load into memory + simultaneously. If 0, determined automatically. #type:int +pin = 0 # If empty or None, processes will not be pinned to cores. Otherwise, set to the starting + core number, or "N:K" to start with N and step by K. #metavar:CORES +pin-io = 0 # If not 0, pins the I/O & Montblanc process to a separate core, or cores + (if --montblanc-threads is specified). Ignored if --dist-pin is not set. #type:bool +pin-main = io # If set, pins the main process to a separate core. If set to "io", pins it to the same + core as the I/O process, if I/O process is pinned. Ignored if --dist-pin is not set. + #options:0|1|io + +[out] +_Help = Options for output products +name = cubical # Base name of output files. #metavar:BASENAME #type:str +mode = sc # Operational mode. + [so] solve only; + [sc] solve and generate corrected visibilities; + [sr] solve and generate corrected residuals; + [ss] solve and generate uncorrected residuals; + [ac] apply solutions, generate corrected visibilities; + [ar] apply solutions, generate corrected residuals; + [as] apply solutions, generate uncorrected residuals; + #options:so|sc|sr|ss|ac|ar|as #metavar:MODE +column = CORRECTED_DATA # Output MS column name (if applicable). #metavar:COLUMN +model-column = # If set, model visibilities will be written to the specified column. #metavar:COLUMN +weight-column = # If set weights from the Robust Solver will be written to the specified colum. + # This should be set only if we are using the robust solver. +reinit-column = 0 # Reinitialize output MS column. Useful if the column is in a half-filled + or corrupt state. #type:bool +subtract-model = 0 # Which model to subtract, if generating residuals. #metavar:MODEL + #type:int +subtract-dirs = : # Which model directions to subtract, if generating residuals. ":" + subtracts all. Can also be specified as "N", "N:M", ":N", "N:", "N,M,K". + #metavar:SLICE. #metavar:DIRS +plots = 1 # Generate summary plots. Use 'show' to show summary plots interactively. +casa-gaintables = 1 # Export gaintables to CASA caltable format. Tables are exported to same + directory as set for cubical databases. #type:bool + +[log] +_Help = Options related to logging +memory = 1 # Log memory usage. #type:bool +boring = 0 # Disable progress bars and some console output. #type:bool +append = 0 # Append to log file if it exists. #type:bool +verbose = 0 # Default console output verbosity level. #metavar:LEVEL(s) + Can either be a single number, or a sequence of "name=level,name=level,..." + assignments. +file-verbose = None # Default logfile output verbosity level. #metavar:LEVEL(s) + Can either be a single number, or a sequence of "name=level,name=level,..." + assignments. If None, then this simply follows the console level. + +[debug] +_Help = Debugging options for the discerning masochist +pdb = 0 # Jump into pdb on any exception. #type:bool +panic-amplitude = 0 # Throw an error if a visibility amplitude in the results exceeds the given value. + Useful for troubleshooting. #type:float +stop-before-solver = 0 # Invoke pdb before entering the solver. #type:bool +escalate-warnings = 0 # Escalate warnings (e.g. np.ComplexWarning) into proper exceptions. #type:bool + +[misc] +_Help = Miscellaneous options +random-seed = None # Seed random number generator with explicit seed. Useful for reproducibility + of the random-based optimizations (sparsification, etc.). #metavar:N +parset-version = 0.1 # Parset version number, for migration purposes. Can't be specified on command + line. #no_cmdline:1 + +[JONES-TEMPLATE] +_Help = Options for {LABEL}-Jones term +_NameTemplate = {LABEL} # Designates this section as a "template", and gives the string for its name. +_ExpandedFrom = --sol-jones # Identifies --section-option that determines what templated sections to + instantiate +_OtherTemplates = _Help:label # Colon-separated list of options which need to have their values templated. +label = {LABEL} # Jones label ("G", "dE"). Substituted automatically. #no_cmdline:1 #no_print:1 +solvable = 1 # Set to 0 (and specify -load-from or -xfer-from) to load a non-solvable + term is loaded from disk. Not to be confused with --sol-jones, which + determines the active Jones terms. + #type:bool +type = complex-2x2 # Type of Jones matrix to solve for. Note that if multiple Jones terms are + enabled, then only complex-2x2 is supported. + #options:complex-2x2|complex-diag|phase-diag|robust-2x2|f-slope|t-slope|tf-plane +load-from = # Load solutions from given database. The DB must define solutions + on the same time/frequency grid (i.e. should normally come from + calibrating the same pointing/observation). By default, the Jones + matrix label is used to form up parameter names, but his may be + overridden by adding an explicit "//LABEL" to the database filename. + #metavar:FILENAME[//LABEL] +xfer-from = # Transfer solutions from given database. Similar to -load-from, but + solutions will be interpolated onto the required time/frequency grid, + so they can originate from a different field (e.g. from a calibrator). + #metavar:FILENAME[//LABEL] +save-to = {data[ms]}/{JONES}-field_{sel[field]}-ddid_{sel[ddid]}.parmdb # Save solutions to given database. + #metavar:FILENAME +dd-term = 0 # Determines whether this term is direction dependent. --model-ddes must + be enabled. #type:bool +fix-dirs = # For DD terms, makes the listed directions non-solvable. + #metavar:DIR1[,DIR2,...] +diag-diag = 0 # If true, then data, model and gains are taken to be diagonal. Off-diagonal + terms in data and model are ignored. #type:bool +update-type = full # Determines update type. This does not change the Jones solver type, but + restricts the update rule to pin the solutions within a certain subspace: + 'full' is the default behaviour; + 'diag' pins the off-diagonal terms to 0; + 'phase-diag' also pins the amplitudes of the diagonal terms to unity; + 'amp-diag' also pins the phases to 0. #options:full|phase-diag|diag|amp-diag +time-int = 1 # Time solution interval for this term. #metavar:TIMESLOTS +freq-int = 1 # Frequency solution interval for this term. #metavar:CHANNELS +max-prior-error = .1 # Flag solution intervals where the prior error estimate is above this value. #type:float +max-post-error = .1 # Flag solution intervals where the posterior variance estimate is above this value. #type:float +clip-low = .1 # Amplitude clipping - flag solutions with diagonal amplitudes below this + va + lue. #metavar:AMPL #type:float +clip-high = 10 # Amplitude clipping - flag solutions with any amplitudes above this value. 0 + disables. #metavar:AMPL #type:float +clip-after = 5 # Number of iterations after which to start clipping this gain. #metavar:NITER + #type:int +max-iter = 20 # Maximum number of iterations spent on this term. #metavar:NITER +conv-quorum = 0.99 # Minimum percentage of converged solutions to accept. #metavar:FRACTION +ref-ant = None # Reference antenna - its phase is guaranteed to be zero. #metavar:ANTENNA +prop-flags = default # Flag propagation policy. Determines how flags raised on gains propagate back + into the data. Options are 'never' to never propagate, 'always' to always + propagate, 'default' to only propagate flags from direction-independent gains. + #options:never|always|default +robust-cov = hybrid # Determines how the residuals covariance matrix is computed if the robust-2x2 + solver is selected. Options are 'compute' to compute normaly, 'identity' to set the + covariance to 1 (identity matrix) as in the Robust-t paper, and 'hybrid' which is the default computes + the covaraince matrix, C but sets it to 1 if the elements are greater than 1. + #options:compute|identity|hybrid +robust-npol = 2 # The number of correlations (polarizations) actually present in the visibilities. + This option only applies if the robust-2x2 solver is selected. + #options:2|4 #type:int +robust-int = 5 # Number of iterations after which the v-parameter is recomputed for the robust solver + #type:int +robust-save-weights = 0 # Determines if the appied weights from the robust-2x2 solver are stored. + This option only applies if the robust-2x2 solver is selected. If this option is set + and output-weight-column must be set as well. + #type:bool + +[g] +_Templated = 1 +dd-term = 0 # Determines whether this term is direction dependent. --model-ddes must +time-int = 1 # Time solution interval for this term. 0 means use entire chunk. #metavar:TIMESLOTS +freq-int = 1 # Frequency solution interval for this term. 0 means use entire chunk. #metavar:CHANNELS +clip-low = .1 # Amplitude clipping - flag solutions with diagonal amplitudes below this + value. #metavar:AMPL #type:float +clip-high = 10 # Amplitude clipping - flag solutions with any amplitudes above this value. + #metavar:AMPL #type:float +clip-after = 5 # Number of iterations after which to clip this gain. #metavar:NITER #type:int +conv-quorum = 0.99 # Minimum percentage of converged solutions to accept. #metavar:FRACTION +ref-ant = None # Reference antenna - its phase is guaranteed to be zero. #metavar:ANTENNA + +[g1] +_Templated = 1 +dd-term = 0 # Determines whether this term is direction dependent. --model-ddes must +time-int = 1 # Time solution interval for this term. 0 means use entire chunk. #metavar:TIMESLOTS +freq-int = 1 # Frequency solution interval for this term. 0 means use entire chunk. #metavar:CHANNELS +clip-low = .1 # Amplitude clipping - flag solutions with diagonal amplitudes below this + value. #metavar:AMPL #type:float +clip-high = 10 # Amplitude clipping - flag solutions with any amplitudes above this value. + #metavar:AMPL #type:float +clip-after = 5 # Number of iterations after which to clip this gain. #metavar:NITER #type:int +conv-quorum = 0.99 # Minimum percentage of converged solutions to accept. #metavar:FRACTION +ref-ant = None # Reference antenna - its phase is guaranteed to be zero. #metavar:ANTENNA + +[g2] +_Templated = 1 +dd-term = 0 # Determines whether this term is direction dependent. --model-ddes must +time-int = 1 # Time solution interval for this term. 0 means use entire chunk. #metavar:TIMESLOTS +freq-int = 1 # Frequency solution interval for this term. 0 means use entire chunk. #metavar:CHANNELS +clip-low = .1 # Amplitude clipping - flag solutions with diagonal amplitudes below this + value. #metavar:AMPL #type:float +clip-high = 10 # Amplitude clipping - flag solutions with any amplitudes above this value. + #metavar:AMPL #type:float +clip-after = 5 # Number of iterations after which to clip this gain. #metavar:NITER #type:int +conv-quorum = 0.99 # Minimum percentage of converged solutions to accept. #metavar:FRACTION +ref-ant = None # Reference antenna - its phase is guaranteed to be zero. #metavar:ANTENNA + +[g3] +_Templated = 1 +dd-term = 0 # Determines whether this term is direction dependent. --model-ddes must +time-int = 1 # Time solution interval for this term. 0 means use entire chunk. #metavar:TIMESLOTS +freq-int = 1 # Frequency solution interval for this term. 0 means use entire chunk. #metavar:CHANNELS +clip-low = .1 # Amplitude clipping - flag solutions with diagonal amplitudes below this + value. #metavar:AMPL #type:float +clip-high = 10 # Amplitude clipping - flag solutions with any amplitudes above this value. + #metavar:AMPL #type:float +clip-after = 5 # Number of iterations after which to clip this gain. #metavar:NITER #type:int +conv-quorum = 0.99 # Minimum percentage of converged solutions to accept. #metavar:FRACTION +ref-ant = None # Reference antenna - its phase is guaranteed to be zero. #metavar:ANTENNA + +[b] +_Templated = 1 +dd-term = 0 # Determines whether this term is direction dependent. --model-ddes must +time-int = 1 # Time solution interval for this term. 0 means use entire chunk. #metavar:TIMESLOTS +freq-int = 1 # Frequency solution interval for this term. 0 means use entire chunk. #metavar:CHANNELS +clip-low = .1 # Amplitude clipping - flag solutions with diagonal amplitudes below this + value. #metavar:AMPL #type:float +clip-high = 10 # Amplitude clipping - flag solutions with any amplitudes above this value. + #metavar:AMPL #type:float +clip-after = 5 # Number of iterations after which to clip this gain. #metavar:NITER #type:int +conv-quorum = 0.99 # Minimum percentage of converged solutions to accept. #metavar:FRACTION +ref-ant = None # Reference antenna - its phase is guaranteed to be zero. #metavar:ANTENNA + +[b1] +_Templated = 1 +dd-term = 0 # Determines whether this term is direction dependent. --model-ddes must +time-int = 1 # Time solution interval for this term. 0 means use entire chunk. #metavar:TIMESLOTS +freq-int = 1 # Frequency solution interval for this term. 0 means use entire chunk. #metavar:CHANNELS +clip-low = .1 # Amplitude clipping - flag solutions with diagonal amplitudes below this + value. #metavar:AMPL #type:float +clip-high = 10 # Amplitude clipping - flag solutions with any amplitudes above this value. + #metavar:AMPL #type:float +clip-after = 5 # Number of iterations after which to clip this gain. #metavar:NITER #type:int +conv-quorum = 0.99 # Minimum percentage of converged solutions to accept. #metavar:FRACTION +ref-ant = None # Reference antenna - its phase is guaranteed to be zero. #metavar:ANTENNA + +[b2] +_Templated = 1 +dd-term = 0 # Determines whether this term is direction dependent. --model-ddes must +time-int = 1 # Time solution interval for this term. 0 means use entire chunk. #metavar:TIMESLOTS +freq-int = 1 # Frequency solution interval for this term. 0 means use entire chunk. #metavar:CHANNELS +clip-low = .1 # Amplitude clipping - flag solutions with diagonal amplitudes below this + value. #metavar:AMPL #type:float +clip-high = 10 # Amplitude clipping - flag solutions with any amplitudes above this value. + #metavar:AMPL #type:float +clip-after = 5 # Number of iterations after which to clip this gain. #metavar:NITER #type:int +conv-quorum = 0.99 # Minimum percentage of converged solutions to accept. #metavar:FRACTION +ref-ant = None # Reference antenna - its phase is guaranteed to be zero. #metavar:ANTENNA + +[b3] +_Templated = 1 +dd-term = 0 # Determines whether this term is direction dependent. --model-ddes must +time-int = 1 # Time solution interval for this term. 0 means use entire chunk. #metavar:TIMESLOTS +freq-int = 1 # Frequency solution interval for this term. 0 means use entire chunk. #metavar:CHANNELS +clip-low = .1 # Amplitude clipping - flag solutions with diagonal amplitudes below this + value. #metavar:AMPL #type:float +clip-high = 10 # Amplitude clipping - flag solutions with any amplitudes above this value. + #metavar:AMPL #type:float +clip-after = 5 # Number of iterations after which to clip this gain. #metavar:NITER #type:int +conv-quorum = 0.99 # Minimum percentage of converged solutions to accept. #metavar:FRACTION +ref-ant = None # Reference antenna - its phase is guaranteed to be zero. #metavar:ANTENNA + +[dd] +_Templated = 1 +dd-term = 1 # Determines whether this term is direction dependent. --model-ddes must +clip-low = 0 # Amplitude clipping - flag solutions with diagonal amplitudes below this + value. #metavar:AMPL #type:float +clip-high = 0 # Amplitude clipping - flag solutions with any amplitudes above this value. + #metavar:AMPL #type:float + +[dd1] +_Templated = 1 +dd-term = 1 # Determines whether this term is direction dependent. --model-ddes must +clip-low = 0 # Amplitude clipping - flag solutions with diagonal amplitudes below this + value. #metavar:AMPL #type:float +clip-high = 0 # Amplitude clipping - flag solutions with any amplitudes above this value. + #metavar:AMPL #type:float +[dd2] +_Templated = 1 +dd-term = 1 # Determines whether this term is direction dependent. --model-ddes must +clip-low = 0 # Amplitude clipping - flag solutions with diagonal amplitudes below this + value. #metavar:AMPL #type:float +clip-high = 0 # Amplitude clipping - flag solutions with any amplitudes above this value. + #metavar:AMPL #type:float + +[dd3] +_Templated = 1 +dd-term = 1 # Determines whether this term is direction dependent. --model-ddes must +clip-low = 0 # Amplitude clipping - flag solutions with diagonal amplitudes below this + value. #metavar:AMPL #type:float +clip-high = 0 # Amplitude clipping - flag solutions with any amplitudes above this value. + #metavar:AMPL #type:float + +## * ``DefaultParset.cfg`` now contains comment clauses which are auto-parsed into documentation by ``ReadCfg.py``, and +## automatically converted into OptionParser command-line arguments. +## +## * Command-line arguments are formed as ``--Section-OptionName``. +## +## * Each section can contain a ``_Help = section description`` option. This is turned into a docstring for the section. +## +## * Each option line can contain a comment, preceded by "#". This is turned into a docstring for the option. Note that +## long comments can be continued on the next line simply by indenting the next line. +## +## * Option comments can have embedded attributes of the form ``#attr:value``. These are removed from the docstring +## automatically. All attributes are optional! A few attributes have special meaning, as per below. +## +## * ``#type:TYPE`` forces the option to be interpreted as a specific Python type. E.g. ``#type:bool``, ``#type:str``, +## ``#type:float``. In the absence of an explicit type attribute, the option will be parsed into a valid Python value +## (using ``eval()``), and converted into a string if that fails. This means that e.g. "None" will be interpreted as +## ``None`` and not the string ``"None"``. Note that this mechanism has some pitfalls: for example "all" gets parsed +## into the built-in ``all()`` function (unless surrounded by quotes), so if you want to make sure an option is treated +## as a string, make sure you add a ``#type:str`` attribute. +## +## * ``#options:A|B|C`` specifies that the option is a "selector" with a fixed set of values. See, for example, +## ``#options:None|LOFAR|FITS`` in BeamModel. +## +## * ``#metavar:VAR`` is passed to the metavar argument of ``OptionParser.add_option()``. This makes for a nicer-looking +## command-line help. E.g. ``#metavar:MHz``. +## +## * ``#cmdline-only:1`` specifies that the option can only be changed from the command-line, not via a parset. +## +## * Options can have aliases. *NB: I'm not convinced about this feature. Maybe it makes things too confusing and should +## be removed.* For example, +## +## ``` +## [Caching] +## PSF|CachePSF = 1 # Cache PSF data. May be expensive with large images, hence optional #type:bool +## ``` +## +## means that the command-line option can be specified as ``--Caching-PSF`` or ``--CachePSF``. The second form is the +## "global alias". The GD object in DDFacet will contain both ``GD["Caching"]["PSF"]`` and +## ``GD["Caching"]["CachePSF"]``, set to the same value. diff --git a/stimela/cargo/cab/cubical_ddf/src/run.py b/stimela/cargo/cab/cubical_ddf/src/run.py new file mode 100644 index 00000000..5a781537 --- /dev/null +++ b/stimela/cargo/cab/cubical_ddf/src/run.py @@ -0,0 +1,66 @@ +import os +import sys +import shlex +import shutil +import subprocess +import glob +import yaml + +CONFIG = os.environ["CONFIG"] +INPUT = os.environ["INPUT"] +MSDIR = os.environ["MSDIR"] +OUTPUT = os.environ["OUTPUT"] + +with open(CONFIG, "r") as _std: + cab = yaml.safe_load(_std) + +junk = cab["junk"] + +args = {} +parset = [] + +for param in cab['parameters']: + name = param['name'] + value = param['value'] + + if value is None: + continue + elif value is False: + value = 0 + elif value is True: + value = 1 + elif name == 'parset': + parset = [value] + continue + elif isinstance(value, list): + value = ",".join(map(str, value)) + + args[name] = value + +# available jones terms +joneses = "g b dd".split() +soljones = args.pop("sol-jones") + +for jones in joneses: + if jones.lower() not in soljones.lower(): + jopts = filter(lambda a: a.startswith( + "{0:s}-".format(jones)), args.keys()) + for item in list(jopts): + del args[item] + +opts = ["{0:s}sol-jones {1:s}".format(cab["prefix"], soljones)] + \ + ['{0}{1} {2}'.format(cab['prefix'], name, value) + for name, value in args.items()] + +_runc = " ".join([cab["binary"]] + parset + opts) +try: + subprocess.check_call(shlex.split(_runc)) +finally: + for item in junk: + for dest in [OUTPUT, MSDIR]: # these are the only writable volumes in the container + items = glob.glob("{dest}/{item}".format(**locals())) + for f in items: + if os.path.isfile(f): + os.remove(f) + elif os.path.isdir(f): + shutil.rmtree(f) diff --git a/stimela/cargo/cab/flagstats/parameters.json b/stimela/cargo/cab/flagstats/parameters.json new file mode 100644 index 00000000..118d2bcd --- /dev/null +++ b/stimela/cargo/cab/flagstats/parameters.json @@ -0,0 +1,39 @@ +{ + "task": "flagstats", + "base": "stimela/msutils", + "tag": "1.4.6", + "version" : "1.1.3", + "description": "Extract flag statistics for a set of fields and antennas from an MS", + "prefix": " ", + "binary": "msutils", + "junk": [], + "msdir": true, + "parameters": [ + { + "info": "MS name", + "dtype": "file", + "required": false, + "name": "msname", + "io": "msfile" + }, + { + "info": "Output file (JSON format)", + "default": null, + "name": "outfile", + "io": "output", + "dtype": "file" + }, + { + "info": "Field(s)", + "dtype": "list:str", + "default": true, + "name": "fields" + }, + { + "info": "Antennas", + "dtype": "list:str", + "default": null, + "name": "antennas" + } + ] +} diff --git a/stimela/cargo/cab/flagstats/src/run.py b/stimela/cargo/cab/flagstats/src/run.py new file mode 100644 index 00000000..967f2343 --- /dev/null +++ b/stimela/cargo/cab/flagstats/src/run.py @@ -0,0 +1,49 @@ +import sys +import os +from MSUtils import flag_stats +import inspect +import glob +import shutil +import yaml +import codecs +import json + +CONFIG = os.environ["CONFIG"] +INPUT = os.environ["INPUT"] +OUTPUT = os.environ["OUTPUT"] +MSDIR = os.environ["MSDIR"] + +with open(CONFIG, "r") as _std: + cab = yaml.safe_load(_std) + +junk = cab["junk"] +args = {} +for param in cab['parameters']: + name = param['name'] + value = param['value'] + + if name in ["fields", "antennas"] and value is not None: + try: + value = list(map(int, value)) + except ValueError: + pass + if name == "outfile": + outfile = value + continue + args[name] = value + +try: + stats = flag_stats.antenna_flags_field(**args) +finally: + for item in junk: + for dest in [OUTPUT, MSDIR]: # these are the only writable volumes in the container + items = glob.glob("{dest}/{item}".format(**locals())) + for f in items: + if os.path.isfile(f): + os.remove(f) + elif os.path.isdir(f): + shutil.rmtree(f) + +with codecs.open(outfile, 'w', 'utf8') as stdw: + a = json.dumps(stats, ensure_ascii=False) + stdw.write(a) diff --git a/stimela/cargo/cab/ragavi/src/run.py b/stimela/cargo/cab/ragavi/src/run.py index 58a27f33..91081cef 100644 --- a/stimela/cargo/cab/ragavi/src/run.py +++ b/stimela/cargo/cab/ragavi/src/run.py @@ -34,6 +34,7 @@ args += ['{0}{1} {2}'.format(cab['prefix'], name, value)] _runc = " ".join([cab['binary']] + args) +print("Running", _runc) try: subprocess.check_call(shlex.split(_runc)) finally: diff --git a/stimela/cargo/cab/ragavi_vis/parameters.json b/stimela/cargo/cab/ragavi_vis/parameters.json index 774e5a2d..5b2545ec 100644 --- a/stimela/cargo/cab/ragavi_vis/parameters.json +++ b/stimela/cargo/cab/ragavi_vis/parameters.json @@ -47,7 +47,7 @@ "amplitude", "imaginary", "phase", - "real", + "real" ] }, { diff --git a/stimela/cargo/cab/ragavi_vis/src/run.py b/stimela/cargo/cab/ragavi_vis/src/run.py index 54daf352..71d984f1 100644 --- a/stimela/cargo/cab/ragavi_vis/src/run.py +++ b/stimela/cargo/cab/ragavi_vis/src/run.py @@ -34,6 +34,7 @@ args += ['{0}{1} {2}'.format(cab['prefix'], name, value)] _runc = " ".join([cab['binary']] + args) +print("Running", _runc) try: subprocess.check_call(shlex.split(_runc)) finally: diff --git a/stimela/cargo/cab/shadems_direct/src/run.py b/stimela/cargo/cab/shadems_direct/src/run.py index 6587d9b4..d3dcb3f9 100644 --- a/stimela/cargo/cab/shadems_direct/src/run.py +++ b/stimela/cargo/cab/shadems_direct/src/run.py @@ -26,6 +26,7 @@ for args in args_list: _runc = " ".join([cab["binary"], ms, args]) + print("Running", _runc) try: subprocess.check_call(shlex.split(_runc)) except subprocess.CalledProcessError as exc: diff --git a/stimela/main.py b/stimela/main.py index ef13013d..4d8981f8 100644 --- a/stimela/main.py +++ b/stimela/main.py @@ -100,7 +100,7 @@ def get_cabs(logfile): return cabs_ -def info(cabdir, header=False): +def info(cabdir, header=False, display=True): """ prints out help information about a cab """ # First check if cab exists @@ -109,7 +109,7 @@ def info(cabdir, header=False): raise RuntimeError("Cab could not be found at : {}".format(cabdir)) # Get cab info cab_definition = cab.CabDefinition(parameter_file=pfile) - if header: + if display: cab_definition.display(header) return cab_definition @@ -297,7 +297,7 @@ def pull(argv): base = [] for cab_ in CAB: cabdir = "{:s}/{:s}".format(stimela.CAB_PATH, cab_) - _cab = info(cabdir) + _cab = info(cabdir, display=False) base.append(f"{_cab.base}:{_cab.tag}") base = set(base) diff --git a/stimela/recipe.py b/stimela/recipe.py index f7ed06be..690478e1 100644 --- a/stimela/recipe.py +++ b/stimela/recipe.py @@ -1,6 +1,7 @@ # -*- coding: future_fstrings -*- import os import sys +import pwd, grp import time import stimela from stimela import docker, singularity, utils, cargo, podman, main @@ -29,7 +30,9 @@ } CONT_IO = cab.IODEST -CDIR = os.environ["PWD"] +CDIR = os.getcwd() + +PULLFOLDER = os.environ.get("STIMELA_PULLFOLDER", CDIR) # make dictionary of wrangler actions. First, add all logging levels _actions = {attr: value for attr, value in logging.__dict__.items() if attr.upper() == attr and type(value) is int} @@ -222,9 +225,9 @@ def setup_job(self, image, config, simage = _cab.base.replace("/", "_") if singularity_image_dir is None: singularity_image_dir = os.path.join(CDIR, "stimela_singularity_images") - singularity_image_dir = os.path.abspath(singularity_image_dir) cont.image = '{0:s}/{1:s}_{2:s}{3:s}'.format(singularity_image_dir, simage, _cab.tag, singularity.suffix) + cont.image = os.path.abspath(cont.image) if not os.path.exists(cont.image): main.pull(f"-s -cb {cont.cabname} -pf {singularity_image_dir}".split()) else: @@ -255,6 +258,10 @@ def setup_job(self, image, config, f'{cab.MOUNT}/configfile', perm='ro', noverify=True) cont.add_volume(os.path.join(cabpath, "src"), f"{cab.MOUNT}/code", "ro") + cont.add_volume(os.path.join(self.workdir, "passwd"), "/etc/passwd") + cont.add_volume(os.path.join(self.workdir, "group"), "/etc/group") + cont.RUNSCRIPT = f"/{self.jtype}_run" + if self.jtype == "singularity": cont.RUNSCRIPT = f"/{self.jtype}" if _cab.base.startswith("stimela/casa") or _cab.base.startswith("stimela/simms"): @@ -262,10 +269,7 @@ def setup_job(self, image, config, cont.add_environ("LANG", "en_US.UTF-8") cont.add_environ("LC_ALL", "en_US.UTF-8") cont.execdir = self.workdir - elif self.jtype == "docker": - cont.add_volume("/etc/passwd", "/etc/passwd", "ro") - cont.add_volume("/etc/group", "/etc/group", "ro") - cont.RUNSCRIPT = f"/{self.jtype}_run" + else: cont.RUNSCRIPT = f"/{self.jtype}_run" @@ -445,11 +449,6 @@ def __init__(self, name, data=None, # if it doesn't exist. These config # files can be resued to re-run the # task - self.parameter_file_dir = parameter_file_dir or "stimela_parameter_files" - if not os.path.exists(self.parameter_file_dir): - self.log.info( - 'Config directory cannot be found. Will create ./{}'.format(self.parameter_file_dir)) - os.mkdir(self.parameter_file_dir) self.jobs = [] self.completed = [] @@ -457,7 +456,7 @@ def __init__(self, name, data=None, self.remaining = [] self.pid = os.getpid() - self.singularity_image_dir = singularity_image_dir + self.singularity_image_dir = singularity_image_dir or PULLFOLDER if self.singularity_image_dir and not self.JOB_TYPE: self.JOB_TYPE = "singularity" @@ -471,6 +470,12 @@ def __init__(self, name, data=None, self.workdir = None self.__make_workdir() + self.parameter_file_dir = parameter_file_dir or f'{self.workdir}/stimela_parameter_files' + if not os.path.exists(self.parameter_file_dir): + self.log.info( + f'Config directory cannot be found. Will create {self.parameter_file_dir}') + os.mkdir(self.parameter_file_dir) + def __make_workdir(self): timestamp = str(time.time()).replace(".", "") self.workdir = os.path.join(CDIR, f".stimela_workdir-{timestamp}") @@ -478,6 +483,18 @@ def __make_workdir(self): timestamp = str(time.time()).replace(".", "") self.workdir = os.path.join(CDIR, f".stimela_workdir-{timestamp}") os.mkdir(self.workdir) + # create passwd and group files to be mounted inside the container + template_dir = os.path.join(os.path.dirname(__file__), "cargo/base") + # get current user info + pw = pwd.getpwuid(os.getuid()) + gr = grp.getgrgid(pw.pw_gid) + with open(os.path.join(self.workdir, "passwd"), "wt") as file: + file.write(open(os.path.join(template_dir, "passwd.template"), "rt").read()) + file.write(f"{pw.pw_name}:x:{pw.pw_uid}:{pw.pw_gid}:{pw.pw_gecos}:/:/bin/bash") + with open(os.path.join(self.workdir, "group"), "wt") as file: + file.write(open(os.path.join(template_dir, "group.template"), "rt").read()) + file.write(f"{gr.gr_name}:x:{gr.gr_gid}:") + def add(self, image, name, config=None, input=None, output=None, msdir=None, diff --git a/stimela/singularity.py b/stimela/singularity.py index b9de2731..c08d3192 100644 --- a/stimela/singularity.py +++ b/stimela/singularity.py @@ -105,6 +105,9 @@ def run(self, *args, output_wrangler=None): Run a singularity container instance """ + #emptylocal = os.path.join(self.execdir, ".local") + #os.makedirs(emptylocal, mode=511, exist_ok=True) + #self.add_volume(emptylocal, "${HOME}/.local", "ro") if self.volumes: volumes = " --bind " + " --bind ".join(self.volumes) else: @@ -118,7 +121,8 @@ def run(self, *args, output_wrangler=None): self.status = "running" self._print("Starting container [{0:s}]. Timeout set to {1:d}. The container ID is printed below.".format( self.name, self.time_out)) - utils.xrun(f"cd {self.execdir} && singularity", ["run", "--workdir", self.execdir] \ + + utils.xrun(f"cd {self.execdir} && singularity", ["run", "--workdir", self.execdir, "--cleanenv"] \ + list(args) + [volumes, self.image, self.RUNSCRIPT], log=self.logger, timeout=self.time_out, output_wrangler=output_wrangler, env=self._env, logfile=self.logfile) diff --git a/stimela/tests/acceptance_tests/stimela-test-meerkat.py b/stimela/tests/acceptance_tests/stimela-test-meerkat.py index 5204671c..1100ceaa 100644 --- a/stimela/tests/acceptance_tests/stimela-test-meerkat.py +++ b/stimela/tests/acceptance_tests/stimela-test-meerkat.py @@ -129,7 +129,7 @@ def testEndToEndReduction(self): # # First selfcal round - recipe.add("cab/cubical", "cubical_cal", + recipe.add("cab/cubical_ddf", "cubical_cal", { 'data-ms': MS, 'data-column': "DATA",