Issue 434 422 455 (#456)

* bump dependencies to ddf 0.6.0

* fixes #434

* Fixes #422

* Fixes #455

* Fixes bug introduced by merge of #437 

* Cleans up logs

cubical/data_handler/ms_tile.py

@@ -171,6 +171,7 @@ def load_ddfacet_models(self, uvwco, loaded_models, model_source, cluster, imod,
             ndirs = model_source._nclus
             loaded_models[model_source] = {}
             ddfsim.set_model_provider(model_source)
+            log.info("Predicting targeted directions... This may take some time")
             for idir, clus in enumerate(model_source._cluster_keys):
                 ddfsim.set_direction(clus)
                 model = ddfsim.simulate(self.tile.dh,
@@ -1140,6 +1141,8 @@ def get_chunk_cubes(self, key, ctype=np.complex128, allocator=np.empty, flag_all
 
         # These points correspond to the autocorrelations - they should always
         # be flagged for the purpose of calibration.
+        assert flags.ndim == 4 and flags.shape[2] == flags.shape[3]
+        nants = flags.shape[3]
         flags[..., range(nants), range(nants)] |= FL.SKIPSOL
 
         obs_arr = subset._column_to_cube(data['obvis'], t_dim, f_dim, row_index, freq_slice, ctype,

cubical/degridder/DDFacetSim.py

@@ -322,16 +322,16 @@ def __vis_2_time_map(DicoSols, times):
             dt_start = self.__mjd2dt([np.min(utc_times)])[0].strftime('%Y/%m/%d %H:%M:%S')
             dt_end = self.__mjd2dt([np.max(utc_times)])[0].strftime('%Y/%m/%d %H:%M:%S')
             if provider is None:
-                log.info("Setting E Jones to unity for '{0:s}' direction '{1:s}' facet '{2:d}' between {3:s} and {4:s} UTC for "
-                         "fractional bandwidth {5:.2f}~{6:.2f} MHz".format(
-                         str(self.__model), str(self.__direction), sub_region_index, dt_start, dt_end, 
-                         np.min(chunk_ctr_frequencies*1e-6), np.max(chunk_ctr_frequencies*1e-6)))
+                log(2).print("Setting E Jones to unity for '{0:s}' direction '{1:s}' facet '{2:d}' between {3:s} and {4:s} UTC for "
+                             "fractional bandwidth {5:.2f}~{6:.2f} MHz".format(
+                             str(self.__model), str(self.__direction), sub_region_index, dt_start, dt_end, 
+                             np.min(chunk_ctr_frequencies*1e-6), np.max(chunk_ctr_frequencies*1e-6)))
                 jones_matrix = None
             elif provider == "FITS":
-                log.info("Calculating E Jones for '{0:s}' direction '{1:s}' facet '{2:d}' between {3:s} and {4:s} UTC for "
-                         "fractional bandwidth {5:.2f}~{6:.2f} MHz".format(
-                         str(self.__model), str(self.__direction), sub_region_index, dt_start, dt_end, 
-                         np.min(chunk_ctr_frequencies*1e-6), np.max(chunk_ctr_frequencies*1e-6)))
+                log(2).print("Calculating E Jones for '{0:s}' direction '{1:s}' facet '{2:d}' between {3:s} and {4:s} UTC for "
+                             "fractional bandwidth {5:.2f}~{6:.2f} MHz".format(
+                             str(self.__model), str(self.__direction), sub_region_index, dt_start, dt_end, 
+                             np.min(chunk_ctr_frequencies*1e-6), np.max(chunk_ctr_frequencies*1e-6)))
                 beam_provider = FITSBeamInterpolator(field_centre,
                                                      chunk_ctr_frequencies,
                                                      chunk_channel_widths,  
@@ -438,8 +438,8 @@ def __init_grid_machine(self, src, dh, tile, poltype, freqs, chan_widths):
         gmachines = []
         dname = self.__cachename_compute(src)
         if should_init_cf:
-            log.info("This is the first time predicting for '{0:s}' direction '{1:s}'. "
-                     "Initializing degridder for {2:d} facets - this may take a wee bit of time.".format(
+            log(2).print("This is the first time predicting for '{0:s}' direction '{1:s}'. "
+                         "Initializing degridder for {2:d} facets - this may take a wee bit of time.".format(
                          str(self.__model), str(self.__direction), src.subregion_count))
             DDFacetSim.__initted_CF_directions.append(dname)    
             DDFacetSim.__direction_CFs[dname] = DDFacetSim.__direction_CFs.get(dname, []) + list(DDFacetSim.__ifacet + np.arange(src.subregion_count)) #unique facet index for this subregion
@@ -535,15 +535,15 @@ def simulate(self, dh, tile, tile_subset, poltype, uvwco, freqs, model_type, cha
         region_model = np.zeros((nrow, nfreq, 4), dtype=np.complex64)
         flagged = np.zeros_like(region_model, dtype=np.bool)
         # now we predict for this direction
-        log.info("Computing visibilities in {1:d} facets for direction '{0:s}' for model '{2:s}'...".format(
-            str(self.__direction), src.subregion_count, str(self.__model)))
+        log(2).print("\tComputing visibilities in {1:d} facets for direction '{0:s}' for model '{2:s}'...".format(
+                     str(self.__direction), src.subregion_count, str(self.__model)))
 
         for gm, subregion_index in zip(gmacs, range(src.subregion_count)):
             model = np.zeros((nrow, nfreq, 4), dtype=np.complex64).copy()
             model_image = src.get_degrid_model(subregion_index=subregion_index).astype(dtype=np.complex64).copy() #degridder needs transposes, dont mod the data globally
 
             if not np.any(model_image):
-                log(2).print("Facet {0:d} is empty. Skipping".format(subregion_index))
+                log(2).print("\tFacet {0:d} is empty. Skipping".format(subregion_index))
                 continue
             dname = self.__cachename_compute(src)
             model_image = DDFacetSim.__detaper_model(gm, model_image.view(), self.__direction_CFs[dname][subregion_index]).copy() # degridder don't respect strides must be contiguous

cubical/degridder/DicoSourceProvider.py

@@ -5,7 +5,7 @@
 except ImportError:
     raise ImportError("Cannot import DDFacet")
 
-from regions import DS9Parser
+from regions import Regions, PolygonPixelRegion
 import numpy as np
 from .geometry import BoundingConvexHull, BoundingBox, BoundingBoxFactory
 from cubical.tools import logger, ModColor
@@ -105,14 +105,14 @@ def __read_regions_file(self, fn, padding_factor, max_size, min_nfacet_per_axis)
         """
         clusters = []
         if fn is not None: # dde case
-            with open(fn) as f:
-                parser = DS9Parser(f.read())
-                for regi, reg in enumerate(parser.shapes):
-                    coords = list(map(int, [c.value for c in reg.coord]))
-                    assert len(coords) % 2 == 0, "Number of region coords must be multiple of 2-tuple"
-                    coords = np.array(coords).reshape([len(coords) // 2, 2])
-                    clusters.append(BoundingConvexHull(coords,
-                                                       name="DDE_REG{0:d}".format(regi + 1)))
+            shapes = Regions.read(fn)
+            if not all([type(reg) is PolygonPixelRegion for reg in shapes]):
+                raise RuntimeError("Currently only supports regions of type 'polygon' with 'physical' (pixel) coordinates as input regions")
+            for regi, reg in enumerate(shapes):
+                coords = np.array(list(zip(map(int, reg.vertices.x), 
+                                            map(int, reg.vertices.y))))
+                clusters.append(BoundingConvexHull(coords,
+                                                    name="DDE_REG{0:d}".format(regi + 1)))
         else: # die case
             clusters = [BoundingBox(0,
                                     self.__nx - 1,

cubical/machines/interval_gain_machine.py

@@ -557,14 +557,31 @@ def issue_warning(msg, sort_index=70):
 
             if len(bad_dirs):
                 if log.verbosity() > 1:
+                    def __extract_dirs(d):
+                        if (type(d) is list or type(d) is np.ndarray) and len(d) == 1:
+                            return str(d[0])
+                        elif (type(d) is list or type(d) is np.ndarray):
+                            return ", ".join(d)
+                        else:
+                            return str(d)
+
+                    def __extract_dirs_perc(d):
+                        if (type(d) is list or type(d) is np.ndarray) and len(d) == 1:
+                            return "{0:.3f}%".format(percflagged[d[0]])
+                        elif (type(d) is list or type(d) is np.ndarray):
+                            return ", ".join(["{0:.3f}%".format(percflagged[dd]) for dd in d])
+                        else:
+                            return "UNDETERMINED"
+
                     msg = "{} directions ({}) flagged {}".format(
                         len(bad_dirs),
-                        ", ".join(["dir {0:s}: {1:.3f}% gains affected".format(
-                            str(d), percflagged[d]) for d in bad_dirs]),
-                        why_flagged
-                    )
+                        ", ".join(["dir {0:s}: {1:s} gains affected".format(
+                                  __extract_dirs(d), __extract_dirs_perc(d)) 
+                            for d in bad_dirs]),
+                        why_flagged)
                 else:
-                    msg = "{} directions flagged {}".format(len(bad_dirs), why_flagged)
+                    msg = "{} directions flagged {}. Verbosity level 2 gives a full list of these directions".format(
+                        len(bad_dirs), why_flagged)
                 issue_warning(msg, 50)
 
     def _update_gain_flags(self, flagtype, flags_arr):

setup.py

@@ -88,7 +88,7 @@
       entry_points={'console_scripts': ['gocubical = cubical.main:main']},
       extras_require={
           'lsm-support': ['montblanc >= 0.6.4'],
-          'degridder-support': ['ddfacet >= 0.5.0', 
+          'degridder-support': ['ddfacet >= 0.6.0', 
                                 'regions >= 0.4',
                                 'meqtrees-cattery >= 1.7.0']
       }