WireCell Noise-Filtering Updates (Data + Simulation)

sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/chndb-base.jsonnet

@@ -20,17 +20,21 @@ function(params, anode, field, n, rms_cuts=[])
     // For MicroBooNE, channel groups is a 2D list.  Each element is
     // one group of channels which should be considered together for
     // coherent noise filtering.
-    groups: [std.range(   0 + n * 5638 + g*32,    0 + n * 5638 + (g+1)*32 - 1) for g in std.range(0,149)] +
-            [std.range(4806 + n * 5638 + g*32, 4806 + n * 5638 + (g+1)*32 - 1) for g in std.range(0,25)] ,
+    // 5638 is the number of channels in a single APA, (1984*2 + 1670), including the 6 channel gap.
+    // 4806 is the number of channels for both induction + half of real collection wires + 6 channel gap
+    groups: [std.range(   0 + n * 5638 + g*32,    0 + n * 5638 + (g+1)*32 - 1) for g in std.range(0,150)] +
+            [std.range(4806 + n * 5638 + g*32, 4806 + n * 5638 + (g+1)*32 - 1) for g in std.range(0,26)] ,
 
 
     // Externally determined "bad" channels.
     //
     // Dead channels: 3232:3263 (inclusive) (East V).   4160:4191 (East Y)
+    // Jumpered region: 4800:4805 (inclusive, East Y), 10438:10443 (inclusive, West Y)
+    // No response: 546, 607, 8574
     // Shorted channels:  7169 (West U), 8378 (West V).
     // There are four physically missing wires ( = bad channels) due to combs, in the center of each 1/2 APA.
     // They are 4374 and 5231 (East Y), 10012 and 10869 (West Y).
-    // So in total, there are 76 bad channels.
+    // So in total, there are 88 bad channels.
     // 
     //bad: [],
     bad: [546, 607] + std.range(3232, 3263) + std.range(4160, 4191) + [4374, 4800, 4801, 4802, 4803, 4804, 4805, 5060, 5231, 5636, 5637, 7169, 8378, 8574, 10012, 10869, 10438, 10439, 10440, 10441, 10442, 10443],
@@ -53,15 +57,15 @@ function(params, anode, field, n, rms_cuts=[])
         response_offset: 0.0,  // ticks?
         pad_window_front: 10,  // ticks?
         pad_window_back: 10,  // ticks?
-        decon_limit: 0.02,
+        decon_limit: 0.02, // (SignalProtection, same as upper_decon_limit (default 0.02))
         decon_limit1: 0.09,
-        adc_limit: 15,
+        adc_limit: 10,
         roi_min_max_ratio: 0.8, // default 0.8
         min_rms_cut: 1.0,  // units???
         max_rms_cut: 30.0,  // units???
 
         // parameter used to make "rcrc" spectrum
-        rcrc: 1.1 * wc.millisecond, // 1.1 for collection, 3.3 for induction
+        rcrc: 0.5 * wc.millisecond, // 1.1 for collection, 3.3 for induction
         rc_layers: 1, // default 2
 
         // parameters used to make "config" spectrum

sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/chndb-perfect.jsonnet

@@ -27,10 +27,12 @@ function(params, anode, field, n, rms_cuts=[])
     // Externally determined "bad" channels.
     //
     // Dead channels: 3232:3263 (inclusive) (East V).   4160:4191 (East Y)
+    // Jumpered region: 4800:4805 (inclusive, East Y), 10438:10443 (inclusive, West Y)
+    // No response: 546, 607, 8574
     // Shorted channels:  7169 (West U), 8378 (West V).
     // There are four physically missing wires ( = bad channels) due to combs, in the center of each 1/2 APA.
     // They are 4374 and 5231 (East Y), 10012 and 10869 (West Y).
-    // So in total, there are 76 bad channels.
+    // So in total, there are 88 bad channels.
     // 
     //bad: [],
     bad: [546, 607] + std.range(3232, 3263) + std.range(4160, 4191) + [4374, 4800, 4801, 4802, 4803, 4804, 4805, 5060, 5231, 5636, 5637, 7169, 8378, 8574, 10012, 10869, 10438, 10439, 10440, 10441, 10442, 10443],
@@ -61,7 +63,7 @@ function(params, anode, field, n, rms_cuts=[])
         max_rms_cut: 30.0,  // units???
 
         // parameter used to make "rcrc" spectrum
-        rcrc: 1.1 * wc.millisecond, // 1.1 for collection, 3.3 for induction
+        rcrc: 0.5 * wc.millisecond, // 1.1 for collection, 3.3 for induction
         rc_layers: 1, // default 2
 
         // parameters used to make "config" spectrum

sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/funcs.jsonnet

@@ -50,6 +50,7 @@ local g = import 'pgraph.jsonnet';
             trace: {
               //gauss: 'gauss%d' % n,
               //wiener: 'wiener%d' % n,
+              ['raw%d' % n]: ['raw%d' % n],
               ['gauss%d' % n]: ['gauss%d' % n],
               ['wiener%d' % n]: ['wiener%d' % n],
               ['threshold%d' % n]: ['threshold%d' % n],

sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/nf.jsonnet

@@ -61,7 +61,7 @@ function(params, anode, chndbobj, n, name='', dft=default_dft)
           wc.tn(single),
         ],
         grouped_filters: [
-          //wc.tn(grouped),
+          // wc.tn(grouped),
         ],
         channel_status_filters: [
         ],

sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/params.jsonnet

@@ -86,7 +86,11 @@ base {
     },
 
     daq: super.daq {
-        nticks: 3415,
+        nticks: 3427,
+    },
+
+    nf: super.nf { 
+        nsamples: $.daq.nticks,
     },
 
     adc: super.adc {

sbndcode/WireCell/cfg/pgrapher/experiment/sbnd/wcls-nf-data.jsonnet

@@ -0,0 +1,213 @@
+// This is a main entry point to configure a WC/LS job that applies
+// noise filtering and signal processing to existing RawDigits.  The
+// FHiCL is expected to provide the following parameters as attributes
+// in the "params" structure.
+//
+// epoch: the hardware noise fix expoch: "before", "after", "dynamic" or "perfect"
+// reality: whether we are running on "data" or "sim"ulation.
+// raw_input_label: the art::Event inputTag for the input RawDigit
+//
+// see the .fcl of the same name for an example
+//
+// Manual testing, eg:
+//
+// jsonnet -V reality=data -V epoch=dynamic -V raw_input_label=daq \\
+//         -V signal_output_form=sparse \\
+//         -J cfg cfg/pgrapher/experiment/uboone/wcls-nf-sp.jsonnet
+//
+// jsonnet -V reality=sim -V epoch=perfect -V raw_input_label=daq \\
+//         -V signal_output_form=sparse \\
+//         -J cfg cfg/pgrapher/experiment/uboone/wcls-nf-sp.jsonnet
+
+
+//local epoch = std.extVar('epoch');  // eg "dynamic", "after", "before", "perfect"
+local sigoutform = std.extVar('signal_output_form');  // eg "sparse" or "dense"
+local raw_input_label = std.extVar('raw_input_label');  // eg "daq"
+local use_paramresp = std.extVar('use_paramresp');  // eg "true" or "false"
+
+local g = import 'pgraph.jsonnet';
+local f = import 'pgrapher/experiment/sbnd/funcs.jsonnet';
+local wc = import 'wirecell.jsonnet';
+local tools_maker = import 'pgrapher/common/tools.jsonnet';
+
+local data_params = import 'params.jsonnet';
+
+local params = data_params {
+    daq: super.daq { // <- super.daq overrides default values
+      // Set the waveform sample length, eg, 6000, 15000, "auto"
+      nticks: std.extVar('nticks'),
+    },
+    # provides filename for parametrized per channel electronics response correction
+    # default is: use_parmresp = false
+    files: super.files {
+      chresp: if (use_paramresp == 'true') then "sbnd-params-chresp-ideal-v1.json.bz2" else null,
+    },
+};
+
+
+local tools = tools_maker(params);
+
+
+local mega_anode = {
+  type: 'MegaAnodePlane',
+  name: 'meganodes',
+  data: {
+    anodes_tn: [wc.tn(anode) for anode in tools.anodes],
+  },
+};
+
+local wcls_maker = import 'pgrapher/ui/wcls/nodes.jsonnet';
+local wcls = wcls_maker(params, tools);
+
+// Collect the WC/LS input converters for use below.  Make sure the
+// "name" argument matches what is used in the FHiCL that loads this
+// file.  In particular if there is no ":" in the inputer then name
+// must be the emtpy string.
+local wcls_input = {
+  adc_digits: g.pnode({
+    type: 'wclsRawFrameSource',
+    name: '',
+    data: {
+      art_tag: raw_input_label,
+      frame_tags: ['orig'],  // this is a WCT designator
+      // nticks: params.daq.nticks,
+    },
+  }, nin=0, nout=1),
+
+};
+
+
+// Collect all the wc/ls output converters for use below.  Note the
+// "name" MUST match what is used in theh "outputers" parameter in the
+// FHiCL that loads this file.
+
+local wcls_output = {
+  // The noise filtered "ADC" values.  These are truncated for
+  // art::Event but left as floats for the WCT SP.  Note, the tag
+  // "raw" is somewhat historical as the output is not equivalent to
+  // "raw data".
+  nf_digits: g.pnode({
+    type: 'wclsFrameSaver',
+    name: 'nfsaver',
+    data: {
+      // anode: wc.tn(tools.anode),
+      anode: wc.tn(mega_anode),
+      digitize: true,  // true means save as RawDigit, else recob::Wire
+      frame_tags: ['raw'],
+      nticks: params.daq.nticks,
+      chanmaskmaps: ['bad'],
+    },
+  }, nin=1, nout=1, uses=[mega_anode]),
+
+
+  // The output of signal processing.  Note, there are two signal
+  // sets each created with its own filter.  The "gauss" one is best
+  // for charge reconstruction, the "wiener" is best for S/N
+  // separation.  Both are used in downstream WC code.
+  sp_signals: g.pnode({
+    type: 'wclsFrameSaver',
+    name: 'spsaver',
+    data: {
+      anode: wc.tn(mega_anode),
+      digitize: false,  // true means save as RawDigit, else recob::Wire
+      frame_tags: ['gauss', 'wiener'],
+
+      // this may be needed to convert the decon charge [units:e-] to be consistent with the LArSoft default ?unit? e.g. decon charge * 0.005 --> "charge value" to GaussHitFinder
+      frame_scale: [0.02, 0.02],
+      nticks: params.daq.nticks,
+
+
+      // uncomment the below configs to save summaries and cmm
+      summary_tags: ['wiener'],
+      summary_operator: {wiener: 'set'},
+      summary_scale: [0.02], # summary scale should be the same as frame_scale
+      chanmaskmaps: ['bad'],
+    },
+  }, nin=1, nout=1, uses=[mega_anode]),
+};
+
+local base = import 'pgrapher/experiment/sbnd/chndb-base.jsonnet';
+
+local chndb = [{
+  type: 'OmniChannelNoiseDB',
+  name: 'ocndbperfect%d' % n,
+  data: base(params, tools.anodes[n], tools.field, n){dft:wc.tn(tools.dft)},
+  uses: [tools.anodes[n], tools.field, tools.dft],
+} for n in std.range(0, std.length(tools.anodes) - 1)];
+
+local chsel_pipes = [
+  g.pnode({
+    type: 'ChannelSelector',
+    name: 'chsel%d' % n,
+    data: {
+      channels: std.range(5638 * n, 5638 * (n + 1) - 1),
+      //tags: ['orig%d' % n], // traces tag
+    },
+  }, nin=1, nout=1)
+  for n in std.range(0, std.length(tools.anodes) - 1)
+];
+
+local nf_maker = import 'pgrapher/experiment/sbnd/nf-data.jsonnet'; //added Ewerton 2024-08
+local nf_pipes = [nf_maker(params, tools.anodes[n], chndb[n], n, name='nf%d' % n) for n in std.range(0, std.length(tools.anodes) - 1)];
+
+local sp_maker = import 'pgrapher/experiment/sbnd/sp.jsonnet';
+local sp = sp_maker(params, tools, { sparse: sigoutform == 'sparse' });
+local sp_pipes = [sp.make_sigproc(a) for a in tools.anodes];
+
+local magoutput = 'sbnd-data-check.root';
+local magnify = import 'pgrapher/experiment/sbnd/magnify-sinks.jsonnet';
+local sinks = magnify(tools, magoutput);
+
+local nfsp_pipes = [
+  g.pipeline([
+               chsel_pipes[n],
+              //  sinks.orig_pipe[n],
+
+               nf_pipes[n],
+              //  sinks.raw_pipe[n],
+
+               // sp_pipes[n],
+               // sinks.decon_pipe[n],
+               // sinks.threshold_pipe[n],
+               // sinks.debug_pipe[n], // use_roi_debug_mode=true in sp.jsonnet
+             ],
+             'nfsp_pipe_%d' % n)
+  for n in std.range(0, std.length(tools.anodes) - 1)
+];
+
+local fanpipe = f.fanpipe('FrameFanout', nfsp_pipes, 'FrameFanin', 'sn_mag_nf');
+
+local retagger = g.pnode({
+  type: 'Retagger',
+  data: {
+    // Note: retagger keeps tag_rules an array to be like frame fanin/fanout.
+    tag_rules: [{
+      // Retagger also handles "frame" and "trace" like fanin/fanout
+      // merge separately all traces like gaussN to gauss.
+      frame: {
+        '.*': 'retagger',
+      },
+      merge: {
+        'raw\\d': 'raw',
+        // 'gauss\\d': 'gauss',
+        // 'wiener\\d': 'wiener',
+      },
+    }],
+  },
+}, nin=1, nout=1);
+
+local sink = g.pnode({ type: 'DumpFrames' }, nin=1, nout=0);
+
+
+// local graph = g.pipeline([wcls_input.adc_digits, rootfile_creation_frames, fanpipe, retagger, wcls_output.sp_signals, sink]);
+local graph = g.pipeline([wcls_input.adc_digits, fanpipe, retagger, wcls_output.nf_digits, sink]);
+
+local app = {
+  type: 'TbbFlow',
+  data: {
+    edges: g.edges(graph),
+  },
+};
+
+// Finally, the configuration sequence
+g.uses(graph) + [app]

sbndcode/WireCell/wcsp_data_sbnd.fcl

@@ -57,7 +57,7 @@ sbnd_wcls_sp_data.wcls_main.params :   {
 
 sbnd_wcls_sp_data.wcls_main.structs:   { 
                                         # Set the waveform sample length, eg, 6000, 15000, "auto"
-                                        nticks: 3415
+                                        nticks: 3427
                                        }
 
 END_PROLOG