added OIDC support

ChangeLog.txt

@@ -1,5 +1,9 @@
 ** Release Notes
 
+1.4.41
+  * added --notExpandaInDS to pathena/prun
+  * introduced PANDAMON_URL
+
 1.4.40
   * added OIDC support
 

pandatools/PandaToolsPkgInfo.py

@@ -1 +1 @@
-release_version = "1.4.40"
+release_version = "1.4.41"

pandatools/PathenaScript.py

@@ -154,6 +154,8 @@
                 help='Use differnet version of Athena on remote WN. By defualt the same version which you are locally using is set up on WN. e.g., --athenaTag=AtlasProduction,14.2.24.3')
 group_input.add_argument('--inDS',  action='store', dest='inDS',  default='',
                 type=str, help='Input dataset names. wildcard and/or comma can be used to concatenate multiple datasets')
+group_input.add_argument('--notExpandInDS', action='store_const', const=True, dest='notExpandInDS',default=False,
+                         help='Allow jobs to use files across dataset boundaries in input dataset container')
 group_input.add_argument('--inDsTxt',action='store',dest='inDsTxt',default='',
                 type=str, help='a text file which contains the list of datasets to run over. newlines are replaced by commas and the result is set to --inDS. lines starting with # are ignored')
 action = group_input.add_argument('--inOutDsJson', action='store', dest='inOutDsJson', default='',
@@ -1655,6 +1657,8 @@ def _onExit(dir, files, del_command):
                'expand':True,
                'exclude':'\.log\.tgz(\.\d+)*$',
                }
+    if options.notExpandInDS:
+        del tmpDict['expand']
     if options.inputType != '':
         tmpDict['include'] = options.inputType
     if options.filelist != []:

pandatools/PrunScript.py

@@ -106,6 +106,8 @@
                   help='Displays version')
 group_input.add_argument('--inDS',action='store',dest='inDS',default='',
                   help='Name of an input dataset or dataset container')
+group_input.add_argument('--notExpandInDS', action='store_const', const=True, dest='notExpandInDS',default=False,
+                         help='Allow jobs to use files across dataset boundaries in input dataset container')
 group_input.add_argument('--inDsTxt',action='store',dest='inDsTxt',default='',
                   help='A text file which contains the list of datasets to run over. Newlines are replaced by commas and the result is set to --inDS. Lines starting with # are ignored')
 group_output.add_argument('--outDS',action='store',dest='outDS',default='',
@@ -1534,7 +1536,7 @@ def _onExit(dir, files, del_command):
                }
     if options.useLogAsInput:
         del tmpDict['exclude']
-    if options.loadXML is None:
+    if options.loadXML is None and not options.notExpandInDS:
         tmpDict['expand'] = True
     if options.nSkipFiles != 0:
         tmpDict['offset'] = options.nSkipFiles

pandatools/queryPandaMonUtils.py

@@ -1,3 +1,4 @@
+import os
 import time
 import datetime
 import json
@@ -12,6 +13,10 @@
     from urllib import urlencode
     from urllib2 import urlopen, Request, HTTPError, URLError
 
+try:
+    baseMonURL = os.environ['PANDAMON_URL']
+except Exception:
+    baseMonURL = 'https://bigpanda.cern.ch'
 
 HEADERS = {'Accept': 'application/json', 'Content-Type':'application/json'}
 
@@ -41,7 +46,7 @@ def query_tasks(jeditaskid=None, username=None, limit=10000, taskname=None, stat
         parmas['extra'] = 'metastruct'
     if sync:
         parmas['timestamp'] = timestamp
-    url = 'https://bigpanda.cern.ch/tasks/?{0}'.format(urlencode(parmas))
+    url = baseMonURL + '/tasks/?{0}'.format(urlencode(parmas))
     if verbose:
         sys.stderr.write('query url = {0}\n'.format(url))
         sys.stderr.write('headers   = {0}\n'.format(json.dumps(HEADERS)))
@@ -61,6 +66,8 @@ def query_tasks(jeditaskid=None, username=None, limit=10000, taskname=None, stat
         if verbose:
             sys.stderr.write('resp code = {0}\n'.format(rec))
         res = rep.read().decode('utf-8')
+        if verbose:
+            sys.stderr.write('data = {0}\n'.format(res))
         ret = json.loads(res)
         return timestamp, url, ret
     except Exception as e: