diff --git a/ChangeLog.txt b/ChangeLog.txt index 0da20146..b4d5724e 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -1,5 +1,9 @@ ** Release Notes +1.4.41 + * added --notExpandaInDS to pathena/prun + * introduced PANDAMON_URL + 1.4.40 * added OIDC support diff --git a/pandatools/PandaToolsPkgInfo.py b/pandatools/PandaToolsPkgInfo.py index 88dcecfb..ce8e3ed0 100644 --- a/pandatools/PandaToolsPkgInfo.py +++ b/pandatools/PandaToolsPkgInfo.py @@ -1 +1 @@ -release_version = "1.4.40" +release_version = "1.4.41" diff --git a/pandatools/PathenaScript.py b/pandatools/PathenaScript.py index b7857533..75e36b2d 100644 --- a/pandatools/PathenaScript.py +++ b/pandatools/PathenaScript.py @@ -154,6 +154,8 @@ help='Use differnet version of Athena on remote WN. By defualt the same version which you are locally using is set up on WN. e.g., --athenaTag=AtlasProduction,14.2.24.3') group_input.add_argument('--inDS', action='store', dest='inDS', default='', type=str, help='Input dataset names. wildcard and/or comma can be used to concatenate multiple datasets') +group_input.add_argument('--notExpandInDS', action='store_const', const=True, dest='notExpandInDS',default=False, + help='Allow jobs to use files across dataset boundaries in input dataset container') group_input.add_argument('--inDsTxt',action='store',dest='inDsTxt',default='', type=str, help='a text file which contains the list of datasets to run over. newlines are replaced by commas and the result is set to --inDS. lines starting with # are ignored') action = group_input.add_argument('--inOutDsJson', action='store', dest='inOutDsJson', default='', @@ -1655,6 +1657,8 @@ def _onExit(dir, files, del_command): 'expand':True, 'exclude':'\.log\.tgz(\.\d+)*$', } + if options.notExpandInDS: + del tmpDict['expand'] if options.inputType != '': tmpDict['include'] = options.inputType if options.filelist != []: diff --git a/pandatools/PrunScript.py b/pandatools/PrunScript.py index 9642309b..65dc0920 100644 --- a/pandatools/PrunScript.py +++ b/pandatools/PrunScript.py @@ -106,6 +106,8 @@ help='Displays version') group_input.add_argument('--inDS',action='store',dest='inDS',default='', help='Name of an input dataset or dataset container') +group_input.add_argument('--notExpandInDS', action='store_const', const=True, dest='notExpandInDS',default=False, + help='Allow jobs to use files across dataset boundaries in input dataset container') group_input.add_argument('--inDsTxt',action='store',dest='inDsTxt',default='', help='A text file which contains the list of datasets to run over. Newlines are replaced by commas and the result is set to --inDS. Lines starting with # are ignored') group_output.add_argument('--outDS',action='store',dest='outDS',default='', @@ -1534,7 +1536,7 @@ def _onExit(dir, files, del_command): } if options.useLogAsInput: del tmpDict['exclude'] - if options.loadXML is None: + if options.loadXML is None and not options.notExpandInDS: tmpDict['expand'] = True if options.nSkipFiles != 0: tmpDict['offset'] = options.nSkipFiles diff --git a/pandatools/queryPandaMonUtils.py b/pandatools/queryPandaMonUtils.py index 8b819432..59d46015 100644 --- a/pandatools/queryPandaMonUtils.py +++ b/pandatools/queryPandaMonUtils.py @@ -1,3 +1,4 @@ +import os import time import datetime import json @@ -12,6 +13,10 @@ from urllib import urlencode from urllib2 import urlopen, Request, HTTPError, URLError +try: + baseMonURL = os.environ['PANDAMON_URL'] +except Exception: + baseMonURL = 'https://bigpanda.cern.ch' HEADERS = {'Accept': 'application/json', 'Content-Type':'application/json'} @@ -41,7 +46,7 @@ def query_tasks(jeditaskid=None, username=None, limit=10000, taskname=None, stat parmas['extra'] = 'metastruct' if sync: parmas['timestamp'] = timestamp - url = 'https://bigpanda.cern.ch/tasks/?{0}'.format(urlencode(parmas)) + url = baseMonURL + '/tasks/?{0}'.format(urlencode(parmas)) if verbose: sys.stderr.write('query url = {0}\n'.format(url)) sys.stderr.write('headers = {0}\n'.format(json.dumps(HEADERS))) @@ -61,6 +66,8 @@ def query_tasks(jeditaskid=None, username=None, limit=10000, taskname=None, stat if verbose: sys.stderr.write('resp code = {0}\n'.format(rec)) res = rep.read().decode('utf-8') + if verbose: + sys.stderr.write('data = {0}\n'.format(res)) ret = json.loads(res) return timestamp, url, ret except Exception as e: