process-step-commands

#!/usr/bin/env python3

# Process job reports and create summary file with performance information.
from argparse import ArgumentParser
from os.path import exists, join
from os import fdopen, unlink, makedirs
from xml.sax import parseString, ContentHandler
import re
from hashlib import sha1
from json import dump, dumps
from tempfile import mkstemp
from shutil import move
from cmsutils import getRelValsFiles, readRelValFile


# Schema of the output
#
# {
#  "<hash>": ["<command>", "<type>"]
# }
#
# <command> is the shell command used.
# <type> is the type of the command, either CMSDRIVER or DAS.
# <hash> is the hash of the command, for fast lookup.

if __name__ == "__main__":
  parser = ArgumentParser()
  parser.add_argument("input", help="input file")
  parser.add_argument("output", help="output dir")
  args = parser.parse_args()
  # We start with empty workflows
  results = []
  if not exists:
    parser.error("%s does not exist")

  inFiles, zipFile = getRelValsFiles(args.input, "*/cmdLog", ".*/cmdLog")
  try:
    [makedirs(join(args.output, x)) for x in "0123456789abcdef"]
  except:
    if not exists(args.output):
      parser.error("Unable to create " + args.output)
      exit(0)

  for name in inFiles:
    m = re.match("(.*/|)([0-9.]+)_([^/]*)/cmdLog", name)
    if not m: continue
    xpath, workflowId, workflow = m.groups()
    report = readRelValFile(name,zipFile)
    report += "\n#"
    command = ""
    lumi_cmd = ""
    for l in report.split("\n"):
      if not l.strip(): continue
      if not l.startswith("#"):
        command=command+l+"\n"
        continue
      if not command: continue
      if ("_lumiRanges.log" in command) and (not "--lumiToProcess" in command):
        lumi_cmd = command
        command = ""
        continue
      step = [c for c in command.split(" ") if c.startswith("step")][-1].split("_",1)[0].replace('step','')
      t = "CMSDRIVER"
      if ("das_client" in command) or ("dasgoclient" in command):
        command = lumi_cmd+re.sub(" --cache.*das-cache.file", " --cache .../das-cache.file", command)
        lumi_cmd = ""
        t = "DAS"
      command = command.strip()
      command = re.sub('python(2|3|)\s+', 'python ', command)
      command = re.sub('[^\s\'"]*/monitor_workflow.py','monitor_workflow.py', command)
      h = sha1(command.encode()).hexdigest()[0:16]
      # One file per hash
      (fh, filename) = mkstemp()
      file = fdopen(fh, "w")
      dump({"command": command, "type": t}, file, sort_keys=True)
      file.close()
      try:
        destFile = join(args.output, h[0], h[1:])
        move(filename, destFile)
      except:
        print ("Unable to move " + destFile + " into place.")
        unlink(filename)
      results.append((workflowId + "-" + str(step), h))
      command = ""
  print (dumps(dict(results), sort_keys=True, indent=2, separators=(',', ': ')))