cleanup-old-results

#! /usr/bin/env python3
from os import unlink
from os.path import exists, basename, join
from subprocess import getstatusoutput
from argparse import ArgumentParser
from re import match
from json import load
DATA_DIR='data'

#
# clean up the results in the given directory that are older than the date given as a parameter
# the IB date must be in the same format as in the IBs
#
def clean_up_results(earliest_date ):
  dexp = '20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]-[0-9][0-9][0-9][0-9]'
  e, o = getstatusoutput("find %s -name '%s' -o -name '*%s*'" % (DATA_DIR, dexp, dexp))
  if e:
    print (o)
    return
  for cdir in o.split("\n"):
    m = match("^(.+|)(%s)(.+|)$" % dexp,basename(cdir))
    current_date = m.group(2)
    if current_date < earliest_date:
      if args.dryRun:
        print( 'Not deleting (dry-run): %s' % cdir )
      else:
        print( 'Deleting: %s because it was too old' % cdir )
        getstatusoutput("rm -rf %s" % cdir)
  return

def cleanup_old_releases(data):
  active_rels_file = join(data,'structure.json')
  if not exists(active_rels_file):
    return
  e, o = getstatusoutput("ls %s/CMSSW_*.json | sed 's|.*/||;s|.json$||'" %  data)
  if e:
    print(o)
    return
  all_rels = [r for r in o.split('\n') if r]
  e, o = getstatusoutput("ls %s/CMSSW_*_summary.txt | sed 's|.*/||;s|_summary.txt$||'" %  data)
  all_rels += [x for x in o.split('\n') if x.startswith('CMSSW_')]
  all_rels = list(set(all_rels))
  active_rels = load(open(active_rels_file))['all_release_queues']
  for rel in all_rels:
    if not rel in active_rels:
      if args.dryRun: continue
      for ext in ['.json', '_summary.txt']:
        rel_file = "%s/%s%s" % (data, rel, ext)
        if exists(rel_file):
          unlink(rel_file)
          print("Deleting ",rel_file)
  return

def cleanup_commands():
  e, o = getstatusoutput("find %s/commands -name '*.json' | xargs cat | sed 's|.*: \"||;s|\".*||' | grep '^[0-9a-f]' | sort | uniq" %  DATA_DIR)
  if e:
    print (o)
    return
  active_cmds = [i for i in o.split("\n")]
  print ("Active cmds: %s" % len(active_cmds))
  e , o = getstatusoutput("find %s/commands/objs -type f | sed 's|.*/objs/||'" % DATA_DIR)
  if e:
    print (o)
    return
  for f in o.split("\n"):
    i = f.replace("/","")
    if i not in active_cmds:
      cfile = "%s/commands/objs/%s" % (DATA_DIR, f)
      if args.dryRun:
        print("Not deleting (dry-run): %s" % cfile)
      else:
        print("Deleting unused command: %s" % cfile)
        unlink(cfile)
  return

#-----------------------------------------------------------------------------------
#---- Start
#-----------------------------------------------------------------------------------

if __name__ == "__main__":

  parser = ArgumentParser()
  parser.add_argument("-d","--date", type=str, help="Results older than this date will be deleted. "
                                                    "The date must be in the same format as it is in the IBs"
                                                    , required=True)
  parser.add_argument("-n", "--dry-run", dest="dryRun", help="Don't actually delete anything", action="store_true", default=False)
  args = parser.parse_args()
  if exists(DATA_DIR):
    clean_up_results(args.date)
    cleanup_commands()
    cleanup_old_releases(DATA_DIR)
  if exists('_data'):
    cleanup_old_releases('_data')