diff --git a/scripts/logParser.py b/scripts/logParser.py index 15fe5a32c17df119f078c2f30c140e711273c056..0aa74bf3c631e7dc70aac5b9a4a6e9318f676dd7 100644 --- a/scripts/logParser.py +++ b/scripts/logParser.py @@ -2,7 +2,6 @@ import optparse, sys, math, subprocess, os from collections import OrderedDict -from csv import writer parser = optparse.OptionParser(usage=__doc__) parser.add_option("-i", "--input", default="-", dest="INPUT_FILE", metavar="PATH", help="input logfile") @@ -11,8 +10,6 @@ parser.add_option("-t", "--test", default=False, dest="TEST", action="store_true parser.add_option("-u", "--nocpu", default=False, dest="SKIPCPU", action="store_true", help="Ignore CPU timing information.") parser.add_option("-m", "--mcver", dest="MC_VER", default="mc", help="Specify MCXX campaign") parser.add_option("-c", "--nocolour", action="store_true", dest="NO_COLOUR", default=False, help="Turn off colour for copying to file") -parser.add_option("-x", "--csv", action="store", dest="OUTPUT_CSV", default="test.csv", help="Output csv file") -parser.add_option("-n", "--nocsv", action="store_true", dest="DISABLE_CSV", default=True, help="Do not store csv file") parser.add_option("-s", "--standalone", action="store_true", dest="STANDALONE", default=False, help="Run based on cvmfs location of files (stand-alone, no mcjoboptions locally)") opts, fileargs = parser.parse_args() @@ -230,50 +227,7 @@ def main(): else: if not len(JOsDict["including file \""+MCXX]): JOsErrors.append("including file \""+MCXX) - - DSIDxxx='' - topJO='' - nTopJO=0 - loginfo( '- jobOptions =',"") - for jo in JOsList: - gitexcomm="find -L "+location+" -type f -name "+jo - retcode = subprocess.Popen(gitexcomm, shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE).communicate() - gitJOs=retcode[0].decode() - if not gitJOs: - raise RuntimeError("ERROR: {0} jobOptions not found in git!".format(jo)) - pieces=gitJOs.split('/') - try: - if len(pieces) and pieces[-1].startswith(MCXX): - ##This is top JO - nTopJO=nTopJO+1 - topJO=jo - DSID=pieces[-2] - DSIDxxx=pieces[-3] - except: - raise RuntimeError("Unknown format of jO file: {0}\nShould look something like ./421xxx/421001/mc.Sherpa_Example1.py".format(gitJOs)) - - if nTopJO!=1: - logerr( "","ERROR: !=1 (%i) \"top\" JO files found!"%nTopJO) - raise RuntimeError("!= 1 \"top\" JO file found") - else: - for jo in JOsList: - if jo == topJO: - if os.path.exists(location+"/{0}/{1}/{2}".format(DSIDxxx,DSID,jo)): - loggood("",jo) - else: - logerr("",jo+" ERROR <-- jobOptions not found in git!") - else: - indices = [i for i,s in enumerate(gitJOs) if "/"+jo in s] - if len(indices)==1: - loggood("",jo) - else: - if not len(indices): - logerr("",jo+" - jobOptions not found in git!") - else: - logwarn("",jo+" - multiple instances of jobOptions not found in git!") - for ix in indices: - logwarn("",jo+" --> %s"%gitJOs[ix]) - + #Checking release release="not found" if not len(JOsDict['using release']): @@ -595,30 +549,6 @@ def main(): else: logerr("Errors : "+str(LogCounts.Errors)+" , Warnings : "+str(LogCounts.Warnings)," -> Errors encountered! Not ready for production!") print("") - - - #Write csv file output - cols=['Brief desciption','JobOptions','CoM energy [GeV]','Events (Evgen-only)','Events (FullSim)','Events (Atlfast II)','Priority','Output formats','Cross section [pb]','Effective luminosity [fb-1]','Filter efficiency','Evgen CPU time/job [hr]','Input files','MC-tag','Release','Comments','Evgen tag','Simul tag','Merge tag','Digi tag','Reco tag','Rec Merge tag','Atlfast tag','Atlf Merge tag'] - row=[] - row=pad(row,24,"") - row[cols.index('JobOptions')]=topJO - #row[cols.index('CoM energy [GeV]')]=13000. - #row[cols.index('Events (Evgen-only)')]=opts.TOTAL_EVENTS - if opts.TOTAL_EVENTS: - row[cols.index('Events (Evgen-only)')]=str(opts.TOTAL_EVENTS)+" (CHECK MANUALLY)" - row[cols.index('Cross section [pb]')]=xs_nb*1000. - if opts.TOTAL_EVENTS: - row[cols.index('Effective luminosity [fb-1]')]=eff_lumi_fb - row[cols.index('Filter efficiency')]=filt_eff - row[cols.index('Evgen CPU time/job [hr]')]=cpuPerJob - row[cols.index('Release')]=release - #for n,c in enumerate(cols): - # print c,row[n] - if not opts.DISABLE_CSV: - outCSV=open(opts.OUTPUT_CSV,'w') - outCSVwriter=writer(outCSV) - outCSVwriter.writerow(row) - outCSV.close() return