diff --git a/src/ILCDIRAC/ILCTransformationSystem/scripts/dirac_ilc_production_summary.py b/src/ILCDIRAC/ILCTransformationSystem/scripts/dirac_ilc_production_summary.py index 672f8f836a5b27701f087b22aae36821efee572f..9729047838c97cf7f403f885cc5b263918fda19e 100644 --- a/src/ILCDIRAC/ILCTransformationSystem/scripts/dirac_ilc_production_summary.py +++ b/src/ILCDIRAC/ILCTransformationSystem/scripts/dirac_ilc_production_summary.py @@ -25,6 +25,8 @@ # """Prepare the production summary tables. +Obtains the list of known detectors and their summary-table titles from the /Operations/Transformation/TablesTitles section in the Configuration System. + Options: -P, --prods prodID Productions: greater than with gt1234, range with 32-56, list with 34,56 -p, --precise_detail Precise detail, slow @@ -40,11 +42,13 @@ from collections import defaultdict import os import random import textwrap +import xml.etree.ElementTree as ET +import re from DIRAC.Core.Base.Script import Script from DIRAC.Core.Utilities.List import breakListIntoChunks from DIRAC import S_OK, exit as dexit, S_ERROR - +from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations def _getFileInfo(lfn, fc): """Retrieve the file info.""" @@ -66,10 +70,22 @@ def _getFileInfo(lfn, fc): if addinfo.count("{"): addinfo = eval(addinfo) else: - addinfo = DEncode.decode(addinfo)[0] + addinfo = DEncode.decode(addinfo.encode())[0] xsec = addinfo.get('xsection', {}).get('sum', {}).get('xsection', 0.0) return (float(lumi), int(nbevts), float(xsec)) +def _addDetectorOutputs(detector): + """Adds all the possible outputs to a new detector.""" + detector.update({'SIM': [], + 'REC': [], + 'DST': [], + 'delphes': [], + 'fastsim': [], + 'sim': [], + 'rec': [], + 'phys': [], + }) + return detector def _translate(detail): """Replace whizard naming convention by human conventions.""" @@ -251,15 +267,23 @@ def main(): continue prodtype = res['Value']['Type'] proddetail = res['Value']['Description'] - meta['Datatype'] = 'DST' if prodtype in ('MCReconstruction', 'MCReconstruction_Overlay') \ - else 'gen' if prodtype == 'MCGeneration' \ - else 'SIM' if prodtype == 'MCSimulation' else None + if prodtype in ['Split', 'Merge']: gLogger.warn("Invalid query for %s productions" % prodtype) continue - if not meta['Datatype']: + if prodtype not in ['MCReconstruction', 'MCReconstruction_Overlay', 'MCGeneration', 'MCSimulation']: gLogger.error("Unknown production type %s" % prodtype) continue + + res = fc.getCompatibleMetadata({'ProdID': prodID}) + if not (res['OK'] and res['Value'].get('Datatype')): + gLogger.error(f"Error getting transformation {prodID}: ", res.get("Message", "No 'Datatype' found for this transformation")) + continue + if 'delphes' in res['Value']['Datatype']: + meta['Datatype'] = 'delphes' + else: + meta['Datatype'] = res['Value']['Datatype'][0] + gLogger.verbose('Looking for files: %s' % meta) res = fc.findFilesByMetadata(meta) if not res['OK']: @@ -306,7 +330,7 @@ def main(): nbevts = info[1] * len(lfns) break - if not lumi: + if (not lumi) and (prodtype != 'MCGeneration'): xsec = 0 files = 0 depthDict = defaultdict(set) @@ -348,7 +372,7 @@ def main(): dirmeta['MomProdID'] = 0 if prodtype != 'MCGeneration': - res = trc.getTransformationMetaDataQuery(str(prodID), 'Input') + res = trc.getTransformationMetaQuery(str(prodID), 'Input') if res['OK']: if 'ProdID' in res['Value']: dirmeta['MomProdID'] = res['Value']['ProdID'] @@ -356,17 +380,7 @@ def main(): dirmeta['detail'] = _translate(detail) metadata.append(dirmeta) - detectors = {'ILD': {'SIM': [], 'REC': [], 'title': 'ILD CDR'}, - 'SID': {'SIM': [], 'REC': [], 'title': 'SID CDR'}, - 'sid': {'SIM': [], 'REC': [], 'title': 'sid dbd prod'}, - 'FCCee_o1_v04': {'SIM': [], 'REC': [], 'title': 'FCC studies'}, - 'gen': [], - } - - corres = {'MCGeneration': 'gen', - 'MCSimulation': 'SIM', - 'MCReconstruction': 'REC', - 'MCReconstruction_Overlay': 'REC'} + detectors = {'gen': []} for channel in metadata: if 'DetectorType' not in channel: @@ -377,23 +391,40 @@ def main(): channel['NumberOfEvents'] / channel['nb_files'], channel['NumberOfEvents'], channel['CrossSection'], str(channel['proddetail']))) + gLogger.notice(f'Transformation {channel["ProdID"]} found as generation.') + else: - if channel['DetectorType'] not in detectors: + ops = Operations() + detOptions = ops.getOptions("/Transformations/TablesTitles", listOrdered=False)['Value'] + matchDet = "" + # Check if the DetectorType of the transformation is a version of a known detector of the Configuration System + for det in detOptions: + # Take the longest match in order to not confuse idea and idea lar. + if channel['DetectorType'].startswith(det) and (len(det) > len(matchDet)): + matchDet = det + if matchDet: + detTitle = ops.getValue(f"/Transformations/TablesTitles/{matchDet}", None) + if matchDet not in detectors: + # If the detector is encountered for the first time, we create a new detector in the `detectors` dictionary, using the name found in the Configuration System + gLogger.notice(f'Detector "{matchDet}", matching transformation DetectorType "{channel["DetectorType"]}", found in Operations, with title "{detTitle}". Adding to known detectors.') + detectors[matchDet] = _addDetectorOutputs({'title': detTitle}) + else: gLogger.error("This is unknown detector", channel['DetectorType']) continue - detectors[channel['DetectorType']][corres[channel['prodtype']]].append((channel['detail'], - channel['Energy'], - channel['DetectorType'], - channel['ProdID'], - channel['nb_files'], - channel['NumberOfEvents'] / channel['nb_files'], - channel['NumberOfEvents'], - channel['CrossSection'], - channel['MomProdID'], - str(channel['proddetail']))) + detectors[matchDet][channel['Datatype']].append((channel['detail'], + channel['Energy'], + channel['DetectorType'], + channel['ProdID'], + channel['nb_files'], + channel['NumberOfEvents'] / channel['nb_files'], + channel['NumberOfEvents'], + channel['CrossSection'], + channel['MomProdID'], + str(channel['proddetail']))) + gLogger.notice(f'Transformation {channel["ProdID"]} found for detector {matchDet}') + header_row_tuple_gen = ('Channel', 'Energy', 'ProdID', 'Tasks', 'Average Evts/task', 'Statistics', 'Cross Section (fb)', 'Comment') - header_row_tuple_det = ('Channel', 'Energy', 'Detector', 'ProdID', 'Number of Files', 'Events/File', 'Statistics', 'Cross Section (fb)', 'Origin ProdID', 'Comment') @@ -413,13 +444,13 @@ def main(): of.write(str(table)) gLogger.info("Gen prods") gLogger.info(str(table)) - detectors.pop('gen') + # pylint: disable=no-member, invalid-sequence-index for detName, infos in detectors.items(): - if any(infos.get(pt) for pt in ('SIM', 'REC')): + if any(infos.get(pt) for pt in _addDetectorOutputs({}).keys()): of.write('<h1>%s prods</h1>\n' % infos['title']) - for ptype in ('SIM', 'REC'): + for ptype in _addDetectorOutputs({}).keys(): if infos[ptype]: of.write('<h2>%s</h2>\n' % ptype) table = Table(header_row=header_row_tuple_det) @@ -427,13 +458,12 @@ def main(): of.write(str(table)) gLogger.info('%s prods %s' % (infos['title'], ptype)) gLogger.info(str(table)) - + # pylint: enable=no-member, invalid-sequence-index of.write(textwrap.dedent(""" </body> </html>""")) gLogger.notice("Check ./tables.html in any browser for the results") dexit(0) - if __name__ == "__main__": main()