diff --git a/Tests/productionTest/whizardddsimgaudi.conf b/Tests/productionTest/whizardddsimgaudi.conf new file mode 100644 index 0000000000000000000000000000000000000000..36714f77dd559e2c546c82d520e647b48fcb2627 --- /dev/null +++ b/Tests/productionTest/whizardddsimgaudi.conf @@ -0,0 +1,40 @@ +[whizard2] +Version = 2.8.3 +EvtType = ZHH + +[ddsim] +SteeringFile = fcc_steer.py + +[gaudiapp] +ExecutableName = k4run +SteeringFile = fccRec_lcio_input.py + +[Production Parameters] +machine = ee +prodGroup = several + +softwareVersion = key4hep_230408 +generatorApplication = whizard2 +simulationApplication = ddsim +reconstructionApplication = gaudiapp +generatorSteeringFile = ee_ZHH_1500gev_polp80.sin + +configVersion = key4hep-devel-2 +configPackage = fccConfig +eventsPerJobs = 10 + +numberOfTasks = 1 + +campaign = winter2023 +energies = 1500 +processes = ZHH +detectorModel = CLIC_o4_v14 + +productionLogLevel = VERBOSE +outputSE = CERN-DST-EOS + +finalOutputSE = CERN-SRM +MoveStatus = Stopped +MoveGroupSize = 10 + +ProdTypes = Gen, Sim, Rec diff --git a/docs/Files/ProdGuide/fccProdNotes.rst b/docs/Files/ProdGuide/fccProdNotes.rst index d9496d93e4a24b894bc821eb480421d20bd421c3..69a1a195cec120e31656222243f811d00811fb61 100644 --- a/docs/Files/ProdGuide/fccProdNotes.rst +++ b/docs/Files/ProdGuide/fccProdNotes.rst @@ -101,8 +101,7 @@ We are generating events using Whizard. :language: ini :linenos: -Whizard2 in principle only requires two application specific parameter: ``EvtType`` and ``OutFormat``. -In particular ``OutFormat`` decides the output format of Whizard output. In fact Whizard can give outputs with extension ``lhe, stdhep, slcio, hepmc``. +Whizard2 in principle only requires two application specific parameter: ``EvtType`` and ``Version``. Regarding its version, Whizard would take the version contained in the key4hep ``softwareVersion`` used for the transformation. However, for Whizard, it is recommended to specify the specific Whizard2 version in the application-specific parameters. @@ -179,7 +178,7 @@ DelphesPythia8_EDM4HEP: fast simulation after generation with Pythia8 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ We are using the k4SimDelphes DelphesPythia8_EDM4HEP executable for, in a single step, generating events with Pythia8 and processing them in a fast simulation. -(Here we do the same thing that happens in `Gaudi: Delphes fast simulation after event generation with Pythia8`_., but inside Gaudi instead of using directly the Dephes standalone executables) +(Here we do the same thing that happens in `In Gaudi: Delphes fast simulation after generation with Pythia8`_., but inside Gaudi instead of using directly the Dephes standalone executables). .. literalinclude:: delphespythia.conf :linenos: @@ -232,11 +231,8 @@ DelphesROOT_EDM4HEP: fast simulation after reading ROOT output ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ TODO -Gaudi workflows ---------------- - -Gaudi: Delphes fast simulation after event generation with Pythia8 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +In Gaudi: Delphes fast simulation after generation with Pythia8 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Here we do the same thing that happens in `DelphesPythia8_EDM4HEP: fast simulation after generation with Pythia8`_., but inside Gaudi instead of using directly the Dephes standalone executables. @@ -258,6 +254,23 @@ The Pythia8 card, ``<card_name>.cmd``, is instead read from the local folder, as The choice of using Gaudi as a generator is done with ``generatorApplication = gaudiapp``. -Gaudi full simulation: Particle Gun, Geant4, reconstruction -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -TODO \ No newline at end of file +Full simulations +---------------- + +In Gaudi full simulation: Particle Gun, Geant4, reconstruction +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +TODO + +Full simulation: Whizard, DDSim, Gaudi +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +This is a full simulation happening in three steps (Gen, Sim, Rec), using Whizard, DDSim (with Geant4), Gaudi. + + .. literalinclude:: whizardddsimgaudi.conf + :language: ini + :linenos: + +We are generating events using Whizard (as would be done for `Event generation: Whizard2`_). Whizard2 in principle only requires two application specific parameter: ``EvtType`` and ``Version``. + +For the simulation step we use DDSim. The choice of using Gaudi as a generator is done with ``simulationApplication = ddsim``. Its only application specific parameter is a steering file, in this case ``fcc_steer.py``. + +Gaudi is selected for the reconstruction with ``reconstructionApplication = gaudiapp``, and run with the ``k4run`` executable. Gaudi always requires a steering file, in this case ``SteeringFile = fccRec_lcio_input.py``. \ No newline at end of file diff --git a/src/ILCDIRAC/ILCTransformationSystem/Tests/Test_diracFccMakeProductions.py b/src/ILCDIRAC/ILCTransformationSystem/Tests/Test_diracFccMakeProductions.py index 8a3ee8d4068c71e0ed90b202e25aa18dc6980d09..028fcbf66115203a484507737334954b42eb5658 100644 --- a/src/ILCDIRAC/ILCTransformationSystem/Tests/Test_diracFccMakeProductions.py +++ b/src/ILCDIRAC/ILCTransformationSystem/Tests/Test_diracFccMakeProductions.py @@ -65,6 +65,8 @@ def configDict(): 'configPackage': 'fccConfig', 'machine': 'ee', 'generatorApplication': 'KKMC', + 'simulationApplication': 'ddsim', + 'reconstructionApplication': 'gaudiapp', 'generatorSteeringFile': 'somesteeringfile', 'campaign': 'winter2023', 'datatype': 'lhe', @@ -570,6 +572,42 @@ def test_createGenProduction2(theChain, aTask, pMockMod): retMeta = theChain.createGenerationProduction(aTask) assert retMeta == {} + +def test_createRecoProduction(theChain, aTask, pMockMod): + """Test creating the reco production.""" + theChain._flags._over = True + assert theChain._flags.over + theChain.overlayEvents = '1.4TeV' + theChain._basepath = '/fcc/ee/' + theChain.detectorModel = 'idea' + with patch("ILCDIRAC.Interfaces.API.NewInterface.ProductionJob.ProductionJob", new=pMockMod): + aTask.parameterDict['process'] = 'qqbar' + aTask.meta = {'Energy': '240', + 'EvtType': 'ZH_incl', + 'Machine': 'ee', + 'NumberOfEvents': 1500, + 'ProdID': '1'} + aTask.eventsPerJob = 100 + retMeta = theChain.createReconstructionProduction(aTask, over=False) + assert retMeta == {} + assert theChain.cliRecoOption == '' + + +def test_createSimProduction(theChain, aTask, pMockMod): + """Test creating the simulation production.""" + theChain._basepath = '/fcc/ee/' + theChain.detectorModel = 'idea' + with patch("ILCDIRAC.Interfaces.API.NewInterface.ProductionJob.ProductionJob", new=pMockMod): + aTask.parameterDict['process'] = 'qqbar' + aTask.meta = {'Energy': '240', + 'EvtType': 'ZH_incl', + 'Machine': 'ee', + 'NumberOfEvents': 1500, + 'ProdID': '1'} + aTask.eventsPerJob = 100 + retMeta = theChain.createSimulationProduction(aTask) + assert retMeta == {} + # ------------------------------------------------------------------------- def test_createKKMCApplication(theChain, aTask, cpMock): @@ -721,4 +759,17 @@ def test_createWhizard2Application(theChain, aTask, cpMock): ret = theChain.createWhizard2Application(aTask) assert isinstance(ret, Whizard2) assert ret.datatype == "stdhep" - assert ret.version == 'mySoftwareVersion' \ No newline at end of file + assert ret.version == 'mySoftwareVersion' + +def test_createDDSimApplication(theChain, aTask, cpMock): + """Test creating the ddsim application.""" + from ILCDIRAC.Interfaces.API.NewInterface.Applications import DDSim + parameter = Mock() + parameter.prodConfigFilename = 'filename' + parameter.dumpConfigFile = False + with patch(SCP, new=Mock(return_value=cpMock)): + theChain.loadParameters(parameter) + with patch("ILCDIRAC.Interfaces.API.NewInterface.Applications.DDSim.setDetectorModel", new = Mock()) as setdetectormocked: + ret = theChain.createDDSimApplication(aTask) + assert isinstance(ret, DDSim) + setdetectormocked.assert_called_once_with('myDetectorModel') diff --git a/src/ILCDIRAC/ILCTransformationSystem/scripts/dirac_fcc_make_productions.py b/src/ILCDIRAC/ILCTransformationSystem/scripts/dirac_fcc_make_productions.py index 7d7c2ad3aa2f8cf18f34a7e3bff55677683a4982..66bb8740d15718b420791645511c7e32beecdf36 100644 --- a/src/ILCDIRAC/ILCTransformationSystem/scripts/dirac_fcc_make_productions.py +++ b/src/ILCDIRAC/ILCTransformationSystem/scripts/dirac_fcc_make_productions.py @@ -46,6 +46,8 @@ Then modify the template to describe the productions:: softwareVersion = key4hep_230408 generatorApplication = whizard2 + simulationApplication = ddsim + reconstructionApplication = gaudiapp generatorSteeringFile = ee_ZHH_1500gev_polp80.sin processingAfterGen = delphesapp @@ -119,6 +121,8 @@ Parameters in the steering file :generatorSteeringFile: path to steering file for the generator :generatorApplication: specify which application to use for event generation + :simulationApplication: specify which application to use for event simulation + :reconstructionApplication: specify which application to use for event reconstruction :numberOfTasks: number of production jobs/task to create for Gen transformations (default is 1) :eventsInSplitFiles: For split transformations, number of events in the input files @@ -175,7 +179,7 @@ from six.moves import zip_longest PRODUCTION_PARAMETERS = 'Production Parameters' PP = PRODUCTION_PARAMETERS -APPLICATION_LIST = ['KKMC', 'delphesapp', 'gaudiapp', 'babayaga', 'bhlumi', 'whizard2'] +APPLICATION_LIST = ['KKMC', 'delphesapp', 'gaudiapp', 'babayaga', 'bhlumi', 'whizard2', 'ddsim'] LIST_ATTRIBUTES = ['ignoreMetadata', 'generatorSteeringFile', 'energies', @@ -194,6 +198,8 @@ STRING_ATTRIBUTES = ['configPackage', 'outputSE', 'finalOutputSE', 'generatorApplication', + 'simulationApplication', + 'reconstructionApplication', 'MoveStatus', 'MoveGroupSize', 'prodGroup', @@ -427,14 +433,9 @@ MoveTypes = %(moveTypes)s self.campaign = '' - self.simApplication = 'ddsim' - self.simVersion = None - self.simSteeringFile = 'clic_steer.py' - - self.recoApplication = 'Marlin' - self.recoVersion = None - self.recoSteeringFile = 'clicReconstruction.xml' + self.simulationApplication = 'ddsim' + self.reconstructionApplication = 'gaudiapp' self.ignoreMetadata = [] @@ -606,6 +607,8 @@ prodGroup = %(prodGroup)s softwareVersion = %(softwareVersion)s generatorApplication = %(generatorApplication)s +simulationApplication = %(simulationApplication)s +reconstructionApplication = %(reconstructionApplication)s generatorSteeringFile = %(generatorSteeringFile)s processingAfterGen = %(processingAfterGen)s @@ -647,21 +650,30 @@ overlayEventType = %(overlayEventType)s """ % (pDict) def createGeneratorApplication(self, task): - """Create the selected generator application.""" - genApp = {'kkmc': self.createKKMCApplication, - 'whizard2': self.createWhizard2Application, - 'babayaga': self.createBabayagaApplication, - 'bhlumi': self.createBhlumiApplication, - 'delphesapp': self.createDelphesApplication, - 'gaudiapp': self.createGaudiApplication, - }[self.generatorApplication.lower()](task) + """Create the selected Generator application.""" + genApp = {'kkmc': self.createKKMCApplication, + 'whizard2': self.createWhizard2Application, + 'babayaga': self.createBabayagaApplication, + 'bhlumi': self.createBhlumiApplication, + 'delphesapp': self.createDelphesApplication, + 'gaudiapp': self.createGaudiApplication, + }[self.generatorApplication.lower()](task) return genApp def createSimulationApplication(self, task): """Create the selected Simulation application.""" - - return {'genericapp': self.createGenericApplication, - }[self.simApplication.lower()](task) + simApp = {'ddsim': self.createDDSimApplication, + 'gaudiapp': self.createGaudiApplication, + }[self.simulationApplication.lower()](task) + simApp.datatype = 'sim' + return simApp + + def createReconstructionApplication(self, task, over): + """Create the selected Reconstruction application.""" + recApp = {'gaudiapp': self.createGaudiApplication, + }[self.reconstructionApplication.lower()](task) + recApp.datatype = 'rec' + return recApp def createKKMCApplication(self, task): """create KKMCee Application.""" @@ -746,9 +758,16 @@ overlayEventType = %(overlayEventType)s gaudi.setKeepRecFile(True) return gaudi - def createGenericApplication(self, task): - """Create GenericApplication: not implemented""" - pass + def createDDSimApplication(self, task): + """create DDSim Application.""" + from ILCDIRAC.Interfaces.API.NewInterface.Applications import DDSim + ddsim = DDSim() + ddsim.setVersion(self.softwareVersion) + ddsim.setDetectorModel(self.detectorModel) + ddsim.setNumberOfEvents(task.eventsPerJob) + ddsim.setEnergy(task.meta['Energy']) + self._setApplicationOptions('DDSim', ddsim, task.applicationOptions) + return ddsim def createGenerationProduction(self, task): """Create generation production.""" @@ -819,7 +838,8 @@ overlayEventType = %(overlayEventType)s raise RuntimeError("Error creating Simulation Production: %s" % res['Message']) simProd.setWorkflowName(prodName) # Add the application - res = simProd.append(self.createSimulationApplication(task)) + simApp = self.createSimulationApplication(task) + res = simProd.append(simApp) if not res['OK']: raise RuntimeError("Error creating simulation Production: %s" % res['Message']) simProd.addFinalization(True, True, True, True) @@ -831,12 +851,12 @@ overlayEventType = %(overlayEventType)s if not res['OK']: raise RuntimeError("Error creating simulation production: %s" % res['Message']) - simProd.addMetadataToFinalFiles({'BeamParticle1': parameterDict['pname1'], - 'BeamParticle2': parameterDict['pname2'], - 'EPA_B1': parameterDict['epa_b1'], - 'EPA_B2': parameterDict['epa_b2'], - } - ) + # simProd.addMetadataToFinalFiles({'BeamParticle1': parameterDict['pname1'], + # 'BeamParticle2': parameterDict['pname2'], + # 'EPA_B1': parameterDict['epa_b1'], + # 'EPA_B2': parameterDict['epa_b2'], + # } + # ) res = simProd.finalizeProd() if not res['OK']: @@ -872,11 +892,11 @@ overlayEventType = %(overlayEventType)s # if not res['OK']: # raise RuntimeError("Error appending overlay to reconstruction transformation: %s" % res['Message']) - # # add reconstruction - # res = recProd.append(self.createMarlinApplication(task, over)) - # if not res['OK']: - # raise RuntimeError("Error appending Marlin to reconstruction production: %s" % res['Message']) - # recProd.addFinalization(True, True, True, True) + # add reconstruction + res = recProd.append(self.createReconstructionApplication(task, over)) + if not res['OK']: + raise RuntimeError("Error appending reconstruction application to reconstruction production: %s" % res['Message']) + recProd.addFinalization(True, True, True, True) description = "CLICDet2017 %s" % meta['Energy'] description += "Overlay" if over else "No Overlay" @@ -888,12 +908,12 @@ overlayEventType = %(overlayEventType)s if not res['OK']: raise RuntimeError("Error creating reconstruction production: %s" % res['Message']) - recProd.addMetadataToFinalFiles({'BeamParticle1': parameterDict['pname1'], - 'BeamParticle2': parameterDict['pname2'], - 'EPA_B1': parameterDict['epa_b1'], - 'EPA_B2': parameterDict['epa_b2'], - } - ) + # recProd.addMetadataToFinalFiles({'BeamParticle1': parameterDict['pname1'], + # 'BeamParticle2': parameterDict['pname2'], + # 'EPA_B1': parameterDict['epa_b1'], + # 'EPA_B2': parameterDict['epa_b2'], + # } + # ) res = recProd.finalizeProd() if not res['OK']: @@ -1030,7 +1050,7 @@ overlayEventType = %(overlayEventType)s ('SPLIT', self.createSplitProduction)]: for task in taskDict.get(pType, []): meta = createProduction(task) - # self.addSimTask(taskDict, meta, originalTask=task) + self.addSimTask(taskDict, meta, originalTask=task) taskDict['MOVE_' + pType].append(dict(meta)) for task in taskDict.get('SIM', []): @@ -1038,7 +1058,7 @@ overlayEventType = %(overlayEventType)s continue gLogger.notice("Creating task %s" % task) simMeta = self.createSimulationProduction(task) - # self.addRecTask(taskDict, simMeta, originalTask=task) + self.addRecTask(taskDict, simMeta, originalTask=task) taskDict['MOVE_SIM'].append(dict(simMeta)) for task in taskDict.get('REC', []): @@ -1112,19 +1132,21 @@ overlayEventType = %(overlayEventType)s return def addGenTask(self, taskDict, originalTask): - """Add a gen task with required options.""" + """Add a generator task with required options.""" return self._addTask(taskDict, metaInput=originalTask.meta, originalTask=originalTask, prodType='GEN', applicationName=self.generatorApplication, ) + def addSimTask(self, taskDict, metaInput, originalTask): + """Add a simulation task.""" + return self._addTask(taskDict, metaInput, originalTask, prodType='SIM', + applicationName=self.simulationApplication) + def addRecTask(self, taskDict, metaInput, originalTask): """Add a reconstruction task.""" - return self._addTask(taskDict, metaInput, originalTask, prodType='REC', applicationName='Marlin') - - def addSimTask(self, taskDict, metaInput, originalTask): - """Add a sim task.""" - return self._addTask(taskDict, metaInput, originalTask, prodType='SIM', applicationName='DDSim') + return self._addTask(taskDict, metaInput, originalTask, prodType='REC', + applicationName=self.reconstructionApplication) @staticmethod def addTaskOptions(options, taskList): diff --git a/src/ILCDIRAC/Interfaces/API/NewInterface/ProductionJob.py b/src/ILCDIRAC/Interfaces/API/NewInterface/ProductionJob.py index a2675d75d634e77d14a257db747a305c396d108b..36d030715005ef336285f72aadfd9aefed80fccb 100644 --- a/src/ILCDIRAC/Interfaces/API/NewInterface/ProductionJob.py +++ b/src/ILCDIRAC/Interfaces/API/NewInterface/ProductionJob.py @@ -803,14 +803,28 @@ class ProductionJob(Job): # pylint: disable=too-many-public-methods, too-many-i generatorPath = os.path.join(*currentPathList) self.finalMetaDict[generatorPath] = {'Generator': self.generator} + if self.vo == 'fcc': + self.configureFCCOutputPath(path, application, currentPathList) + + if self.vo == 'ilc': + self.configureILCOutputPath(path, application, currentPathList) + + res = self._updateProdParameters(application) + if not res['OK']: + return res + + self.checked = True + + return S_OK() + + def configureFCCOutputPath(self, path, application, currentPathList): # comment of Lorenzo Valentini, 12-05-2023 # In fcc we have a particular use of Gaudi. Its application is defined with the functions setOutputSimFile/setOutputRecFile. # However, it is sometimes used for other tasks, for which we don't want it to use the file path of the sim/rec transformations. # Up to now, seems like whenever we will be using sim/rec transformations we won't have to define the datatype, which we use to discriminate the two cases. - - # comment of Lorenzo Valentini, 12-05-2023 + # this is the case of a (maybe Gaudi) application used for sim/rec transformations: no datatype defined, and we check the presence of setOutputSimFile - if hasattr(application, "setOutputSimFile") and not application.willBeCut and self.vo == 'fcc' and not application.datatype: + if hasattr(application, "setOutputSimFile") and not application.willBeCut and not application.datatype: detPath = os.path.join(path, application.detector) self.finalMetaDict[detPath] = {"DetectorType": application.detector} if application.keepRecFile: @@ -827,11 +841,62 @@ class ProductionJob(Job): # pylint: disable=too-many-public-methods, too-many-i LOG.notice('Will store the files under', path) self.finalpaths.append(path) - # comment of Lorenzo Valentini, 12-05-2023 - # this is the case of a (maybe Gaudi) application used for dst/rec transformations: no datatype defined, and we check the presence of setOutputRecFile. It would make more sense to just check for setOutputDstFile, - # but we wanted to keep it backward compatibile with the ilc simulations. - # In the case of fcc, we check for the absence of datatype. - elif hasattr(application, "setOutputRecFile") and not application.willBeCut and (self.vo != 'fcc' or (self.vo == 'fcc' and not application.datatype)): + # # comment of Lorenzo Valentini, 12-05-2023 + # # this is the case of a (maybe Gaudi) application used for dst/rec transformations: no datatype defined, and we check the presence of setOutputRecFile. It would make more sense to just check for setOutputDstFile, + # # but we wanted to keep it backward compatibile with the ilc simulations. + # # In the case of fcc, we check for the absence of datatype. + # elif hasattr(application, "setOutputRecFile") and not application.willBeCut and not application.datatype: + # detPath = os.path.join(path, application.detectortype) + # self.finalMetaDict[detPath] = {'DetectorType': application.detectortype} + # if application.keepRecFile: + # path = os.path.join(detPath, 'rec') + # self.finalMetaDict[path] = {'Datatype': 'rec'} + # fname = self.basename + '_rec.slcio' + # application.setOutputRecFile(fname, path) + # LOG.notice('Will store the files under', path) + # self.finalpaths.append(path) + # path = os.path.join(detPath, 'dst') + # self.finalMetaDict[path] = {'Datatype': 'dst'} + # fname = self.basename + '_dst.slcio' + # application.setOutputDstFile(fname, path) + # LOG.notice('Will store the files under', path) + # self.finalpaths.append(path) + + elif hasattr(application, "outputFile") and hasattr(application, 'datatype') and not application.outputFile and not application.willBeCut: + LOG.notice('Adding output meta data for %s' % type(application)) + path = os.path.join(*currentPathList) + + if not application.datatype and self.datatype: + application.datatype = self.datatype + + # here we are adding detector and detectortype to the path (either one or the other) if they are defined. For fcc, we usually set the detector using the attribute application.detector. + if hasattr(application, "detectortype"): + if application.detectortype: + path = os.path.join(path, application.detectortype) + self.finalMetaDict[path] = {"DetectorType": application.detectortype} + if hasattr(application, "detector"): + if application.detector and not application.detectortype: + path = os.path.join(path, application.detector) + self.finalMetaDict[path] = {"DetectorType": application.detector} + if hasattr(application, "detectortype") or hasattr(application, "detector"): + if self.detector and not application.detectortype and not application.detector: + path = os.path.join(path, self.detector) + self.finalMetaDict[path] = {"DetectorType": self.detector} + + path = os.path.join(path, application.datatype) + self.finalMetaDict[path] = {'Datatype': application.datatype} + + LOG.notice('Will store the files under', '%s' % path) + self.finalpaths.append(path) + extension = getattr(application, '_extension', 'stdhep') + + fname = self.basename + "_%s" % (application.datatype.lower()) + "." + extension + application.setOutputFile(fname, path) + + return + + def configureILCOutputPath(self, path, application, currentPathList): + if hasattr(application, "setOutputRecFile") and not application.willBeCut: detPath = os.path.join(path, application.detectortype) self.finalMetaDict[detPath] = {'DetectorType': application.detectortype} if application.keepRecFile: @@ -848,8 +913,6 @@ class ProductionJob(Job): # pylint: disable=too-many-public-methods, too-many-i LOG.notice('Will store the files under', path) self.finalpaths.append(path) - # comment of Lorenzo Valentini, 12-05-2023 - # this is the case of an application with a single outputfile. It works in the same way for both fcc and ilc. elif hasattr(application, "outputFile") and hasattr(application, 'datatype') and not application.outputFile and not application.willBeCut: LOG.notice('Adding output meta data for %s' % type(application)) path = os.path.join(*currentPathList) @@ -857,11 +920,6 @@ class ProductionJob(Job): # pylint: disable=too-many-public-methods, too-many-i if not application.datatype and self.datatype: application.datatype = self.datatype - # comment of Lorenzo Valentini, 12-05-2023 - # here we are adding detector and detectortype to the path, if they are defined. For fcc, we usually set the detector using the attribute application.detector. - # Anyway, seems like for ilc we wanted to "be ready" for both cases (detector, detectortype), so I did something similar for fcc. - # The code for ilc is not modified, to keep it backward compatible. However seems like in that case we would add to the path both detector and detectortype. - # For fcc we don't need to see the detector in the path two times, so I added a check to avoid that. if hasattr(application, "detectortype"): if application.detectortype: path = os.path.join(path, application.detectortype) @@ -870,10 +928,7 @@ class ProductionJob(Job): # pylint: disable=too-many-public-methods, too-many-i path = os.path.join(path, self.detector) self.finalMetaDict[path] = {"DetectorType": self.detector} if hasattr(application, "detector"): - if application.detector and self.vo != 'fcc': - path = os.path.join(path, application.detector) - self.finalMetaDict[path] = {"DetectorType": application.detector} - if application.detector and self.vo == 'fcc' and not application.detectortype: + if application.detector: path = os.path.join(path, application.detector) self.finalMetaDict[path] = {"DetectorType": application.detector} path = os.path.join(path, application.datatype) @@ -887,13 +942,7 @@ class ProductionJob(Job): # pylint: disable=too-many-public-methods, too-many-i fname = self.basename + "_%s" % (application.datatype.lower()) + "." + extension application.setOutputFile(fname, path) - res = self._updateProdParameters(application) - if not res['OK']: - return res - - self.checked = True - - return S_OK() + return def _updateProdParameters(self, application): """Update the prod parameters stored in the production parameters visible from the web.""" diff --git a/src/ILCDIRAC/Interfaces/API/NewInterface/Tests/Test_ProductionJob.py b/src/ILCDIRAC/Interfaces/API/NewInterface/Tests/Test_ProductionJob.py index 3f65207ebe2f43de4d533d52327e8761599fc514..0d70cf68e2fe19391bbf0ee8946c534007f6b596 100644 --- a/src/ILCDIRAC/Interfaces/API/NewInterface/Tests/Test_ProductionJob.py +++ b/src/ILCDIRAC/Interfaces/API/NewInterface/Tests/Test_ProductionJob.py @@ -42,6 +42,7 @@ from Tests.Utilities.GeneralUtils import assertEqualsImproved, assertDiracFailsW assertDiracSucceeds from Tests.Utilities.FileUtils import FileUtil import six +import os __RCSID__ = "$Id$" @@ -801,9 +802,38 @@ class ProductionJobJobSpecificParamsTest(ProductionJobTestCase): self.assertIn(args[1], opsDict) return opsDict[args[1]] - def test_jobSpecificParams_detector_and_datatype(self): + @parameterized.expand([('ilc', 250, 250, '/vo/prod/250gev/fake/'), + ('ilc', 250, None, '/vo/prod/250gev/fake/'), + ('fcc', None, 250, '/vo/prod/250gev/fake/'), + ]) + def test_jobSpecificParams_energyandevents(self, vo, selfenergy, appenergy, finalPaths): + """Testing different combinations of energies and paths""" self.myapp.setOutputSE.return_value = S_OK(True) self.prodJob.evttype = '' + self.prodJob.basepath = '/vo/prod/' + self.prodJob.basename = 'basename' + self.prodJob.generator = 'whatever' + self.prodJob.vo = vo + self.prodJob.energy = selfenergy + self.prodJob.slicesize = 2 + self.myapp.energy = appenergy + self.myapp.numberOfEvents = 1 + self.myapp.detector = 'awesome' + self.myapp.eventType = 'fake' + self.myapp.detectortype = 'awesome' + self.myapp.willBeCut = False + self.myapp.outputFile = False + self.myapp.extraCLIArguments = '' + self.myapp.keepRecFile = False + with patch('%s.ProductionJob._updateProdParameters' % MODULE_NAME, new=Mock(return_value=S_OK())), \ + patch('%s.ProductionJob.configureILCOutputPath' % MODULE_NAME, new=Mock(return_value=S_OK())), \ + patch('%s.ProductionJob.configureFCCOutputPath' % MODULE_NAME, new=Mock(return_value=S_OK())): + assertDiracSucceeds(self.prodJob.append(self.myapp), self) + + self.assertIn(finalPaths, self.prodJob.finalMetaDict.keys()) + + def test_configureFCCOutputPath(self): + self.myapp.setOutputSE.return_value = S_OK(True) self.myapp.willBeCut = False self.myapp.outputFile = False self.myapp.extraCLIArguments = '' @@ -811,26 +841,53 @@ class ProductionJobJobSpecificParamsTest(ProductionJobTestCase): hasAttrMock.side_effect = self.hasAttrMocks with patch('%s.hasattr' % MODULE_NAME, new=hasAttrMock, create=True), \ patch('%s.ProductionJob._updateProdParameters' % MODULE_NAME, new=Mock(return_value=S_OK())): - assertDiracSucceeds(self.prodJob.append(self.myapp), self) + self.prodJob.configureFCCOutputPath("/a/a", self.myapp, [self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType]) self.prodJob.evttype = '' hasAttrMock = Mock() hasAttrMock.side_effect = self.hasAttrMocks_2 with patch('%s.hasattr' % MODULE_NAME, new=hasAttrMock, create=True): self.myapp.detectortype = 'application_detectortype' - assertDiracSucceeds(self.prodJob.append(self.myapp), self) + self.prodJob.configureFCCOutputPath("/a/a", self.myapp, [self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType]) @parameterized.expand([(True, "ilc", None, 'awesome', 'awesome', ['/vo/prod/250gev/fake/awesome/REC', '/vo/prod/250gev/fake/awesome/DST']), (True, "ilc", None, None, 'awesome', ['/vo/prod/250gev/fake/awesome/REC', '/vo/prod/250gev/fake/awesome/DST']), (False, "ilc", None, 'awesome', 'awesome', ['/vo/prod/250gev/fake/awesome/DST']), (False, "ilc", "delphes", 'awesome', 'awesome', ['/vo/prod/250gev/fake/awesome/DST']), - (True, "fcc", None, 'awesome', 'awesome', ['/vo/prod/250gev/fake/awesome/REC', '/vo/prod/250gev/fake/awesome/DST']), - (False, "fcc", None, 'awesome', 'awesome', ['/vo/prod/250gev/fake/awesome/DST']), - (False, "fcc", "delphes", 'awesome', 'awesome', ['/vo/prod/250gev/fake/awesome/delphes']), + ]) + def test_configureILCOutputPath_rec_and_dst(self, keep, vo, datatype, detector, detectortype, finalPaths): + """Testing different combinations of vo, transformation, detector, datatype""" + self.myapp.setOutputSE.return_value = S_OK(True) + self.prodJob.evttype = '' + self.prodJob.basepath = '/vo/prod/' + self.prodJob.basename = 'basename' + self.prodJob.generator = 'whatever' + self.prodJob.vo = vo + self.myapp.detector = detector + self.myapp.datatype = datatype + self.myapp.eventType = 'fake' + self.myapp.detectortype = detectortype + self.myapp.willBeCut = False + self.myapp.outputFile = False + self.myapp.extraCLIArguments = '' + self.myapp.keepRecFile = keep + self.myapp.setOutputRecFile = Mock(name='RecFile') + self.myapp.setOutputDstFile = Mock(name='DstFile') + del self.myapp.setOutputSimFile + with patch('%s.ProductionJob._updateProdParameters' % MODULE_NAME, new=Mock(return_value=S_OK())): + self.prodJob.configureILCOutputPath(os.path.join(self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType), self.myapp, []) + if keep: + self.myapp.setOutputRecFile.assert_called_once() + else: + self.myapp.setOutputRecFile.assert_not_called() + self.myapp.setOutputDstFile.assert_called_once() + assert self.prodJob.finalpaths == finalPaths + + @parameterized.expand([(False, "fcc", "delphes", 'awesome', 'awesome', ['/vo/prod/250gev/fake/awesome/delphes']), (False, "fcc", "delphes", 'awesome', None, ['/vo/prod/250gev/fake/awesome/delphes']), (False, "fcc", "delphes", None, 'awesome', ['/vo/prod/250gev/fake/awesome/delphes']), ]) - def test_jobSpecificParams_rec_and_dst(self, keep, vo, datatype, detector, detectortype, finalPaths): + def test_configureFCCOutputPath_rec_and_dst(self, keep, vo, datatype, detector, detectortype, finalPaths): """Testing different combinations of vo, transformation, detector, datatype""" self.myapp.setOutputSE.return_value = S_OK(True) self.prodJob.evttype = '' @@ -850,14 +907,8 @@ class ProductionJobJobSpecificParamsTest(ProductionJobTestCase): self.myapp.setOutputDstFile = Mock(name='DstFile') del self.myapp.setOutputSimFile with patch('%s.ProductionJob._updateProdParameters' % MODULE_NAME, new=Mock(return_value=S_OK())): - assertDiracSucceeds(self.prodJob.append(self.myapp), self) - if vo != 'fcc': - if keep: - self.myapp.setOutputRecFile.assert_called_once() - else: - self.myapp.setOutputRecFile.assert_not_called() - self.myapp.setOutputDstFile.assert_called_once() - elif not datatype: + self.prodJob.configureFCCOutputPath(os.path.join(self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType), self.myapp, [self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType]) + if not datatype: if keep: self.myapp.setOutputRecFile.assert_called_once() else: @@ -866,7 +917,6 @@ class ProductionJobJobSpecificParamsTest(ProductionJobTestCase): else: self.myapp.setOutputRecFile.assert_not_called() self.myapp.setOutputDstFile.assert_not_called() - assert self.prodJob.finalpaths == finalPaths @parameterized.expand([(True, "fcc", None, 'awesome', 'awesome', ['/vo/prod/250gev/fake/awesome/rec', '/vo/prod/250gev/fake/awesome/sim']), @@ -876,7 +926,7 @@ class ProductionJobJobSpecificParamsTest(ProductionJobTestCase): (False, "fcc", "delphes", 'awesome', None, ['/vo/prod/250gev/fake/awesome/delphes']), (False, "fcc", "delphes", None, 'awesome', ['/vo/prod/250gev/fake/awesome/delphes']), ]) - def test_jobSpecificParams_rec_and_sim(self, keep, vo, datatype, detector, detectortype, finalPaths): + def test_configureFCCOutputPath_rec_and_sim(self, keep, vo, datatype, detector, detectortype, finalPaths): """Testing different combinations of vo, transformation, detector, datatype""" self.myapp.setOutputSE.return_value = S_OK(True) self.prodJob.evttype = '' @@ -896,7 +946,7 @@ class ProductionJobJobSpecificParamsTest(ProductionJobTestCase): self.myapp.setOutputSimFile = Mock(name='SimFile') del self.myapp.setOutputDstFile with patch('%s.ProductionJob._updateProdParameters' % MODULE_NAME, new=Mock(return_value=S_OK())): - assertDiracSucceeds(self.prodJob.append(self.myapp), self) + self.prodJob.configureFCCOutputPath(os.path.join(self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType), self.myapp, [self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType]) if not datatype: if keep: self.myapp.setOutputRecFile.assert_called_once() @@ -909,16 +959,38 @@ class ProductionJobJobSpecificParamsTest(ProductionJobTestCase): assert self.prodJob.finalpaths == finalPaths + def test_configureFCCOutputPath_selfdetector(self): + """Testing different combination of detector information""" + self.myapp.setOutputSE.return_value = S_OK(True) + self.prodJob.evttype = '' + self.prodJob.basepath = '/vo/prod/' + self.prodJob.basename = 'basename' + self.prodJob.vo = "fcc" + self.prodJob.detector = 'idea' + self.myapp.detector = '' + self.myapp.datatype = "sim" + self.myapp.eventType = 'fakeEvent' + self.myapp.detectortype = '' + self.myapp.willBeCut = False + self.myapp.outputFile = False + self.myapp.extraCLIArguments = '' + with patch('%s.ProductionJob._updateProdParameters' % MODULE_NAME, new=Mock(return_value=S_OK())): + self.prodJob.configureFCCOutputPath(os.path.join(self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType), self.myapp, [self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType]) + assert self.prodJob.finalpaths == ['/vo/prod/250gev/fakeEvent/idea/sim'] + - @parameterized.expand([('detector', 'detectortype', ['/vo/prod/250gev/fake/generator/detectortype/detector/datatype']), + @parameterized.expand([('detector', 'detector', 'detectortype', ['/vo/prod/250gev/fake/generator/detectortype/detector/datatype']), + ('detector', False, False, ['/vo/prod/250gev/fake/generator/detector/datatype']), + (False, 'detector', False, ['/vo/prod/250gev/fake/generator/detector/datatype']), ]) - def test_jobSpecificParams_rec_and_dst__2(self, detector, detectortype, finalPaths): + def test_configureILCOutputPath_rec_and_dst__2(self, selfdetector, detector, detectortype, finalPaths): """Testing different combinations of vo, transformation, detector, datatype""" self.myapp.setOutputSE.return_value = S_OK(True) self.prodJob.evttype = '' self.prodJob.basepath = '/vo/prod/' self.prodJob.basename = 'basename' self.prodJob.generator = 'generator' + self.prodJob.detector = selfdetector self.prodJob.vo = "ilc" self.myapp.detector = detector self.myapp.datatype = "datatype" @@ -930,41 +1002,35 @@ class ProductionJobJobSpecificParamsTest(ProductionJobTestCase): self.myapp.keepRecFile = False del self.myapp.setOutputRecFile with patch('%s.ProductionJob._updateProdParameters' % MODULE_NAME, new=Mock(return_value=S_OK())): - assertDiracSucceeds(self.prodJob.append(self.myapp), self) + self.prodJob.configureILCOutputPath(os.path.join(self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType), self.myapp, [self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType, self.prodJob.generator]) assert self.prodJob.finalpaths == finalPaths - - @parameterized.expand([(250, 250, ['/vo/prod/250gev/fake/awesome/DST']), - (250, None, ['/vo/prod/250gev/fake/awesome/DST']), - (None, 250, ['/vo/prod/250gev/fake/awesome/DST']), + @parameterized.expand([('SIM', False, ['/vo/prod/250gev/fake/generator/detectortype/detector/SIM']), + (False, 'SIM', ['/vo/prod/250gev/fake/generator/detectortype/detector/SIM']) ]) - def test_jobSpecificParams_energyandevents(self, selfenergy, appenergy, finalPaths): - """Testing different combinations of energies and paths""" + def test_configureILCOutputPath_rec_and_sim(self, selfdata, appdata, finalPaths): + """Testing different combinations of vo, transformation, detector, datatype""" self.myapp.setOutputSE.return_value = S_OK(True) self.prodJob.evttype = '' self.prodJob.basepath = '/vo/prod/' self.prodJob.basename = 'basename' - self.prodJob.generator = 'whatever' + self.prodJob.generator = 'generator' self.prodJob.vo = "ilc" - self.prodJob.energy = selfenergy - self.prodJob.slicesize = 2 - self.myapp.energy = appenergy - self.myapp.numberOfEvents = 1 - self.myapp.detector = 'awesome' + self.prodJob.datatype = selfdata + self.myapp.detector = "detector" + self.myapp.datatype = appdata self.myapp.eventType = 'fake' - self.myapp.detectortype = 'awesome' + self.myapp.detectortype = "detectortype" self.myapp.willBeCut = False self.myapp.outputFile = False self.myapp.extraCLIArguments = '' self.myapp.keepRecFile = False - self.myapp.setOutputRecFile = Mock(name='RecFile') - self.myapp.setOutputDstFile = Mock(name='DstFile') + del self.myapp.setOutputRecFile with patch('%s.ProductionJob._updateProdParameters' % MODULE_NAME, new=Mock(return_value=S_OK())): - assertDiracSucceeds(self.prodJob.append(self.myapp), self) - self.myapp.setOutputRecFile.assert_not_called() - self.myapp.setOutputDstFile.assert_called_once() + self.prodJob.configureILCOutputPath(os.path.join(self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType), self.myapp, [self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType, self.prodJob.generator]) assert self.prodJob.finalpaths == finalPaths + def test_setters_1(self): import random self.prodJob.setDryRun(True)