From e968882189ae32cdbb6620aa2e29510b3a6b120e Mon Sep 17 00:00:00 2001
From: Andre Sailer <andre.philippe.sailer@cern.ch>
Date: Wed, 25 Oct 2023 15:57:31 +0200
Subject: [PATCH] Merge branch 'v34-fullsimddsim' into 'Rel-v34r0'

DDSim, FCC full simulation

See merge request CLICdp/iLCDirac/ILCDIRAC!478

(cherry picked from commit 3eb5cb623fa957bb8058807ebb5d317bd8fa613e)

d638f359 separating the process of creation of output paths for ilc and fcc
2c135292 Adding DDSim and enabling the full simulations (Gen. Sim, Rec)
---
 Tests/productionTest/whizardddsimgaudi.conf   |  40 +++++
 docs/Files/ProdGuide/fccProdNotes.rst         |  35 +++--
 .../Tests/Test_diracFccMakeProductions.py     |  53 ++++++-
 .../scripts/dirac_fcc_make_productions.py     | 118 ++++++++------
 .../API/NewInterface/ProductionJob.py         | 101 ++++++++----
 .../NewInterface/Tests/Test_ProductionJob.py  | 144 +++++++++++++-----
 6 files changed, 366 insertions(+), 125 deletions(-)
 create mode 100644 Tests/productionTest/whizardddsimgaudi.conf

diff --git a/Tests/productionTest/whizardddsimgaudi.conf b/Tests/productionTest/whizardddsimgaudi.conf
new file mode 100644
index 000000000..36714f77d
--- /dev/null
+++ b/Tests/productionTest/whizardddsimgaudi.conf
@@ -0,0 +1,40 @@
+[whizard2]
+Version = 2.8.3
+EvtType = ZHH
+
+[ddsim]
+SteeringFile = fcc_steer.py
+
+[gaudiapp]
+ExecutableName = k4run
+SteeringFile = fccRec_lcio_input.py
+
+[Production Parameters]
+machine = ee
+prodGroup = several
+
+softwareVersion = key4hep_230408
+generatorApplication = whizard2
+simulationApplication = ddsim
+reconstructionApplication = gaudiapp
+generatorSteeringFile = ee_ZHH_1500gev_polp80.sin
+
+configVersion = key4hep-devel-2
+configPackage = fccConfig
+eventsPerJobs = 10
+
+numberOfTasks = 1
+
+campaign = winter2023
+energies = 1500
+processes = ZHH
+detectorModel = CLIC_o4_v14
+
+productionLogLevel = VERBOSE
+outputSE = CERN-DST-EOS
+
+finalOutputSE = CERN-SRM
+MoveStatus = Stopped
+MoveGroupSize = 10
+
+ProdTypes = Gen, Sim, Rec
diff --git a/docs/Files/ProdGuide/fccProdNotes.rst b/docs/Files/ProdGuide/fccProdNotes.rst
index d9496d93e..69a1a195c 100644
--- a/docs/Files/ProdGuide/fccProdNotes.rst
+++ b/docs/Files/ProdGuide/fccProdNotes.rst
@@ -101,8 +101,7 @@ We are generating events using Whizard.
     :language: ini
     :linenos:
 
-Whizard2 in principle only requires two application specific parameter: ``EvtType`` and ``OutFormat``.
-In particular ``OutFormat`` decides the output format of Whizard output. In fact Whizard can give outputs with extension ``lhe, stdhep, slcio, hepmc``.
+Whizard2 in principle only requires two application specific parameter: ``EvtType`` and ``Version``.
 Regarding its version, Whizard would take the version contained in the key4hep ``softwareVersion`` used for the transformation.
 However, for Whizard, it is recommended to specify the specific Whizard2 version in the application-specific parameters.
 
@@ -179,7 +178,7 @@ DelphesPythia8_EDM4HEP: fast simulation after generation with Pythia8
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 We are using the k4SimDelphes DelphesPythia8_EDM4HEP executable for, in a single step, generating events with Pythia8 and processing them in a fast simulation.
-(Here we do the same thing that happens in `Gaudi: Delphes fast simulation after event generation with Pythia8`_., but inside Gaudi instead of using directly the Dephes standalone executables)
+(Here we do the same thing that happens in `In Gaudi: Delphes fast simulation after generation with Pythia8`_., but inside Gaudi instead of using directly the Dephes standalone executables).
 
   .. literalinclude:: delphespythia.conf
     :linenos:
@@ -232,11 +231,8 @@ DelphesROOT_EDM4HEP: fast simulation after reading ROOT output
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 TODO
 
-Gaudi workflows
----------------
-
-Gaudi: Delphes fast simulation after event generation with Pythia8
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+In Gaudi: Delphes fast simulation after generation with Pythia8
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Here we do the same thing that happens in `DelphesPythia8_EDM4HEP: fast simulation after generation with Pythia8`_., but inside Gaudi instead of using directly the Dephes standalone executables.
 
@@ -258,6 +254,23 @@ The Pythia8 card, ``<card_name>.cmd``, is instead read from the local folder, as
 
 The choice of using Gaudi as a generator is done with ``generatorApplication = gaudiapp``.
 
-Gaudi full simulation: Particle Gun, Geant4, reconstruction
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-TODO
\ No newline at end of file
+Full simulations
+----------------
+
+In Gaudi full simulation: Particle Gun, Geant4, reconstruction
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+TODO
+
+Full simulation: Whizard, DDSim, Gaudi
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+This is a full simulation happening in three steps (Gen, Sim, Rec), using Whizard, DDSim (with Geant4), Gaudi.
+
+  .. literalinclude:: whizardddsimgaudi.conf
+    :language: ini
+    :linenos:
+
+We are generating events using Whizard (as would be done for `Event generation: Whizard2`_). Whizard2 in principle only requires two application specific parameter: ``EvtType`` and ``Version``.
+
+For the simulation step we use DDSim. The choice of using Gaudi as a generator is done with ``simulationApplication = ddsim``. Its only application specific parameter is a steering file, in this case ``fcc_steer.py``.
+
+Gaudi is selected for the reconstruction with ``reconstructionApplication = gaudiapp``, and run with the ``k4run`` executable. Gaudi always requires a steering file, in this case ``SteeringFile = fccRec_lcio_input.py``.
\ No newline at end of file
diff --git a/src/ILCDIRAC/ILCTransformationSystem/Tests/Test_diracFccMakeProductions.py b/src/ILCDIRAC/ILCTransformationSystem/Tests/Test_diracFccMakeProductions.py
index 8a3ee8d40..028fcbf66 100644
--- a/src/ILCDIRAC/ILCTransformationSystem/Tests/Test_diracFccMakeProductions.py
+++ b/src/ILCDIRAC/ILCTransformationSystem/Tests/Test_diracFccMakeProductions.py
@@ -65,6 +65,8 @@ def configDict():
           'configPackage': 'fccConfig',
           'machine': 'ee',
           'generatorApplication': 'KKMC',
+          'simulationApplication': 'ddsim',
+          'reconstructionApplication': 'gaudiapp',
           'generatorSteeringFile': 'somesteeringfile',
           'campaign': 'winter2023',
           'datatype': 'lhe',
@@ -570,6 +572,42 @@ def test_createGenProduction2(theChain, aTask, pMockMod):
     retMeta = theChain.createGenerationProduction(aTask)
   assert retMeta == {}
 
+
+def test_createRecoProduction(theChain, aTask, pMockMod):
+  """Test creating the reco production."""
+  theChain._flags._over = True
+  assert theChain._flags.over
+  theChain.overlayEvents = '1.4TeV'
+  theChain._basepath = '/fcc/ee/'
+  theChain.detectorModel = 'idea'
+  with patch("ILCDIRAC.Interfaces.API.NewInterface.ProductionJob.ProductionJob", new=pMockMod):
+    aTask.parameterDict['process'] = 'qqbar'
+    aTask.meta = {'Energy': '240',
+                  'EvtType': 'ZH_incl',
+                  'Machine': 'ee',
+                  'NumberOfEvents': 1500,
+                  'ProdID': '1'}
+    aTask.eventsPerJob = 100
+    retMeta = theChain.createReconstructionProduction(aTask, over=False)
+  assert retMeta == {}
+  assert theChain.cliRecoOption == ''
+
+
+def test_createSimProduction(theChain, aTask, pMockMod):
+  """Test creating the simulation production."""
+  theChain._basepath = '/fcc/ee/'
+  theChain.detectorModel = 'idea'
+  with patch("ILCDIRAC.Interfaces.API.NewInterface.ProductionJob.ProductionJob", new=pMockMod):
+    aTask.parameterDict['process'] = 'qqbar'
+    aTask.meta = {'Energy': '240',
+                  'EvtType': 'ZH_incl',
+                  'Machine': 'ee',
+                  'NumberOfEvents': 1500,
+                  'ProdID': '1'}
+    aTask.eventsPerJob = 100
+    retMeta = theChain.createSimulationProduction(aTask)
+  assert retMeta == {}
+
 # -------------------------------------------------------------------------
 
 def test_createKKMCApplication(theChain, aTask, cpMock):
@@ -721,4 +759,17 @@ def test_createWhizard2Application(theChain, aTask, cpMock):
   ret = theChain.createWhizard2Application(aTask)
   assert isinstance(ret, Whizard2)
   assert ret.datatype == "stdhep"
-  assert ret.version == 'mySoftwareVersion'
\ No newline at end of file
+  assert ret.version == 'mySoftwareVersion'
+
+def test_createDDSimApplication(theChain, aTask, cpMock):
+  """Test creating the ddsim application."""
+  from ILCDIRAC.Interfaces.API.NewInterface.Applications import DDSim
+  parameter = Mock()
+  parameter.prodConfigFilename = 'filename'
+  parameter.dumpConfigFile = False
+  with patch(SCP, new=Mock(return_value=cpMock)):
+    theChain.loadParameters(parameter)
+  with patch("ILCDIRAC.Interfaces.API.NewInterface.Applications.DDSim.setDetectorModel", new = Mock()) as setdetectormocked:
+    ret = theChain.createDDSimApplication(aTask)
+  assert isinstance(ret, DDSim)
+  setdetectormocked.assert_called_once_with('myDetectorModel')
diff --git a/src/ILCDIRAC/ILCTransformationSystem/scripts/dirac_fcc_make_productions.py b/src/ILCDIRAC/ILCTransformationSystem/scripts/dirac_fcc_make_productions.py
index 7d7c2ad3a..66bb8740d 100644
--- a/src/ILCDIRAC/ILCTransformationSystem/scripts/dirac_fcc_make_productions.py
+++ b/src/ILCDIRAC/ILCTransformationSystem/scripts/dirac_fcc_make_productions.py
@@ -46,6 +46,8 @@ Then modify the template to describe the productions::
 
   softwareVersion = key4hep_230408
   generatorApplication = whizard2
+  simulationApplication = ddsim
+  reconstructionApplication = gaudiapp
   generatorSteeringFile = ee_ZHH_1500gev_polp80.sin
   processingAfterGen = delphesapp
 
@@ -119,6 +121,8 @@ Parameters in the steering file
 
   :generatorSteeringFile: path to steering file for the generator
   :generatorApplication: specify which application to use for event generation
+  :simulationApplication: specify which application to use for event simulation
+  :reconstructionApplication: specify which application to use for event reconstruction
   :numberOfTasks: number of production jobs/task to create for Gen transformations (default is 1)
 
   :eventsInSplitFiles: For split transformations, number of events in the input files
@@ -175,7 +179,7 @@ from six.moves import zip_longest
 
 PRODUCTION_PARAMETERS = 'Production Parameters'
 PP = PRODUCTION_PARAMETERS
-APPLICATION_LIST = ['KKMC', 'delphesapp', 'gaudiapp', 'babayaga', 'bhlumi', 'whizard2']
+APPLICATION_LIST = ['KKMC', 'delphesapp', 'gaudiapp', 'babayaga', 'bhlumi', 'whizard2', 'ddsim']
 LIST_ATTRIBUTES = ['ignoreMetadata',
                    'generatorSteeringFile',
                    'energies',
@@ -194,6 +198,8 @@ STRING_ATTRIBUTES = ['configPackage',
                      'outputSE',
                      'finalOutputSE',
                      'generatorApplication',
+                     'simulationApplication',
+                     'reconstructionApplication',
                      'MoveStatus',
                      'MoveGroupSize',
                      'prodGroup',
@@ -427,14 +433,9 @@ MoveTypes = %(moveTypes)s
 
     self.campaign = ''
 
-    self.simApplication = 'ddsim'
-    self.simVersion = None
-    self.simSteeringFile = 'clic_steer.py'
-
-    self.recoApplication = 'Marlin'
-    self.recoVersion = None
-    self.recoSteeringFile = 'clicReconstruction.xml'
+    self.simulationApplication = 'ddsim'
 
+    self.reconstructionApplication = 'gaudiapp'
 
     self.ignoreMetadata = []
 
@@ -606,6 +607,8 @@ prodGroup = %(prodGroup)s
 softwareVersion = %(softwareVersion)s
 
 generatorApplication = %(generatorApplication)s
+simulationApplication = %(simulationApplication)s
+reconstructionApplication = %(reconstructionApplication)s
 generatorSteeringFile = %(generatorSteeringFile)s
 processingAfterGen = %(processingAfterGen)s
 
@@ -647,21 +650,30 @@ overlayEventType = %(overlayEventType)s
 """ % (pDict)
 
   def createGeneratorApplication(self, task):
-    """Create the selected generator application."""
-    genApp =  {'kkmc': self.createKKMCApplication,
-               'whizard2': self.createWhizard2Application,
-               'babayaga': self.createBabayagaApplication,
-               'bhlumi': self.createBhlumiApplication,               
-               'delphesapp': self.createDelphesApplication,
-               'gaudiapp': self.createGaudiApplication,
-             }[self.generatorApplication.lower()](task)
+    """Create the selected Generator application."""
+    genApp = {'kkmc': self.createKKMCApplication,
+              'whizard2': self.createWhizard2Application,
+              'babayaga': self.createBabayagaApplication,
+              'bhlumi': self.createBhlumiApplication,
+              'delphesapp': self.createDelphesApplication,
+              'gaudiapp': self.createGaudiApplication,
+              }[self.generatorApplication.lower()](task)
     return genApp
 
   def createSimulationApplication(self, task):
     """Create the selected Simulation application."""
-
-    return {'genericapp': self.createGenericApplication,
-            }[self.simApplication.lower()](task)
+    simApp = {'ddsim': self.createDDSimApplication,
+              'gaudiapp': self.createGaudiApplication,
+              }[self.simulationApplication.lower()](task)
+    simApp.datatype = 'sim'
+    return simApp
+
+  def createReconstructionApplication(self, task, over):
+    """Create the selected Reconstruction application."""
+    recApp = {'gaudiapp': self.createGaudiApplication,
+              }[self.reconstructionApplication.lower()](task)
+    recApp.datatype = 'rec'
+    return recApp
 
   def createKKMCApplication(self, task):
     """create KKMCee Application."""
@@ -746,9 +758,16 @@ overlayEventType = %(overlayEventType)s
       gaudi.setKeepRecFile(True)
     return gaudi
 
-  def createGenericApplication(self, task):
-    """Create GenericApplication: not implemented"""
-    pass
+  def createDDSimApplication(self, task):
+    """create DDSim Application."""
+    from ILCDIRAC.Interfaces.API.NewInterface.Applications import DDSim
+    ddsim = DDSim()
+    ddsim.setVersion(self.softwareVersion)
+    ddsim.setDetectorModel(self.detectorModel)
+    ddsim.setNumberOfEvents(task.eventsPerJob)
+    ddsim.setEnergy(task.meta['Energy'])
+    self._setApplicationOptions('DDSim', ddsim, task.applicationOptions)
+    return ddsim
 
   def createGenerationProduction(self, task):
     """Create generation production."""
@@ -819,7 +838,8 @@ overlayEventType = %(overlayEventType)s
       raise RuntimeError("Error creating Simulation Production: %s" % res['Message'])
     simProd.setWorkflowName(prodName)
     # Add the application
-    res = simProd.append(self.createSimulationApplication(task))
+    simApp = self.createSimulationApplication(task)
+    res = simProd.append(simApp)
     if not res['OK']:
       raise RuntimeError("Error creating simulation Production: %s" % res['Message'])
     simProd.addFinalization(True, True, True, True)
@@ -831,12 +851,12 @@ overlayEventType = %(overlayEventType)s
     if not res['OK']:
       raise RuntimeError("Error creating simulation production: %s" % res['Message'])
 
-    simProd.addMetadataToFinalFiles({'BeamParticle1': parameterDict['pname1'],
-                                       'BeamParticle2': parameterDict['pname2'],
-                                       'EPA_B1': parameterDict['epa_b1'],
-                                       'EPA_B2': parameterDict['epa_b2'],
-                                     }
-                                   )
+    # simProd.addMetadataToFinalFiles({'BeamParticle1': parameterDict['pname1'],
+    #                                    'BeamParticle2': parameterDict['pname2'],
+    #                                    'EPA_B1': parameterDict['epa_b1'],
+    #                                    'EPA_B2': parameterDict['epa_b2'],
+    #                                  }
+    #                                )
 
     res = simProd.finalizeProd()
     if not res['OK']:
@@ -872,11 +892,11 @@ overlayEventType = %(overlayEventType)s
     #   if not res['OK']:
     #     raise RuntimeError("Error appending overlay to reconstruction transformation: %s" % res['Message'])
 
-    # # add reconstruction
-    # res = recProd.append(self.createMarlinApplication(task, over))
-    # if not res['OK']:
-    #   raise RuntimeError("Error appending Marlin to reconstruction production: %s" % res['Message'])
-    # recProd.addFinalization(True, True, True, True)
+    # add reconstruction
+    res = recProd.append(self.createReconstructionApplication(task, over))
+    if not res['OK']:
+      raise RuntimeError("Error appending reconstruction application to reconstruction production: %s" % res['Message'])
+    recProd.addFinalization(True, True, True, True)
 
     description = "CLICDet2017 %s" % meta['Energy']
     description += "Overlay" if over else "No Overlay"
@@ -888,12 +908,12 @@ overlayEventType = %(overlayEventType)s
     if not res['OK']:
       raise RuntimeError("Error creating reconstruction production: %s" % res['Message'])
 
-    recProd.addMetadataToFinalFiles({'BeamParticle1': parameterDict['pname1'],
-                                       'BeamParticle2': parameterDict['pname2'],
-                                       'EPA_B1': parameterDict['epa_b1'],
-                                       'EPA_B2': parameterDict['epa_b2'],
-                                     }
-                                   )
+    # recProd.addMetadataToFinalFiles({'BeamParticle1': parameterDict['pname1'],
+    #                                    'BeamParticle2': parameterDict['pname2'],
+    #                                    'EPA_B1': parameterDict['epa_b1'],
+    #                                    'EPA_B2': parameterDict['epa_b2'],
+    #                                  }
+    #                                )
 
     res = recProd.finalizeProd()
     if not res['OK']:
@@ -1030,7 +1050,7 @@ overlayEventType = %(overlayEventType)s
                                     ('SPLIT', self.createSplitProduction)]:
       for task in taskDict.get(pType, []):
         meta = createProduction(task)
-        # self.addSimTask(taskDict, meta, originalTask=task)
+        self.addSimTask(taskDict, meta, originalTask=task)
         taskDict['MOVE_' + pType].append(dict(meta))
 
     for task in taskDict.get('SIM', []):
@@ -1038,7 +1058,7 @@ overlayEventType = %(overlayEventType)s
         continue
       gLogger.notice("Creating task %s" % task)
       simMeta = self.createSimulationProduction(task)
-      # self.addRecTask(taskDict, simMeta, originalTask=task)
+      self.addRecTask(taskDict, simMeta, originalTask=task)
       taskDict['MOVE_SIM'].append(dict(simMeta))
 
     for task in taskDict.get('REC', []):
@@ -1112,19 +1132,21 @@ overlayEventType = %(overlayEventType)s
     return
 
   def addGenTask(self, taskDict, originalTask):
-    """Add a gen task with required options."""
+    """Add a generator task with required options."""
     return self._addTask(taskDict, metaInput=originalTask.meta,
                          originalTask=originalTask, prodType='GEN',
                          applicationName=self.generatorApplication,
                          )
 
+  def addSimTask(self, taskDict, metaInput, originalTask):
+    """Add a simulation task."""
+    return self._addTask(taskDict, metaInput, originalTask, prodType='SIM',
+                         applicationName=self.simulationApplication)
+
   def addRecTask(self, taskDict, metaInput, originalTask):
     """Add a reconstruction task."""
-    return self._addTask(taskDict, metaInput, originalTask, prodType='REC', applicationName='Marlin')
-
-  def addSimTask(self, taskDict, metaInput, originalTask):
-    """Add a sim task."""
-    return self._addTask(taskDict, metaInput, originalTask, prodType='SIM', applicationName='DDSim')
+    return self._addTask(taskDict, metaInput, originalTask, prodType='REC',
+                         applicationName=self.reconstructionApplication)
 
   @staticmethod
   def addTaskOptions(options, taskList):
diff --git a/src/ILCDIRAC/Interfaces/API/NewInterface/ProductionJob.py b/src/ILCDIRAC/Interfaces/API/NewInterface/ProductionJob.py
index a2675d75d..36d030715 100644
--- a/src/ILCDIRAC/Interfaces/API/NewInterface/ProductionJob.py
+++ b/src/ILCDIRAC/Interfaces/API/NewInterface/ProductionJob.py
@@ -803,14 +803,28 @@ class ProductionJob(Job):  # pylint: disable=too-many-public-methods, too-many-i
         generatorPath = os.path.join(*currentPathList)
         self.finalMetaDict[generatorPath] = {'Generator': self.generator}
 
+    if self.vo == 'fcc':
+      self.configureFCCOutputPath(path, application, currentPathList)
+
+    if self.vo == 'ilc':
+      self.configureILCOutputPath(path, application, currentPathList)
+
+    res = self._updateProdParameters(application)
+    if not res['OK']:
+      return res
+
+    self.checked = True
+
+    return S_OK()
+  
+  def configureFCCOutputPath(self, path, application, currentPathList):
     # comment of Lorenzo Valentini, 12-05-2023
     # In fcc we have a particular use of Gaudi. Its application is defined with the functions setOutputSimFile/setOutputRecFile.
     # However, it is sometimes used for other tasks, for which we don't want it to use the file path of the sim/rec transformations.
     # Up to now, seems like whenever we will be using sim/rec transformations we won't have to define the datatype, which we use to discriminate the two cases.
-    
-    # comment of Lorenzo Valentini, 12-05-2023
+
     # this is the case of a (maybe Gaudi) application used for sim/rec transformations: no datatype defined, and we check the presence of setOutputSimFile
-    if hasattr(application, "setOutputSimFile") and not application.willBeCut and self.vo == 'fcc' and not application.datatype: 
+    if hasattr(application, "setOutputSimFile") and not application.willBeCut and not application.datatype:
       detPath = os.path.join(path, application.detector)
       self.finalMetaDict[detPath] = {"DetectorType": application.detector}
       if application.keepRecFile:
@@ -827,11 +841,62 @@ class ProductionJob(Job):  # pylint: disable=too-many-public-methods, too-many-i
       LOG.notice('Will store the files under', path)
       self.finalpaths.append(path)
 
-    # comment of Lorenzo Valentini, 12-05-2023
-    # this is the case of a (maybe Gaudi) application used for dst/rec transformations: no datatype defined, and we check the presence of setOutputRecFile. It would make more sense to just check for setOutputDstFile,
-    # but we wanted to keep it backward compatibile with the ilc simulations.
-    # In the case of fcc, we check for the absence of datatype.
-    elif hasattr(application, "setOutputRecFile") and not application.willBeCut and (self.vo != 'fcc' or (self.vo == 'fcc' and not application.datatype)):
+    # # comment of Lorenzo Valentini, 12-05-2023
+    # # this is the case of a (maybe Gaudi) application used for dst/rec transformations: no datatype defined, and we check the presence of setOutputRecFile. It would make more sense to just check for setOutputDstFile,
+    # # but we wanted to keep it backward compatibile with the ilc simulations.
+    # # In the case of fcc, we check for the absence of datatype.
+    # elif hasattr(application, "setOutputRecFile") and not application.willBeCut and not application.datatype:
+    #   detPath = os.path.join(path, application.detectortype)
+    #   self.finalMetaDict[detPath] = {'DetectorType': application.detectortype}
+    #   if application.keepRecFile:
+    #     path = os.path.join(detPath, 'rec')
+    #     self.finalMetaDict[path] = {'Datatype': 'rec'}
+    #     fname = self.basename + '_rec.slcio'
+    #     application.setOutputRecFile(fname, path)
+    #     LOG.notice('Will store the files under', path)
+    #     self.finalpaths.append(path)
+    #   path = os.path.join(detPath, 'dst')
+    #   self.finalMetaDict[path] = {'Datatype': 'dst'}
+    #   fname = self.basename + '_dst.slcio'
+    #   application.setOutputDstFile(fname, path)
+    #   LOG.notice('Will store the files under', path)
+    #   self.finalpaths.append(path)
+
+    elif hasattr(application, "outputFile") and hasattr(application, 'datatype') and not application.outputFile and not application.willBeCut:
+      LOG.notice('Adding output meta data for %s' % type(application))
+      path = os.path.join(*currentPathList)
+
+      if not application.datatype and self.datatype:
+        application.datatype = self.datatype
+
+      # here we are adding detector and detectortype to the path (either one or the other) if they are defined. For fcc, we usually set the detector using the attribute application.detector.
+      if hasattr(application, "detectortype"):
+        if application.detectortype:
+          path = os.path.join(path, application.detectortype)
+          self.finalMetaDict[path] = {"DetectorType": application.detectortype}
+      if hasattr(application, "detector"):
+        if application.detector and not application.detectortype:
+          path = os.path.join(path, application.detector)
+          self.finalMetaDict[path] = {"DetectorType": application.detector}
+      if hasattr(application, "detectortype") or hasattr(application, "detector"):
+        if self.detector and not application.detectortype and not application.detector:
+          path = os.path.join(path, self.detector)
+          self.finalMetaDict[path] = {"DetectorType": self.detector}
+
+      path = os.path.join(path, application.datatype)
+      self.finalMetaDict[path] = {'Datatype': application.datatype}
+
+      LOG.notice('Will store the files under', '%s' % path)
+      self.finalpaths.append(path)
+      extension = getattr(application, '_extension', 'stdhep')
+
+      fname = self.basename + "_%s" % (application.datatype.lower()) + "." + extension
+      application.setOutputFile(fname, path)
+
+    return
+
+  def configureILCOutputPath(self, path, application, currentPathList):
+    if hasattr(application, "setOutputRecFile") and not application.willBeCut:
       detPath = os.path.join(path, application.detectortype)
       self.finalMetaDict[detPath] = {'DetectorType': application.detectortype}
       if application.keepRecFile:
@@ -848,8 +913,6 @@ class ProductionJob(Job):  # pylint: disable=too-many-public-methods, too-many-i
       LOG.notice('Will store the files under', path)
       self.finalpaths.append(path)
 
-    # comment of Lorenzo Valentini, 12-05-2023
-    # this is the case of an application with a single outputfile. It works in the same way for both fcc and ilc. 
     elif hasattr(application, "outputFile") and hasattr(application, 'datatype') and not application.outputFile and not application.willBeCut:
       LOG.notice('Adding output meta data for %s' % type(application))
       path = os.path.join(*currentPathList)
@@ -857,11 +920,6 @@ class ProductionJob(Job):  # pylint: disable=too-many-public-methods, too-many-i
       if not application.datatype and self.datatype:
         application.datatype = self.datatype
 
-      # comment of Lorenzo Valentini, 12-05-2023
-      # here we are adding detector and detectortype to the path, if they are defined. For fcc, we usually set the detector using the attribute application.detector.
-      # Anyway, seems like for ilc we wanted to "be ready" for both cases (detector, detectortype), so I did something similar for fcc.
-      # The code for ilc is not modified, to keep it backward compatible. However seems like in that case we would add to the path both detector and detectortype.
-      # For fcc we don't need to see the detector in the path two times, so I added a check to avoid that.
       if hasattr(application, "detectortype"):
         if application.detectortype:
           path = os.path.join(path, application.detectortype)
@@ -870,10 +928,7 @@ class ProductionJob(Job):  # pylint: disable=too-many-public-methods, too-many-i
           path = os.path.join(path, self.detector)
           self.finalMetaDict[path] = {"DetectorType": self.detector}
       if hasattr(application, "detector"):
-        if application.detector and self.vo != 'fcc':
-          path = os.path.join(path, application.detector)
-          self.finalMetaDict[path] = {"DetectorType": application.detector}
-        if application.detector and self.vo == 'fcc' and not application.detectortype:
+        if application.detector:
           path = os.path.join(path, application.detector)
           self.finalMetaDict[path] = {"DetectorType": application.detector}
       path = os.path.join(path, application.datatype)
@@ -887,13 +942,7 @@ class ProductionJob(Job):  # pylint: disable=too-many-public-methods, too-many-i
       fname = self.basename + "_%s" % (application.datatype.lower()) + "." + extension
       application.setOutputFile(fname, path)
 
-    res = self._updateProdParameters(application)
-    if not res['OK']:
-      return res
-
-    self.checked = True
-
-    return S_OK()
+    return
 
   def _updateProdParameters(self, application):
     """Update the prod parameters stored in the production parameters visible from the web."""
diff --git a/src/ILCDIRAC/Interfaces/API/NewInterface/Tests/Test_ProductionJob.py b/src/ILCDIRAC/Interfaces/API/NewInterface/Tests/Test_ProductionJob.py
index 3f65207eb..0d70cf68e 100644
--- a/src/ILCDIRAC/Interfaces/API/NewInterface/Tests/Test_ProductionJob.py
+++ b/src/ILCDIRAC/Interfaces/API/NewInterface/Tests/Test_ProductionJob.py
@@ -42,6 +42,7 @@ from Tests.Utilities.GeneralUtils import assertEqualsImproved, assertDiracFailsW
     assertDiracSucceeds
 from Tests.Utilities.FileUtils import FileUtil
 import six
+import os
 
 __RCSID__ = "$Id$"
 
@@ -801,9 +802,38 @@ class ProductionJobJobSpecificParamsTest(ProductionJobTestCase):
     self.assertIn(args[1], opsDict)
     return opsDict[args[1]]
 
-  def test_jobSpecificParams_detector_and_datatype(self):
+  @parameterized.expand([('ilc', 250, 250, '/vo/prod/250gev/fake/'),
+                         ('ilc', 250, None, '/vo/prod/250gev/fake/'),
+                         ('fcc', None, 250, '/vo/prod/250gev/fake/'),
+                         ])
+  def test_jobSpecificParams_energyandevents(self, vo, selfenergy, appenergy, finalPaths):
+    """Testing different combinations of energies and paths"""
     self.myapp.setOutputSE.return_value = S_OK(True)
     self.prodJob.evttype = ''
+    self.prodJob.basepath = '/vo/prod/'
+    self.prodJob.basename = 'basename'
+    self.prodJob.generator = 'whatever'
+    self.prodJob.vo = vo
+    self.prodJob.energy = selfenergy
+    self.prodJob.slicesize = 2
+    self.myapp.energy = appenergy
+    self.myapp.numberOfEvents = 1
+    self.myapp.detector = 'awesome'
+    self.myapp.eventType = 'fake'
+    self.myapp.detectortype = 'awesome'
+    self.myapp.willBeCut = False
+    self.myapp.outputFile = False
+    self.myapp.extraCLIArguments = ''
+    self.myapp.keepRecFile = False
+    with patch('%s.ProductionJob._updateProdParameters' % MODULE_NAME, new=Mock(return_value=S_OK())), \
+        patch('%s.ProductionJob.configureILCOutputPath' % MODULE_NAME, new=Mock(return_value=S_OK())), \
+        patch('%s.ProductionJob.configureFCCOutputPath' % MODULE_NAME, new=Mock(return_value=S_OK())):
+      assertDiracSucceeds(self.prodJob.append(self.myapp), self)
+
+    self.assertIn(finalPaths, self.prodJob.finalMetaDict.keys())
+
+  def test_configureFCCOutputPath(self):
+    self.myapp.setOutputSE.return_value = S_OK(True)
     self.myapp.willBeCut = False
     self.myapp.outputFile = False
     self.myapp.extraCLIArguments = ''
@@ -811,26 +841,53 @@ class ProductionJobJobSpecificParamsTest(ProductionJobTestCase):
     hasAttrMock.side_effect = self.hasAttrMocks
     with patch('%s.hasattr' % MODULE_NAME, new=hasAttrMock, create=True), \
          patch('%s.ProductionJob._updateProdParameters' % MODULE_NAME, new=Mock(return_value=S_OK())):
-      assertDiracSucceeds(self.prodJob.append(self.myapp), self)
+      self.prodJob.configureFCCOutputPath("/a/a", self.myapp, [self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType])
     self.prodJob.evttype = ''
 
     hasAttrMock = Mock()
     hasAttrMock.side_effect = self.hasAttrMocks_2
     with patch('%s.hasattr' % MODULE_NAME, new=hasAttrMock, create=True):
       self.myapp.detectortype = 'application_detectortype'
-      assertDiracSucceeds(self.prodJob.append(self.myapp), self)
+      self.prodJob.configureFCCOutputPath("/a/a", self.myapp, [self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType])
 
   @parameterized.expand([(True, "ilc", None, 'awesome', 'awesome', ['/vo/prod/250gev/fake/awesome/REC', '/vo/prod/250gev/fake/awesome/DST']),
                          (True, "ilc", None, None, 'awesome', ['/vo/prod/250gev/fake/awesome/REC', '/vo/prod/250gev/fake/awesome/DST']),
                          (False, "ilc", None, 'awesome', 'awesome', ['/vo/prod/250gev/fake/awesome/DST']),
                          (False, "ilc", "delphes", 'awesome', 'awesome', ['/vo/prod/250gev/fake/awesome/DST']),
-                         (True, "fcc", None, 'awesome', 'awesome', ['/vo/prod/250gev/fake/awesome/REC', '/vo/prod/250gev/fake/awesome/DST']),
-                         (False, "fcc", None, 'awesome', 'awesome', ['/vo/prod/250gev/fake/awesome/DST']),
-                         (False, "fcc", "delphes", 'awesome', 'awesome', ['/vo/prod/250gev/fake/awesome/delphes']),
+                         ])
+  def test_configureILCOutputPath_rec_and_dst(self, keep, vo, datatype, detector, detectortype, finalPaths):
+    """Testing different combinations of vo, transformation, detector, datatype"""
+    self.myapp.setOutputSE.return_value = S_OK(True)
+    self.prodJob.evttype = ''
+    self.prodJob.basepath = '/vo/prod/'
+    self.prodJob.basename = 'basename'
+    self.prodJob.generator = 'whatever'
+    self.prodJob.vo = vo
+    self.myapp.detector = detector
+    self.myapp.datatype = datatype
+    self.myapp.eventType = 'fake'
+    self.myapp.detectortype = detectortype
+    self.myapp.willBeCut = False
+    self.myapp.outputFile = False
+    self.myapp.extraCLIArguments = ''
+    self.myapp.keepRecFile = keep
+    self.myapp.setOutputRecFile = Mock(name='RecFile')
+    self.myapp.setOutputDstFile = Mock(name='DstFile')
+    del self.myapp.setOutputSimFile
+    with patch('%s.ProductionJob._updateProdParameters' % MODULE_NAME, new=Mock(return_value=S_OK())):
+      self.prodJob.configureILCOutputPath(os.path.join(self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType), self.myapp, [])
+    if keep:
+      self.myapp.setOutputRecFile.assert_called_once()
+    else:
+      self.myapp.setOutputRecFile.assert_not_called()
+    self.myapp.setOutputDstFile.assert_called_once()
+    assert self.prodJob.finalpaths == finalPaths
+
+  @parameterized.expand([(False, "fcc", "delphes", 'awesome', 'awesome', ['/vo/prod/250gev/fake/awesome/delphes']),
                          (False, "fcc", "delphes", 'awesome', None, ['/vo/prod/250gev/fake/awesome/delphes']),
                          (False, "fcc", "delphes", None, 'awesome', ['/vo/prod/250gev/fake/awesome/delphes']),
                          ])
-  def test_jobSpecificParams_rec_and_dst(self, keep, vo, datatype, detector, detectortype, finalPaths):
+  def test_configureFCCOutputPath_rec_and_dst(self, keep, vo, datatype, detector, detectortype, finalPaths):
     """Testing different combinations of vo, transformation, detector, datatype"""
     self.myapp.setOutputSE.return_value = S_OK(True)
     self.prodJob.evttype = ''
@@ -850,14 +907,8 @@ class ProductionJobJobSpecificParamsTest(ProductionJobTestCase):
     self.myapp.setOutputDstFile = Mock(name='DstFile')
     del self.myapp.setOutputSimFile
     with patch('%s.ProductionJob._updateProdParameters' % MODULE_NAME, new=Mock(return_value=S_OK())):
-      assertDiracSucceeds(self.prodJob.append(self.myapp), self)
-    if vo != 'fcc':
-      if keep:
-        self.myapp.setOutputRecFile.assert_called_once()
-      else:
-        self.myapp.setOutputRecFile.assert_not_called()
-      self.myapp.setOutputDstFile.assert_called_once()
-    elif not datatype:
+      self.prodJob.configureFCCOutputPath(os.path.join(self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType), self.myapp, [self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType])
+    if not datatype:
       if keep:
         self.myapp.setOutputRecFile.assert_called_once()
       else:
@@ -866,7 +917,6 @@ class ProductionJobJobSpecificParamsTest(ProductionJobTestCase):
     else:
       self.myapp.setOutputRecFile.assert_not_called()
       self.myapp.setOutputDstFile.assert_not_called()
-
     assert self.prodJob.finalpaths == finalPaths
 
   @parameterized.expand([(True, "fcc", None, 'awesome', 'awesome', ['/vo/prod/250gev/fake/awesome/rec', '/vo/prod/250gev/fake/awesome/sim']),
@@ -876,7 +926,7 @@ class ProductionJobJobSpecificParamsTest(ProductionJobTestCase):
                          (False, "fcc", "delphes", 'awesome', None, ['/vo/prod/250gev/fake/awesome/delphes']),
                          (False, "fcc", "delphes", None, 'awesome', ['/vo/prod/250gev/fake/awesome/delphes']),
                          ])
-  def test_jobSpecificParams_rec_and_sim(self, keep, vo, datatype, detector, detectortype, finalPaths):
+  def test_configureFCCOutputPath_rec_and_sim(self, keep, vo, datatype, detector, detectortype, finalPaths):
     """Testing different combinations of vo, transformation, detector, datatype"""
     self.myapp.setOutputSE.return_value = S_OK(True)
     self.prodJob.evttype = ''
@@ -896,7 +946,7 @@ class ProductionJobJobSpecificParamsTest(ProductionJobTestCase):
     self.myapp.setOutputSimFile = Mock(name='SimFile')
     del self.myapp.setOutputDstFile
     with patch('%s.ProductionJob._updateProdParameters' % MODULE_NAME, new=Mock(return_value=S_OK())):
-      assertDiracSucceeds(self.prodJob.append(self.myapp), self)
+      self.prodJob.configureFCCOutputPath(os.path.join(self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType), self.myapp, [self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType])
     if not datatype:
       if keep:
         self.myapp.setOutputRecFile.assert_called_once()
@@ -909,16 +959,38 @@ class ProductionJobJobSpecificParamsTest(ProductionJobTestCase):
 
     assert self.prodJob.finalpaths == finalPaths
 
+  def test_configureFCCOutputPath_selfdetector(self):
+    """Testing different combination of detector information"""
+    self.myapp.setOutputSE.return_value = S_OK(True)
+    self.prodJob.evttype = ''
+    self.prodJob.basepath = '/vo/prod/'
+    self.prodJob.basename = 'basename'
+    self.prodJob.vo = "fcc"
+    self.prodJob.detector = 'idea'
+    self.myapp.detector = ''
+    self.myapp.datatype = "sim"
+    self.myapp.eventType = 'fakeEvent'
+    self.myapp.detectortype = ''
+    self.myapp.willBeCut = False
+    self.myapp.outputFile = False
+    self.myapp.extraCLIArguments = ''
+    with patch('%s.ProductionJob._updateProdParameters' % MODULE_NAME, new=Mock(return_value=S_OK())):
+      self.prodJob.configureFCCOutputPath(os.path.join(self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType), self.myapp, [self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType])
+    assert self.prodJob.finalpaths == ['/vo/prod/250gev/fakeEvent/idea/sim']
+
 
-  @parameterized.expand([('detector', 'detectortype', ['/vo/prod/250gev/fake/generator/detectortype/detector/datatype']),
+  @parameterized.expand([('detector', 'detector', 'detectortype', ['/vo/prod/250gev/fake/generator/detectortype/detector/datatype']),
+                        ('detector', False, False, ['/vo/prod/250gev/fake/generator/detector/datatype']),
+                        (False, 'detector', False, ['/vo/prod/250gev/fake/generator/detector/datatype']),
                          ])
-  def test_jobSpecificParams_rec_and_dst__2(self, detector, detectortype, finalPaths):
+  def test_configureILCOutputPath_rec_and_dst__2(self, selfdetector, detector, detectortype, finalPaths):
     """Testing different combinations of vo, transformation, detector, datatype"""
     self.myapp.setOutputSE.return_value = S_OK(True)
     self.prodJob.evttype = ''
     self.prodJob.basepath = '/vo/prod/'
     self.prodJob.basename = 'basename'
     self.prodJob.generator = 'generator'
+    self.prodJob.detector = selfdetector
     self.prodJob.vo = "ilc"
     self.myapp.detector = detector
     self.myapp.datatype = "datatype"
@@ -930,41 +1002,35 @@ class ProductionJobJobSpecificParamsTest(ProductionJobTestCase):
     self.myapp.keepRecFile = False
     del self.myapp.setOutputRecFile
     with patch('%s.ProductionJob._updateProdParameters' % MODULE_NAME, new=Mock(return_value=S_OK())):
-      assertDiracSucceeds(self.prodJob.append(self.myapp), self)
+      self.prodJob.configureILCOutputPath(os.path.join(self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType), self.myapp, [self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType, self.prodJob.generator])
     assert self.prodJob.finalpaths == finalPaths
 
-
-  @parameterized.expand([(250, 250, ['/vo/prod/250gev/fake/awesome/DST']),
-                         (250, None, ['/vo/prod/250gev/fake/awesome/DST']),
-                         (None, 250, ['/vo/prod/250gev/fake/awesome/DST']),
+  @parameterized.expand([('SIM', False, ['/vo/prod/250gev/fake/generator/detectortype/detector/SIM']),
+                         (False, 'SIM', ['/vo/prod/250gev/fake/generator/detectortype/detector/SIM'])
                          ])
-  def test_jobSpecificParams_energyandevents(self, selfenergy, appenergy, finalPaths):
-    """Testing different combinations of energies and paths"""
+  def test_configureILCOutputPath_rec_and_sim(self, selfdata, appdata, finalPaths):
+    """Testing different combinations of vo, transformation, detector, datatype"""
     self.myapp.setOutputSE.return_value = S_OK(True)
     self.prodJob.evttype = ''
     self.prodJob.basepath = '/vo/prod/'
     self.prodJob.basename = 'basename'
-    self.prodJob.generator = 'whatever'
+    self.prodJob.generator = 'generator'
     self.prodJob.vo = "ilc"
-    self.prodJob.energy = selfenergy
-    self.prodJob.slicesize = 2
-    self.myapp.energy = appenergy
-    self.myapp.numberOfEvents = 1
-    self.myapp.detector = 'awesome'
+    self.prodJob.datatype = selfdata
+    self.myapp.detector = "detector"
+    self.myapp.datatype = appdata
     self.myapp.eventType = 'fake'
-    self.myapp.detectortype = 'awesome'
+    self.myapp.detectortype = "detectortype"
     self.myapp.willBeCut = False
     self.myapp.outputFile = False
     self.myapp.extraCLIArguments = ''
     self.myapp.keepRecFile = False
-    self.myapp.setOutputRecFile = Mock(name='RecFile')
-    self.myapp.setOutputDstFile = Mock(name='DstFile')
+    del self.myapp.setOutputRecFile
     with patch('%s.ProductionJob._updateProdParameters' % MODULE_NAME, new=Mock(return_value=S_OK())):
-      assertDiracSucceeds(self.prodJob.append(self.myapp), self)
-    self.myapp.setOutputRecFile.assert_not_called()
-    self.myapp.setOutputDstFile.assert_called_once()
+      self.prodJob.configureILCOutputPath(os.path.join(self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType), self.myapp, [self.prodJob.basepath, self.prodJob.getEnergyPath(), self.myapp.eventType, self.prodJob.generator])
     assert self.prodJob.finalpaths == finalPaths
 
+
   def test_setters_1(self):
     import random
     self.prodJob.setDryRun(True)
-- 
GitLab