diff --git a/src/ILCDIRAC/Core/scripts/dirac_ilc_add_cvmfs_software.py b/src/ILCDIRAC/Core/scripts/dirac_ilc_add_cvmfs_software.py index 2e9504f5fae1880fb8573e75fe1c0838cbbceb0c..8a902abdec35cab8582ee7c7e76da9cc9c2623e2 100644 --- a/src/ILCDIRAC/Core/scripts/dirac_ilc_add_cvmfs_software.py +++ b/src/ILCDIRAC/Core/scripts/dirac_ilc_add_cvmfs_software.py @@ -46,7 +46,7 @@ Options: :since: Feb 18, 2015 """ from __future__ import absolute_import -import os +import os, sys import subprocess from DIRAC.Core.Base.Script import Script @@ -200,23 +200,44 @@ class CVMFSAdder(object): :returns: None """ - for root, dirs, _files in os.walk(os.path.join(self.parameter["basepath"], "lcgeo")): + detectorBasePath = os.path.join(self.parameter["basepath"], "lcgeo") + gLogger.notice('Looking for detectors in:', repr(detectorBasePath)) + for root, dirs, _files in os.walk(detectorBasePath): for direct in dirs: if root.endswith("compact"): # the main xml file must have the same name as the folder xmlPath = os.path.join(root, direct, direct + ".xml") if os.path.exists(xmlPath): - self.detmodels[direct] = xmlPath - + gLogger.notice('Found', xmlPath) + relPath = xmlPath[len(detectorBasePath):].strip('/') + self.detmodels[direct] = relPath # for key4hep - for root, dirs, _files in os.walk(self.getFromInitScript('lcgeo_DIR')): + theEnvVar = None + for envVar in ['K4GEO', 'LCGEO', 'k4geo_DIR', 'lcgeo_DIR']: + try: + detectorBasePath = self.getFromInitScript(envVar) + gLogger.notice('Using the environment variable %s to obtain detectors XML locations.' %envVar) + gLogger.notice('Looking for detectors in:', repr(detectorBasePath)) + theEnvVar = envVar + break + except: + continue + if not theEnvVar: + gLogger.error('None of the following environment variables exists:', 'LCGEO', 'K4GEO', 'lcgeo_DIR', 'k4geo_DIR') + sys.exit(1) + + gLogger.notice('Looking for detectors in:', repr(detectorBasePath)) + for root, dirs, _files in os.walk(detectorBasePath): for direct in dirs: if root.endswith("compact"): # the main xml file must have the same name as the folder xmlPath = os.path.join(root, direct, direct + ".xml") if os.path.exists(xmlPath): - self.detmodels[direct] = xmlPath - + gLogger.notice('Found', xmlPath) + relPath = xmlPath[len(detectorBasePath):].strip('/') + self.detmodels[direct] = relPath + gLogger.notice('Found %d detectors' % len(self.detmodels)) + self.detmodels['EnvironmentVariable'] = envVar def findFCCDetectorModels(self): """find all detector models in fccdetectors and fill the self.detmodels dictionary with Detmodel as @@ -243,6 +264,7 @@ key and path as value. xmlPath = os.path.join(root, mainFile) if os.path.exists(xmlPath): gLogger.notice('Found', xmlPath) + relPath = xmlPath[len(detectorBasePath):].strip('/') self.detmodels[detModel] = xmlPath gLogger.notice('Found %d detectors' % len(self.detmodels)) @@ -382,7 +404,7 @@ key and path as value. def getFromInitScript(self, variable): """Return value of the variable as set by the init script.""" command = "source %s > /dev/null; echo $%s" % (self.cliParams.initScriptLocation, variable) - variableValue = subprocess.check_output(command, shell=True).strip() + variableValue = subprocess.check_output(command, shell=True, encoding="utf8").strip() return variableValue @Script() diff --git a/src/ILCDIRAC/Workflow/Modules/Test/Test_DDSimWorkflow.py b/src/ILCDIRAC/Workflow/Modules/Test/Test_DDSimWorkflow.py index 127afa6fd55e6441db186d506496228ff06fea15..e869878d198306fdd199741170efe889b3b3beeb 100644 --- a/src/ILCDIRAC/Workflow/Modules/Test/Test_DDSimWorkflow.py +++ b/src/ILCDIRAC/Workflow/Modules/Test/Test_DDSimWorkflow.py @@ -646,52 +646,59 @@ class TestDDSimAnalysisASI(TestDDSimAnalysis): class TestDDSimAnalysisDetXMLCS(TestDDSimAnalysis): """tests for _getDetectorXML.""" - @patch("%s.getSoftwareFolder" % DD4H_NAME, new=Mock(return_value=S_OK("/win32"))) + @patch("%s.getEnvironmentScript" % DD4H_NAME, new=Mock(return_value=S_OK("ddsiming.sh"))) + @patch("%s.shellCall" % DD4H_NAME, new=Mock(return_value={'OK': True, 'Value': (0, '/cvmfs/sw.hsf.org/spackages7/lcgeo/0.18/x86_64-centos7-gcc11.2.0-opt/6ciib/share/lcgeo/compact\n', '')})) def test_DDSim_getDetectorXML(self): """DDSim.getDetectorXML from CS.................................................................""" gLogger.setLevel("ERROR") - xmlPath = "/path/to/camelot.xml" - self.ddsim.detectorModel = "camelot" - self.ddsim.ops.getOptionsDict = Mock(return_value=S_OK(dict(camelot=xmlPath))) + xmlPath = "/cvmfs/sw.hsf.org/spackages7/lcgeo/0.18/x86_64-centos7-gcc11.2.0-opt/6ciib/share/lcgeo/compact/CLIC/compact/CLIC_o3_v14/CLIC_o3_v14.xml" + self.ddsim.detectorModel = "CLIC_o3_v14" + self.ddsim.ops.getOptionsDict = Mock(return_value=S_OK(dict(CLIC_o3_v14=xmlPath))) + self.ddsim.ops.getValue = Mock(return_value='LCGEO') self.ddsim.workflow_commons = dict() res = self.ddsim._getDetectorXML() - self.assertEqual(res['Value'], xmlPath) + self.assertEqual(res['Value'], "/cvmfs/sw.hsf.org/spackages7/lcgeo/0.18/x86_64-centos7-gcc11.2.0-opt/6ciib/share/lcgeo/compact/CLIC/compact/CLIC_o3_v14/CLIC_o3_v14.xml") - @patch("%s.getSoftwareFolder" % DD4H_NAME, new=Mock(return_value=S_OK("/win32"))) + @patch("%s.getEnvironmentScript" % DD4H_NAME, new=Mock(return_value=S_OK("ddsiming.sh"))) + @patch("%s.shellCall" % DD4H_NAME, new=Mock(return_value={'OK': True, 'Value': (0, '/cvmfs/sw.hsf.org/spackages7/lcgeo/0.18/x86_64-centos7-gcc11.2.0-opt/6ciib/share/lcgeo/compact\n', '')})) def test_DDSim_getDetectorXML_relPatg(self): """DDSim.getDetectorXML from CS with relative path..............................................""" gLogger.setLevel("ERROR") - xmlPath = "rel/path/to/camelot.xml" - self.ddsim.detectorModel = "camelot" - self.ddsim.ops.getOptionsDict = Mock(return_value=S_OK(dict(camelot=xmlPath))) + xmlPath = "CLIC/compact/CLIC_o3_v14/CLIC_o3_v14.xml" + self.ddsim.detectorModel = "CLIC_o3_v14" + self.ddsim.ops.getOptionsDict = Mock(return_value=S_OK(dict(CLIC_o3_v14=xmlPath))) + self.ddsim.ops.getValue = Mock(return_value='LCGEO') self.ddsim.workflow_commons = dict() res = self.ddsim._getDetectorXML() - self.assertEqual(res['Value'], os.path.join("/win32", xmlPath)) + self.assertEqual(res['Value'], "/cvmfs/sw.hsf.org/spackages7/lcgeo/0.18/x86_64-centos7-gcc11.2.0-opt/6ciib/share/lcgeo/compact/CLIC/compact/CLIC_o3_v14/CLIC_o3_v14.xml") + @patch("%s.getEnvironmentScript" % DD4H_NAME, new=Mock(return_value=S_OK("ddsiming.sh"))) @patch("%s.getSoftwareFolder" % DD4H_NAME, new=Mock(return_value=S_OK("/win32"))) - def test_DDSim_getDetectorXML_Fail(self): - """DDSim.getDetectorXML Failure.................................................................""" + def test_DDSim_getDetectorXML_oldRelPatg(self): + """DDSim.getDetectorXML from CS with relative path..............................................""" gLogger.setLevel("ERROR") - xmlPath = "/path/to/camelot.xml" - self.ddsim.detectorModel = "fortressOfSolitude" - self.ddsim.ops.getOptionsDict = Mock(return_value=S_OK(dict(camelot=xmlPath))) + xmlPath = "detectors/ILD_o1_v05/ILD_o1_v05.xml" + self.ddsim.detectorModel = "ILD_o1_v05" + self.ddsim.ops.getOptionsDict = Mock(return_value=S_OK(dict(ILD_o1_v05=xmlPath))) + self.ddsim.ops.getValue = Mock(return_value=None) self.ddsim.workflow_commons = dict() res = self.ddsim._getDetectorXML() - self.assertFalse(res['OK']) - self.assertEqual(res['Message'], "Detector model was not found") + self.assertEqual(res['Value'], "/win32/detectors/ILD_o1_v05/ILD_o1_v05.xml") - @patch("%s.getSoftwareFolder" % DD4H_NAME, new=Mock(return_value=S_ERROR("Windows not supported"))) - def test_DDSim_getDetectorXML_NoSoftFolder(self): - """DDSim.getDetectorXML Error no SoftwareFolder.................................................""" + @patch("%s.getEnvironmentScript" % DD4H_NAME, new=Mock(return_value=S_OK("ddsiming.sh"))) + @patch("%s.shellCall" % DD4H_NAME, new=Mock(return_value={'OK': True, 'Value': (0, '/cvmfs/sw.hsf.org/spackages7/lcgeo/0.18/x86_64-centos7-gcc11.2.0-opt/6ciib/share/lcgeo/compact\n', '')})) + def test_DDSim_getDetectorXML_Fail(self): + """DDSim.getDetectorXML Failure.................................................................""" gLogger.setLevel("ERROR") - xmlPath = "/path/to/camelot.xml" - self.ddsim.detectorModel = "camelot" - self.ddsim.ops.getOptionsDict = Mock(return_value=S_OK(dict(camelot=xmlPath))) + xmlPath = "/cvmfs/sw.hsf.org/spackages7/lcgeo/0.18/x86_64-centos7-gcc11.2.0-opt/6ciib/share/lcgeo/compact/CLIC/compact/CLIC_o3_v14/CLIC_o3_v14.xml" + self.ddsim.detectorModel = "CLIC_o3_v1412345_unexistent" + self.ddsim.ops.getOptionsDict = Mock(return_value=S_OK(dict(CLIC_o3_v14=xmlPath))) + self.ddsim.ops.getValue = Mock(return_value='LCGEO') self.ddsim.workflow_commons = dict() res = self.ddsim._getDetectorXML() - self.assertEqual(res['Message'], "Windows not supported") + self.assertFalse(res['OK']) + self.assertEqual(res['Message'], "Detector model was not found") - @patch("%s.getSoftwareFolder" % DD4H_NAME, new=Mock(return_value=S_OK("/win32"))) def test_DDSim_getDetectorXML_NoDetModels(self): """DDSim.getDetectorXML Error no detectorModels.................................................""" gLogger.setLevel("ERROR") diff --git a/src/ILCDIRAC/Workflow/Modules/Test/Test_GaudiAppModule.py b/src/ILCDIRAC/Workflow/Modules/Test/Test_GaudiAppModule.py index d18895393dc7d909d4fe7ff4f623f741359228a5..1f783aa94de27d9fc87615f32ab9eadff1e1cbd4 100644 --- a/src/ILCDIRAC/Workflow/Modules/Test/Test_GaudiAppModule.py +++ b/src/ILCDIRAC/Workflow/Modules/Test/Test_GaudiAppModule.py @@ -224,17 +224,6 @@ class TestGaudiAppModuleRunit(TestGaudiAppModule): #NB: resolveIFpaths = {"OK": False} will cause RunIt() to be interrupted earlier self.assertEqual(res['Message'], "just a test") - @patch("%s.getEnvironmentScript" % MODULE_NAME, new=Mock(return_value=S_OK("setup.sh"))) - def test_GaudiApp_runIt_success_getdetectormodel(self): - """gaudi.runit success using the getdetectormodel function................................................................""" - self.gaudi.platform = "Windows" - self.gaudi.applicationLog = self.logFileName - self.gaudi.executableName = self.executableName - self.gaudi.InputFile = "inputfile" - self.gaudi.detectorModel = "detectormodel" - res = self.gaudi.runIt() - self.assertIn("Could not find gaudiapp", res['Message']) - @patch("%s.getEnvironmentScript" % MODULE_NAME, new=Mock(return_value=S_OK("setup.sh"))) @patch("%s.shellCall" % MODULE_NAME, new=Mock(return_value=S_OK((0, "AllGood")))) def test_GaudiApp_runIt_fail_getdetectormodelmocking(self): diff --git a/src/ILCDIRAC/Workflow/Utilities/DD4hepMixin.py b/src/ILCDIRAC/Workflow/Utilities/DD4hepMixin.py index 5f3cbcbae7d39bae2b7a26146c004d8e571e1d7b..cee55a840f209d1cf37df953993d1b8b322385a0 100644 --- a/src/ILCDIRAC/Workflow/Utilities/DD4hepMixin.py +++ b/src/ILCDIRAC/Workflow/Utilities/DD4hepMixin.py @@ -27,10 +27,13 @@ from __future__ import absolute_import import os import tarfile +import subprocess +from DIRAC.Core.Utilities.Subprocess import shellCall + from DIRAC import S_OK, S_ERROR, gLogger -from ILCDIRAC.Core.Utilities.CombinedSoftwareInstallation import getSoftwareFolder, unzip_file_into_dir +from ILCDIRAC.Core.Utilities.CombinedSoftwareInstallation import unzip_file_into_dir, getEnvironmentScript, getSoftwareFolder LOG = gLogger.getSubLogger(__name__) @@ -63,15 +66,32 @@ class DD4hepMixin(object): LOG.error("Failed to get list of DetectorModels from the ConfigSystem", detectorModels['Message']) return S_ERROR("Failed to get list of DetectorModels from the ConfigSystem") - softwareFolder = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion) - if not softwareFolder['OK']: - return softwareFolder - softwareRoot = softwareFolder['Value'] + res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript) + if not res['OK']: + LOG.error("Could not obtain the environment script: ", res["Message"]) + return res + envScriptPath = res["Value"] + + # getting the value of the detectors XML directory. + envVariableName = self.ops.getValue(f"/DDSimDetectorModels/{self.applicationVersion}/EnvironmentVariable", None) if self.detectorModel in detectorModels['Value']: - detModelPath = detectorModels['Value'][self.detectorModel] - if not detModelPath.startswith("/"): - detModelPath = os.path.join(softwareRoot, detModelPath) + + if envVariableName or detectorModels['Value'][self.detectorModel].startswith("/"): + res = shellCall(0, "source %s > /dev/null; echo $%s" % (envScriptPath, envVariableName), callbackFunction=None, bufferLimit=20971520) + if not res['OK']: + LOG.error("Failed to obtain the detector XML location: ", res["Message"]) + return res + softwareRoot = res['Value'][1].strip() + + else: + softwareFolder = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion) + if not softwareFolder['OK']: + LOG.error("Failed to obtain the detector XML location: ", res["Message"]) + return softwareFolder + softwareRoot = softwareFolder['Value'] + + detModelPath = os.path.join(softwareRoot, detectorModels['Value'][self.detectorModel]) LOG.info("Found path for DetectorModel %s in CS: %s " % (self.detectorModel, detModelPath)) return S_OK(detModelPath)