From b689d09449c2fee480b4ce9a9e257a54b81c2722 Mon Sep 17 00:00:00 2001
From: Emmanuel Le Guirriec <emmanuel.le.guirriec@cern.ch>
Date: Tue, 11 Dec 2018 11:33:36 +0000
Subject: [PATCH] Merge branch '21.2-add-frozen-derivation-test' into '21.2'
First version of the frozen derivation test script
See merge request atlas/athena!16488
(cherry picked from commit 830e8d306b780740082dfcc8dc3c03ac745d6efe [formerly 8b307f6fb2c9f3e683260cfa6e05264c5c0d8278])
6c0f0b46 First version of the frozen derivation test script
ee5ceb38 Add option to enable the user to run the test in data or mc
Former-commit-id: 3ecbb60ed59808e08dee2416be46ac6fc16e6fbf
---
.../python/iconfTool/__init__.py | 0
.../python/iconfTool/utils/__init__.py | 0
.../DerivationFrameworkCore/CMakeLists.txt | 2 +-
.../scripts/frozen_derivation_test.py | 307 ++++++++++++++++++
4 files changed, 308 insertions(+), 1 deletion(-)
mode change 100644 => 100755 Control/AthenaConfiguration/python/iconfTool/__init__.py
mode change 100755 => 100644 Control/AthenaConfiguration/python/iconfTool/utils/__init__.py
create mode 100755 PhysicsAnalysis/DerivationFramework/DerivationFrameworkCore/scripts/frozen_derivation_test.py
diff --git a/Control/AthenaConfiguration/python/iconfTool/__init__.py b/Control/AthenaConfiguration/python/iconfTool/__init__.py
old mode 100644
new mode 100755
diff --git a/Control/AthenaConfiguration/python/iconfTool/utils/__init__.py b/Control/AthenaConfiguration/python/iconfTool/utils/__init__.py
old mode 100755
new mode 100644
diff --git a/PhysicsAnalysis/DerivationFramework/DerivationFrameworkCore/CMakeLists.txt b/PhysicsAnalysis/DerivationFramework/DerivationFrameworkCore/CMakeLists.txt
index 547e982716c..f5ed44409d3 100644
--- a/PhysicsAnalysis/DerivationFramework/DerivationFrameworkCore/CMakeLists.txt
+++ b/PhysicsAnalysis/DerivationFramework/DerivationFrameworkCore/CMakeLists.txt
@@ -31,4 +31,4 @@ atlas_add_component( DerivationFrameworkCore
# Install files from the package:
atlas_install_python_modules( python/*.py )
atlas_install_joboptions( share/*.py )
-
+atlas_install_scripts( scripts/frozen_derivation_test.py )
diff --git a/PhysicsAnalysis/DerivationFramework/DerivationFrameworkCore/scripts/frozen_derivation_test.py b/PhysicsAnalysis/DerivationFramework/DerivationFrameworkCore/scripts/frozen_derivation_test.py
new file mode 100755
index 00000000000..f3ae6d70299
--- /dev/null
+++ b/PhysicsAnalysis/DerivationFramework/DerivationFrameworkCore/scripts/frozen_derivation_test.py
@@ -0,0 +1,307 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2002-2018 CERN for the benefit of the ATLAS collaboration
+
+import glob
+import logging
+import multiprocessing
+import os
+import subprocess
+import sys
+import uuid
+
+# Setting logging options
+fmt = '%(asctime)s :: %(levelname)-8s :: %(message)s'
+datefmt = '%m/%d/%Y %H:%M'
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG,
+ format=fmt,
+ datefmt=datefmt,
+ filename='./frozen_derivation_test.log',
+ filemode='w')
+console = logging.StreamHandler()
+formatter = logging.Formatter(fmt,datefmt=datefmt)
+console.setFormatter(formatter)
+logger.addHandler(console)
+
+###
+# Function to check if the derivation job succeeded or not
+###
+def check_job_success(logFile):
+ ''' Return true/false depending on whether the provided log file has "successful run" pattern. '''
+
+ # Simply tail the log and look for the famous "successful run"
+ cmd = ' tail -n 20 ' + logFile + ' | grep "\\"successful run\\"" | wc -l '
+
+ result = subprocess.check_output(['/bin/bash', '-c',cmd]).rstrip()
+ logger.debug('Job success check for the log file %s is %s', logFile, result)
+
+ return (result == '1')
+
+####
+## Function to run the clean test
+####
+def run_clean_test(inputFile,extraArgs,release,cleanDir,uniqId):
+ ''' Runs the clean derivation test and checks if the job succeeded or not. '''
+
+ logger.info('Starting clean test...')
+
+ # Build the actual command
+ cmd = ( ' mkdir -p ' + cleanDir + '/clean_derivation_test_' + uniqId + ';'
+ ' cd ' + cleanDir + '/clean_derivation_test_' + uniqId + ';'
+ ' source $AtlasSetup/scripts/asetup.sh ' + release + ' >& /dev/null;'
+ ' Reco_tf.py --inputAODFile %s %s > derivation_test.log 2>&1;'%(inputFile,extraArgs) )
+
+ # Execute the actual command
+ subprocess.call(cmd,shell=True)
+
+ # Check the job status
+ logFile = cleanDir + '/clean_derivation_test_' + uniqId + '/log.AODtoDAOD'
+ logger.debug('Attempting to read the clean log file from %s', logFile)
+ result = check_job_success(logFile)
+
+ logger.info('Finished clean test w/ status %s...', 'SUCCESS' if result else 'FAILURE')
+ pass
+
+####
+## Function to run the patched test
+####
+def run_patched_test(inputFile,extraArgs,pwd,release):
+ ''' Runs the patched derivation test and checks if the job succeeded or not. '''
+
+ logger.info('Starting patched test...')
+
+ # Build the actual command
+ cmd = ' cd '+pwd+';'
+ if not release: # This is the CI escape
+ pass
+ elif 'WorkDir_DIR' in os.environ:
+ cmake_build_dir = (os.environ['WorkDir_DIR'])
+ logger.debug('Using the local cmake build directory %s'%(cmake_build_dir))
+ cmd += ( ' source $AtlasSetup/scripts/asetup.sh '+release+' >& /dev/null;'
+ ' source '+cmake_build_dir+'/setup.sh;' )
+ else :
+ cmd += ( ' source $AtlasSetup/scripts/asetup.sh '+release+' >& /dev/null;' )
+ cmd += ( ' mkdir -p patched_derivation_test; cd patched_derivation_test;'
+ ' Reco_tf.py --inputAODFile %s %s > derivation_test.log 2>&1;'%(inputFile,extraArgs) )
+
+ # Execute the actual command
+ subprocess.call(cmd,shell=True)
+
+ # Check the job status
+ logFile = pwd + '/patched_derivation_test/log.AODtoDAOD'
+ logger.debug('Attempting to read the patched log file from %s', logFile )
+ result = check_job_success(logFile)
+
+ logger.info('Finished patched test w/ status %s...', 'SUCCESS' if result else 'FAILURE' )
+ pass
+
+####
+## Function to run the diff-root
+####
+def compare_files(cleanHeadDir,uniqId,pwd,isPatchedOnly):
+ ''' Compares the clean and patched DAOD files to see if the contents differ. '''
+
+ cleanDir = cleanHeadDir + '/clean_derivation_test_' + uniqId
+ if isPatchedOnly:
+ cleanDir = cleanHeadDir
+
+ logger.debug('Reading the reference file from location '+cleanDir)
+
+ comparison_command = ( 'acmd.py diff-root ' + cleanDir + '/DAOD_PHYSVAL.pool.root '
+ 'patched_derivation_test/DAOD_PHYSVAL.pool.root '
+ '--error-mode resilient '
+ '--entries 20 > patched_derivation_test/diff-root.DAOD.log 2>&1' )
+ output,error = subprocess.Popen(['/bin/bash', '-c', comparison_command], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+
+ # We want to catch/print both container additions/subtractions as well as
+ # changes in these containers. `allGood_return_code` is meant to catch
+ # other issues found in the diff (sync issues etc.)
+ passed_all_test=True
+ allGood_return_code=False
+
+ logFile = pwd + '/patched_derivation_test/diff-root.DAOD.log'
+ logger.debug('Attempting to read the diff-log file from %s', logFile )
+ with open(logFile) as f:
+ for line in f:
+ if 'WARNING' in line: # Catches container addition/subtractions
+ logger.error(line)
+ passed_all_test=False
+ if 'leaves differ' in line: # Catches changes in branches
+ logger.error(line)
+ passed_all_test=False
+ if 'INFO all good.' in line:
+ allGood_return_code = True
+
+ _decision = passed_all_test and allGood_return_code
+ if _decision:
+ logger.info('Passed!')
+ else:
+ logger.error('Your tag breaks changes the AOD content! See patched_derivation_test/diff-root.DAOD.log file for more information.')
+
+ return _decision
+
+####
+## Function to get release information
+####
+def get_release_setup():
+ ''' This function tries to read environment variables to figure out the appropriate ATLAS setup '''
+
+ # Read in environment variables
+ current_nightly = os.environ['AtlasBuildStamp']
+ release_base=os.environ['AtlasBuildBranch']
+ release_head=os.environ['AtlasVersion']
+ platform=os.environ['AthDerivation_PLATFORM']
+ project=os.environ['AtlasProject']
+
+ # Search through the main nightly installation location
+ builds_dir_searchStr='/cvmfs/atlas-nightlies.cern.ch/repo/sw/'+release_base+'/[!latest_]*/'+project+'/'+release_head
+ sorted_list = sorted(glob.glob(builds_dir_searchStr), key=os.path.getmtime)
+
+ latest_nightly = ''
+ for folder in reversed(sorted_list):
+ if not glob.glob(folder+'/../../'+release_base+'__'+project+'*-opt*.log') : continue
+ latest_nightly = folder.split('/')[-3]
+ break
+
+ if current_nightly != latest_nightly:
+ logger.info('Please be aware that you are not testing your tags in the latest available nightly, which is %s',latest_nightly)
+
+ setup='%s,%s,%s,AthDerivation'%(release_base,platform.replace('-', ','),current_nightly)
+
+ logger.info('Your tags will be tested in environment %s',setup)
+
+ return setup
+
+
+####
+## Main Function
+####
+def main():
+ ''' Main function that parses user inputs, prepares and runs the tests '''
+
+ from optparse import OptionParser
+ parser=OptionParser(usage='%prog [options]',
+ version='%prog 0.1.0')
+
+ parser.add_option('--log-level',
+ type='choice',
+ action='store',
+ dest='logLevel',
+ choices=['critical','error','warning','info','debug'],
+ default='info',
+ help='Logging level (default: info)')
+
+ parser.add_option('-c',
+ '--clean-dir',
+ type='string',
+ dest='cleanDir',
+ default='/tmp/',
+ help='Specify the head directory for running the clean derivation tests '
+ '(default: /tmp/${USER}). If provided w/ -p (--patched) option, '
+ 'the reference results are expected to be in this location.')
+
+ parser.add_option('-p',
+ '--patched',
+ action='store_true',
+ dest='patched',
+ default=False,
+ help='Flag to avoid clean run. '
+ 'In this mode only patched test will run and the '
+ 'changes will be compared against predefined '
+ ' reference files (default: false)')
+
+ parser.add_option('--mode',
+ type='choice',
+ action='store',
+ dest='mode',
+ choices=['data','mc'],
+ default='mc',
+ help='Test mode (default: mc)')
+
+ (options,args) = parser.parse_args()
+
+ logger.info(76*'-')
+
+ # Set logging level
+ logger.setLevel(options.logLevel.upper())
+
+ # Check if patched only
+ isPatchedOnly = options.patched
+
+ # Check if the clean directory exists
+ cleanHeadDir = options.cleanDir
+ if str(cleanHeadDir) == '/tmp/':
+ cleanHeadDir += os.environ['USER']
+
+ if os.path.exists(cleanHeadDir):
+ logger.info('The head directory for the output of the clean tests will be %s', cleanHeadDir)
+ else:
+ logger.error('The head directory for the output of the clean tests doesn\'t exist!')
+ logger.error('Please check that %s exists and you have enough permissions to read/write.', cleanHeadDir)
+ logger.error('If not please specify a folder that satisfies the requirements via -c or --clean-dir options.')
+ sys.exit(-1)
+
+ # Check if an ATLAS project is setup
+ if 'AtlasVersion' not in os.environ:
+ logging.error('Please setup the an ATLAS release!')
+ sys.exit(-1)
+
+ # Get the release information
+ mySetup = ''
+ try:
+ mySetup = get_release_setup()
+ except KeyError as error:
+ logger.warning('Cannot find the key %s',error)
+ logger.warning('This might be benign if running in CI enviroment')
+ cleanSetup = mySetup
+
+ # Get the PWD and create UUID to avoid possible overlap between runs
+ myPwd = os.getcwd()
+ logger.debug('Current working directory is %s',myPwd)
+ uniqId = str(uuid.uuid4())
+ logger.debug('Created uuid %s',uniqId)
+
+ # Run the tests
+ logger.info('%s %s %s',25*'-',' Run AthDerivation Test ',25*'-')
+ inputFile = ( '/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/DerivationFrameworkART/{}'
+ ''.format('AOD.14795494._005958.pool.root.1' if options.mode == 'mc' else 'data18_13TeV.00348403.physics_Main.merge.AOD.f920_m1947._lb0829._0001.1') )
+ extraArgs = ( '--outputDAODFile pool.root --maxEvents 20 --reductionConf PHYSVAL --preExec \'rec.doApplyAODFix.set_Value_and_Lock(True); '
+ 'from BTagging.BTaggingFlags import BTaggingFlags;BTaggingFlags.CalibrationTag = "{}"; '
+ 'from AthenaCommon.AlgSequence import AlgSequence; topSequence = AlgSequence(); '
+ 'topSequence += CfgMgr.xAODMaker__DynVarFixerAlg("InDetTrackParticlesFixer", Containers = ["InDetTrackParticlesAux." ] )\''
+ ''.format('BTagCalibRUN12-08-40' if options.mode == 'mc' else 'BTagCalibRUN12Onl-08-40') )
+
+ myprocesses = {}
+ if not isPatchedOnly:
+ myprocesses['clean'] = multiprocessing.Process(target=run_clean_test,kwargs={'inputFile':inputFile,'extraArgs' : extraArgs,'release':cleanSetup,'cleanDir':cleanHeadDir,'uniqId':uniqId})
+ myprocesses['patched'] = multiprocessing.Process(target=run_patched_test,kwargs={'inputFile':inputFile,'extraArgs' : extraArgs,'pwd':myPwd,'release':mySetup})
+ myprocesses['clean'].start()
+ myprocesses['patched'].start()
+ else:
+ myprocesses['patched'] = multiprocessing.Process(target=run_patched_test,kwargs={'inputFile':inputFile,'extraArgs' : extraArgs,'pwd':myPwd,'release':mySetup})
+ myprocesses['patched'].start()
+
+ for process in myprocesses:
+ myprocesses[process].join()
+
+ # Post processing
+ logger.info('%s %s %s',25*'-',' Run xAOD content check ',25*'-')
+
+ all_tests_passed = True
+
+ if not compare_files(cleanHeadDir,uniqId,myPwd,isPatchedOnly):
+ all_tests_passed = False
+
+ logger.info('%s %s %s',25*'-',' Summary of all tests ',25*'-')
+ if all_tests_passed:
+ logger.info(' All Tests: Passed (0)')
+ logger.info(76*'-')
+ sys.exit(0)
+ else:
+ logger.info(' All Tests: FAILED (-1)')
+ logger.info(76*'-')
+ sys.exit(-1)
+
+if __name__ == '__main__':
+ main()
--
GitLab