Merge branch '21.2-add-frozen-derivation-test' into '21.2'

First version of the frozen derivation test script

See merge request atlas/athena!16488

(cherry picked from commit 830e8d30 [formerly 8b307f6fb2c9f3e683260cfa6e05264c5c0d8278])

6c0f0b46 First version of the frozen derivation test script
ee5ceb38 Add option to enable the user to run the test in data or mc

Former-commit-id: 3ecbb60e

PhysicsAnalysis/DerivationFramework/DerivationFrameworkCore/CMakeLists.txt

@@ -31,4 +31,4 @@ atlas_add_component( DerivationFrameworkCore
 # Install files from the package:
 atlas_install_python_modules( python/*.py )
 atlas_install_joboptions( share/*.py )
-
+atlas_install_scripts( scripts/frozen_derivation_test.py )

PhysicsAnalysis/DerivationFramework/DerivationFrameworkCore/scripts/frozen_derivation_test.py

+#!/usr/bin/env python
+
+# Copyright (C) 2002-2018 CERN for the benefit of the ATLAS collaboration
+
+import glob
+import logging
+import multiprocessing
+import os
+import subprocess
+import sys
+import uuid
+
+# Setting logging options
+fmt = '%(asctime)s :: %(levelname)-8s :: %(message)s'
+datefmt = '%m/%d/%Y %H:%M'
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG,
+                    format=fmt,
+                    datefmt=datefmt,
+                    filename='./frozen_derivation_test.log',
+                    filemode='w')
+console = logging.StreamHandler()
+formatter = logging.Formatter(fmt,datefmt=datefmt)
+console.setFormatter(formatter)
+logger.addHandler(console)
+
+###
+# Function to check if the derivation job succeeded or not
+###
+def check_job_success(logFile):
+    ''' Return true/false depending on whether the provided log file has "successful run" pattern. '''
+
+    # Simply tail the log and look for the famous "successful run"
+    cmd = ' tail -n 20 ' + logFile + ' | grep "\\"successful run\\"" | wc -l '
+
+    result = subprocess.check_output(['/bin/bash', '-c',cmd]).rstrip()
+    logger.debug('Job success check for the log file %s is %s', logFile, result)
+
+    return (result == '1')
+
+####
+## Function to run the clean test 
+####
+def run_clean_test(inputFile,extraArgs,release,cleanDir,uniqId):
+    ''' Runs the clean derivation test and checks if the job succeeded or not. '''
+   
+    logger.info('Starting clean test...')
+
+    # Build the actual command
+    cmd = ( ' mkdir -p ' + cleanDir + '/clean_derivation_test_' + uniqId + ';'
+            ' cd '       + cleanDir + '/clean_derivation_test_' + uniqId + ';'
+            ' source $AtlasSetup/scripts/asetup.sh ' + release + ' >& /dev/null;'
+            ' Reco_tf.py --inputAODFile %s %s > derivation_test.log 2>&1;'%(inputFile,extraArgs) )
+
+    # Execute the actual command
+    subprocess.call(cmd,shell=True)
+
+    # Check the job status
+    logFile = cleanDir + '/clean_derivation_test_' + uniqId + '/log.AODtoDAOD'
+    logger.debug('Attempting to read the clean log file from %s', logFile)
+    result = check_job_success(logFile)
+
+    logger.info('Finished clean test w/ status %s...', 'SUCCESS' if result else 'FAILURE')
+    pass
+
+####
+## Function to run the patched test 
+####
+def run_patched_test(inputFile,extraArgs,pwd,release):
+    ''' Runs the patched derivation test and checks if the job succeeded or not. '''
+
+    logger.info('Starting patched test...')
+
+    # Build the actual command
+    cmd = ' cd '+pwd+';'
+    if not release: # This is the CI escape
+        pass
+    elif 'WorkDir_DIR' in os.environ:
+        cmake_build_dir = (os.environ['WorkDir_DIR'])
+        logger.debug('Using the local cmake build directory %s'%(cmake_build_dir))
+        cmd += ( ' source $AtlasSetup/scripts/asetup.sh '+release+'  >& /dev/null;'
+                 ' source '+cmake_build_dir+'/setup.sh;' )
+    else :
+        cmd += ( ' source $AtlasSetup/scripts/asetup.sh '+release+'  >& /dev/null;' )
+    cmd += ( ' mkdir -p patched_derivation_test; cd patched_derivation_test;'
+             ' Reco_tf.py --inputAODFile %s %s > derivation_test.log 2>&1;'%(inputFile,extraArgs) )
+
+    # Execute the actual command
+    subprocess.call(cmd,shell=True)
+
+    # Check the job status
+    logFile = pwd + '/patched_derivation_test/log.AODtoDAOD'
+    logger.debug('Attempting to read the patched log file from %s', logFile )
+    result = check_job_success(logFile)
+
+    logger.info('Finished patched test w/ status %s...', 'SUCCESS' if result else 'FAILURE' )
+    pass
+
+####
+## Function to run the diff-root 
+####
+def compare_files(cleanHeadDir,uniqId,pwd,isPatchedOnly):
+    ''' Compares the clean and patched DAOD files to see if the contents differ. '''
+
+    cleanDir = cleanHeadDir + '/clean_derivation_test_' + uniqId
+    if isPatchedOnly:
+        cleanDir = cleanHeadDir
+
+    logger.debug('Reading the reference file from location '+cleanDir)
+
+    comparison_command = ( 'acmd.py diff-root ' + cleanDir + '/DAOD_PHYSVAL.pool.root '
+                           'patched_derivation_test/DAOD_PHYSVAL.pool.root ' 
+                           '--error-mode resilient ' 
+                           '--entries 20 > patched_derivation_test/diff-root.DAOD.log 2>&1' )
+    output,error = subprocess.Popen(['/bin/bash', '-c', comparison_command], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+
+    # We want to catch/print both container additions/subtractions as well as
+    # changes in these containers.  `allGood_return_code` is meant to catch
+    # other issues found in the diff (sync issues etc.)
+    passed_all_test=True
+    allGood_return_code=False
+
+    logFile = pwd + '/patched_derivation_test/diff-root.DAOD.log'
+    logger.debug('Attempting to read the diff-log file from %s', logFile )
+    with open(logFile) as f:
+        for line in f:
+            if 'WARNING' in line: # Catches container addition/subtractions
+                logger.error(line)
+                passed_all_test=False
+            if 'leaves differ' in line: # Catches changes in branches
+                logger.error(line)
+                passed_all_test=False
+            if 'INFO all good.' in line:
+                allGood_return_code = True
+
+    _decision = passed_all_test and allGood_return_code
+    if _decision:
+        logger.info('Passed!')
+    else:
+        logger.error('Your tag breaks changes the AOD content! See patched_derivation_test/diff-root.DAOD.log file for more information.')
+
+    return _decision
+
+####
+## Function to get release information
+####
+def get_release_setup():
+    ''' This function tries to read environment variables to figure out the appropriate ATLAS setup '''
+
+    # Read in environment variables
+    current_nightly = os.environ['AtlasBuildStamp']
+    release_base=os.environ['AtlasBuildBranch']
+    release_head=os.environ['AtlasVersion']
+    platform=os.environ['AthDerivation_PLATFORM']
+    project=os.environ['AtlasProject']
+
+    # Search through the main nightly installation location
+    builds_dir_searchStr='/cvmfs/atlas-nightlies.cern.ch/repo/sw/'+release_base+'/[!latest_]*/'+project+'/'+release_head
+    sorted_list = sorted(glob.glob(builds_dir_searchStr), key=os.path.getmtime)
+
+    latest_nightly = ''
+    for folder in reversed(sorted_list):
+        if not glob.glob(folder+'/../../'+release_base+'__'+project+'*-opt*.log') : continue
+        latest_nightly = folder.split('/')[-3]
+        break
+
+    if current_nightly != latest_nightly:
+        logger.info('Please be aware that you are not testing your tags in the latest available nightly, which is %s',latest_nightly)
+
+    setup='%s,%s,%s,AthDerivation'%(release_base,platform.replace('-', ','),current_nightly)
+
+    logger.info('Your tags will be tested in environment %s',setup)
+
+    return setup
+
+
+####
+## Main Function
+####
+def main():
+    ''' Main function that parses user inputs, prepares and runs the tests '''
+
+    from optparse import OptionParser
+    parser=OptionParser(usage='%prog [options]',
+                        version='%prog 0.1.0')
+
+    parser.add_option('--log-level',
+                      type='choice',
+                      action='store',
+                      dest='logLevel',
+                      choices=['critical','error','warning','info','debug'],
+                      default='info',
+                      help='Logging level (default: info)')
+
+    parser.add_option('-c',
+                      '--clean-dir',
+                      type='string',
+                      dest='cleanDir',
+                      default='/tmp/',
+                      help='Specify the head directory for running the clean derivation tests '
+                           '(default: /tmp/${USER}). If provided w/ -p (--patched) option, '
+                           'the reference results are expected to be in this location.')
+
+    parser.add_option('-p',
+                      '--patched',
+                      action='store_true',
+                      dest='patched',
+                      default=False,
+                      help='Flag to avoid clean run. ' 
+                           'In this mode only patched test will run and the '
+                           'changes will be compared against predefined '
+                           ' reference files (default: false)')
+
+    parser.add_option('--mode',
+                      type='choice',
+                      action='store',
+                      dest='mode',
+                      choices=['data','mc'],
+                      default='mc',
+                      help='Test mode (default: mc)')
+
+    (options,args) = parser.parse_args()
+
+    logger.info(76*'-')
+
+    # Set logging level
+    logger.setLevel(options.logLevel.upper())
+
+    # Check if patched only
+    isPatchedOnly = options.patched
+
+    # Check if the clean directory exists
+    cleanHeadDir = options.cleanDir
+    if str(cleanHeadDir) == '/tmp/':
+        cleanHeadDir += os.environ['USER']
+
+    if os.path.exists(cleanHeadDir):
+        logger.info('The head directory for the output of the clean tests will be %s', cleanHeadDir)
+    else:
+        logger.error('The head directory for the output of the clean tests doesn\'t exist!')
+        logger.error('Please check that %s exists and you have enough permissions to read/write.', cleanHeadDir)
+        logger.error('If not please specify a folder that satisfies the requirements via -c or --clean-dir options.')
+        sys.exit(-1)    
+
+    # Check if an ATLAS project is setup
+    if 'AtlasVersion' not in os.environ:
+        logging.error('Please setup the an ATLAS release!')
+        sys.exit(-1)
+
+    # Get the release information
+    mySetup = ''
+    try:
+        mySetup = get_release_setup()
+    except KeyError as error:
+        logger.warning('Cannot find the key %s',error)
+        logger.warning('This might be benign if running in CI enviroment')
+    cleanSetup = mySetup
+
+    # Get the PWD and create UUID to avoid possible overlap between runs
+    myPwd = os.getcwd()
+    logger.debug('Current working directory is %s',myPwd)
+    uniqId = str(uuid.uuid4())
+    logger.debug('Created uuid %s',uniqId)
+
+    # Run the tests
+    logger.info('%s %s %s',25*'-',' Run AthDerivation Test ',25*'-')
+    inputFile  = ( '/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/DerivationFrameworkART/{}'
+                   ''.format('AOD.14795494._005958.pool.root.1' if options.mode == 'mc' else 'data18_13TeV.00348403.physics_Main.merge.AOD.f920_m1947._lb0829._0001.1') )
+    extraArgs  = ( '--outputDAODFile pool.root --maxEvents 20 --reductionConf PHYSVAL --preExec \'rec.doApplyAODFix.set_Value_and_Lock(True); '
+                   'from BTagging.BTaggingFlags import BTaggingFlags;BTaggingFlags.CalibrationTag = "{}"; '
+                   'from AthenaCommon.AlgSequence import AlgSequence; topSequence = AlgSequence(); '
+                   'topSequence += CfgMgr.xAODMaker__DynVarFixerAlg("InDetTrackParticlesFixer", Containers = ["InDetTrackParticlesAux." ] )\''
+                   ''.format('BTagCalibRUN12-08-40' if options.mode == 'mc' else 'BTagCalibRUN12Onl-08-40') )
+
+    myprocesses = {}
+    if not isPatchedOnly:
+        myprocesses['clean']   = multiprocessing.Process(target=run_clean_test,kwargs={'inputFile':inputFile,'extraArgs' : extraArgs,'release':cleanSetup,'cleanDir':cleanHeadDir,'uniqId':uniqId})
+        myprocesses['patched'] = multiprocessing.Process(target=run_patched_test,kwargs={'inputFile':inputFile,'extraArgs' : extraArgs,'pwd':myPwd,'release':mySetup})
+        myprocesses['clean'].start()
+        myprocesses['patched'].start()
+    else:
+        myprocesses['patched'] = multiprocessing.Process(target=run_patched_test,kwargs={'inputFile':inputFile,'extraArgs' : extraArgs,'pwd':myPwd,'release':mySetup})
+        myprocesses['patched'].start()
+
+    for process in myprocesses:
+        myprocesses[process].join()
+
+    # Post processing
+    logger.info('%s %s %s',25*'-',' Run xAOD content check ',25*'-')
+    
+    all_tests_passed = True
+
+    if not compare_files(cleanHeadDir,uniqId,myPwd,isPatchedOnly):
+        all_tests_passed = False
+
+    logger.info('%s %s %s',25*'-','  Summary of all tests  ',25*'-')
+    if all_tests_passed:
+        logger.info(' All Tests: Passed (0)')
+        logger.info(76*'-')
+        sys.exit(0)
+    else:
+        logger.info(' All Tests: FAILED (-1)')
+        logger.info(76*'-')
+        sys.exit(-1)
+
+if __name__ == '__main__':
+    main()