Commit 958558a0 authored by Maxime Fernoux's avatar Maxime Fernoux
Browse files

Fix conflicts

parents 317cb20f e2c40243
Pipeline #4171221 failed with stages
in 10 minutes and 8 seconds
......@@ -2,8 +2,8 @@ variables:
# we can use the cern DOCKER_CACHE to avoid overloading dockerhub
# see here: https://clouddocs.web.cern.ch/containers/tutorials/registry.html#dockerhub-cache
#
BUILD_IMAGE: registry.cern.ch/docker.io/atlas/analysisbase:22.2.73
ATHANALYSIS_IMAGE: registry.cern.ch/docker.io/atlas/athanalysis:22.2.73
BUILD_IMAGE: registry.cern.ch/docker.io/atlas/analysisbase:22.2.75
ATHANALYSIS_IMAGE: registry.cern.ch/docker.io/atlas/athanalysis:22.2.75
ATHENA_IMAGE: registry.cern.ch/docker.io/atlas/centos7-atlasos-dev:latest-gcc11
DATA_URL: https://dguest-ci.web.cern.ch/dguest-ci/ci/ftag/dumper
......@@ -129,11 +129,6 @@ test_ca_minimal:
script:
- test-dumper minimal
test_upgrade:
<<: *run-athanalysis
script:
- test-dumper upgrade
test_reduced_ca:
<<: *run-athanalysis
script:
......@@ -147,7 +142,20 @@ test_multi_ca:
test_ca_trigger_emtopo:
<<: *run-athanalysis
script:
- test-dumper trigger-emtopo
- test-dumper -d reduced trigger-emtopo
artifacts:
paths:
- reduced
expire_in: 1 hour
test_ca_trigger_emtopo_full:
<<: *run-athanalysis
script:
- test-dumper -d full trigger-topo-fullprec
artifacts:
paths:
- full
expire_in: 1 hour
# Athena
......@@ -170,6 +178,15 @@ test_ca_trigger:
script:
- test-dumper trigger
test_ca_trigger_all:
<<: *run-athena
script:
- test-dumper trigger-all
test_upgrade:
<<: *run-athena
script:
- test-dumper upgrade
# various unit tests
test_singlebtag_submit:
......@@ -186,6 +203,16 @@ test_configs:
script:
- test-configs-single-b -v
###################################
## compare outputs
###################################
compare_precision:
stage: run
needs: [test_ca_trigger_emtopo_full, test_ca_trigger_emtopo]
script:
h5diff -p 0.0005 full/output.h5 reduced/output.h5
###################################
## build images
###################################
......
......@@ -127,8 +127,9 @@ if (NOT XAOD_STANDALONE)
bin/ca-dump-retag
bin/ca-dump-single-btag
bin/ca-dump-single-btag-fixedcone
bin/ca-dump-trigger-btag
bin/ca-dump-trigger-pflow
bin/ca-dump-trigger-emtopo
bin/ca-dump-trigger-all
bin/ca-dump-trigger-workingpoints
bin/ca-dump-minimal-btag
bin/ca-dump-upgrade
......
......@@ -11,6 +11,7 @@ dataset dumper.
"""
import sys
import json
from AthenaConfiguration.ComponentFactory import CompFactory
from AthenaConfiguration.MainServicesConfig import MainServicesCfg as getConfig
......@@ -21,7 +22,7 @@ from BTagTrainingPreprocessing import dumper
def run():
args = dumper.base_parser(__doc__, multiconfig=True).parse_args()
args = dumper.base_parser(__doc__).parse_args()
from AthenaConfiguration.AllConfigFlags import ConfigFlags as cfgFlags
dumper.update_cfgFlags(cfgFlags, args)
......@@ -35,12 +36,15 @@ def run():
output = CompFactory.H5FileSvc(path=str(args.output))
ca.addService(output)
for cfg in args.config_files:
with open(args.config_file) as f:
config_keys = json.load(f).keys()
for cfg in config_keys:
ca.addEventAlgo(CompFactory.SingleBTagAlg(
name=f'{cfg.stem}Dumper',
name=f'{cfg}Dumper',
output=output,
configFileName=str(cfg),
group=cfg.stem,
configFileName=str(args.config_file),
group=cfg,
))
return ca.run()
......
#!/usr/bin/env python3
"""
Dump some ftag trigger info!
This runs quite a few algorithms upstream of the dumping code, to
extract trigger jets from the trigger decisions, truth label them, and
compare them to offline reconstructed jets.
"""
from BTagTrainingPreprocessing import trigger as trig
from BTagTrainingPreprocessing import dumper
from AthenaConfiguration.ComponentFactory import CompFactory
import sys
def get_args():
pflow_chain = 'HLT_j20_0eta290_020jvt_boffperf_pf_ftf_L1J15'
emtopo_chain = 'HLT_j80_roiftf_preselj20_L1J20'
dh = dict(help='(default: %(default)s)')
parser = dumper.base_parser(__doc__)
parser.add_argument('-p','--pflow-chain', default=pflow_chain, **dh)
parser.add_argument('-e','--emtopo-chain', default=emtopo_chain, **dh)
return parser.parse_args()
def run():
args = get_args()
from AthenaConfiguration.AllConfigFlags import ConfigFlags as cfgFlags
dumper.update_cfgFlags(cfgFlags, args)
cfgFlags.lock()
#########################################################################
################### Build the component accumulator #####################
#########################################################################
#
ca = dumper.getMainConfig(cfgFlags, args)
ca.merge(trig.pflowDumper(cfgFlags, chain=args.pflow_chain))
ca.merge(trig.emtopoDumper(cfgFlags, chain=args.emtopo_chain))
output = CompFactory.H5FileSvc(path=str(args.output))
ca.addService(output)
ca.addEventAlgo(CompFactory.SingleBTagAlg(
'pflowDumper',
output=output,
configFileName=str(args.config_file),
group='pflow'))
ca.addEventAlgo(CompFactory.SingleBTagAlg(
'topoDumper',
output=output,
configFileName=str(args.config_file),
group='emtopo'))
#########################################################################
########################### Run everything ##############################
#########################################################################
return ca.run()
if __name__ == '__main__':
code = run()
sys.exit(0 if code.isSuccess() else 1)
......@@ -20,6 +20,10 @@ def get_args():
parser = dumper.base_parser(__doc__)
parser.add_argument('-t','--threads', type=int, default=0)
parser.add_argument('-n','--chain', default=default_chain, **dh)
parser.add_argument(
'-p', '--force-full-precision',
action='store_true',
help='force all half-precision outputs to full precision')
return parser.parse_args()
def run():
......@@ -41,7 +45,9 @@ def run():
ca = dumper.getMainConfig(cfgFlags, args)
ca.merge(trig.emtopoDumper(cfgFlags, chain=args.chain))
ca.merge(dumper.getDumperConfig(args.config_file, args.output))
ca.merge(dumper.getDumperConfig(
args.config_file, args.output,
force_full_precision=args.force_full_precision))
#########################################################################
########################### Run everything ##############################
......
......@@ -15,6 +15,7 @@ from AthenaPoolCnvSvc.PoolReadConfig import PoolReadCfg
from AthenaConfiguration.ComponentFactory import CompFactory
from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator
from DerivationFrameworkMCTruth.TruthDerivationToolsConfig import DFCommonTruthClassificationToolCfg
import sys
......@@ -22,6 +23,10 @@ import sys
def getSingleBTagConfig(config, output):
ca = ComponentAccumulator()
# decorate with promptLepton information
DFCommonTruthClassifier = ca.getPrimaryAndMerge(DFCommonTruthClassificationToolCfg())
ca.addEventAlgo(CompFactory.DerivationFramework.CommonAugmentation(
'promptLeptonAugmenter', AugmentationTools = [DFCommonTruthClassifier]))
ca.addEventAlgo(CompFactory.FlavorTagDiscriminants.PoorMansIpAugmenterAlg(
'PoorMansIpAugmenterAlg', prefix='btagIp_', primaryVertexContainer='PrimaryVertices'))
......
......@@ -18,8 +18,9 @@ BREAK="----------------------------------------------"
declare -A SCRIPTS_BY_MODE=(
[single-btag]=dump-single-btag
[retag]=ca-dump-retag
[trigger]=ca-dump-trigger-btag
[trigger]=ca-dump-trigger-pflow
[trigger-wp]=ca-dump-trigger-workingpoints
[trigger-all]=ca-dump-trigger-all
[upgrade]=ca-dump-upgrade
[upgrade-HI]=ca-dump-upgrade-HI
)
......@@ -28,6 +29,7 @@ declare -A CONFIGS_BY_MODE=(
[retag]=EMPFlow.json
[trigger]=trigger.json
[trigger-wp]=trigger_wp.json
[trigger-all]=trigger_all.json
[upgrade]=upgrade.json
[upgrade-HI]=upgrade-HI.json
)
......@@ -36,6 +38,7 @@ declare -A INPUTS_BY_MODE=(
[retag]=single-btag.txt
[trigger]=trigger.txt
[trigger-wp]=trigger-workingpoints.txt
[trigger-all]=trigger.txt
[upgrade]=upgrade.txt
[upgrade-HI]=upgrade-HI.txt
)
......
# Samples from tuning campaign, see
# Samples from 21st round validation of Sample A, see
#
# https://its.cern.ch/jira/browse/ATLMCPROD-9424
# https://its.cern.ch/jira/browse/ATR-25472
#
mc16_13TeV.410470.PhPy8EG_A14_ttbar_hdamp258p75_nonallhad.recon.AOD.e6337_e5984_s3126_d1677_r12711
mc16_13TeV.427080.Pythia8EvtGen_A14NNPDF23LO_flatpT_Zprime.recon.AOD.e5362_e5984_s3126_d1677_r12711
#
# Similar samples from jet tuning campaign, see
#
# https://its.cern.ch/jira/browse/ATR-22610
# https://its.cern.ch/jira/browse/ATLMCPROD-9423
#
# JZ1 to JZ4 is probably pretty ok, based on the slicing here:
#
# https://gitlab.cern.ch/atlas-physics/pmg/infrastructure/mc15joboptions/-/blob/ffe400a1b619910790f3c64825ac82a0139f95bf/common/Filters/JetFilter_JZX_Fragment.py
mc16_13TeV.800036.Py8EG_A14N23LO_jetjet_JZ1WwithSW.recon.AOD.e7914_s3126_d1677_r12711
mc16_13TeV.364702.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ2WithSW.recon.AOD.e7142_s3126_d1677_r12711
mc16_13TeV.364703.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ3WithSW.recon.AOD.e7142_s3126_d1677_r12711
mc16_13TeV.364704.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ4WithSW.recon.AOD.e7142_s3126_d1677_r12711
# PhPy8EG_A14_ttbar_hdamp258p75_SingleLep with pileup
mc21.601229.PhPy8EG_A14_ttbar_hdamp258p75_SingleLep.recon.AOD.e8357_e7400_s3775_r13614_r13614_tid29004489_00
\ No newline at end of file
......@@ -21,7 +21,7 @@ class DumperHelpFormatter(
...
def base_parser(description, add_help=True, multiconfig=False):
def base_parser(description, add_help=True):
parser = argparse.ArgumentParser(description=description, formatter_class=DumperHelpFormatter, add_help=add_help)
parser.add_argument(
"input_files", nargs="+",
......@@ -29,14 +29,9 @@ def base_parser(description, add_help=True, multiconfig=False):
parser.add_argument(
"-o", "--output", default=Path("output.h5"), type=Path,
help="output filename")
if multiconfig:
parser.add_argument(
"-c", "--config-files", required=True, type=Path, nargs='+',
help="job configuration files")
else:
parser.add_argument(
"-c", "--config-file", required=True, type=Path,
help="job configuration file")
parser.add_argument(
"-c", "--config-file", required=True, type=Path,
help="job configuration file")
parser.add_argument(
"-m", "--max-events",
type=int, nargs="?",
......@@ -102,7 +97,7 @@ def getMainConfig(flags, args):
return ca
def getDumperConfig(config, output_path):
def getDumperConfig(config, output_path, force_full_precision=False):
ca = ComponentAccumulator()
output = CompFactory.H5FileSvc(path=str(output_path))
......@@ -111,6 +106,7 @@ def getDumperConfig(config, output_path):
btagAlg = CompFactory.SingleBTagAlg(f'{config.stem}DatasetDumper')
btagAlg.output = output
btagAlg.configFileName = str(config)
btagAlg.forceFullPrecision = force_full_precision
ca.addEventAlgo(btagAlg)
return ca
......@@ -4,10 +4,11 @@ from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator
# needed for trigger decision tool
from AthenaServices.MetaDataSvcConfig import MetaDataSvcCfg
# only works in full athena for now
# tools!
from TrigDecisionTool.TrigDecisionToolConfig import TrigDecisionToolCfg
from ParticleJetTools.JetParticleAssociationAlgConfig import (
JetParticleAssociationAlgCfg, JetParticleFixedConeAssociationAlgCfg)
from math import inf
##############################################################################
###################### internal functions ####################################
......@@ -46,88 +47,20 @@ def getDRTool(cfgFlags):
)
return dRTool
def getTrackAssociationAlgs(
cfgFlags, jc,
bc='',
tpc='HLT_IDTrack_FS_FTF',
an='FsTracks',
isFixedCone=False):
acc = ComponentAccumulator()
Associator = CompFactory.JetParticleShrinkingConeAssociation
trackOnJetDecorator = f'{jc}.{an}ForBTagging'
suffix='_fixedCone' if isFixedCone else ''
# the formula here is R = p1 + exp(p2 + p3 * pt)
# default values for shrinking cone from:
# https://gitlab.cern.ch/atlas/athena/-/blob/master/PhysicsAnalysis/AnalysisCommon/ParticleJetTools/python/JetParticleAssociationAlgConfig.py#L12
assoc_tool = Associator(
'FSTrackConeTool'+suffix,
coneSizeFitPar1= 0.5 if isFixedCone else +0.239,
coneSizeFitPar2=-inf if isFixedCone else -1.220, # exp(-inf) -> 0
coneSizeFitPar3= 0 if isFixedCone else -1.64e-5, # fixed cone
JetContainer=jc,
InputParticleContainer=tpc,
OutputDecoration=trackOnJetDecorator.split('.')[-1],
)
assoc_alg = CompFactory.JetDecorationAlg(
'FSTrackAssociationAlg'+suffix,
JetContainer=jc,
Decorators=[assoc_tool])
acc.addEventAlgo(assoc_alg)
if bc:
copier = CompFactory.FlavorTagDiscriminants.BTagTrackLinkCopyAlg(
'FSTrackCopier'+suffix,
jetTracks=trackOnJetDecorator,
btagTracks=f'{bc}.{an}',
jetLinkName=f'{bc}.jetLink'
)
acc.addEventAlgo(copier)
return acc
# We do imports in this function because it's expensive to set up and
# only works in Athena. It should only be required for the btagIp_*
# variables.
def getTrackAugmentation(
cfgFlags,
tpc='HLT_IDTrack_FS_FTF',
pvc='HLT_IDVertex_FS'):
try:
from BTagging.BTagTrackAugmenterAlgConfig import (
BTagTrackAugmenterAlgCfg as TrackAugCfg)
return TrackAugCfg(
cfgFlags,
TrackCollection=tpc,
PrimaryVertexCollectionName=pvc)
except ModuleNotFoundError as err:
print(f'WARNING: problem setting up track augmentation: {err}')
return ComponentAccumulator()
def triggerJetGetterCfg(cfgFlags, chain, temp_jets, temp_btag=''):
ca = ComponentAccumulator()
# This is (was?) needed for TDT
ca.merge(MetaDataSvcCfg(cfgFlags))
# We have two ways to set up the trigger decision tool, because it
# works differently in full Athena and in AthAnalysis.
try:
# First the Athena way. This sets up the trigger decision tool
# the way Tim Martin recommends.
trigDecTool = TrigDecisionToolCfg(cfgFlags).getPrimary()
except RuntimeError:
# This is the fallback way since the above function fails in
# AthAnalysis. As of 2021.03.25 this version only works for
# the first file, then crashes in one or another terrible way.
trigDecTool = CompFactory.Trig.TrigDecisionTool('TrigDecisionTool')
trigDecTool.NavigationFormat = "TrigComposite" # Run 3 style
tdtca = TrigDecisionToolCfg(cfgFlags)
ca.merge(tdtca)
# Local component to move trigger elements into collections so we
# can access them inside offline code
jetGetter = CompFactory.TriggerJetGetterAlg(
'TriggerJetGetterAlg')
jetGetter.triggerDecisionTool = trigDecTool
f'TriggerJetGetter_{chain}')
jetGetter.triggerDecisionTool = tdtca.getPrimary()
jetGetter.bJetChain = chain
jetGetter.outputJets = temp_jets
jetGetter.outputBTag = temp_btag
......@@ -174,18 +107,24 @@ def offlineBTagMatcherCfg(
def jetMatcherCfg(
flags,
target_jets,
source_jets='HLT_AntiKt4EMPFlowJets_subresjesgscIS_ftf',
fast_dips=['fastDIPS20211215']):
source_jets,
fast_dips=[],
float_map={},
iparticles=[],
dR='deltaRToMatchedJet',
):
ca = ComponentAccumulator()
fd_vars = {f'{t}_p{x}':f'{t}_p{x}' for x in 'cub' for t in fast_dips}
ca.addEventAlgo(CompFactory.JetMatcherAlg(
name='pflow_matcher',
iparticlesToCopy={x:x for x in iparticles}
matcher = CompFactory.JetMatcherAlg(
name=f'Matcher_{target_jets}_from_{source_jets}',
targetJet=target_jets,
sourceJet=source_jets,
floatsToCopy=fd_vars,
iparticlesToCopy={x:x for x in ['TracksForMinimalJetTag']},
drMatchMax=0.001,
))
floatsToCopy=(fd_vars | float_map),
iparticlesToCopy=iparticlesToCopy,
dR=dR
)
ca.addEventAlgo(matcher)
return ca
......@@ -194,6 +133,10 @@ def jetMatcherCfg(
##############################################################################
def pflowDumper(cfgFlags, chain):
# this thing only exists in full athena, thus the import here
from BTagging.BTagTrackAugmenterAlgConfig import BTagTrackAugmenterAlgCfg
ca = ComponentAccumulator()
# build labeling collections
......@@ -214,26 +157,44 @@ def pflowDumper(cfgFlags, chain):
cfgFlags,
btag_key=temp_btag,
jet_key=temp_jets))
# match to associated pflow jets
ca.merge(jetMatcherCfg(cfgFlags, temp_jets))
ca.merge(jetMatcherCfg(
cfgFlags, temp_jets,
source_jets='HLT_AntiKt4EMPFlowJets_subresjesgscIS_ftf',
fast_dips=['fastDIPS20211215'],
iparticles=['TracksForMinimalJetTag','GhostTrack_ftf'],
dR='deltaRToPFlowJet'
))
# associate fullscan tracks to the jets
fs_tracks = 'HLT_IDTrack_FS_FTF'
fs_vertices = 'HLT_IDVertex_FS'
ca.merge(getTrackAugmentation(
cfgFlags,
tpc=fs_tracks,
pvc=fs_vertices,
))
ca.merge(BTagTrackAugmenterAlgCfg(
cfgFlags,
TrackCollection=fs_tracks,
PrimaryVertexCollectionName=fs_vertices))
ca.addEventAlgo(CompFactory.FlavorTagDiscriminants.PoorMansIpAugmenterAlg(
name='_'.join(['PoorMansIpAugmenter',fs_tracks, fs_vertices]),
trackContainer=fs_tracks,
primaryVertexContainer=fs_vertices
))
ca.merge(getTrackAssociationAlgs(cfgFlags, temp_jets, temp_btag ))
ca.merge(getTrackAssociationAlgs(
cfgFlags, temp_jets, temp_btag, an='FsTracks_fixedCone',isFixedCone=True
))
primaryVertexContainer=fs_vertices))
ca.merge(JetParticleAssociationAlgCfg(
cfgFlags,
JetCollection=temp_jets,
InputParticleCollection=fs_tracks,
OutputParticleDecoration='FsTracks'))
ca.merge(JetParticleFixedConeAssociationAlgCfg(
cfgFlags,
fixedConeRadius=0.5,
JetCollection=temp_jets,
InputParticleCollection=fs_tracks,
OutputParticleDecoration='FsTracksFixedCone'))
for pt_threshold in [10, 20]:
ca.merge(JetParticleAssociationAlgCfg(
cfgFlags,
JetCollection=temp_jets,
InputParticleCollection=fs_tracks,
OutputParticleDecoration=f'FsTracks{pt_threshold}GeV',
MinimumJetPt=pt_threshold*1e3))
return ca
......@@ -255,19 +216,36 @@ def emtopoDumper(cfgFlags, chain):
# associate fullscan tracks to the jets
tracks = 'HLT_IDTrack_JetSuper_FTF'
ca.addEventAlgo(CompFactory.FlavorTagDiscriminants.PoorMansIpAugmenterAlg(
trackContainer=tracks
))
ca.merge(getTrackAssociationAlgs(
cfgFlags, temp_jets,
tpc=tracks,
an='superTracks'
))
ca.merge(getTrackAssociationAlgs(
cfgFlags, temp_jets,
tpc=tracks,
an='superTracksFixedCone',
isFixedCone=True
))
fs_tracks = 'HLT_IDTrack_FS_FTF'
for tr in [tracks, fs_tracks]:
ca.addEventAlgo(
CompFactory.FlavorTagDiscriminants.PoorMansIpAugmenterAlg(
name=f'PoormanIp_{tr}',
trackContainer=tr
))
ca.merge(JetParticleAssociationAlgCfg(
cfgFlags,
JetCollection=temp_jets,
InputParticleCollection=fs_tracks,
OutputParticleDecoration='FsTracks'))
ca.merge(JetParticleAssociationAlgCfg(
cfgFlags,
JetCollection=temp_jets,
InputParticleCollection=tracks,
OutputParticleDecoration='superTracks'))
ca.merge(JetParticleFixedConeAssociationAlgCfg(
cfgFlags,
fixedConeRadius=0.5,
JetCollection=temp_jets,
InputParticleCollection=fs_tracks,
OutputParticleDecoration='superTracksFixedCone'))
for pt_threshold in [10, 20]:
ca.merge(JetParticleAssociationAlgCfg(
cfgFlags,
JetCollection=temp_jets,
InputParticleCollection=tracks,
OutputParticleDecoration=f'superTracks{pt_threshold}GeV',
MinimumJetPt=pt_threshold*1e3))
return ca
......@@ -17,11 +17,16 @@ BTagJetWriter::BTagJetWriter(
H5::Group& output_file,
const BTagJetWriterConfig& config)
{
using H5Utils::Compression;
Compression f = Compression::STANDARD;
Compression h = config.force_full_precision ?