Merge branch 'main-athfile' into 'main'

Removed obsolete acmd utilities: filter-files and merge-files

See merge request !70796

Control/AthenaPython/python/ConfigLib.py

@@ -15,7 +15,6 @@ __author__ = "Sebastien Binet <binet@cern.ch>"
 __all__ = [
     'read_file',
     'copy_file',
-    #'merge_files',
     'AutoCfg',
     ]
 

Tools/PyUtils/python/scripts/__init__.py

@@ -11,9 +11,6 @@ acmdlib.register('diff-root', 'PyUtils.scripts.diff_root_files')
 acmdlib.register('dump-root', 'PyUtils.scripts.dump_root_file')
 acmdlib.register('gen-klass', 'PyUtils.scripts.gen_klass')
 
-acmdlib.register('merge-files', 'PyUtils.scripts.merge_files')
-acmdlib.register('filter-files', 'PyUtils.scripts.filter_files')
-
 acmdlib.register('cmake.depends', 'PyUtils.scripts.cmake_depends')
 acmdlib.register('cmake.new-skeleton', 'PyUtils.scripts.cmake_newskeleton')
 acmdlib.register('cmake.new-pkg', 'PyUtils.scripts.cmake_newpkg')

Tools/PyUtils/python/scripts/filter_files.py

-# Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
-
-# @file PyUtils.scripts.filter_files
-# @purpose take a bunch of input (pool/bs) files and produce a filtered one
-#          autoconfiguration is (attempted to be) performed
-# @author Sebastien Binet
-# @date March 2010
-from __future__ import with_statement
-
-__doc__ = "filter multiple input (pool/bs) files"
-__author__ = "Sebastien Binet"
-
-
-### imports -------------------------------------------------------------------
-import PyUtils.acmdlib as acmdlib
-
-@acmdlib.command(
-    name='filter-files'
-    )
-@acmdlib.argument(
-    '-o', '--output',
-    required=True,
-    help="Name of the filtered output file"
-    )
-@acmdlib.argument(
-    'files',
-    nargs='+',
-    help='path to the input (pool/bs) files'
-    )
-@acmdlib.argument(
-    '-s', '--selection',
-    required=True,
-    help='comma separated list of tuples (run,event) numbers to select or an ascii file containg a list of such run+event numbers to select'
-    )
-def main(args):
-    """filter multiple input (pool/bs) files"""
-
-    exitcode = 0
-
-    import PyUtils.Logging as L
-    msg = L.logging.getLogger('filter-files')
-    msg.setLevel(L.logging.INFO)
-
-    msg.info(':'*40)
-
-    import os.path as osp
-    args.files = [ osp.expandvars(osp.expanduser(fname))
-                   for fname in args.files ]
-
-    args.selection = osp.expandvars(osp.expanduser(args.selection))
-    
-    msg.info('input files: %s', args.files)
-    msg.info('output file: %s', args.output)
-    msg.info('selection:   %s', args.selection)
-
-    import os
-    if os.path.exists(args.selection):
-        selection = []
-        with open(args.selection, 'r') as s:
-            for line in s:
-                if line.strip().startswith('#'):
-                    continue
-                l = line.strip().split()
-                if len(l)==1: # assume this is only the event number
-                    runnbr, evtnbr = None, int(l[0])
-                elif len(l)==2: # a pair (run,evt) number
-                    runnbr, evtnbr = int(l[0]), int(l[1])
-                else:
-                    raise RuntimeError(
-                        'file [%s] has invalid format at line:\n%r' %
-                        (args.selection, line)
-                        )
-                selection.append((runnbr, evtnbr))
-    else:
-        try:
-            args.selection = eval(args.selection)
-        except Exception as err:
-            msg.error('caught:\n%s', err)
-            msg.error('.. while trying to parse selection-string')
-            import traceback
-            traceback.print_exc()
-            return 1
-        
-        selection = []
-        for item in args.selection:
-            if not isinstance(item, (tuple, list, int)):
-                raise TypeError('type: %r' % type(item))
-
-            if isinstance(item, (tuple, list)):
-                if len(item) == 1:
-                    runnbr, evtnbr = None, int(item[0])
-                elif len(item) == 2:
-                    runnbr, evtnbr = int(item[0]), int(item[1])
-                else:
-                    raise RuntimeError(
-                        'item [%s] has invalid arity (%s)' %
-                        (item, len(item))
-                        )
-            else:
-                runnbr, evtnbr = None, int(item)
-            selection.append((runnbr, evtnbr))
-
-    # put back the massaged selection into our workspace
-    args.selection = selection[:]
-
-    from PyUtils.MetaReader import read_metadata
-    metadata = read_metadata(args.files[0], None, 'lite')[args.files[0]]
-    
-    if metadata['file_type'] == 'BS':
-        # optimization: run directly 'AtlCopyBSEvent.exe
-        import subprocess
-        cmd = ' '.join([
-            'AtlCopyBSEvent',
-            '-e %(evt-list)s',
-            '%(run-list)s',
-            '--out %(output)s',
-            '%(files)s',
-            ])
-        evt_list = [str(i) for _,i in args.selection]
-        run_list = [str(i) for i,_ in args.selection if i is not None]
-        cmd = cmd % {
-            'evt-list': ','.join(evt_list),
-            'run-list': '' if len(run_list)==0 else '-r '+','.join(run_list),
-            'output': args.output,
-            'files':  ' '.join(args.files),
-            }
-        return subprocess.call(cmd.split())
-    
-    import AthenaCommon.ChapPy as api
-    app = api.AthenaApp()
-    app << """
-    import AthenaCommon.Constants as Lvl
-    from AthenaCommon.AthenaCommonFlags import jobproperties as jp
-    acf = jp.AthenaCommonFlags
-    acf.FilesInput = %(files)s
-
-    # events to process
-    acf.EvtMax = EvtMax = theApp.EvtMax = -1
-
-    # configuration
-    import AthenaPython.ConfigLib as apcl
-    cfg = apcl.AutoCfg(
-        name='filter-files',
-        input_files=acf.FilesInput(),
-        output_file='%(output)s')
-
-    for type_name in ('evgen',
-                      'hits',
-                      'rdo',
-                      'esd',
-                      'aod',
-                      'tag',
-                      'usr',):
-        res = getattr(cfg, 'is_' + type_name)()
-        if res:
-            cfg.msg.info('input file type is ['+type_name+']')
-            break
-    else:
-        cfg.msg.info('input file stream is of unknown type')
-        cfg.msg.info('autoconfiguration might not work...')
-        pass
-        
-    # add the filtering algorithm
-    # get a handle on the job main sequence
-    import AthenaCommon.AlgSequence as acas
-    job = acas.AlgSequence()
-    
-    ## filter configuration ##
-    ##  -> we use the special sequence 'AthMasterSeq' which
-    ##      is run before any other algorithm (which are usually in the
-    ##      'TopAlg' sequence
-    seq = acas.AthSequencer('AthMasterSeq')
-    
-    import GaudiSequencer.PyComps as gspc
-    seq += gspc.PyEvtFilter(
-       'filter_pyalg',
-       # the store-gate key. leave as an empty string to take any eventinfo instance
-       evt_info=None,
-       OutputLevel=Lvl.INFO)
-    seq.filter_pyalg.evt_list = %(selection)s   
-
-
-    cfg.configure_job()
-
-    if (cfg.is_rdo() or
-        cfg.is_esd() or
-        cfg.is_aod()):
-        # main jobos
-        from RecExConfig.RecFlags import rec
-        rec.runUnsupportedLegacyReco=True
-        include ('RecExCond/RecExCommon_flags.py')
-        include ('RecExCommon/RecExCommon_topOptions.py')
-
-    """ % args.__dict__
-    
-    stdout = None
-    exitcode = app.run(stdout=stdout)
-    return exitcode

Tools/PyUtils/python/scripts/merge_files.py

-# Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration
-
-# @file PyUtils.scripts.merge_files
-# @purpose take a bunch of input (pool/bs) files and produce a single one
-#          autoconfiguration is (attempted to be) performed
-# @author Sebastien Binet
-# @date February 2010
-
-__doc__ = "merge multiple input (pool/bs) files"
-__author__ = "Sebastien Binet"
-
-
-### imports -------------------------------------------------------------------
-import PyUtils.acmdlib as acmdlib
-
-@acmdlib.command(
-    name='merge-files'
-    )
-@acmdlib.argument(
-    '-o', '--output',
-    required=True,
-    help="Name of the merged output file"
-    )
-@acmdlib.argument(
-    'files',
-    nargs='+',
-    help='path to the input (pool/bs) files'
-    )
-@acmdlib.argument(
-    '--evts',
-    type=int,
-    default=-1,
-    help="number of events to process"
-    )
-@acmdlib.argument(
-    '--logfile',
-    default='<stdout>',
-    help = "Path to a file where to put athena job's logfile"
-    )
-@acmdlib.argument(
-    "--dump-jobo",
-    dest    = "dump_jobo",
-    default = None,
-    help    = "tell application to save the automatically generated "\
-    "joboption under some name for (mainly) debugging "\
-    "and/or customization purposes.")
-
-def main(args):
-    """merge multiple input (pool/bs) files"""
-
-    exitcode = 0
-
-    import PyUtils.Logging as L
-    msg = L.logging.getLogger('merge-files')
-    msg.setLevel(L.logging.INFO)
-
-    msg.info(':'*40)
-
-    import os.path as osp
-    args.files = [ osp.expandvars(osp.expanduser(fname))
-                   for fname in args.files ]
-
-    msg.info('input files: %s', args.files)
-    msg.info('output file: %s', args.output)
-    msg.info('evts to process: %s', args.evts)
-    msg.info('log-files: %s', args.logfile)
-
-    import AthenaCommon.ChapPy as api
-    app = api.AthenaApp(dump_jobo=args.dump_jobo)
-    app << """
-    from AthenaCommon.AthenaCommonFlags import jobproperties as jp
-    acf = jp.AthenaCommonFlags
-    acf.FilesInput = %(files)s
-
-    # events to process
-    acf.EvtMax = EvtMax = theApp.EvtMax = %(evts)s
-
-    # configuration
-    import AthenaPython.ConfigLib as apcl
-    cfg = apcl.AutoCfg(
-        name='merge-files',
-        input_files=acf.FilesInput(),
-        output_file='%(output)s')
-
-    for type_name in ('evgen',
-                      'hits',
-                      'rdo',
-                      'esd',
-                      'aod',
-                      'tag',
-                      'usr',):
-        res = getattr(cfg, 'is_' + type_name)()
-        if res:
-            cfg.msg.info('input file type is ['+type_name+']')
-            break
-    else:
-        cfg.msg.info('input file stream is of unknown type')
-        cfg.msg.info('autoconfiguration might not work...')
-        pass
-
-    # See ATLASRECTS-7082
-    rec.doTruth.set_Value_and_Lock(False)
-
-    cfg.configure_job()
-
-    if (cfg.is_rdo() or
-        cfg.is_esd() or
-        cfg.is_aod()):
-        # main jobos
-        include ('RecExCond/RecExCommon_flags.py')
-        # FIXME: work-around for bug #56185
-        from AthenaCommon.DetFlags import DetFlags
-        DetFlags.makeRIO.all_setOff()
-        # FIXME -- end
-        from RecExConfig.RecFlags import rec
-        rec.runUnsupportedLegacyReco=True
-        include ('RecExCommon/RecExCommon_topOptions.py')
-        
-    """ % args.__dict__
-
-    stdout = args.logfile
-    if stdout.lower() in ('<stdout>', 'stdout',):
-        stdout = None
-    else:
-        stdout = open(stdout, 'w')
-
-    exitcode = app.run(stdout=stdout)
-    return exitcode