diff --git a/Database/APR/TrigCollQuery/TrigCollQuery/TrigCollQuery.h b/Database/APR/TrigCollQuery/TrigCollQuery/TrigCollQuery.h new file mode 100644 index 0000000000000000000000000000000000000000..7a49afdd1fec9b63f14a81e844bb9ef0a38b7ed4 --- /dev/null +++ b/Database/APR/TrigCollQuery/TrigCollQuery/TrigCollQuery.h @@ -0,0 +1,172 @@ +/* + Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration +*/ + +#ifndef TRIGCOLLQUERY_TRIGCOLLQUERY_H +#define TRIGCOLLQUERY_TRIGCOLLQUERY_H + +#include <map> +#include <string> +#include <vector> +#include <set> + +class TrigCollQuery +{ +public: + TrigCollQuery(); + ~TrigCollQuery(); + + /// read in the map of trigger bits + void readTriggerMap( std::string filename = "" ); + + /// replace trigger names in a collection query with bitwise operands + std::string triggerQueryRemap( const std::string& query, + const std::string& tech ); + + /// use "CTPWord" attribute name for L1 trigger + /// - for backward compatibility wotj early 2008 tags (FDR2) + void useCTPWord( bool use=true ); + + /// restrict trigger query to given runs + void queryRuns( const std::string& runs ); + + /// set (get) the name of the RunNumber column + std::string runNumberColumn( const std::string& column = "" ); + + /// set/get the URL for the trigger decoding web service + std::string triggerWebService( const std::string& url = "" ); + + /// set/get the directory URL where trigger XML configuration files are + std::string XMLConfigDir( const std::string& url = "" ); + + std::string defaultXMLConfigDir(); + + /// set paths to certificate when using trigger web decoding + void setCertPath( const std::string& certpath, const std::string& keypath ); + + /// set MC project + void setDataPeriod( const std::string& period ) { m_dataPeriod = period; } + /// set AMI tag + void setAMITag( const std::string& tag ) { m_AMITag = tag; } + + void setDebug() { m_outputLevel = 1; } + +protected: + typedef std::map<std::string, unsigned> TriggerMap; + + /// parse the XML file with trigger bit positions + int parseXML( const std::string& filepath, TriggerMap& trigMap ); + + /// read Web server directory looking for configuration files + int readWebDir( const std::string& _url, std::set<std::string>& files ); + + /// initialize Xersecs + bool initXML(); + + + + struct TrigValRange { + TrigValRange( unsigned sr, unsigned er, unsigned bitpos, const std::string& attribute="" ) + : startrun(sr), endrun(er), bitPosition(bitpos), + attributeName( attribute ) + { } + + bool operator<(const TrigValRange& rhs) const + { + return bitPosition < rhs.bitPosition; + } + + unsigned startrun, endrun, bitPosition; + std::string attributeName; + }; + + typedef std::vector< TrigValRange > TrigValRangeVect; + + /// Query the web service for trigger bit mappings + /// trigger : coma separated list of trigger names + /// useHighBits : when true use the upper half of the word for 64 but EF trigger attributes, + void WSQueryTriggerPositions( const std::string& triggers, bool useHighBits ); + + /// Build list of trigger bit positions for query run ranges + /// trigger : trigger name + /// highBits : use to query upper half of a 64bit trigger word attribute (currently EF only) + TrigValRangeVect makeTrigValRangeVect( const std::string& trigger, bool highBits ); + + /* + tvRange : run range with trigger bit position constant (contains bit position and attribute name) + condidtion : true for testing bit value of 1, false for 0 + tech : POOL Collection technology (ROOT or Relational) + */ + /// Produce trigger condition query fragment for a given run-range + std::string addTriggerCondition( const TrigValRange& tvRange, + bool condition, + bool firstPass, + const std::string& tech, + int firstRange, + int valRangeGrouping); + + typedef std::vector< std::pair<unsigned,unsigned> > RunsVect; + + // properties - filled in by athena joboptions + // (declared here so TrigCollQueryTool can be considered an interface) + + /// location of the trigger map XML configuration files. can be directory or URL + std::string m_triggerMapDir; + + /// the name of the attribute wher RunNumber is kept in the collection + std::string m_runNumberColumnName; + + /// list of run ranges to query (empty = query all runs) + std::string m_queryRuns; + + /// compatibility flag for the old name of CTPWord attribute + bool m_useCTPWord; + + bool m_xmlInitialized; + + /// runs option string (e.g. as specified on the command line) + std::string m_runsStr; + /// decoded runs from m_runsStr + RunsVect m_runs; + + /// URL of the trigger decoding web service + std::string m_triggerWebService; + + /// true (defualt) - use web service, false - use XML config files + bool m_usingWebService; + + /// paths to certificate files + std::string m_certpath; + std::string m_keypath; + + /// data period used by the trigger decoding web service to identify MC project + std::string m_dataPeriod; + /// AMI tag used by the trigger decoding web service for MC projects + std::string m_AMITag; + + /// output level setting, so far only non-zero recognized + int m_outputLevel; + + struct TriggerRunMap { + unsigned startrun, endrun; + TriggerMap triggerPos; + }; + + class TriggerMapVect : public std::vector< TriggerRunMap* > { + public: + ~TriggerMapVect() { + for( size_t i = 0; i<this->size(); i++ ) delete (*this)[i]; + } + }; + + TriggerMapVect m_triggerMapVect; + + typedef std::map< std::string, TrigValRangeVect* > TriggerRangesMap; + TriggerRangesMap m_triggerRangesMap; + +}; + + +#endif + + diff --git a/Database/APR/TrigCollQuery/TrigCollQuery/TrigCollQueryTool.h b/Database/APR/TrigCollQuery/TrigCollQuery/TrigCollQueryTool.h new file mode 100644 index 0000000000000000000000000000000000000000..3662ab19b2f5f549a034f9eb850dae457e474d2f --- /dev/null +++ b/Database/APR/TrigCollQuery/TrigCollQuery/TrigCollQueryTool.h @@ -0,0 +1,58 @@ +/* + Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration +*/ + +#ifndef TRIGCOLLQUERY_TRIGCOLLQUERYTOOL_H +#define TRIGCOLLQUERY_TRIGCOLLQUERYTOOL_H + + +#include "AthenaKernel/IAthenaSelectorTool.h" +#include "GaudiKernel/AlgTool.h" + +#include "TrigCollQuery.h" + + +class TrigCollQueryTool + : virtual public IAthenaSelectorTool, + public AlgTool, + public TrigCollQuery +{ +public: + TrigCollQueryTool(const std::string& name, + const std::string& type, + const IInterface* parent); + + +/// Gaudi boilerplate + // do I need that ? + // static const InterfaceID& interfaceID(); + + /// Initialize AlgTool + virtual StatusCode initialize(); + + /// Called at the end of initialize + virtual StatusCode postInitialize(); + + /// Called at the beginning of execute + virtual StatusCode preNext(); + + /// Called at the end of execute + virtual StatusCode postNext(); + + /// Called at the beginning of finalize + virtual StatusCode preFinalize(); + + /// Finalize AlgTool + virtual StatusCode finalize(); + + virtual ~TrigCollQueryTool(); + +protected: + /// property - path to Trigger XML files + std::string m_XMLDir; +}; + + +#endif + + diff --git a/Database/APR/TrigCollQuery/cmt/requirements b/Database/APR/TrigCollQuery/cmt/requirements new file mode 100755 index 0000000000000000000000000000000000000000..b0ca01fe317f7f8a6ff4946af91fd4104252904d --- /dev/null +++ b/Database/APR/TrigCollQuery/cmt/requirements @@ -0,0 +1,25 @@ +package TrigCollQuery + +author Marcin Nowak <Marcin.Nowak@cern.ch> + +use AtlasPolicy AtlasPolicy-01-* +use GaudiInterface GaudiInterface-* External +use AthenaKernel AthenaKernel-* Control + + +# library TrigDecision *.cxx +apply_pattern dual_use_library files="TrigCollQuery.cxx TrigCollQueryTool.cxx" + +apply_pattern declare_joboptions files="-s=../share *.py" + +#apply_pattern declare_python_modules files="__init__.py TrigCollQueryConfig.py" + +application runCollUtil runCollUtil.cxx +macro_append runCollUtil_dependencies " TrigCollQueryLib" + +application decodeTriggerQuery decodeTriggerQuery.cxx +macro_append decodeTriggerQuery_dependencies " TrigCollQueryLib" + +private +use AtlasXercesC AtlasXercesC-* External +use AtlasBoost AtlasBoost-* External diff --git a/Database/APR/TrigCollQuery/doc/mainpage.h b/Database/APR/TrigCollQuery/doc/mainpage.h new file mode 100644 index 0000000000000000000000000000000000000000..824d2241f74e969db5436635e83ab63952f997b9 --- /dev/null +++ b/Database/APR/TrigCollQuery/doc/mainpage.h @@ -0,0 +1,83 @@ +/* + Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration +*/ + +/** + +@mainpage TrigCollQuery Package + +The package allowing Athena EventSelect and POOL Collection utilities to query TAG collections using trigger names, even though trigger information is stored in bit arrays. + +@author Marcin.Nowak@cern.ch + +@section IntroductionTrigCollQuery Introduction +Due to a large number of triggers, storing each trigger in a separate attribute in TAG collections is impractical (and sometimes even impossible). Because of that, trigger information is stored in bit arrays, which in turn are stored as groups of 32-bit integer attributes in TAG collection. + +TrigCollQuery package provides tools that allow Athena EventSelector and the standalone POOL Collection utilities to query TAG collections using well know trigger names without worrying about bit positions and the names of attributes storing the bit arrays. + + +@section TrigCollQueryOverview Class Overview + The TrigCollQuery package contains of following classes: + + - TrigCollQuery : The main class that can read the XML trigger configuration file and do the transformation of trigger query string replacing trigger names with bit operations. + + - TrigCollQueryTool : Implementation of IAthenaSelectorTool interface on top of TrigCollQuery. Allows usage of the trigger queries by Athena EventSelector. + + +@section TrigCollQueryExecutables Tools +The TrigCollQuery package contains one command line tool - runCollUtil.exe. This tool is a wrapper for all standalone POOL Collection utilities. When put in front of a command that normally executes one of the POOL utilites, it will look for a -query option and search it for any trigger references, replacing them by bit operations. +If the first option (before the executable name of the collection utility) is -usectpword, queries for L1 trigger decision will use the old name of the L1 trigger attribute - "CTPWord". The current attribute name is "L1PassedTrigMaskTAV" + +@section QueryUsageTrigCollQuery Syntax of Trigger Queries +To query for trigged bit set, add the following statement to the query string: + +TriggerPassed(TriggerName1[,TriggerName2,...]) + +Correspondingly, use a similar syntax to select events that that have not passed given triggers: + +TriggerNotPassed(TriggerName1[,TriggerName2,...]) + +Both statements may appear more than once in the query string, but grouping them together may result in a more efficient query (if the trigger bits are stored in the same word/attribute). + + + +@section EventSelectorUsageTrigCollQuery Trigger-based TAG Event selection in Athena +To use a trigger condition in an TAG event selection query in an Athena job, add the following lines to the jonOptions file: + +include("TrigCollQuery/TrigCollQuery_jobOptions.py")<br> +ServiceMgr.EventSelector.Query = "NLooseElectron>1 && TriggerPassed(L2_e105)"<br> + + +TrigCollQueryTool properties: + + - TrigBitsMapFile : the name of the XML file with trigger bit mappings. The file will be searched for in the $XMLPATH. <br> Example:<br> + MyQueryTool.TrigBitsMapFile = "TriggerMenuXML/ChainTagMap.xml" + + - UseCTPWord : Boolean flag that tells the tool to use the old "CTPWord" attribute name for L1 trigger + +@section UtilityUsageTrigCollQuery Querying Triggers using POOL command-line Collection Utilities +To be able to query triggers when using POOL command line collection utilities like CollAppend, one needs to parse the query and translate trigger names into bit positions before passing it on to the utility. This is achieved using a "wrapper" application called runCollUtil.exe. Simply add the wrapper at the start of the command that invokes a POOL collection utility. + +Example:<br> +runCollUtil.exe CollListAttrib -src fdr08_run1_MinBias_o1_r24_t3 RootCollection -query 'TriggerPassed(L2_e12)' + + +@section TrigCollQueryTriggerMap The XML Trigger Configuration File +Mapping of the trigger names to bit locations in the TAGs is done according to trigger configuration stored by default in TriggerMenuXML/ChainTagMap.xml file. This file is part of Athena release since version 14.2.20 and is provided by Trigger/TriggerCommon/TriggerMenuXML package. It is currently assumed there are 3 trigger levels: L1, L2 and EF, and trigger bits are stored in TAG attributes called CTPWord, L2PassedTrigMask and EFPassedTrigMask. + + +@section ExtrasTrigCollQuery Extra Pages + + - @ref UsedTrigCollQuery + - @ref RequirementsTrigCollQuery +*/ + +/** +@page UsedTrigCollQuery Used Packages +@htmlinclude used_packages.html +*/ + +/** +@page RequirementsTrigCollQuery Requirements +@include requirements +*/ diff --git a/Database/APR/TrigCollQuery/share/TrigCollQuery_jobOptions.py b/Database/APR/TrigCollQuery/share/TrigCollQuery_jobOptions.py new file mode 100644 index 0000000000000000000000000000000000000000..791b03e5652ac33bc209d042edb89776712f74ac --- /dev/null +++ b/Database/APR/TrigCollQuery/share/TrigCollQuery_jobOptions.py @@ -0,0 +1,107 @@ + +import PyUtils.Logging as L +msg = L.logging.getLogger('TrigCollQuery') +msg.setLevel(L.logging.INFO) + + +class TrigTAGFilePeeker: + def __init__(self, run_attr_name='RunNumber'): + self.run_attr_name = run_attr_name + self.all_runs = set() + self.file_cache = {} + + def get_runs_from_tagfile(self, fname): + # check own cache for this file + if fname in self.file_cache.keys(): + return self.file_cache[fname] + # check file type with AthFile - this should avoid reopening files more times than necessary + msg.debug("Checking file %s" % fname) + import PyUtils.AthFile as athFile + fileinfos = athFile.fopen(fname).fileinfos + if not 'TAG' in fileinfos['stream_names']: + return [] + # this is a TAG file, open it and read all run numbers + # fileinfos have only the run number from the first TAG + import PyUtils.Helpers as H + with H.restricted_ldenviron(projects=['AtlasCore']): + import re + with H.ShutUp(filters=[re.compile('TClass::TClass:0: RuntimeWarning: no dictionary for.*'), + re.compile('.*duplicate entry.*')]): + msg.debug("Opening TAG file %s" % fname) + import PyUtils.RootUtils as ru + f = ru.import_root().TFile.Open(fname, "read") + if f is None or not f: + msg.warning("Failed to open TAG file %s" % fname) + return [] + coll_tree = f.Get('POOLCollectionTree') + run_numbers = set() + if coll_tree is not None: + for row in xrange(0, coll_tree.GetEntries()): + coll_tree.GetEntry(row) + run_numbers.add( getattr(coll_tree, self.run_attr_name) ) + del coll_tree + f.Close() + del f + self.file_cache[fname] = run_numbers + msg.info("TAG file: %s, found runs: %s" % (fname, str(run_numbers))) + return run_numbers + + def get_runs_from_all_taginput(self): + from AthenaCommon.AthenaCommonFlags import athenaCommonFlags + # ifiles = athenaCommonFlags.PoolTAGInput() # not set correctly! + ifiles = athenaCommonFlags.FilesInput() + if ifiles is None or ifiles == []: + ifiles = ServiceMgr.EventSelector.InputCollections + + self.all_runs = set() + for file in ifiles: + runs = self.get_runs_from_tagfile(file) + self.all_runs |= set(runs) + return self.all_runs + + def get_all_runs_as_str(self): + run_string = ','.join( [str(run) for run in self.get_runs_from_all_taginput()] ) + return run_string + + +from TrigCollQuery.TrigCollQueryConf import TrigCollQueryTool +Tool = TrigCollQueryTool("TrigCollQueryTool") + +from AthenaCommon.AppMgr import ServiceMgr +ServiceMgr.EventSelector.HelperTools += [ Tool ] + +if ServiceMgr.EventSelector.Query.find('Trigger'): + msg.info('Found Trigger keyword in the query string, scanning TAG input files for run numbers') + peeker = TrigTAGFilePeeker() + run_string = peeker.get_all_runs_as_str() + Tool.QueryRuns = run_string + msg.info("Total runs from TAGs: " + run_string) + del peeker + +del msg + + +# Example usage of Trigger selection in the EventSelector: +# +#ServiceMgr.EventSelector.Query = "NLooseElectron>1 && TriggerPassed(L2_e105)" + + +# TrigCollQuery properties: +# see: https://twiki.cern.ch/twiki/bin/view/Atlas/TagTriggerQueries +# +#Tool.QueryRuns = "119267-119269,119274-119280,119302-119325" + +# this is the default in .cpp +#Tool.TriggerMapWebService = "https://atlas-tagservices.cern.ch/tagservices/triggerDecoder/getDecodedTriggerInfo.php" + +# Needed for MC, don't use for Data +#Tool.Project = "" +#Tool.AMITag = "" + +# default value +#Tool.RunNumberAttribute = "RunNumber" + +# outdated - these files are old, probably useless +#Tool.TriggerMapDir = "/afs/cern.ch/user/a/attrgcnf/TagChainMaps" + +#Tool.UseCTPWord = False diff --git a/Database/APR/TrigCollQuery/src/TrigCollQuery.cxx b/Database/APR/TrigCollQuery/src/TrigCollQuery.cxx new file mode 100644 index 0000000000000000000000000000000000000000..0ed5ee733b5350aaf24ea9c878c187a38bfcde8c --- /dev/null +++ b/Database/APR/TrigCollQuery/src/TrigCollQuery.cxx @@ -0,0 +1,875 @@ +/* + Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration +*/ + +#include "TrigCollQuery/TrigCollQuery.h" + +#include "boost/tokenizer.hpp" + +#include <iostream> +#include <sstream> +#include <fstream> +#include <memory> +#include <set> +#include <iomanip> + +using namespace std; + + +const std::string TriggerPassed( "TriggerPassed" ); +const std::string TriggerNotPassed( "TriggerNotPassed" ); +const std::string TriggerPassedLowbits( "TriggerPassedInclusive" ); +const std::string TriggerNotPassedLowbits( "TriggerNotPassedInclusive" ); + +const std::string L1prefix( "L1_" ); +const std::string L2prefix( "L2_" ); +const std::string EFprefix( "EF_" ); + +const std::string L1word_old( "CTPWord" ); +const std::string L1word( "L1PassedTrigMaskTAV" ); +const std::string L1wordAP( "L1PassedTrigMaskTAP" ); +const std::string L1wordBP( "L1PassedTrigMaskTBP" ); +const std::string L2word( "L2PassedTrigMask" ); +const std::string EFword( "EFPassedTrigMask" ); + +const std::string root_bitAndStr( " & " ); +const std::string root_logicAndStr( " && " ); +const std::string root_logicOrStr( " || " ); +const std::string root_equalsStr( "==" ); +const std::string root_notEqualsStr( "!=" ); + +const std::string sql_bitAndStr( " bitand(" ); +const std::string sql_logicAndStr( " AND " ); +const std::string sql_logicOrStr( " OR " ); +const std::string sql_equalsStr( " = " ); +const std::string sql_notEqualsStr( " != " ); + +const std::string XMLDir( "TriggerMenuXML" ); +const std::string MapFileName( "ChainTagMap" ); +const std::string RunNumberAttribute( "RunNumber" ); + + +class SyntaxError : public runtime_error { +public: + SyntaxError() : std::runtime_error("Syntax error in the trigger condition") + {} +}; + +class UnknownTrigger : public runtime_error { +public: + UnknownTrigger( const std::string& msg ) + : std::runtime_error( std::string("Unknown trigger name: ") + msg ) + {} +}; + + + +TrigCollQuery::TrigCollQuery() : + m_runNumberColumnName( RunNumberAttribute ) + , m_xmlInitialized( false ) + , m_usingWebService( false ) + , m_outputLevel( 0 ) +{ + useCTPWord(false); + triggerWebService("https://atlas-tagservices.cern.ch/tagservices/triggerDecoder/getDecodedTriggerInfo.php"); +} + + + +void TrigCollQuery::useCTPWord( bool use ) { + m_useCTPWord = use; +} + +std::string +TrigCollQuery::triggerWebService( const std::string& url ) { + if( url.length() ) { + m_triggerWebService = url; + m_usingWebService = true; + } + return m_triggerWebService; +} + +std::string +TrigCollQuery::XMLConfigDir( const std::string& url ) { + if( url.length() ) { + m_triggerMapDir = url; + m_usingWebService = false; + } + return m_triggerMapDir; +} + +std::string +TrigCollQuery::defaultXMLConfigDir() { + return "/afs/cern.ch/user/a/attrgcnf/TagChainMaps/"; +} + + +void +TrigCollQuery::setCertPath( const std::string& certpath, const std::string& keypath ) { + m_certpath = certpath; + m_keypath = keypath; +} + + + +typedef boost::tokenizer<boost::char_separator<char> > Tizer; + + +std::string +TrigCollQuery::triggerQueryRemap( const std::string& query, const std::string& tech ) +{ + string mappedQuery; + boost::char_separator<char> sep("", " |&!(,)"); + Tizer tizer( query, sep ); + + for( Tizer::iterator token = tizer.begin(); token != tizer.end(); ++token ) { + if( *token == TriggerPassed || *token == TriggerNotPassed + || *token == TriggerPassedLowbits || *token == TriggerNotPassedLowbits ) + { + bool trigCond = ( *token == TriggerPassed || *token == TriggerPassedLowbits ); + bool useHighBits = ( *token != TriggerPassedLowbits && *token != TriggerNotPassedLowbits ); + if( ++token == tizer.end() || *token != "(" ) throw SyntaxError(); + + vector<string> triggers_vec; + string triggers_str; + string trigsep; + // collect trigger names used in this operator + do { + if( ++token == tizer.end() ) throw SyntaxError(); + if( *token == ")" ) break; + triggers_vec.push_back( *token ); + triggers_str += trigsep + *token; + trigsep = ","; + if( ++token == tizer.end() ) throw SyntaxError(); + } while( *token == "," ); + + if( m_usingWebService ) { + WSQueryTriggerPositions( triggers_str, useHighBits ); + } + mappedQuery += " ("; + bool firstPass = true; + for( vector<string>::const_iterator trigger = triggers_vec.begin(), end = triggers_vec.end(); + trigger != end; ++trigger ) + { + TrigValRangeVect trigValRangeVect; + if( m_usingWebService ) { + TriggerRangesMap::const_iterator mi = m_triggerRangesMap.find( *trigger ); + if( mi != m_triggerRangesMap.end() ) + trigValRangeVect = *(mi->second); + } else + trigValRangeVect = makeTrigValRangeVect( *trigger, useHighBits ); + + if( !trigValRangeVect.size() ) { + // add a FALSE statement because the trigger was not active + // using condition '0 & 1 != 0' for TriggerPassed + // note: this will result in TRUE for NotPassed + trigValRangeVect.push_back( TrigValRange(0,99999999,0, "0") ); + } + std::sort( trigValRangeVect.begin(), trigValRangeVect.end() ); + int firstValRange = 1; // open bracket + int valRangeGrouping = 0; + for( size_t r = 0; r < trigValRangeVect.size(); r++ ) { + if( r+1 < trigValRangeVect.size() + && trigValRangeVect[r].bitPosition == trigValRangeVect[r+1].bitPosition + && trigValRangeVect[r].attributeName == trigValRangeVect[r+1].attributeName ) { + // group run ranges with the same bit position + valRangeGrouping = (valRangeGrouping? 2 : 1); // 1 == first, >1 == next + } else { + valRangeGrouping = (valRangeGrouping? 3 : 0); // 3 means end of group + } + mappedQuery += addTriggerCondition( trigValRangeVect[r], + trigCond, firstPass, tech, + firstValRange, valRangeGrouping ); + if( valRangeGrouping == 3 ) + valRangeGrouping = 0; // reset group after its end + firstValRange = 2; // add "or" in the next pass + } + mappedQuery += ")"; // closing braket (to match the one added by firstValRange==1?) + firstPass = false; + } + // end of TriggerPassed(), close the parens + mappedQuery += ")"; + + } else { + mappedQuery += *token; + } + } + + return mappedQuery; +} + + +TrigCollQuery::TrigValRangeVect +TrigCollQuery::makeTrigValRangeVect( const std::string& trigger, bool useHighBits ) +{ + bool discontinuous = false; + TrigValRangeVect trigValRangeVect; + // check all trigger maps for this trigger position + for( size_t m=0; m < m_triggerMapVect.size(); m++ ) { + TriggerRunMap *map = m_triggerMapVect[m]; + TriggerMap::const_iterator ti = map->triggerPos.find( trigger ); + if( ti == map->triggerPos.end() ) { + // trigger not defined in this map - this is a "hole" + discontinuous = true; + continue; + } + // bit position for the trigger + unsigned bitPos = ti->second; + if( trigValRangeVect.size()) { + TrigValRange& lastRange = trigValRangeVect[ trigValRangeVect.size()-1 ]; + if( lastRange.bitPosition == bitPos && !discontinuous ) { + //extend the last range (only if there was no hole) + lastRange.endrun = map->endrun; + continue; + } + if( lastRange.bitPosition != bitPos && lastRange.endrun >= map->startrun ) { + //truncate the last range + lastRange.endrun = map->startrun - 1; + } + } + discontinuous = false; + trigValRangeVect.push_back( TrigValRange(map->startrun, map->endrun, bitPos) ); + } + if( !trigValRangeVect.size() ) + throw UnknownTrigger(trigger); + if( !discontinuous ) { + // extend last range to infinity + trigValRangeVect[ trigValRangeVect.size()-1 ].endrun = 99999999; + } + + string wordName; + if( trigger.compare(0, 3, L1prefix) == 0 ) { + wordName = (m_useCTPWord? L1word_old : L1word ); + useHighBits = false; + } + if( trigger.compare(0, 3, L2prefix) == 0 ) { + wordName = L2word; + useHighBits = false; + } + if( trigger.compare(0, 3, EFprefix) == 0 ) { + wordName = EFword; + } + + // convert (triggerName, bit position) to (attributeName, bitpos) + for( TrigValRangeVect::iterator it = trigValRangeVect.begin(), end = trigValRangeVect.end(); + it != end; ++it ) { + ostringstream stream; + stream << wordName << unsigned(it->bitPosition/32); + it->attributeName = stream.str(); + it->bitPosition = (it->bitPosition % 32) + (useHighBits? 32 : 0); + } + + return trigValRangeVect; // inefficient but easy +} + + + + +void TrigCollQuery::queryRuns( const std::string& runs ) +{ + // cout << endl << ">>> TrigCollQuery QueryRuns:" << runs << endl << endl; + m_runsStr = runs; + + boost::char_separator<char> sep(","); + Tizer tizer( runs, sep ); + + for( Tizer::iterator token = tizer.begin(); token != tizer.end(); ++token ) { + size_t dash_idx = token->find('-'); + if( dash_idx != string::npos ) { + int startrun = atoi(token->c_str()); + int endrun = ( dash_idx+1 >= token->size() ? 99999999 : atoi( token->substr( dash_idx+1 ).c_str() ) ); + m_runs.push_back( pair<int,int>(startrun, endrun) ); + } else { + int run = atoi(token->c_str()); + m_runs.push_back( pair<int,int>(run, run) ); + } + // cout<< "** Run range: " << m_runs.back().first << ", " << m_runs.back().second << endl; + } +} + + +/// set (get) the name of the RunNumber column +std::string TrigCollQuery::runNumberColumn( const std::string& column ) +{ + if( column.size() ) { + m_runNumberColumnName = column; + } + return m_runNumberColumnName; +} + + +#include <boost/regex.hpp> + +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <dirent.h> + +void TrigCollQuery::readTriggerMap( std::string filename ) +{ + const string logName = "TrigCollQuery"; + if( m_runs.empty() ) queryRuns("1-99999999"); + + string xmlpath; + if( !m_triggerMapDir.empty() ) { + // Athena property, if set, overrides environment + xmlpath = m_triggerMapDir; + } else { + const char *pathenv_name = "XMLPATH"; + char* xmlpath_env = getenv( pathenv_name ); + if( xmlpath_env ) + xmlpath = xmlpath_env; + xmlpath += ":."; + } + + // cout << endl << ">>> TrigCollQuery searching XMLPATH:" << xmlpath << endl << endl; + Tizer paths( xmlpath, boost::char_separator<char>(":") ); + Tizer::iterator path; + boost::regex const xmlmap_filenameRE( ".*"+MapFileName+"_(\\d{8})-(\\d{8})\\.xml$"); + for( path = paths.begin(); path != paths.end(); ++path ) { + set<string> files; + // Fill "files" with filenames in the given element of XMLPATH + boost::cmatch what; + if( *path == "http" ) { + ++path; + if( path == paths.end() ) break; + readWebDir( string("http:")+*path, files ); + } else { + const std::string dirpath( *path + "/" + XMLDir ); + DIR *d = opendir( dirpath.c_str() ); + if( !d ) { + // cout << logName << " Can't open XMLPATH component: " << dirpath << endl; + continue; + } else { + // Reading all files from directory into a set (to sort them) + // cout << logName << " Searching XMLPATH path: " << dirpath << endl; + struct dirent *de; + while( (de = readdir(d)) != 0 ) { + // cout << " found file " << de->d_name << endl; + files.insert( dirpath + "/" + de->d_name ); + } + closedir( d ); + } + } + + set<string>::const_iterator fi = files.begin(); + while( fi != files.end() ) { + if( boost::regex_match(fi->c_str(), what, xmlmap_filenameRE) ) { + TriggerRunMap *trm = new TriggerRunMap; + trm->startrun = atoi(what[1].first); + trm->endrun = atoi(what[2].first); + for( RunsVect::const_iterator ri = m_runs.begin(); ri != m_runs.end(); ++ri ) { + // cout << "==== Comparing " << ri->first << " <= " << trm->endrun << " && " << ri->second << " >= " << trm->startrun << endl; + if( ri->first <= trm->endrun && ri->second >= trm->startrun ) { + // cout << " parsing: " << *fi << ": RUNS: " << trm->startrun << "-" << trm->endrun << endl; + if( parseXML( *fi, trm->triggerPos ) ) { + cerr << logName << " Failed reading trigger map file " << *fi << endl; + delete trm; trm = 0; + } else { + m_triggerMapVect.push_back( trm ); + continue; + } + } + } + } + ++fi; + } + if( files.size() ) break; // read only one location + } + if( m_triggerMapVect.empty() ) { + cout << logName << " Warning: Failed to retrieve run-dependent trigger mapping information for the given run range" << endl + << " Looking for the old (static and possibly obsolete) trigger map file" << endl; + + if( !filename.size() ) + filename = XMLDir + "/" + MapFileName + ".xml"; + struct stat statbuf; + string fullpath; + for( path = paths.begin(); path != paths.end(); ++path ) { + fullpath = *path + "/" + filename; + if( 0 == stat( fullpath.c_str(), &statbuf ) ) { + // cout << logName << " Found trigger mappings file: " << fullpath << endl; + break; + } + } + if( path == paths.end() ) { + cerr << logName << " Warning: File " << filename << " not found in " << xmlpath << endl; + cerr << logName << " No trigger mapping information available - trigger queries will not work" << endl; + return; + } + TriggerRunMap *trm = new TriggerRunMap; + trm->startrun = 1; + trm->endrun = 99999999; + + if( !parseXML( fullpath, trm->triggerPos ) ) { + cout << logName << " XML trigger map " << fullpath << " read in " << endl; + m_triggerMapVect.push_back( trm ); + } else { + delete trm; + cerr << logName << " Error reading XML trigger map! " << fullpath << endl; + } + } +} + + + + + +const string DocType = "TAGMAP"; +const string LevelElementName = "LEVEL"; +const string LevelAttributeName = "level"; + +const string TriggerElementName = "MAPELEMENT"; +const string TriggerAttributeName = "chainname"; +const string BitAttributeName = "tagindex"; + + + +#include <xercesc/parsers/XercesDOMParser.hpp> +#include <xercesc/dom/DOM.hpp> +#include <xercesc/sax/HandlerBase.hpp> +#include <xercesc/util/XMLString.hpp> +#include <xercesc/util/PlatformUtils.hpp> +#include <xercesc/framework/URLInputSource.hpp> +#include <xercesc/util/BinInputStream.hpp> +#include <xercesc/util/BinFileInputStream.hpp> + +#include <boost/scoped_array.hpp> + +XERCES_CPP_NAMESPACE_USE + +typedef std::basic_string<XMLCh> XercesString; + +// Converts from a narrow-character string to a wide-character string. +inline XercesString fromNative(const char* str) +{ + boost::scoped_array<XMLCh> ptr( XMLString::transcode(str) ); + return XercesString(ptr.get( )); +} + +// Converts from a narrow-character string to a wide-charactr string. +inline XercesString fromNative(const std::string& str) +{ + return fromNative(str.c_str( )); +} + +// Converts from a wide-character string to a narrow-character string. +inline std::string toNative(const XMLCh* str) +{ + boost::scoped_array<char> ptr( XMLString::transcode(str) ); + return std::string(ptr.get( )); +} + +// Converts from a wide-character string to a narrow-character string. +inline std::string toNative(const XercesString& str) +{ + return toNative(str.c_str( )); +} + + + + +std::string TrigCollQuery::addTriggerCondition( const TrigValRange& trigValRange, + bool condition, + bool firstPass, + const std::string& tech, + int firstRange, + int valRangeGrouping ) +{ + //cout << "Adding condition for " << trigValRange.attributeName << " bit " << trigValRange.bitPosition << endl; + uint64_t mask = (1ULL << trigValRange.bitPosition); + + string nextOper, logicAnd, logicOr, logicEquals, comparison, bitAndOperation; + if( tech == "RelationalCollection" ) { + logicAnd = sql_logicAndStr; + logicOr = sql_logicOrStr; + logicEquals = sql_equalsStr; + comparison = condition? sql_notEqualsStr : sql_equalsStr; + bitAndOperation = sql_bitAndStr + trigValRange.attributeName + ", "; + } else { + logicAnd = root_logicAndStr; + logicOr = root_logicOrStr; + logicEquals = root_equalsStr; + comparison = condition? root_notEqualsStr : root_equalsStr; + bitAndOperation = "(" + trigValRange.attributeName + root_bitAndStr; + } + + ostringstream out; + // add AND between different triggers + if( !firstPass && firstRange < 2 ) out << logicAnd; + // add OR between run ranges for the same trigger + if( firstRange == 1 ) out << "( "; // closing bracket must be added by calling method + if( firstRange == 2 ) out << logicOr; + // opening bracket for the start of run range (0 means it's not compressed) + if( valRangeGrouping == 0 ) out << "( "; + // extra bracket at the start of a commpresed run range + if( valRangeGrouping == 1 ) out << "( ( "; + + // add run range + if( trigValRange.startrun == trigValRange.endrun ) { + out << m_runNumberColumnName << logicEquals << trigValRange.startrun; + nextOper = logicAnd; + } else { + if( trigValRange.startrun > 1 ) { + out << m_runNumberColumnName << ">=" << trigValRange.startrun; + nextOper = logicAnd; + } + if( trigValRange.endrun < 99999999 ) { + out << nextOper << m_runNumberColumnName << "<=" << trigValRange.endrun; + nextOper = logicAnd; + } + } + // finish here if just creating compressed run range for the same bit + if( valRangeGrouping == 1 || valRangeGrouping == 2 ) + return out.str(); + + // close compressed run range for same bit pos + if( valRangeGrouping == 3) out << ")"; + + // Finally the real bitwise AND operation for the trigger + out << nextOper << bitAndOperation; + // give the mask in hex to avoid sign issues in ROOT + if( tech == "RootCollection" ) out << hex << "0x"; + out << mask << ")" << dec << comparison << " 0) "; + + // else throw std::runtime_error(string("Unknown collection technology: ") + tech); + return out.str(); +} + + + + +int TrigCollQuery::parseXML( const std::string& xmlfilename, TriggerMap& trigMap ) { + + if( !initXML() ) return 1; + + std::auto_ptr<XercesDOMParser> parser( new XercesDOMParser() ); + parser->setDoSchema( false ); + parser->setValidationScheme(XercesDOMParser::Val_Never); + auto_ptr<ErrorHandler> eh( new HandlerBase() ); + parser->setErrorHandler( eh.get() ); + + try { + parser->parse( xmlfilename.c_str() ); + DOMDocument *doc = parser->getDocument(); + + DOMElement* elementRoot = doc->getDocumentElement(); + if( !elementRoot ) throw(std::runtime_error( "empty XML document" )); + + DOMNodeList *nodelist = elementRoot->getChildNodes(); + for( XMLSize_t xx = 0; xx < nodelist->getLength(); ++xx ) + { + DOMNode* currentNode = nodelist->item(xx); + if( currentNode->getNodeType() == DOMNode::ELEMENT_NODE ) { + // Found node which is an Element. Re-cast node as element + // static_cast should be safe as we confirmed the NoneType (right?) + DOMElement* currentElement = static_cast< xercesc::DOMElement* >( currentNode ); + if( toNative(currentElement->getTagName()) == LevelElementName ) { + // Already tested node as type element and of name "ApplicationSettings". + // Read attributes of element "ApplicationSettings". + string level = toNative( currentElement->getAttribute( fromNative(LevelAttributeName).c_str()) ); + + DOMNodeList *triggerlist = currentNode->getChildNodes(); + for( XMLSize_t t = 0; t < triggerlist->getLength(); ++t ) { + DOMNode* currentNode = triggerlist->item(t); + if( currentNode->getNodeType() == DOMNode::ELEMENT_NODE ) { + // Found node which is an Element. Re-cast node as element + DOMElement* currentElement = static_cast< xercesc::DOMElement* >( currentNode ); + if( toNative(currentElement->getTagName()) == TriggerElementName ) { + string trigger = toNative( currentElement->getAttribute( fromNative(TriggerAttributeName).c_str() ) ); + string bit = toNative( currentElement->getAttribute( fromNative(BitAttributeName).c_str() ) ); + // cout << "Trigger: " << chainname << " bit=" << bit << endl; + istringstream bitnumber( bit ); + bitnumber >> trigMap[ trigger ]; + } + } + } + } + } + } + + return 0; + + } + catch (const XMLException& toCatch) { + cout << "Error! Exception: " << toNative(toCatch.getMessage()) << endl; + } + catch (const DOMException& toCatch) { + cout << "Error! Exception: " << toNative(toCatch.getMessage()) << endl; + } + catch( const SAXException &e ) { + cout << "Error! Exception: " << toNative(e.getMessage()) << endl; + } + catch( exception &e ) { + cout << "Exception cought: " << e.what() << endl; + } + catch(...) { + cout << "Unexpected Exception " << endl ; + } + return -1; +} + + + + +int TrigCollQuery::readWebDir( const std::string& _url, set<string>& files ) +{ + if( !initXML() ) return 1; + + string url( _url ); + if( url[ url.size()-1 ] != '/' ) + url += '/'; + try { + URLInputSource dir( url.c_str() ); + BinInputStream * dircont; + try { + dircont = dir.makeStream (); + }catch( const XMLException& e) { + cout << toNative(e.getMessage()) << endl; + return -1; + } + + vector<XMLByte> buff; + int iniBuffSize = 20000; + buff.resize( iniBuffSize ); + + int total_read_bytes = 0; + do{ + int rb = dircont->readBytes( &buff[total_read_bytes], iniBuffSize); + total_read_bytes += rb; + // cout << "*** read " << rb << " bytes" << endl; + if( rb == 0 ) + break; + if( buff.size() > 10*1024*1024 ) { + cerr << "Exceeded 10MB while reading from " << url << " Trigger configuration may be incomplete" << endl; + break; + } + buff.resize( buff.size() + rb ); + }while(true); + + boost::regex const xmlmap_filenameRE( "a href=\"("+MapFileName+"_\\d{8}-\\d{8}\\.xml)"); + boost::cmatch what; + const char *start = (char*)&buff[0], *end = (char*)&buff[ buff.size() ]; + boost::match_flag_type flags = boost::match_default; + while( regex_search( start, end, what, xmlmap_filenameRE, flags)) { + string filename( what[1].first, what[1].second ); + filename = url + filename; + files.insert( filename ); + // cout << " > " << "found:" << filename << endl; + // update search position: + start = what[0].second; + // update flags: + flags |= boost::match_prev_avail; + flags |= boost::match_not_bob; + } + + return 0; + } + catch (const XMLException& toCatch) { + cout << "Error! Exception: " << toNative(toCatch.getMessage()) << endl; + } + catch (const DOMException& toCatch) { + cout << "Error! Exception: " << toNative(toCatch.getMessage()) << endl; + } + catch( const SAXException &e ) { + cout << "Error! Exception: " << toNative(e.getMessage()) << endl; + } + catch( exception &e ) { + cout << "Exception cought: " << e.what() << endl; + } + catch(...) { + cout << "Unexpected Exception " << endl ; + } + return -1; +} + + + +#include <cstdio> +#include <cstdlib> +#include <unistd.h> +#include <xercesc/framework/MemBufInputSource.hpp> + + +/// Read trigger bit mappings for triggers present in the query (only) +void TrigCollQuery::WSQueryTriggerPositions( const string& triggers, bool useHighBits ) +{ + m_triggerRangesMap.clear(); + + if( m_certpath.empty() ) { + const char* proxy = getenv("X509_USER_PROXY"); + if( proxy ) { + m_certpath = m_keypath = proxy; + } else { + // check the default proxy location + ostringstream px; + px << "/tmp/x509up_u" << getuid(); + if( access(px.str().c_str(), R_OK) ) + throw runtime_error( "Certificate path not specified and X509_USER_PROXY not set (run voms-proxy-init)" ); + m_certpath = m_keypath = px.str(); + } + } +// if( m_dataPeriod.empty() ) +// throw runtime_error( "Project (data period/MC) not set" ); + + string command = string("curl -s -S --key ") + m_keypath + " --sslv3 --cert " + m_certpath + + " -k --url \"" + m_triggerWebService + + "?trignms=" + triggers + + "&runrange=" + m_runsStr + + "&phys_raw=" + (useHighBits? "phys" : "raw") + + "&filename_tag=" + m_dataPeriod + + "&ami_tag=" + m_AMITag + + "\" 2>&1"; + + if( m_outputLevel ) { + cout << " -------- executing command:" << endl << command << endl; + } + + FILE* fp = popen(command.c_str(), "r"); + if( !fp ) throw runtime_error("Failed to execute CURL command: " + command); + string output; + const size_t readsize(1000); + char buffer[readsize+1]; + size_t readbytes = 0; + do { + readbytes = fread(buffer, 1, readsize, fp); + buffer[readbytes] = 0; // string terminator + output += buffer; + } while( readbytes == readsize ); + + if( ferror(fp) ) { + throw runtime_error("reading CURL output: " + output); + } + int rc = pclose(fp); + if( rc ) { + cerr << "Error executing the CURL commnad: " << command << endl + << "pclose() returned code 0x" << hex << rc << endl; + throw runtime_error("executing the CURL commnad: " + output); + } + // cout << "------------ CURL output:" << endl << output << endl; + + if( output.find("404 Not Found") != string::npos ) + throw runtime_error(output); + + if( !initXML() ) throw runtime_error("XML init failed"); + MemBufInputSource xmlSource((const XMLByte*)output.c_str(), output.length(), "id"); + + std::auto_ptr<XercesDOMParser> parser( new XercesDOMParser() ); + parser->setDoSchema( false ); + parser->setValidationScheme(XercesDOMParser::Val_Never); + auto_ptr<ErrorHandler> eh( new HandlerBase() ); + parser->setErrorHandler( eh.get() ); + + try { + parser->parse( xmlSource ); + //parser->parse( "test.xml" ); + DOMDocument *doc = parser->getDocument(); + // cout << "========= XML document parsed" << endl; + + DOMElement* elementRoot = doc->getDocumentElement(); + if( !elementRoot ) + throw runtime_error( "empty XML document" ); + string rootTagName = toNative(elementRoot->getTagName()); + if( rootTagName == "ERROR" ) + throw runtime_error( output ); + + DOMNodeList *nodelist = elementRoot->getChildNodes(); + for( XMLSize_t xx = 0; xx < nodelist->getLength(); ++xx ) + { + DOMNode* currentNode = nodelist->item(xx); + if( currentNode->getNodeType() == DOMNode::ELEMENT_NODE ) { + DOMElement* currentElement = static_cast< xercesc::DOMElement* >( currentNode ); + + // cout << ">> Tag Name:" << toNative(currentElement->getTagName()) << endl; + if( toNative(currentElement->getTagName()) == "trigger" ) { + // Read attributes of element + string triggerName = toNative( currentElement->getAttribute( fromNative("name").c_str()) ); + DOMNodeList *triggernodes = currentNode->getChildNodes(); + for( XMLSize_t t = 0; t < triggernodes->getLength(); ++t ) { + DOMNode* currentNode = triggernodes->item(t); + if( currentNode->getNodeType() == DOMNode::ELEMENT_NODE ) { + DOMElement* currentElement = static_cast< xercesc::DOMElement* >( currentNode ); + // cout << ">> Tag Name:" << toNative(currentElement->getTagName()) << endl; + if( toNative(currentElement->getTagName()) == "run" ) { + istringstream runNumberStr( toNative( currentElement->getAttribute( fromNative("number").c_str()) ) ); + unsigned long long runNumber; + runNumberStr >> runNumber; + DOMNodeList *runnodes = currentNode->getChildNodes(); + for( XMLSize_t r = 0; r < runnodes->getLength(); ++r ) { + DOMNode* currentNode = runnodes->item(r); + if( currentNode->getNodeType() == DOMNode::ELEMENT_NODE ) { + DOMElement* currentElement = static_cast< xercesc::DOMElement* >( currentNode ); + // cout << ">> Tag Name:" << toNative(currentElement->getTagName()) << endl; + if( toNative(currentElement->getTagName()) == "decoded_trigger" ) { + string word = toNative( currentElement->getAttribute( fromNative("word").c_str() ) ); + if( word == "undefined" ) + continue; + if( word == "unknown" ) + throw runtime_error(string("Unknown trigger: ") + triggerName); + istringstream bitstr( toNative( currentElement->getAttribute( fromNative("bit").c_str() ) ) ); + unsigned bitpos = 0; + bitstr >> bitpos; + // cout << "Trigger " << triggerName << " run=" << runNumber << " word=" << word << " bit=" << bitpos << endl; + if( m_triggerRangesMap.find(triggerName) == m_triggerRangesMap.end() ) { + m_triggerRangesMap[triggerName] = new TrigValRangeVect; + } + TrigValRangeVect* tv = m_triggerRangesMap[triggerName]; + if( tv->size() && tv->back().endrun+1 == runNumber + && tv->back().bitPosition == bitpos && tv->back().attributeName == word ) + // expand range of existing entry + tv->back().endrun++; + else + tv->push_back( TrigValRange(runNumber, runNumber, bitpos, word) ); + } + } + } + } + + } + } + + } + } + } + + } + catch (const XMLException& toCatch) { + throw runtime_error(string("XML Exception: ") + toNative(toCatch.getMessage()) ); + } + catch (const DOMException& toCatch) { + throw runtime_error(string("XML DOM Exception: ") + toNative(toCatch.getMessage()) ); + } + catch( const SAXException &e ) { + throw runtime_error(string("XML SAX Exception: ") + toNative(e.getMessage()) ); + } +} + + + + + +bool TrigCollQuery::initXML() +{ + if( ! m_xmlInitialized ) { + try { + XMLPlatformUtils::Initialize(); + } + catch (const XMLException& toCatch) { + cerr << "Error during initialization! :" << toNative(toCatch.getMessage()) << endl; + } + m_xmlInitialized = true; + } + return m_xmlInitialized; +} + + +TrigCollQuery::~TrigCollQuery() +{ + if( m_xmlInitialized ) { + try { + XMLPlatformUtils::Terminate(); + } catch (const XMLException& toCatch) {} + } + for( TriggerRangesMap::const_iterator it = m_triggerRangesMap.begin(), end = m_triggerRangesMap.end(); + it != end; ++it ) + delete it->second; +} diff --git a/Database/APR/TrigCollQuery/src/TrigCollQueryTool.cxx b/Database/APR/TrigCollQuery/src/TrigCollQueryTool.cxx new file mode 100644 index 0000000000000000000000000000000000000000..d8126d218035482c336fd0412b4047438dd33354 --- /dev/null +++ b/Database/APR/TrigCollQuery/src/TrigCollQueryTool.cxx @@ -0,0 +1,143 @@ +/* + Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration +*/ + +#include "TrigCollQuery/TrigCollQueryTool.h" + +#include "GaudiKernel/ISvcLocator.h" +#include "GaudiKernel/MsgStream.h" +#include "GaudiKernel/ServiceHandle.h" +#include "GaudiKernel/IJobOptionsSvc.h" + +#include "boost/tokenizer.hpp" + +#include <iostream> +#include <sstream> + +#include <map> + +using namespace std; + +typedef boost::tokenizer<boost::char_separator<char> > Tizer; + + + +TrigCollQueryTool::TrigCollQueryTool(const std::string& name, + const std::string& type, + const IInterface* parent) + : AlgTool(name, type, parent) +{ + // properties - default values set in TrigCollQuery::TrigCollQuery() + declareProperty( "TriggerMapDir", m_XMLDir ); + declareProperty( "UseCTPWord", m_useCTPWord ); + declareProperty( "QueryRuns", m_queryRuns ); + declareProperty( "RunNumberAttribute", m_runNumberColumnName ); + declareProperty( "TriggerMapWebService", m_triggerWebService ); + declareProperty( "Project", m_dataPeriod ); + declareProperty( "AMITag", m_AMITag ); + + declareInterface<IAthenaSelectorTool>(this); +} + + +TrigCollQueryTool::~TrigCollQueryTool() +{} + + + + +/// Initialize AlgTool +StatusCode TrigCollQueryTool::initialize() +{ + MsgStream log( msgSvc(), name() ); + log << MSG::DEBUG << "Initialize" << endreq; + + queryRuns( m_queryRuns ); + if( !m_XMLDir.empty() ) { + log << MSG::DEBUG << "Using Trigger configuration from XML files: " << m_XMLDir << endreq; + XMLConfigDir( m_XMLDir ); + readTriggerMap( ); + } + + return StatusCode::SUCCESS; +} + + +/// Called at the end of initialize +StatusCode TrigCollQueryTool::postInitialize() +{ + MsgStream log( msgSvc(), name() ); + log << MSG::DEBUG << "postInitialization start" << endreq; + + const IService* parentSvc = dynamic_cast<const IService*>(this->parent()); + if (parentSvc == 0) { + log << MSG::ERROR << "Unable to get parent Service" << endreq; + return(StatusCode::FAILURE); + } + const IProperty* propertyServer = dynamic_cast<const IProperty*>(parentSvc); + if( propertyServer == 0 ) { + log << MSG::ERROR << "Unable to get Property Server from the parent service" << endreq; + return(StatusCode::FAILURE); + } + StringProperty queryProperty("Query", ""); + StatusCode status = propertyServer->getProperty(&queryProperty); + if (!status.isSuccess()) { + log << MSG::ERROR << "Unable to get Query property from " << parentSvc->name() << endreq; + return(StatusCode::FAILURE); + } + log << MSG::INFO << "EventSelector property Query is: " << queryProperty.toString() << endreq; + StringProperty collProperty("CollectionType", ""); + status = propertyServer->getProperty(&collProperty); + if (!status.isSuccess()) { + log << MSG::ERROR << "Unable to get CollectionType property from " << parentSvc->name() << endreq; + return(StatusCode::FAILURE); + } + log << MSG::DEBUG << "EventSelector property CollectionType is: " << collProperty.toString() << endreq; + try { + string remappedQ = triggerQueryRemap(queryProperty.toString(), collProperty.toString()); + log << MSG::INFO << "Remappeed Trigger Query: " << remappedQ << endreq; + queryProperty.setValue(remappedQ); + } catch( runtime_error &e ) { + log << MSG::ERROR << "Trigger Query Error: " << e.what() << endreq; + return StatusCode::FAILURE; + } + status = const_cast<IProperty*>(propertyServer)->setProperty(queryProperty); + if (!status.isSuccess()) { + log << MSG::ERROR << "Unable to set Query property from " << parentSvc->name() << endreq; + return(StatusCode::FAILURE); + } + + log << MSG::DEBUG << "postInitialization end" << endreq; + return StatusCode::SUCCESS; +} + + + +/// Called at the beginning of execute +StatusCode TrigCollQueryTool::preNext() +{ + return StatusCode::SUCCESS; +} + + +/// Called at the end of execute +StatusCode TrigCollQueryTool::postNext() +{ + return StatusCode::SUCCESS; +} + + +/// Called at the beginning of finalize +StatusCode TrigCollQueryTool::preFinalize() +{ + return StatusCode::SUCCESS; +} + + +/// Finalize AlgTool +StatusCode TrigCollQueryTool::finalize() +{ + return StatusCode::SUCCESS; +} + + diff --git a/Database/APR/TrigCollQuery/src/components/TrigCollQuery_entries.cxx b/Database/APR/TrigCollQuery/src/components/TrigCollQuery_entries.cxx new file mode 100755 index 0000000000000000000000000000000000000000..4b826ae21eb9219489b4d7ba07631ebc2f8a6c3b --- /dev/null +++ b/Database/APR/TrigCollQuery/src/components/TrigCollQuery_entries.cxx @@ -0,0 +1,10 @@ +#include "TrigCollQuery/TrigCollQueryTool.h" +#include "GaudiKernel/DeclareFactoryEntries.h" + +DECLARE_TOOL_FACTORY( TrigCollQueryTool ) + +DECLARE_FACTORY_ENTRIES(TrigCollQuery) { + DECLARE_TOOL( TrigCollQueryTool ) +} + + diff --git a/Database/APR/TrigCollQuery/src/components/TrigCollQuery_load.cxx b/Database/APR/TrigCollQuery/src/components/TrigCollQuery_load.cxx new file mode 100755 index 0000000000000000000000000000000000000000..f78a4ef4cef1557a74ea0482829ded6013f55bb8 --- /dev/null +++ b/Database/APR/TrigCollQuery/src/components/TrigCollQuery_load.cxx @@ -0,0 +1,4 @@ +#include "GaudiKernel/LoadFactoryEntries.h" + +LOAD_FACTORY_ENTRIES(TrigCollQuery) + diff --git a/Database/APR/TrigCollQuery/src/decodeTriggerQuery.cxx b/Database/APR/TrigCollQuery/src/decodeTriggerQuery.cxx new file mode 100644 index 0000000000000000000000000000000000000000..61ae3ef8d0dded2ec53cfc734c119069eef2646a --- /dev/null +++ b/Database/APR/TrigCollQuery/src/decodeTriggerQuery.cxx @@ -0,0 +1,174 @@ +/* + Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration +*/ + +#include "TrigCollQuery/TrigCollQuery.h" + +#include <iostream> +using namespace std; + +#include <libgen.h> +#include <unistd.h> +#include <stdio.h> +#include <cstdlib> +#include <algorithm> + +string app; +TrigCollQuery queryRemapper; + + +void usage() { + cerr << "Usage: " << app << " OPTIONS -runs <runlist> -query <query>" << endl + << " OPTIONS: -help -d -syntax <syntax> -runnumbercol <col_name> -xmlcfg <url> -trigsvc <url> -project <MC_project> -amitag <MC_tag> -cert <file> -key <file>" << endl; +} + +void help() { + usage(); + cerr << +" -runs <runs> : decode triggers for given runs \n" +" example: 1,3,5-12,22 \n" +" -query : trigger query to be decoded.\n" +" example: TriggerPassed(TrigName1, TrigName2, ...) && TriggerNotPassed(TrigNameX) \n" +" -syntax : query syntax: ROOT (C++) or Relational. Default is ROOT \n" +" \n" +" -help : print out this help \n" +" -debug (-d) : print debugging messages \n" +" -xmlcfg <url> : read trigger configuration XML files from directory <url> \n" +" do not use trigger configuration web service \n" +" 'default' will use " << queryRemapper.defaultXMLConfigDir() << "\n" +" -trigsvc <url>: use web service <url> for trigger configuration \n" +" 'default' will use " << queryRemapper.triggerWebService() << "\n" +" Options relevant to the use of the trigger web service:\n" +" -project <name>: MC project name \n" +" -amitag <tag> : AMI tag for MC project \n" +" -cert <file> : path to certificate in PEM format, if X509_USER_PROXY is not defined \n" +" (proxy can he created by voms-proxy-init) \n" +" note: for non interactive use the certificate should be without a passphrase \n" +" -key <file> : path to certificate key in PEM format \n" +"\n" +" -runnumbercol <attribute_name> \n" +" : name of the attribute where run number is kept (default is RunNumber)\n" +" \n" +" More informatin on WWW page: \n" +" https://twiki.cern.ch/twiki/bin/viewauth/Atlas/TagTriggerQueries \n"; +} + +void printError(const string& err="", int exitcode=0) { + cerr << app << ": ERROR - " << err << endl; + usage(); + if( exitcode ) + exit(exitcode); +} + + + +int main(int argc, const char *argv[]) +{ + app = basename( const_cast<char*>(argv[0]) ); + string query; + string srcTech("Root"); + bool useXMLConfig(false); + string certpath, keypath; + int i = 1; + while( argc > i ) { + string option = argv[i]; + if( option == "-help" ) { + help(); + exit(0); + } + if( option == "-d" || option == "-debug" ) { + i++; + queryRemapper.setDebug(); + } else if( option == "-xmlcfg" ) { + i++; + if( argc <= i ) + printError( "Missing parameter for -xmlcfg option", 2 ); + string url = argv[i]; + if( url == "default" ) + url = queryRemapper.defaultXMLConfigDir(); + queryRemapper.XMLConfigDir( url ); + useXMLConfig = true; + cout << "Setting Trigger XML Config location to: " << url << endl; + i++; + } else if( option == "-trigsvc" ) { + i++; + if( argc <= i ) + printError( "Missing parameter for -trigsvc option", 2 ); + string url = argv[i]; + if( url == "default" ) + url = queryRemapper.triggerWebService(); + queryRemapper.triggerWebService( url ); + i++; + } else if( option == "-project" ) { + i++; + if( argc <= i ) + printError( "Missing parameter for -project option", 2 ); + queryRemapper.setDataPeriod( argv[i] ); + i++; + } else if( option == "-amitag" ) { + i++; + if( argc <= i ) + printError( "Missing parameter for -amitag option", 2 ); + queryRemapper.setAMITag( argv[i] ); + i++; + } else if( option == "-cert" ) { + i++; + if( argc <= i ) + printError( "Missing parameter for -cert option", 2 ); + certpath = argv[i]; + if( keypath.empty() ) keypath = certpath; + i++; + } else if( option == "-key" ) { + i++; + if( argc <= i ) + printError( "Missing parameter for -key option", 2 ); + keypath = argv[i]; + i++; + } else if( option == "-runnumbercol" ) { + i++; + if( argc <= i ) + printError( "Missing parameter for -runnumbercol option", 2 ); + queryRemapper.runNumberColumn( argv[i] ); + i++; + } else if( option == "-runs" ) { + i++; + if( argc <= i ) + printError( "Missing parameter for -runs option", 2 ); + queryRemapper.queryRuns( argv[i] ); + i++; + } else if( option == "-syntax" ) { + i++; + if( argc <= i ) + printError( "Missing parameter for -syntax option", 2 ); + srcTech = argv[i]; + i++; + // cout << "Setting input collection technology to " << srcTech << endl; + } else if( option == "-query" ) { + i++; + if( argc <= i ) + printError( "Missing parameter for -query option", 2 ); + query = argv[i]; + i++; + } + } + + std::transform(srcTech.begin(), srcTech.end(), srcTech.begin(), ::toupper); + if( srcTech.substr(0,3) == "ROO" ) + srcTech = "RootCollection"; + else if( srcTech.substr(0,3) == "REL" || srcTech.substr(0,3) == "SQL" ) + srcTech = "RelationalCollection"; + else + printError("Unrecongnized query syntax. Use ROOT or RELATIONAL", 30); + + try { + if( useXMLConfig ) { + // cout << "Calling read XML Config" << endl; + queryRemapper.readTriggerMap(); + } else { + queryRemapper.setCertPath( certpath, keypath ); + } + cout << queryRemapper.triggerQueryRemap( query, srcTech ) << endl; + }catch( exception &e ) { + printError(e.what(), 3); + } +} diff --git a/Database/APR/TrigCollQuery/src/runCollUtil.cxx b/Database/APR/TrigCollQuery/src/runCollUtil.cxx new file mode 100644 index 0000000000000000000000000000000000000000..ad94c3a26191dc64373c19a11891451f78c18e0b --- /dev/null +++ b/Database/APR/TrigCollQuery/src/runCollUtil.cxx @@ -0,0 +1,218 @@ +/* + Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration +*/ + +#include "TrigCollQuery/TrigCollQuery.h" + +#include <iostream> +using namespace std; + +#include <libgen.h> +#include <unistd.h> +#include <stdio.h> +#include <cstdlib> + +string app; +TrigCollQuery queryRemapper; + + +void usage() { + cerr << "Usage: " << app << " [-help] [-d] -runs <runlist> OPTIONS POOLUtilityName POOLoptions" << endl + << " OPTIONS: [-runnumbercol <col_name>] [ -xmlcfg <url>| -trigsvc <url> [-project <MC_project> -amitag <MC_tag>] [-cert <file> -key <file>] ]" << endl; +} + +void help() { + usage(); + cerr << +" -help : print out this help \n" +" -debug (-d) : print some debugging messages \n" +" -runs <runs> : decode trigger queries for given runs \n" +" example: 1,3,5-12,22 \n" +" -xmlcfg <url> : read trigger configuration XML files from directory <url> \n" +" do not use trigger configuration web service \n" +" 'default' will use " << queryRemapper.defaultXMLConfigDir() << "\n" +" -trigsvc <url>: use web service <url> for trigger configuration \n" +" 'default' will use " << queryRemapper.triggerWebService() << "\n" +" Options relevant to the use of the trigger web service:\n" +" -project <name>: MC project name \n" +" -amitag <tag> : AMI tag for MC project \n" +" -cert <file> : path to certificate in PEM format, if X509_USER_PROXY is not defined \n" +" (proxy can he created by voms-proxy-init) \n" +" note: for non interactive use the certificate should be without a passphrase \n" +" -key <file> : path to certificate key in PEM format \n" +"\n" +" -runnumbercol <attribute_name> \n" +" : name of the attribute where run number is kept (default is RunNumber)\n" +" \n" +" POOLUtilityName - name of the POOL Collection utility to run \n" +" POOLoptions - standard options for the POOL Utility. Expected options: \n" +" - src : required to find out storage technology for query format. \n" +" Note: PHYSICAL_NAME assumes SQL query, LOGICAL_NAME is not supported \n" +" - query : query scanned for trigger names. Use TriggerPassed(TrigName1,...) \n" +" expressions \n" +" \n" +" More informatin on WWW page: \n" +" https://twiki.cern.ch/twiki/bin/viewauth/Atlas/TagTriggerQueries \n"; +} + +void printError(const string& err="", int exitcode=0) { + cerr << app << ": ERROR - " << err << endl; + usage(); + if( exitcode ) + exit(exitcode); +} + + + +int main(int argc, const char *argv[]) +{ + app = basename( const_cast<char*>(argv[0]) ); + string remappedQ; + // number of items to skip from the beginning of the command line + // when invoking a collection tool + // default = 1 - just the name of the QueryTool itlself + int skipArgs = 1; + + bool useXMLConfig(false); + int i = 1; + bool privateOptions = true; + string certpath, keypath; + while( privateOptions && argc > i ) { + string option = argv[i]; + if( option == "-help" ) { + help(); + exit(0); + } + if( option == "-d" || option == "-debug" ) { + i++, skipArgs++; + queryRemapper.setDebug(); + } else if( option == "-xmlcfg" ) { + i++, skipArgs++; + if( argc <= i ) + printError( "Missing parameter for -xmlcfg option", 2 ); + string url = argv[i]; + if( url == "default" ) + url = queryRemapper.defaultXMLConfigDir(); + queryRemapper.XMLConfigDir( url ); + useXMLConfig = true; + cout << "Setting Trigger XML Config location to: " << url << endl; + i++, skipArgs++; + } else if( option == "-trigsvc" ) { + i++, skipArgs++; + if( argc <= i ) + printError( "Missing parameter for -trigsvc option", 2 ); + string url = argv[i]; + if( url == "default" ) + url = queryRemapper.triggerWebService(); + queryRemapper.triggerWebService( url ); + i++, skipArgs++; + } else if( option == "-project" ) { + i++, skipArgs++; + if( argc <= i ) + printError( "Missing parameter for -project option", 2 ); + queryRemapper.setDataPeriod( argv[i] ); + i++, skipArgs++; + } else if( option == "-amitag" ) { + i++, skipArgs++; + if( argc <= i ) + printError( "Missing parameter for -amitag option", 2 ); + queryRemapper.setAMITag( argv[i] ); + i++, skipArgs++; + } else if( option == "-cert" ) { + i++, skipArgs++; + if( argc <= i ) + printError( "Missing parameter for -cert option", 2 ); + certpath = argv[i]; + if( keypath.empty() ) keypath = certpath; + i++, skipArgs++; + } else if( option == "-key" ) { + i++, skipArgs++; + if( argc <= i ) + printError( "Missing parameter for -key option", 2 ); + keypath = argv[i]; + i++, skipArgs++; + } else if( option == "-usectpword" ) { + queryRemapper.useCTPWord(true); + skipArgs++; + i++; + cout << "Using CTPWord atribute for L1 trigger" << endl; + } else if( option == "-runnumbercol" ) { + i++, skipArgs++; + if( argc <= i ) + printError( "Missing parameter for -runnumbercol option", 2 ); + queryRemapper.runNumberColumn( argv[i] ); + i++, skipArgs++; + } else if( option == "-runs" ) { + i++, skipArgs++; + if( argc <= i ) + printError( "Missing parameter for -runs option", 2 ); + queryRemapper.queryRuns( argv[i] ); + i++, skipArgs++; + } else + privateOptions = false; + } + + bool mixedTech = false; + int ii = i; + string srcTech; + while( argc > ii ) { + string option( argv[ii] ); + if( option == "-src" ) { + while( argc > ii+2 && *argv[ii+1] != '-' ) { + if( !srcTech.empty() && srcTech != argv[ii+2] ) { + // error: tech mismatch + mixedTech = true; + break; + } + ii += 2; + srcTech = argv[ii]; + if( srcTech == "PHYSICAL_NAME" ) { + // Assume this is Relational + srcTech = "RelationalCollection"; + } + cout << "Setting input collection technology to " << srcTech << endl; + } + } + ++ii; + } + if( mixedTech ) + printError("can not mix up imput ROOT and Relational collection types when querying triggers", 1 ); + if( srcTech.empty() ) + printError("unable to determine collection technology", 10 ); + + while( argc > i ) { + string option( argv[i] ); + if( option == "-query" ) { + if( ++i >= argc ) + printError("missing query parameter", 2 ); + try { + if( useXMLConfig ) { + // cout << "Calling read XML Config" << endl; + queryRemapper.readTriggerMap(); + } else { + queryRemapper.setCertPath( certpath, keypath ); + } + remappedQ = queryRemapper.triggerQueryRemap( argv[i], srcTech ); + argv[i] = remappedQ.c_str(); + }catch( exception &e ) { + printError(e.what(), 3); + } + } + i++; + } + + // cout << ">>> skipArgs is " << skipArgs << ", argc is " << argc << endl; + if( skipArgs >= argc ) + printError("Utility to execute not specified ", 5 ); + + cout << "new command: "; + for( i = skipArgs; i < argc; i++ ) { + cout << argv[i] << " "; + } + cout << endl; + + execvp( argv[skipArgs], (char* const*)(argv+skipArgs) ); + + perror( (app+": Error - execvp() failed").c_str() ); + return 6; +}