Merge branch 'malexand-addstreamconfig-master' into 'master'

Add TurboStream and StrippingStream properties

See merge request !394

DaVinciTests/tests/options/DVTestTurbo2016.py

+###############################################################################
+# (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+'''Test accessing candidates on Turbo mdst.'''
+
+from Configurables import DaVinci, LoKi__HDRFilter
+from GaudiConf import IOHelper
+from PhysSelPython.Wrappers import AutomaticData, ValidBPVSelection, SelectionSequence
+
+# 2018 Turbo data.
+IOHelper('ROOT').inputFiles([
+    'PFN:root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/swtest/lhcb/LHCb/Collision18/CHARMTWOBODY.MDST/00075136/0000/00075136_00000005_1.charmtwobody.mdst',
+    'PFN:root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/swtest/lhcb/LHCb/Collision18/CHARMTWOBODY.MDST/00075136/0000/00075136_00000012_1.charmtwobody.mdst',
+    'PFN:root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/swtest/lhcb/LHCb/Collision18/CHARMTWOBODY.MDST/00075136/0000/00075136_00000026_1.charmtwobody.mdst',
+])
+
+# Configure the data settings.
+DaVinci().TurboStream = 'Charmtwobody'
+DaVinci().DataType = '2018'
+DaVinci().DDDBtag = 'dddb-20190206-3'
+DaVinci().CondDBtag = 'cond-20191004-3'
+DaVinci().EvtMax = 1000
+
+# Get particles from an HLT2 line and check that they have a valid BPV
+line = 'Hlt2CharmHadDstp2D0Pip_D02KmPipTurbo'
+data = AutomaticData(line + '/Particles')
+validbpv = ValidBPVSelection([data])
+selseq = SelectionSequence(line + 'Seq', TopSelection=validbpv)
+DaVinci().UserAlgorithms.append(selseq.sequence())
+DaVinci().EventPreFilters = [
+    LoKi__HDRFilter(
+        'PreFilter',
+        Code='HLT_PASS({0!r})'.format(line + 'Decision'),
+        Location='Hlt2/DecReports')
+]

DaVinciTests/tests/options/DV_MCuDST.py

@@ -64,7 +64,7 @@ daVinci = DaVinci (
     InputType     = 'MDST'                    , 
     DataType      = "2016"                    , 
     Simulation    = True                      ,
-    RootInTES     = '/Event/AllStreams'       ,
+    StrippingStream='AllStreams'              ,
     ##
     HistogramFile = "DVHistos.root"           ,
     ##

DaVinciTests/tests/options/dtfdict.py

@@ -48,7 +48,7 @@ DaVinci().Lumi = True
 DaVinci().DataType = '2012'
 
 #for MDST
-DaVinci(RootInTES = '/Event/Bhadron', InputType = 'MDST')
+DaVinci(StrippingStream = 'Bhadron', InputType = 'MDST')
 
 DaVinci().EvtMax = 1000
 DaVinci().TupleFile = 'test_DFTDict.root'

DaVinciTests/tests/qmtest/davinci.qms/davinci_turbo_2016_read.qmt

+<?xml version="1.0" ?>
+<!--
+    (c) Copyright 2020 CERN for the benefit of the LHCb Collaboration
+
+    This software is distributed under the terms of the GNU General Public
+    Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".
+
+    In applying this licence, CERN does not waive the privileges and immunities
+    granted to it by virtue of its status as an Intergovernmental Organization
+    or submit itself to any jurisdiction.
+-->
+<!DOCTYPE extension  PUBLIC '-//QM/2.3/Extension//EN'  'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'>
+<extension class="GaudiTest.GaudiExeTest" kind="test">
+<argument name="program"><text>gaudirun.py</text></argument>
+<argument name="timeout"><integer>3600</integer></argument>
+<argument name="args"><set>
+  <text>$DAVINCITESTSROOT/tests/options/DVTestTurbo2016.py</text> 
+</set></argument>
+<argument name="validator"><text>
+findReferenceBlock("""
+SELECT:Hlt2Char...SUCCESS Number of counters : 1
+ |    Counter                                      |     #     |    sum     | mean/eff^* | rms/err^*  |     min     |     max     |
+ |*"#passed"                                       |       281 |        281 |( 100.0000 +- 0.000000)%|   -------   |   -------   |""", stdout, result, causes)
+findReferenceBlock("""VALIDBPV_Hlt2Ch...SUCCESS Number of counters : 5
+ |    Counter                                      |     #     |    sum     | mean/eff^* | rms/err^*  |     min     |     max     |
+ | "# Hlt2CharmHadDstp2D0Pip_D02KmPipTurbo"        |       281 |        297 |     1.0569 |    0.24661 |      1.0000 |      3.0000 |
+ | "# input particles"                             |       281 |        297 |     1.0569 |    0.24661 |      1.0000 |      3.0000 |
+ |*"#accept"                                       |       281 |        281 |( 100.0000 +- 0.000000)% |
+ | "#passed"                                       |       281 |        297 |     1.0569 |    0.24661 |      1.0000 |      3.0000 |
+ |*"efficiency"                                    |       297 |        297 |( 100.0000 +- 0.000000)%|   -------   |   -------   |""", stdout, result, causes)
+countErrorLines({"FATAL":0})
+</text></argument>
+</extension>

Phys/DaVinci/python/DaVinci/Configuration.py

@@ -72,6 +72,10 @@ class DaVinci(LHCbConfigurableUser) :
         , "ProductionType"     : "None"          # Sets up options needed in various production modes
         , "RootInTES"          : ""              # RootInTES property propagated to MainSequence in case of MDST input type
         , "Detectors"          : ['Velo','PuVeto','Rich1','Rich2','TT','IT','OT','Spd','Prs','Ecal','Hcal','Muon','Magnet','Tr']
+        , "TurboStream":
+        None  # Name of the Turbo stream for the data. Mutually exclusive with StrippingStream
+        , "StrippingStream":
+        None  # Name of the Stripping stream of the data. Mutually exclusive with TurboStream
        }
 
     _propertyDocDct = {  
@@ -103,6 +107,10 @@ class DaVinci(LHCbConfigurableUser) :
         , "ProductionType"     : """ Enables special settings for running in production (e.g. stripping) """
         , "RootInTES"          : """ RootInTES (for uDst input type) """
         , "Detectors"          : """ List of detectors """ 
+        , "TurboStream":
+        """ Name of the Turbo stream for the data. Mutually exclusive with StrippingStream """
+        , "StrippingStream":
+        """ Name of the Stripping stream for the data. Mutually exclusive with TurboStream """
         }
 
     __used_configurables__ = [
@@ -120,12 +128,73 @@ class DaVinci(LHCbConfigurableUser) :
     __known_default_detectors =  ['Velo', 'PuVeto', 'Rich1', 'Rich2', 'TT', 'IT', 'OT', 'Spd', 'Prs', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr']
     __known_upgrade_detectors__ = ['VP', 'UT', 'FT', 'Rich1Pmt', 'Rich2Pmt', 'Ecal', 'Hcal', 'Muon', 'Magnet', 'Tr' ]
 
+    __known_TurboStreams = {
+        'Allstreams', 'Turbo', 'Charmcharged', 'Charmkshh', 'Charmmultibody',
+        'Charmspecparked', 'Charmspecprescaled', 'Charmspec', 'Charmtwobody',
+        'Leptons', 'Charminclbaryon', 'Charmxsec'
+    }
+
+    __known_DstStrippingStreams = {
+        'BhadronCompleteEvent', 'CharmCompleteEvent', 'Dimuon', 'EW',
+        'Semileptonic', 'Calibration', 'MiniBias', 'Radiative', 'AllStreams',
+        'ALL', 'IFT', 'CharmToBeSwum'
+    }
+
+    __known_MdstStrippingStreams = {
+        'Leptonic', 'Charm', 'PID', 'Bhadron', 'ALLMDST', 'AllStreams'
+    }
+
+
     ## Known monitoring sequences run by default
     KnownMonitors        = []
 
     mainSeq = GaudiSequencer("DaVinciUserSequence")
     moniSeq = GaudiSequencer("MonitoringSequence")
 
+    def strippingRootInTES(self):
+        '''Get the RootInTES for stripping data given the stream and InputType. StrippingStream must be set.'''
+        if not self.isPropertySet('StrippingStream'):
+            raise ValueError('StrippingStream is not set!')
+
+        if self.getProp('InputType').upper() == 'MDST':
+            return '/Event/' + self.getProp('StrippingStream')
+        return ''
+
+    def turboRootInTES(self):
+        '''Get the RootInTES for Turbo data given the stream, DataType, and Simulation flag. TurboStream and
+        DataType must be set.'''
+        if not self.isPropertySet('TurboStream'):
+            raise ValueError('TurboStream is not set!')
+        stream = self.getProp('TurboStream')
+        if not self.isPropertySet('DataType'):
+            raise ValueError('DataType is not set!')
+
+        stream = stream.capitalize()
+        if int(self.getProp('DataType')) <= 2016 or self.getProp('Simulation'):
+            return '/Event/Turbo'
+        return '/Event/' + stream + '/Turbo'
+
+    def getRootInTES(self):
+        '''Get the RootInTES given the TurboStream, DataType & Simulation flag, or StrippingStream & InputType.
+        Either TurboStream & DataType or StrippingStream must be set.'''
+
+        if self.isPropertySet('TurboStream') and self.isPropertySet(
+                'StrippingStream'):
+            raise ValueError(
+                'Both TurboStream and StrippingStream are set! You can\'t '
+                'run on both - set one or the other')
+
+        if self.isPropertySet('TurboStream'):
+            return self.turboRootInTES()
+        elif self.isPropertySet('StrippingStream'):
+            return self.strippingRootInTES()
+        raise ValueError(
+            'You must set at least TurboStream & DataType or StrippingStream!')
+
+    def isTurbo(self):
+        '''Check whether DaVinci is configured for Turbo data.'''
+        return self.getProp('Turbo') or self.isPropertySet('TurboStream')
+
 ################################################################################
 # Check Options are OK
 #
@@ -178,6 +247,49 @@ class DaVinci(LHCbConfigurableUser) :
         if self.getProp("MergeGenFSR") and not self.getProp("Simulation"):
             raise TypeError("Cannot MergeGenFSR on real data");
 
+        # Turbo data.
+        if self.isPropertySet('TurboStream'):
+            stream = self.getProp('TurboStream')
+            # First letter is always capital, others lowercase.
+            stream = stream.capitalize()
+            log.info('Using TurboStream: ' + stream)
+            if not stream in self.__known_TurboStreams:
+                log.warning('TurboStream {0!r} isn\'t in the list of know Turbo streams: {1!r}'\
+                            .format(stream, self.__known_TurboStreams))
+            log.info('Set Turbo = True')
+            self.setProp('Turbo', True)
+            log.info('Set InputType = "MDST"')
+            self.setProp('InputType', 'MDST')
+            rootInTes = self.turboRootInTES()
+            log.info('Set RootInTES = ' + repr(rootInTes))
+            self.setProp('RootInTES', rootInTes)
+
+        # Stripping data.
+        elif self.isPropertySet('StrippingStream'):
+            # InputType could be set accordingly, if not for the fact that AllStreams is used for both
+            # DST and mDST MC.
+            stream = self.getProp('StrippingStream')
+            log.info('Using StrippingStream = ' + stream)
+            if not stream in self.__known_DstStrippingStreams.union(
+                    self.__known_MdstStrippingStreams):
+                log.warning('Unknown StrippingStream: ' + repr(stream))
+            log.info('Set Turbo = False')
+            self.setProp('Turbo', False)
+            if self.getProp('InputType').upper() == 'MDST':
+                if not stream in self.__known_MdstStrippingStreams:
+                    log.warning((
+                        'InputType is MDST but StrippingStream {0!r} isn\'t in the list '
+                        'of known MDST Stripping streams: {1!r}').format(
+                            stream, self.__known_MdstStrippingStreams))
+            else:
+                if not stream in self.__known_DstStrippingStreams:
+                    log.warning((
+                        'InputType is DST but StrippingStream {0!r} isn\'t in the list '
+                        'of known DST Stripping streams: {1!r}').format(
+                            stream, self.__known_DstStrippingStreams))
+            rootInTes = self.strippingRootInTES()
+            log.info('Set RootInTES = ' + repr(rootInTes))
+            self.setProp('RootInTES', rootInTes)
             
         if 'MDST' == self.getProp('InputType').upper() :
             if not self.getProp('RootInTES') :