Event/EventPacker/src/component/BufferPackerBaseAlg.h

@@ -56,7 +56,7 @@ namespace DataPacking::Buffer {
       // if ( this->msgLevel( MSG::DEBUG ) ) this->debug() << "Input location " << m_data << " empty." << endmsg;
       // return {0};
       //}
-      if ( m_encodingKey == 0 ) {
+      if ( m_encodingKey == 0u ) {
         throw GaudiException( "encoding key is zero", __PRETTY_FUNCTION__, StatusCode::FAILURE );
         ++m_zero_key;
       }

Event/EventPacker/src/component/BufferRelationPackerBaseAlg.h

@@ -101,7 +101,7 @@ namespace DataPacking::Buffer {
       buffer.reserve( prels->data().size() );
 
       static const Gaudi::StringKey PackedObjectLocations{"PackedObjectLocations"};
-      if ( m_encodingKey == 0 ) ++m_zero_key;
+      if ( m_encodingKey == 0u ) ++m_zero_key;
       const auto& map  = m_annsvc->s2i( m_encodingKey, PackedObjectLocations );
       auto        iloc = map.find( m_rels.fullKey().key() );
       if ( iloc == map.end() ) {

Hlt/HltDAQ/src/component/HltDecReportsDecoder.cpp

@@ -158,11 +158,12 @@ namespace LHCb::Hlt::DAQ {
       outputSummary.setTaskID( pop( data ) );
     }
     // --------------------------------- get configuration --------------------
-    if ( key != 0 && m_forcedKey != 0 ) ++m_overrule_nonzero_key;
+    if ( key != 0 && m_forcedKey != 0u ) ++m_overrule_nonzero_key;
     if ( key == 0 ) key = outputSummary.configuredTCK();
-    if ( key == 0 && m_forcedKey == 0 ) ++m_not_overruled_zero_key;
+    if ( key == 0 && m_forcedKey == 0u ) ++m_not_overruled_zero_key;
 
-    const auto& tbl = m_svc->i2s( m_forcedKey == 0 ? key : m_forcedKey.value(), selectionID_for( m_sourceID.value() ) );
+    const auto& tbl =
+        m_svc->i2s( m_forcedKey == 0u ? key : m_forcedKey.value(), selectionID_for( m_sourceID.value() ) );
 
     // ---------------- loop over decisions in the bank body; insert them into the output container
     int err = 0;

Hlt/HltDAQ/src/component/HltPackedBufferDecoder.cpp

@@ -114,7 +114,6 @@ namespace LHCb::Hlt::DAQ {
         if ( msgLevel( MSG::DEBUG ) ) {
           debug() << "Rawbank size " << rawBank->size() << endmsg;
           debug() << "Payload size " << payload.size() << endmsg;
-          debug() << "Compression " << compression << endmsg;
         }
       }
 

PyConf/python/PyConf/application.py

@@ -89,8 +89,8 @@ def configured_ann_svc(name='HltANNSvc',
              for k, v in key_registry.items()})
 
     if 'Overrule' in kwargs:
-        log.info('Overrule specified for keys {}'.format(
-            kwargs['Overrule'].keys()))
+        log.info('Overrule specified for keys {}'.format(', '.join(
+            "0x{:08x}".format(k) for k in kwargs['Overrule'].keys())))
     else:
         log.info('No Overrule specified - relying solely on repository')
 

PyConf/python/PyConf/filecontent_metadata.py

@@ -49,7 +49,7 @@ def get_keys_from_jsonfile(json_file):
     return keys
 
 
-def update_json_file(fname, d, **kwargs):
+def update_decoding_keys_in_json_file(fname, d, **kwargs):
     try:
         with open(fname, '+x') as f:
             json.dump(d, f, **kwargs)
@@ -59,12 +59,14 @@ def update_json_file(fname, d, **kwargs):
                 txt = f.read()
             prev = json.loads(txt) if len(txt) > 1 else dict(
             )  # json has at least '{}' as content
-            if any(prev[k] != d[k] for k in prev.keys() & d.keys()):
+            prev_keys = prev.get('DecodingKeys', dict())
+            if any(prev_keys[k] != d[k] for k in prev_keys.keys() & d.keys()):
                 raise RuntimeError(
                     'key collision while merging dictionaries: {} vs {}'.
-                    format(prev, d))
+                    format(prev_keys, d))
+            prev_keys.update(d)
             with open(fname, 'wt', encoding='utf-8') as f:
-                json.dump({**prev, **d}, f, **kwargs)
+                json.dump(prev, f, **kwargs)
 
 
 def _flush_keys_to_git(key_and_tables):
@@ -84,8 +86,7 @@ def _flush_keys_to_git(key_and_tables):
                     _dict_from_json(d), _dict_from_json(table))
                 continue
 
-            log.info("Adding new key {} to git repository at {}".format(
-                key, path))
+            log.info("Adding key {} to git repository at {}".format(key, path))
 
             oid = _get_hash_for_text(table, write_to_odb=True, path=path)
             if oid[:8] != key:
@@ -95,10 +96,9 @@ def _flush_keys_to_git(key_and_tables):
             branch_name = 'key-' + key
             _commit_oid_to_branch(path, 'master', branch_name, oid)
 
-            log.info('****  to publish this encoding table, please do `git -C {0} push origin {1}\n' \
-                'and then in the readback application  configure the GitANNSvc to use the branch {1} by adding \n' \
-                '    from PyConf.application import metainfo_repos\n' \
-                '    metainfo_repos.global_bind( extra_central_tags = [  \'{1}\' ] )'.format(path,branch_name))
+            log.warning(
+                'Created new key {0} - to publish the corresponding decoding table, please do `git -C {1} push origin {2}`'
+                .format(key, path, branch_name))
 
             # to make sure we can continue, merge the just created branch into master,
             _merge_branch_to_master(path, branch_name)
@@ -129,8 +129,8 @@ class __KeyRegistry(object):
     def __del__(self):
         if not self.__locked and not self.empty():
             log.error(
-                "OOPS -- registry not locked, and not empty -- someone forgot to flush"
-            )
+                "OOPS -- registry not locked, and not empty -- got new keys {} -- someone forgot to flush"
+                .format(self.keys()))
 
     @staticmethod
     def valid_key(key):
@@ -141,9 +141,8 @@ class __KeyRegistry(object):
         i = self.__d.get(key, None)
         if i is not None:  # already registered
             if i == text: return  # same, so done
-            else:
-                raise RuntimeError(
-                    "cannot modify registered item for key {}".format(key))
+            raise RuntimeError(
+                "cannot modify registered item for key {}".format(key))
         if self.__locked:
             raise RuntimeError("trying to add new key to a locked registry")
         log.info('adding new key {} to registry'.format(key))
@@ -177,12 +176,10 @@ class __KeyRegistry(object):
 
     def append_to_jsonfile(self, fname, **kwargs):
         self.lock()  # prevent future updates that would not be flushed...
-        update_json_file(
-            fname, {
-                'DecodingKeys':
-                {'0x{}'.format(k): json.loads(v)
-                 for k, v in self.__d.items()}
-            }, **kwargs)
+        update_decoding_keys_in_json_file(
+            fname,
+            {'0x{}'.format(k): json.loads(v)
+             for k, v in self.__d.items()}, **kwargs)
 
     def flush_to_git(self):
         self.lock()  # prevent future updates that would not be flushed...
@@ -395,6 +392,16 @@ def _make_key(cfg):
             return key
         if d == cfg:
             log.info("****  known key = {}".format(key))
+            # to make sure that, when writing keys to the manifest, _all_ keys encountered are written
+            # also pre-exisiting keys are registered at this point.
+            #
+            # This eg. avoids the scenario that in a first run, a key is (still) new and would be
+            # added, but then in any subsequent run the key is already in the (local) git repository,
+            # thus it is no longer new, and is thus subsequently _not_ added to the manifest...
+            #  -- which implies that the key has to be published to the global reco if someone else
+            #     wants to process the data, or the local repo must be made configured for use by the
+            #     job processing the data.
+            key_registry.add(key, table)
             return key
 
         # we've got a hash collision - add a space to get a different key, and try again

VP/VPDAQ/src/VPRetinaSPmixer.cpp

@@ -160,90 +160,12 @@ LHCb::RawEvent VPRetinaSPmixer::operator()( const LHCb::RawEvent& rawEvent ) con
 
   } else {
 
-    // make list of pairs of banks to combine, there are 208 sensors and 104 pairs
-    std::array<std::pair<LHCb::RawBank const*, LHCb::RawBank const*>, 104> bankPairs;
-    bankPairs.fill( {nullptr, nullptr} ); // fill with iterators to the end of the bank
     // Loop over VP RawBanks
     for ( auto iterBank : tBanks ) {
-
-      const unsigned int sensor = iterBank->sourceID();
-      const unsigned int pos    = sensor / 2; // integer division to get pair pos
-      if ( pos > 103 ) {
-        warning() << "Bad sensorID (" << sensor << ") should be 0->207" << endmsg;
-        return result;
-      }
-      if ( ( sensor & 0x1U ) == 0x0U ) {
-        // first sensor in pair [even number]
-        bankPairs[pos].first = iterBank;
-      } else {
-        // second sensor in pair
-        bankPairs[pos].second = iterBank;
-      }
-    }
-
-    for ( auto iPair : bankPairs ) {
-      uint32_t                   sensor0 = 0;
-      LHCb::span<const uint32_t> bank0   = {};
-      uint32_t                   sensor1 = 0;
-      LHCb::span<const uint32_t> bank1   = {};
-      if ( iPair.first ) {
-        sensor0 = iPair.first->sourceID();
-        bank0   = iPair.first->range<uint32_t>();
-      }
-      if ( iPair.second ) {
-        sensor1 = iPair.second->sourceID();
-        bank1   = iPair.second->range<uint32_t>();
-      }
-
-      std::vector<uint32_t> mixedSP;
-      unsigned int          pairBankSize = bank0[0] + bank1[0];
-      mixedSP.reserve( pairBankSize + 1 );
-      mixedSP.push_back( pairBankSize ); // set bank size (element 0) to sum of length of both inputs
-      // check if the isolated cluster bits should be set (set to zero if too many clusters)
-      uint32_t icf_over_mask = 0xffffffffU; // default mask is everything passes through
-      if ( pairBankSize > m_icf_cut ) {
-        icf_over_mask = 0x7fffffffU; // mask to zero bit 31
-      }
-
-      // skip entry 0 (bank size) when copying words into the combined bank
-      for ( auto isp_word = begin( bank0 ) + 1; isp_word != end( bank0 ); ++isp_word ) {
-        mixedSP.push_back( ( *isp_word ) & icf_over_mask );
-      }
-
-      // skip entry 0 (bank size) when copying words into the combined bank
-      for ( auto isp_word = begin( bank1 ) + 1; isp_word != end( bank1 ); ++isp_word ) {
-        mixedSP.push_back( ( *isp_word ) & icf_over_mask );
-      }
-
-      uint32_t n = mixedSP[0];
-      if ( n > 1 ) { // check there are at least 2 SPs
-        for ( uint32_t i = n - 1; i > 0; --i ) {
-          // pick random int from 0 to i inclusive
-          uint32_t j = static_cast<int>( m_rndm.shoot() * ( i + 1 ) );
-          std::swap( mixedSP[i + 1], mixedSP[j + 1] );
-        }
-      }
-
-      assert( mixedSP.size() == 1 + mixedSP[0] );
-
-      // see comment above for bit definitions
-      uint32_t bankID;
-      if ( ( sensor0 & 0x4 ) == 0x0 ) { // c-side has even module numbers (bit 2 is first of module index)
-        bankID = 0x1800;                // C-side
-      } else {
-        bankID = 0x1000; // A-side
-      }
-      bankID |= ( sensor0 >> 1 ); // left shift by 1 as final bit in each superpixel
-
-      result.addBank( bankID, LHCb::RawBank::VP, VPRetinaCluster::c_SPBankVersion, mixedSP );
-
+      result.addBank( iterBank->sourceID(), LHCb::RawBank::VP, VPRetinaCluster::c_SPBankVersion,
+                      iterBank->range<uint32_t>() );
       ++nBanks;
-
-      if ( msgLevel( MSG::VERBOSE ) ) {
-        verbose() << format( "Added bank with sourceID 0x%x containing sensors %i and %i", bankID, sensor0, sensor1 )
-                  << endmsg;
-      }
-    } // loop over all banks
+    }
   }
 
   if ( msgLevel( MSG::DEBUG ) ) debug() << "Added " << nBanks << " raw banks of retina clusters to TES" << endmsg;