diff --git a/Event/xAOD/xAODCore/Root/FloatCompressor.cxx b/Event/xAOD/xAODCore/Root/FloatCompressor.cxx index d2f4f0925cc7caac04f253c336d775c9dbfa8709..cf2c6bccf8f333630067b7b50be045b5139e1710 100644 --- a/Event/xAOD/xAODCore/Root/FloatCompressor.cxx +++ b/Event/xAOD/xAODCore/Root/FloatCompressor.cxx @@ -22,6 +22,49 @@ namespace xAOD { // SEEE EEEE EMMM MMMM MMMM MMMM MMMM MMMM // F F 8 0 0 0 7 F + // Definition: + // + // Assume that we'd like to keep only 7 bits in the mantissa + // In this case the memory layout of the bits will be: + // + // Sign | Exp (8 bits) | Frac (23 bits) + // S EEEEEEEE FFFFFFLRTTTTTTTTTTTTTTT + // + // where + // + // S : Sign bit + // E : Exponent bits + // F : Fraction bits + // L : Least significant bit (lsb) for 7 bits mantissa precision + // R : Rounding bit + // T : Sticky bits (i.e any bit after lsb + 1) + // + // In the current implementation there are essentially 4 cases: + // + // Case 1: L = 0 and R = 0 + // In this case there'll be rounding down + // + // Case 2: L = 1 and R = 0 + // In this case there'll be rounding down + // + // Case 3: L = 0 and R = 1 + // In this case there'll be rounding up + // + // Note: This scenario can be different than bfloat16 implementaion + // of TensorFlow, where they round down if all the Ts are zero. + // Otherwise, they also round up. + // + // Case 4: L = 1 and R = 1 + // In this case there'll be rounding up + // + // In all cases, we do an extra check to avoid overflow. + // + // From a technical point of view, the rounding is computed + // to be the half of the lsb(=1) and added to the original value + // as long as the new value doesn't overflow. Then the + // undesired bits are masked. We never go below 5 bits in the + // mantissa. + // Adjust the received bit number to some reasonable value: if( m_mantissaBits < 5 ) { m_mantissaBits = 5; @@ -39,6 +82,10 @@ namespace xAOD { // Set the Magic numbers m_rounding = 0x1 << ( 32 - (1 + 8 + m_mantissaBits) - 1 ); + // The part below is taken from AthenaPoolCnvSvc/Compressor + // and would work the same as long as the user doesn't + // compress lower than 3 mantissa bits, which is not allowed + // in any case. m_vmax = 0x7f7 << 20; m_vmax |= 0x000fffff xor (m_rounding); } @@ -61,7 +108,7 @@ namespace xAOD { //safety-check if value (omitting the sign-bit) is lower than vmax //(avoid overflow) - if( ( fi.ivalue & 0x7ffffff ) < m_vmax ) { + if( ( fi.ivalue & 0x7fffffff ) < m_vmax ) { fi.ivalue += m_rounding; }