From a06cd7935a8f3537a6fbdc84d592d80668d4e2fc Mon Sep 17 00:00:00 2001
From: amete <serhanmete@gmail.com>
Date: Wed, 12 Aug 2020 14:06:27 +0200
Subject: [PATCH 1/5] Switch to using ThinningInfo for the lossy float
compression inside copyAuxStoreThinned
---
.../AthContainers/tools/copyAuxStoreThinned.h | 12 ++
.../share/copyAuxStoreThinned_test.ref | 48 ++++++++
.../AthContainers/src/copyAuxStoreThinned.cxx | 81 ++++++++------
.../test/copyAuxStoreThinned_test.cxx | 105 +++++++++++++++++-
4 files changed, 212 insertions(+), 34 deletions(-)
diff --git a/Control/AthContainers/AthContainers/tools/copyAuxStoreThinned.h b/Control/AthContainers/AthContainers/tools/copyAuxStoreThinned.h
index faeb2498d2a5..ac10b6af72bf 100644
--- a/Control/AthContainers/AthContainers/tools/copyAuxStoreThinned.h
+++ b/Control/AthContainers/AthContainers/tools/copyAuxStoreThinned.h
@@ -40,6 +40,18 @@ void copyAuxStoreThinned (const SG::IConstAuxStore& orig,
SG::IAuxStore& copy,
const SG::ThinningInfo* info);
+/**
+ * @brief Helper method to apply lossy float compression
+ * @param dst Pointer to the start of the vector's data
+ * @param idst Index of element in vector
+ * @param eltSize Element size for the auxid
+ * @param typeName Type name for the auxid
+ * @param nmantissa Compression level to be used for the auxid
+ */
+void lossyFloatCompress (void* dst, size_t dst_index,
+ const std::size_t& eltSize,
+ const std::string& typeName,
+ const unsigned int& nmantissa);
} // namespace SG
diff --git a/Control/AthContainers/share/copyAuxStoreThinned_test.ref b/Control/AthContainers/share/copyAuxStoreThinned_test.ref
index a5bce3fd2565..0f41f8db3453 100644
--- a/Control/AthContainers/share/copyAuxStoreThinned_test.ref
+++ b/Control/AthContainers/share/copyAuxStoreThinned_test.ref
@@ -1 +1,49 @@
test1
+
+Testing lossy float compression...
+
+Iteration [0]
+
+Original value 0.512345 (binary : 0 01111110 00000110010100100001011)
+Compressed value 0.511719 (binary : 0 01111110 00000110000000000000000)
+Original value at [0] 0.512345 (binary : 0 01111110 00000110010100100001011)
+Compressed value at [0] 0.511719 (binary : 0 01111110 00000110000000000000000)
+Original value at [1] 0.123456 (binary : 0 01111011 11111001101011010000000)
+Compressed value at [1] 0.123535 (binary : 0 01111011 11111010000000000000000)
+
+Iteration [1]
+
+Original value 1.51234 (binary : 0 01111111 10000011001010010000101)
+Compressed value 1.51562 (binary : 0 01111111 10000100000000000000000)
+Original value at [0] 1.51234 (binary : 0 01111111 10000011001010010000101)
+Compressed value at [0] 1.51562 (binary : 0 01111111 10000100000000000000000)
+Original value at [1] 2.12346 (binary : 0 10000000 00001111110011010110100)
+Compressed value at [1] 2.125 (binary : 0 10000000 00010000000000000000000)
+
+Iteration [2]
+
+Original value 2.51235 (binary : 0 10000000 01000001100101001000011)
+Compressed value 2.51562 (binary : 0 10000000 01000010000000000000000)
+Original value at [0] 2.51235 (binary : 0 10000000 01000001100101001000011)
+Compressed value at [0] 2.51562 (binary : 0 10000000 01000010000000000000000)
+Original value at [1] 4.12346 (binary : 0 10000001 00000111111001101011010)
+Compressed value at [1] 4.125 (binary : 0 10000001 00001000000000000000000)
+
+Iteration [3]
+
+Original value 3.51235 (binary : 0 10000000 11000001100101001000011)
+Compressed value 3.51562 (binary : 0 10000000 11000010000000000000000)
+Original value at [0] 3.51235 (binary : 0 10000000 11000001100101001000011)
+Compressed value at [0] 3.51562 (binary : 0 10000000 11000010000000000000000)
+Original value at [1] 6.12346 (binary : 0 10000001 10000111111001101011010)
+Compressed value at [1] 6.125 (binary : 0 10000001 10001000000000000000000)
+
+Iteration [4]
+
+Original value 4.51234 (binary : 0 10000001 00100000110010100100001)
+Compressed value 4.5 (binary : 0 10000001 00100000000000000000000)
+Original value at [0] 4.51234 (binary : 0 10000001 00100000110010100100001)
+Compressed value at [0] 4.5 (binary : 0 10000001 00100000000000000000000)
+Original value at [1] 8.12346 (binary : 0 10000010 00000011111100110101101)
+Compressed value at [1] 8.125 (binary : 0 10000010 00000100000000000000000)
+
diff --git a/Control/AthContainers/src/copyAuxStoreThinned.cxx b/Control/AthContainers/src/copyAuxStoreThinned.cxx
index 6043683eed10..2bd46dfe8b21 100644
--- a/Control/AthContainers/src/copyAuxStoreThinned.cxx
+++ b/Control/AthContainers/src/copyAuxStoreThinned.cxx
@@ -18,6 +18,7 @@
#include "AthContainersInterfaces/IAuxStoreCompression.h"
#include "AthenaKernel/ThinningDecisionBase.h"
#include "CxxUtils/no_sanitize_undefined.h"
+#include "CxxUtils/FloatCompressor.h"
#include <vector>
@@ -48,7 +49,7 @@ void copyAuxStoreThinned NO_SANITIZE_UNDEFINED
const ThinningDecisionBase* dec = info ? info->m_decision : nullptr;
size_t nremaining = dec ? dec->thinnedSize() : size;
-
+
// Access the auxiliary type registry:
SG::AuxTypeRegistry& r = SG::AuxTypeRegistry::instance();
@@ -62,17 +63,8 @@ void copyAuxStoreThinned NO_SANITIZE_UNDEFINED
sel_auxids = iio->getSelectedAuxIDs();
}
- // Get the auxiliary IDs of the variables that should be compressed
- SG::auxid_set_t comp_auxids_high;
- SG::auxid_set_t comp_auxids_low;
- const IAuxStoreCompression* icomp = dynamic_cast<const IAuxStoreCompression*> (&orig);
- if (icomp != nullptr) {
- comp_auxids_high = icomp->getCompressedAuxIDs(true); // High compression
- comp_auxids_low = icomp->getCompressedAuxIDs(false); // Low compression
- }
-
copy.resize (nremaining);
-
+
// Loop over all the variables of the original container:
for (SG::auxid_t auxid : auxids) {
// Skip null auxids (happens if we don't have the dictionary)
@@ -110,12 +102,11 @@ void copyAuxStoreThinned NO_SANITIZE_UNDEFINED
}
}
- // Get the element size and the type name for the current auxid
- const size_t eltSize{r.getEltSize(auxid)};
- const std::string typeName{r.getTypeName(auxid)};
- const bool compressed_high{comp_auxids_high.test(auxid)};
- const bool compressed_low{comp_auxids_low.test(auxid)};
- const bool compressed{compressed_high || compressed_low};
+ // Get the element size, type name and compression info for the current auxid
+ // This information is later passed to the helper function lossyFloatCompress
+ const size_t eltSize{ r.getEltSize(auxid) };
+ const std::string typeName{ r.getTypeName(auxid) };
+ const unsigned int nmantissa{ info != nullptr ? info->compression(auxid) : 0 };
// Create the target variable:
void* dst = copy.getData (auxid, nremaining, nremaining);
@@ -124,27 +115,51 @@ void copyAuxStoreThinned NO_SANITIZE_UNDEFINED
for (std::size_t isrc = 0, idst = 0; isrc < size; ++isrc) {
if (!dec || !dec->thinned(isrc)) {
r.copyForOutput (auxid, dst, idst, src, isrc);
- // Compression BEGINS
- if (compressed) {
- // Get the pointer to the memory
- void* eltPtr = reinterpret_cast<char*>(dst) + idst*eltSize;
-
- // Here comes the actual compression
- // By now we should only have either float or std::vector<float>
- if(typeName.find("vector") == std::string::npos) {
- *(float*) eltPtr = icomp->getCompressedValue(*(float*) eltPtr, compressed_high);
- } else {
- std::vector<float> &vals = *(reinterpret_cast<std::vector<float>*>(eltPtr));
- for(auto &val: vals) {
- val = icomp->getCompressedValue(val, compressed_high);
- }
- }
- } // Compression ENDS
+ // Apply lossy float compression here (in-place)
+ // Maybe it would be better to do this via the registry during copy
+ lossyFloatCompress(dst, idst, eltSize,typeName, nmantissa);
++idst;
}
}
}
}
+/**
+ * @brief Helper method to apply lossy float compression
+ * @param dst Pointer to the start of the vector's data
+ * @param idst Index of element in vector
+ * @param eltSize Element size for the auxid
+ * @param typeName Type name for the auxid
+ * @param nmantissa Compression level to be used for the auxid
+ */
+void lossyFloatCompress (void* dst, size_t idst,
+ const std::size_t& eltSize,
+ const std::string& typeName,
+ const unsigned int& nmantissa)
+{
+ // Check if there is anything to be done
+ if (nmantissa <= 0) return;
+
+ // Setup all possible compressors: done only once
+ static const std::vector<CxxUtils::FloatCompressor> compressors = [] {
+ std::vector<CxxUtils::FloatCompressor> result;
+ for (std::size_t idx = 0; idx <= 23; ++idx)
+ result.emplace_back(idx);
+ return result;
+ }();
+
+ // Get the pointer to the memory
+ void* eltPtr = reinterpret_cast<char*>(dst) + idst*eltSize;
+
+ // This is where we apply in-place lossy float compression
+ if(typeName == "float") {
+ *(float*) eltPtr = compressors[nmantissa].reduceFloatPrecision(*(float*) eltPtr);
+ } else if (typeName == "std::vector<float>"){
+ std::vector<float> &vals = *(reinterpret_cast<std::vector<float>*>(eltPtr));
+ for(auto &val: vals) {
+ val = compressors[nmantissa].reduceFloatPrecision(val);
+ }
+ }
+}
} // namespace SG
diff --git a/Control/AthContainers/test/copyAuxStoreThinned_test.cxx b/Control/AthContainers/test/copyAuxStoreThinned_test.cxx
index 10ba51c6541a..c05328709313 100644
--- a/Control/AthContainers/test/copyAuxStoreThinned_test.cxx
+++ b/Control/AthContainers/test/copyAuxStoreThinned_test.cxx
@@ -17,6 +17,7 @@
#include "AthContainers/AuxTypeRegistry.h"
#include "AthContainers/PackedContainer.h"
#include "AthenaKernel/ThinningDecisionBase.h"
+#include "CxxUtils/FloatCompressor.h"
#include <vector>
#include <iostream>
#include <cassert>
@@ -58,7 +59,7 @@ void compare (const SG::PackedParameters& a,
assert (a.scale() == b.scale());
}
-
+
void compare (const SG::AuxStoreInternal& a,
const SG::AuxStoreInternal& b,
bool thinned = false,
@@ -173,10 +174,112 @@ void test1()
src.suppress (ftyp);
copyAuxStoreThinned (src, dst2, &info);
compare (src, dst2, true, ftyp);
+
+}
+
+void test2()
+{
+ std::cout << std::endl;
+ std::cout << "Testing lossy float compression..." << std::endl;
+ std::cout << std::endl;
+
+ // Prepare the necessary bits and pieces
+ SG::ThinningDecisionBase dec;
+ SG::ThinningInfo info;
+ info.m_decision = &dec;
+ dec.resize (5);
+ dec.buildIndexMap();
+ AuxStoreTest src;
+
+ // We want two types: float and std::vector<float>
+ SG::auxid_t ftyp = SG::AuxTypeRegistry::instance().getAuxID<float> ("aFloat");
+ SG::auxid_t fvtyp = SG::AuxTypeRegistry::instance().getAuxID<std::vector<float>> ("aVecFloat");
+
+ // Get a handle on the underlying data
+ float* fptr = reinterpret_cast<float*> (src.getData (ftyp, 5, 5));
+ std::vector<float>* fvptr = reinterpret_cast<std::vector<float>*> (src.getData (fvtyp, 5, 5));
+
+ // Fill some random valus
+ for (int i=0; i < 5; i++) {
+ fptr[i] = i + 0.512345;
+ fvptr[i].push_back(i + 0.512345);
+ fvptr[i].push_back(i*2 + 0.123456);
+ }
+
+
+ // Compress the values w/ nmantissa bits
+ const unsigned int nmantissa = 7;
+ info.m_compression[nmantissa].insert(ftyp);
+ info.m_compression[nmantissa].insert(fvtyp);
+ SG::AuxStoreInternal dst1;
+ copyAuxStoreThinned (src, dst1, &info);
+
+ // Retrieve the compressed values
+ float* rfptr = reinterpret_cast<float*> (dst1.getData (ftyp, 5, 5));
+ std::vector<float>* rfvptr = reinterpret_cast<std::vector<float>*> (dst1.getData (fvtyp, 5, 5));
+
+ // Instentiate a new compressor and check against the result
+ const CxxUtils::FloatCompressor fc( nmantissa );
+ CxxUtils::FloatCompressor::floatint_t uni;
+ uint32_t result{0};
+
+ // Helper to pring int in binary w/ a nicer format
+ auto intToBinStr = [] (uint32_t val)
+ {
+ std::string s = std::bitset<32>(val).to_string();
+ return (s.substr(0,1) + " " + s.substr(1,8) + " " + s.substr(9,32));
+ };
+
+ // Check the compresion results
+ for (int i = 0; i < 5; i++) {
+ std::cout << "Iteration [" << i << "]" << std::endl;
+ std::cout << std::endl;
+
+ // Testing float
+
+ // Print the original value
+ uni.fvalue = fptr[i];
+ std::cout << "Original value " << std::dec << std::setw( 15 ) << uni.fvalue
+ << " (binary : " << intToBinStr(uni.ivalue) << ")" << std::endl;
+ // Print the compressed value
+ uni.fvalue = rfptr[i];
+ std::cout << "Compressed value " << std::dec << std::setw( 15 ) << uni.fvalue
+ << " (binary : " << intToBinStr(uni.ivalue) << ")" << std::endl;
+ result = uni.ivalue;
+ // Check against by-hand compression
+ uni.fvalue = fc.reduceFloatPrecision( fptr[i] );
+ assert( result == uni.ivalue );
+
+ // Testing std::vector<float>
+
+ std::vector<float>* invec = fvptr + i;
+ std::vector<float>* outvec = rfvptr + i;
+ assert(invec != nullptr && outvec != nullptr);
+ assert(outvec->size() == 2 && outvec->size() == 2);
+
+ for(int j = 0; j < 2; j++) {
+ // Print the original value
+ uni.fvalue = invec->at(j);
+ std::cout << "Original value at [" << j << "] " << std::dec << std::setw( 8 ) << uni.fvalue
+ << " (binary : " << intToBinStr(uni.ivalue) << ")" << std::endl;
+ // Print the compressed value
+ uni.fvalue = outvec->at(j);
+ std::cout << "Compressed value at [" << j << "] " << std::dec << std::setw( 8 ) << uni.fvalue
+ << " (binary : " << intToBinStr(uni.ivalue) << ")" << std::endl;
+ result = uni.ivalue;
+ // Check against by-hand compression
+ uni.fvalue = fc.reduceFloatPrecision( invec->at(j) );
+ assert( result == uni.ivalue );
+ }
+
+ std::cout << std::endl;
+ }
+
}
int main()
{
test1();
+ test2();
return 0;
}
--
GitLab
From 89280c194c21bf468518a25018283962a96b55a0 Mon Sep 17 00:00:00 2001
From: amete <serhanmete@gmail.com>
Date: Wed, 12 Aug 2020 14:20:15 +0200
Subject: [PATCH 2/5] Minor fixes
---
Control/AthContainers/src/copyAuxStoreThinned.cxx | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/Control/AthContainers/src/copyAuxStoreThinned.cxx b/Control/AthContainers/src/copyAuxStoreThinned.cxx
index 2bd46dfe8b21..b71c6978cfa5 100644
--- a/Control/AthContainers/src/copyAuxStoreThinned.cxx
+++ b/Control/AthContainers/src/copyAuxStoreThinned.cxx
@@ -117,7 +117,7 @@ void copyAuxStoreThinned NO_SANITIZE_UNDEFINED
r.copyForOutput (auxid, dst, idst, src, isrc);
// Apply lossy float compression here (in-place)
// Maybe it would be better to do this via the registry during copy
- lossyFloatCompress(dst, idst, eltSize,typeName, nmantissa);
+ lossyFloatCompress(dst, idst, eltSize, typeName, nmantissa);
++idst;
}
}
@@ -138,7 +138,7 @@ void lossyFloatCompress (void* dst, size_t idst,
const unsigned int& nmantissa)
{
// Check if there is anything to be done
- if (nmantissa <= 0) return;
+ if (nmantissa == 0) return;
// Setup all possible compressors: done only once
static const std::vector<CxxUtils::FloatCompressor> compressors = [] {
--
GitLab
From d16ac9108d04af97c05fcf1dae8a3d208ba47ad3 Mon Sep 17 00:00:00 2001
From: amete <serhanmete@gmail.com>
Date: Wed, 12 Aug 2020 18:34:14 +0200
Subject: [PATCH 3/5] Bugfix vector size check
---
Control/AthContainers/test/copyAuxStoreThinned_test.cxx | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Control/AthContainers/test/copyAuxStoreThinned_test.cxx b/Control/AthContainers/test/copyAuxStoreThinned_test.cxx
index c05328709313..db2a83780a5b 100644
--- a/Control/AthContainers/test/copyAuxStoreThinned_test.cxx
+++ b/Control/AthContainers/test/copyAuxStoreThinned_test.cxx
@@ -255,7 +255,7 @@ void test2()
std::vector<float>* invec = fvptr + i;
std::vector<float>* outvec = rfvptr + i;
assert(invec != nullptr && outvec != nullptr);
- assert(outvec->size() == 2 && outvec->size() == 2);
+ assert(invec->size() == 2 && outvec->size() == 2);
for(int j = 0; j < 2; j++) {
// Print the original value
--
GitLab
From 3677d9fa7a08ad68ddbce6cf485c69481648640f Mon Sep 17 00:00:00 2001
From: amete <serhanmete@gmail.com>
Date: Thu, 13 Aug 2020 10:57:54 +0200
Subject: [PATCH 4/5] A few minor improvements
---
.../AthContainers/tools/copyAuxStoreThinned.h | 2 +-
Control/AthContainers/src/copyAuxStoreThinned.cxx | 10 +++++++---
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/Control/AthContainers/AthContainers/tools/copyAuxStoreThinned.h b/Control/AthContainers/AthContainers/tools/copyAuxStoreThinned.h
index ac10b6af72bf..daff78e7b32a 100644
--- a/Control/AthContainers/AthContainers/tools/copyAuxStoreThinned.h
+++ b/Control/AthContainers/AthContainers/tools/copyAuxStoreThinned.h
@@ -48,7 +48,7 @@ void copyAuxStoreThinned (const SG::IConstAuxStore& orig,
* @param typeName Type name for the auxid
* @param nmantissa Compression level to be used for the auxid
*/
-void lossyFloatCompress (void* dst, size_t dst_index,
+void lossyFloatCompress (void* dst, std::size_t dst_index,
const std::size_t& eltSize,
const std::string& typeName,
const unsigned int& nmantissa);
diff --git a/Control/AthContainers/src/copyAuxStoreThinned.cxx b/Control/AthContainers/src/copyAuxStoreThinned.cxx
index b71c6978cfa5..3af0822cd6d2 100644
--- a/Control/AthContainers/src/copyAuxStoreThinned.cxx
+++ b/Control/AthContainers/src/copyAuxStoreThinned.cxx
@@ -132,18 +132,22 @@ void copyAuxStoreThinned NO_SANITIZE_UNDEFINED
* @param typeName Type name for the auxid
* @param nmantissa Compression level to be used for the auxid
*/
-void lossyFloatCompress (void* dst, size_t idst,
+void lossyFloatCompress (void* dst, std::size_t idst,
const std::size_t& eltSize,
const std::string& typeName,
const unsigned int& nmantissa)
{
// Check if there is anything to be done
- if (nmantissa == 0) return;
+ // Total number of explicit mantissa bits for a 32 bit float is 23
+ // 0 is used to denote no compression should be applied to the variable
+ static const unsigned int NMANTISSA_MAX = 23;
+ if (nmantissa == 0 || nmantissa >= NMANTISSA_MAX) return;
// Setup all possible compressors: done only once
static const std::vector<CxxUtils::FloatCompressor> compressors = [] {
std::vector<CxxUtils::FloatCompressor> result;
- for (std::size_t idx = 0; idx <= 23; ++idx)
+ result.reserve(NMANTISSA_MAX);
+ for (std::size_t idx = 0; idx < NMANTISSA_MAX; ++idx)
result.emplace_back(idx);
return result;
}();
--
GitLab
From 645e640aca5a472b3a8b4a806d42e6449ff0a857 Mon Sep 17 00:00:00 2001
From: amete <serhanmete@gmail.com>
Date: Thu, 13 Aug 2020 17:56:16 +0200
Subject: [PATCH 5/5] Fix the typo in the comments
---
Control/AthContainers/test/copyAuxStoreThinned_test.cxx | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Control/AthContainers/test/copyAuxStoreThinned_test.cxx b/Control/AthContainers/test/copyAuxStoreThinned_test.cxx
index db2a83780a5b..9d75255af87d 100644
--- a/Control/AthContainers/test/copyAuxStoreThinned_test.cxx
+++ b/Control/AthContainers/test/copyAuxStoreThinned_test.cxx
@@ -223,7 +223,7 @@ void test2()
CxxUtils::FloatCompressor::floatint_t uni;
uint32_t result{0};
- // Helper to pring int in binary w/ a nicer format
+ // Helper to print int in binary w/ a nicer format
auto intToBinStr = [] (uint32_t val)
{
std::string s = std::bitset<32>(val).to_string();
--
GitLab