Fix some bugs in H5 writers

Fix some bugs in H5 writers

PhysicsAnalysis/AnalysisCommon/HDF5Utils/CMakeLists.txt

@@ -74,5 +74,4 @@ atlas_install_scripts( test/test-h5-output )
 
 atlas_add_test( test_hdf5_writer
   SCRIPT test-hdf5-writer
-  LINK_LIBRARIES HDF5Utils
   POST_EXEC_SCRIPT test-h5-output)

PhysicsAnalysis/AnalysisCommon/HDF5Utils/HDF5Utils/Writer.h

@@ -230,8 +230,8 @@ namespace H5Utils {
         buffer() {
         hsize_t offset = 0;
         for (const auto& arg: args) {
-          const size_t this_dim_max = extent.at(N-1);
-          if (offset > this_dim_max) return;
+          const size_t this_dim_max = extent.at(extent.size() - N);
+          if (offset >= this_dim_max) return;
           DataFlattener<N-1, F, decltype(arg), M> in(filler, arg, extent);
           buffer.insert(buffer.end(), in.buffer.begin(), in.buffer.end());
           for (const auto& in_ele: in.element_offsets){

PhysicsAnalysis/AnalysisCommon/HDF5Utils/test/test-h5-output

 #!/usr/bin/env bash
 set -eu
-noerror.sh
-h5ls output.h5
+
+# check that one number we set is there
+h5dump -o raw.txt -y -d '4d[0,1,2,3,4]' output.h5 > /dev/null
+EIGHTY_SIX=$(cat raw.txt | xargs | paste -s | cut -d , -f 2)
+if (( $EIGHTY_SIX == 86 )); then
+    echo "test magic number is correct!"
+else
+    echo "test magic number is incorrect ($EIGHTY_SIX != 86)"
+    exit 1
+fi

PhysicsAnalysis/AnalysisCommon/HDF5Utils/util/test-hdf5-writer.cxx

@@ -26,13 +26,7 @@ using consumer_t = H5Utils::Consumers<const out_t&>;
 
 consumer_t getConsumers() {
   consumer_t consumers;
-  auto half = H5Utils::Compression::HALF_PRECISION;
-  consumers.add("half" , [](const out_t& o) { return o.ftype; }, 0, half);
-  consumers.add("dhalf", [](const out_t& o) { return o.dtype; }, 0, half);
 #define ADD(NAME) consumers.add(#NAME, [](const out_t& o){ return o.NAME;}, 0)
-  ADD(ftype);
-  ADD(dtype);
-  ADD(btype);
   ADD(ctype);
   ADD(stype);
   ADD(itype);
@@ -43,7 +37,13 @@ consumer_t getConsumers() {
   ADD(uitype);
   ADD(ultype);
   ADD(ulltype);
+  ADD(ftype);
+  ADD(dtype);
+  ADD(btype);
 #undef ADD
+  auto half = H5Utils::Compression::HALF_PRECISION;
+  consumers.add("half" , [](const out_t& o) { return o.ftype; }, 0, half);
+  consumers.add("dhalf", [](const out_t& o) { return o.dtype; }, 0, half);
   return consumers;
 }
 
@@ -80,7 +80,7 @@ auto nestOutputs(int offset, size_t length) {
     nestOutputs<N-1>(std::declval<int>(),std::declval<size_t>()));
   std::vector<ret_t> ret;
   for (size_t n = 0; n < length; n++) {
-    ret.push_back(nestOutputs<N-1>(n + offset, length));
+    ret.push_back(nestOutputs<N-1>(n + offset, length + 1));
   }
   return ret;
 }
@@ -92,17 +92,20 @@ auto nestOutputs<1>(int offset, size_t length) {
 //-------------------------------------------------------------------------
 // main routine
 
-int main(int, char*[]) {
-  // make the output file
-  H5::H5File out_file("output.h5", H5F_ACC_TRUNC);
+void fill(H5::Group& out_file, size_t iterations) {
+
+  const int deflate = 7;
 
   // scalar output
   using scalar_writer_t = H5Utils::Writer<0, consumer_t::input_type>;
   scalar_writer_t::configuration_type scalar_config;
   scalar_config.name = "scalar";
+  scalar_config.deflate = deflate;
   consumer_t consumers = getConsumers();
   scalar_writer_t scalar(out_file, consumers, scalar_config);
-  scalar.fill(getOutputs(1, 1, 0.5).at(0));
+  for (size_t n = 0; n < iterations; n++) {
+    scalar.fill(getOutputs(1 + n, 1, 0.5).at(0));
+  }
 
   // 1d output
   using d1_t = H5Utils::Writer<1, consumer_t::input_type>;
@@ -110,8 +113,11 @@ int main(int, char*[]) {
   d1_config.name = "1d";
   d1_config.extent = {10};
   d1_config.chunks = {5};
+  d1_config.deflate = deflate;
   d1_t d1(out_file, consumers, d1_config);
-  d1.fill(getOutputs(0, 10, 0.5));
+  for (size_t n = 0; n < iterations; n++) {
+    d1.fill(getOutputs(n, 10, 0.5));
+  }
 
   // 4d output
   using d4_t = H5Utils::Writer<4, consumer_t::input_type>;
@@ -119,9 +125,24 @@ int main(int, char*[]) {
   d4_config.name = "4d";
   d4_config.extent = {2,3,4,5};
   d4_config.chunks = {1,2,1,2};
+  d4_config.deflate = deflate;
   d4_t d4(out_file, consumers, d4_config);
-  d4.fill(nestOutputs<4>(0,3));
+  for (size_t n = 0; n < iterations; n++) {
+    auto vals = nestOutputs<4>(n, 2);
+    // store some specific value to make sure we have the indexing
+    // right
+    vals.at(1).at(2).at(3).at(4).stype = 86;
+    d4.fill(vals);
+  }
+}
 
+int main(int nargs, char* argv[]) {
+  H5::H5File out_file("output.h5", H5F_ACC_TRUNC);
+  size_t iterations = 1;
+  if (nargs > 2) {
+    return 1;
+  }
+  if (nargs > 1) iterations = std::atoi(argv[1]);
+  fill(out_file, iterations);
   return 0;
 }
-