Merge branch 'vector_column' into 'main'

support for std::vector<T> columns in columnar tools

See merge request !78448

PhysicsAnalysis/Columnar/ColumnarCore/ColumnarCore/ColumnAccessor.h

@@ -49,7 +49,7 @@ namespace columnar
   /// specialize this template to provide support for the specific types
   /// and modes they support.  The primary way in which users specify
   /// alternate accessors is by wrapping the column type, e.g.
-  /// `VectorColumn<float>` for a column that contains a vector of
+  /// `std::vector<float>` for a column that contains a vector of
   /// floats per object.
   ///
   /// Originally all accessor classes were separate and unique
@@ -61,7 +61,7 @@ namespace columnar
   ///   that covers what before where multiple classes.
   /// * There are some accessor configuration that are supported now and
   ///   would have required a separate template class before, e.g.
-  ///   something like `VectorColumn<RetypeColumn<...>>` wasn't
+  ///   something like `std::vector<RetypeColumn<...>>` wasn't
   ///   supported before, but now it is.
   /// * Overall I find it easier to ensure consistency across
   ///   specializations with this approach, as well as making it easier

PhysicsAnalysis/Columnar/ColumnarCore/ColumnarCore/VectorColumn.h

+/*
+  Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
+*/
+
+/// @author Nils Krumnack
+
+
+#ifndef COLUMNAR_CORE_VECTOR_COLUMN_H
+#define COLUMNAR_CORE_VECTOR_COLUMN_H
+
+#include <ColumnarCore/ColumnAccessor.h>
+#include <ColumnarCore/VectorConvertView.h>
+#include <span>
+#include <vector>
+
+namespace columnar
+{
+  /// @file VectorColumn.h
+  ///
+  /// `std::vector` specialization for `AccessorTemplate`
+  /// ===================================================
+  ///
+  /// Per-object vectors get handled differently depending on the
+  /// framework, so there are different accessor specializations in
+  /// different environments.  So far (11 Dec 24) only input accessors
+  /// are supported, and they will generally return an object that
+  /// behaves like `std::span<const CT>`
+
+
+
+  // in xAOD mode we can do a straightforward conversion from
+  // std::vector to std::span, as xAODs natively use std::vector
+  template<typename CT>
+    requires (ColumnTypeTraits<CT,ColumnarModeXAOD>::isNativeType)
+  struct ColumnTypeTraits<std::vector<CT>,ColumnarModeXAOD> final
+  {
+    using CM = ColumnarModeXAOD;
+    using ElementType = typename ColumnTypeTraits<CT,CM>::ColumnType;
+    using ColumnType = NativeColumn<std::vector<ElementType>>;
+    using UserType = std::span<const ElementType>;
+    static constexpr bool isNativeType = false;
+    static constexpr bool useConvertInput = true;
+    static constexpr bool useConvertWithDataInput = false;
+    static ColumnInfo& updateColumnInfo (ColumnarTool<CM>& /*columnarTool*/, ColumnInfo& info) {return info;}
+    static std::span<const ElementType> convertInput (const std::vector<ElementType>& value) {return std::span<const ElementType> (value);}
+  };
+
+
+
+
+  // the column accessor for std::vector in ColumnarModeArray
+  //
+  // This is one of the more tricky accessors, as it need to connect to
+  // two underlying columns.  The first column is the offset column, the
+  // second is the data column.
+  template<ContainerId CI,typename CT>
+    requires (ColumnTypeTraits<CT,ColumnarModeArray>::isNativeType)
+  class AccessorTemplate<CI,std::vector<CT>,ColumnAccessMode::input,ColumnarModeArray> final
+  {
+    /// Public Members
+    /// ==============
+  public:
+
+    static constexpr ColumnAccessMode CAM = ColumnAccessMode::input;
+    using CM = ColumnarModeArray;
+    using ElementType = typename ColumnTypeTraits<CT,CM>::ColumnType;
+
+    AccessorTemplate () = default;
+
+    AccessorTemplate (ColumnarTool<CM>& columnarTool, const std::string& name, ColumnInfo&& info = {})
+    {
+      auto myinfo = info;
+      myinfo.offsetName = columnarTool.objectName (CI);
+      myinfo.isOffset = true;
+      info.offsetName = columnarTool.objectName (CI) + "." + name + ".offset";
+      m_offsetData = std::make_unique<ColumnAccessorDataArray> (&m_offsetIndex, &m_offsetData, &typeid (ColumnarOffsetType), ColumnAccessMode::input);
+      columnarTool.addColumn (info.offsetName, m_offsetData.get(), std::move (myinfo));
+      m_dataData = std::make_unique<ColumnAccessorDataArray> (&m_dataIndex, &m_dataData, &typeid (ElementType), ColumnAccessMode::input);
+      columnarTool.addColumn (columnarTool.objectName(CI) + "." + name + ".data", m_dataData.get(), std::move (info));
+    }
+
+    AccessorTemplate (AccessorTemplate&& that)
+    {
+      moveAccessor (m_offsetIndex, m_offsetData, that.m_offsetIndex, that.m_offsetData);
+      moveAccessor (m_dataIndex, m_dataData, that.m_dataIndex, that.m_dataData);
+    }
+
+    AccessorTemplate& operator = (AccessorTemplate&& that)
+    {
+      if (this != &that)
+      {
+        moveAccessor (m_offsetIndex, m_offsetData, that.m_offsetIndex, that.m_offsetData);
+        moveAccessor (m_dataIndex, m_dataData, that.m_dataIndex, that.m_dataData);
+      }
+      return *this;
+    }
+
+    std::span<const ElementType> operator () (ObjectId<CI,CM> id) const noexcept
+    {
+      auto *offset = static_cast<const ColumnarOffsetType*>(id.getData()[m_offsetIndex]);
+      auto *data = static_cast<const ElementType*>(id.getData()[m_dataIndex]);
+      return std::span<const ElementType> (&data[offset[id.getIndex()]], offset[id.getIndex()+1]-offset[id.getIndex()]);
+    }
+
+    bool isAvailable (ObjectId<CI,CM> id) const noexcept
+    {
+      auto *data = static_cast<const ElementType*>(id.getData()[m_dataIndex]);
+      return data != nullptr;
+    }
+
+    /// Private Members
+    /// ===============
+  private:
+
+    unsigned m_offsetIndex = 0u;
+    std::unique_ptr<ColumnAccessorDataArray> m_offsetData;
+    unsigned m_dataIndex = 0u;
+    std::unique_ptr<ColumnAccessorDataArray> m_dataData;
+  };
+
+
+  /// a @ref std::vector accessor for types that can be implemented via
+  /// conversions
+  ///
+  /// This is a bit more involved in that it needs to wrap the
+  /// underlying view, and apply the conversion on access.  That means I
+  /// need to have a custom view and iterator object to handle it.
+  /// Furthermore, since some conversions need a data pointer and others
+  /// don't, I need to have two different implementations of the view
+  /// and iterator.
+  template<ContainerId CI,typename CT,typename CM>
+    requires (ColumnTypeTraits<CT,CM>::useConvertInput || ColumnTypeTraits<CT,CM>::useConvertWithDataInput)
+  class AccessorTemplate<CI,std::vector<CT>,ColumnAccessMode::input,CM> final
+  {
+    /// Public Members
+    /// ==============
+  public:
+
+    AccessorTemplate () = default;
+
+    AccessorTemplate (ColumnarTool<CM>& columnarTool, const std::string& name, ColumnInfo&& info = {})
+      : m_accessor(columnarTool,name,std::move(ColumnTypeTraits<CT,CM>::updateColumnInfo(columnarTool, info)))
+    {
+    }
+
+    auto operator () (ObjectId<CI,CM> id) const
+    {
+      if constexpr (ColumnTypeTraits<CT,CM>::useConvertWithDataInput)
+        return detail::VectorConvertView ([data = id.getData()](const auto& value) {return ColumnTypeTraits<CT,CM>::convertInput (data, value);}, m_accessor(id));
+      else
+        return detail::VectorConvertView ([](const auto& value) {return ColumnTypeTraits<CT,CM>::convertInput(value);}, m_accessor(id));
+    }
+
+    [[nodiscard]] bool isAvailable (ObjectId<CI,CM> id) const noexcept
+    {
+      return m_accessor.isAvailable (id);
+    }
+
+    /// Private Members
+    /// ===============
+
+  private:
+
+    ColumnAccessor<CI,std::vector<typename ColumnTypeTraits<CT,CM>::ColumnType>,CM> m_accessor;
+  };
+}
+
+#endif

PhysicsAnalysis/Columnar/ColumnarCore/ColumnarCore/VectorConvertView.h

+/*
+  Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
+*/
+
+/// @author Nils Krumnack
+
+
+#ifndef COLUMNAR_CORE_VECTOR_CONVERT_VIEW_H
+#define COLUMNAR_CORE_VECTOR_CONVERT_VIEW_H
+
+#include <ColumnarCore/ColumnAccessor.h>
+
+namespace columnar
+{
+  namespace detail
+  {
+  /// @file VectorConvertView.h
+  ///
+  /// `std::vector` views and iterators that incorporate a type conversion
+  /// ====================================================================
+  ///
+  /// If the underlying column is an `std::vector`, but the access
+  /// requires a type conversion (e.g. `ElementLink` or an enum), then I
+  /// can't just use a `std::span` to access the data.  Instead I rely
+  /// on this view and iterator to handle the conversion on access.
+  /// Presumably with C++23 this could be replaced with a range adaptor,
+  /// but for now I need to do it manually.
+
+
+    /// an iterator that does converts members using a passed in function
+    template<typename FunctionType,typename IteratorType>
+    class VectorConvertIterator final
+    {
+      /// Public Members
+      /// ==============
+    public:
+
+      VectorConvertIterator (const FunctionType& val_function, IteratorType&& val_iterator)
+        : m_function (val_function), m_iterator (std::move (val_iterator))
+      {}
+
+      [[nodiscard]] bool operator == (const VectorConvertIterator& that) const noexcept {
+        return m_iterator == that.m_iterator;}
+      [[nodiscard]] bool operator != (const VectorConvertIterator& that) const noexcept {
+        return m_iterator != that.m_iterator;}
+      
+      VectorConvertIterator& operator ++ () noexcept {
+        ++ m_iterator; return *this;}
+
+      [[nodiscard]] decltype(auto) operator * () const noexcept {
+        return m_function (*m_iterator);}
+
+      /// Private Members
+      /// ===============
+    private:
+
+      [[no_unique_address]] FunctionType m_function;
+      IteratorType m_iterator;
+    };
+    template<typename FunctionType,typename IteratorType> VectorConvertIterator (FunctionType&&,IteratorType&&) -> VectorConvertIterator<std::remove_cv_t<FunctionType>,std::remove_cv_t<IteratorType>>;
+
+
+
+    /// a range view that does converts members using a passed in function
+    template<typename FunctionType,typename ViewType>
+    class VectorConvertView final
+    {
+      /// Common Public Members
+      /// =====================
+    public:
+
+      explicit VectorConvertView (const FunctionType& val_function, const ViewType& val_view) noexcept
+        : m_function (val_function), m_view (val_view)
+      {}
+
+      [[nodiscard]] std::size_t size () const noexcept
+      {
+        return m_view.size();
+      }
+
+      [[nodiscard]] decltype(auto) operator[] (std::size_t index) const
+      {
+        if (index >= m_view.size()) [[unlikely]]
+          throw std::out_of_range ("VectorConvertView::operator[]: index out of range");
+        return m_function (m_view[index]);
+      }
+
+      [[nodiscard]] auto begin () const noexcept
+      {
+        return VectorConvertIterator (m_function, m_view.begin());
+      }
+
+      [[nodiscard]] auto end () const noexcept
+      {
+        return VectorConvertIterator (m_function, m_view.end());
+      }
+
+      /// Private Members
+      /// ===============
+    private:
+
+      [[no_unique_address]] FunctionType m_function;
+      ViewType m_view;
+    };
+    template<typename FunctionType,typename ViewType> VectorConvertView (FunctionType&&,ViewType&&) -> VectorConvertView<std::remove_cv_t<FunctionType>,std::remove_cv_t<ViewType>>;
+  }
+}
+
+#endif

PhysicsAnalysis/Columnar/ColumnarCore/test/gt_AccessorsArray.cxx

@@ -16,6 +16,7 @@ ATLAS_NO_CHECK_FILE_THREAD_SAFETY;
 #include <AsgTesting/UnitTest.h>
 #include <ColumnarCore/ColumnAccessor.h>
 #include <ColumnarCore/ObjectColumn.h>
+#include <ColumnarCore/VectorColumn.h>
 #include <ColumnarEventInfo/EventInfoDef.h>
 #include <ColumnarCore/ParticleDef.h>
 
@@ -133,6 +134,52 @@ namespace columnar
       EXPECT_EQ (objectRange.endIndex(), 6);
     }
   }
+
+
+  TEST (AccessorTest, vectorEventAccessor)
+  {
+    MyTool tool;
+    MyAccessor<std::vector<uint32_t>,ContainerId::eventInfo> eventAccessor {tool, "var1"};
+    MyAccessor<std::vector<RetypeColumn<uint64_t,uint32_t>>,ContainerId::eventInfo> eventRetypeAccessor {tool, "var1"};
+    {
+      auto columns = tool.getColumnInfo();
+      EXPECT_EQ (columns.size(), 3);
+      auto& columnOffset = columns[1];
+      EXPECT_EQ (columnOffset.name, "EventInfo.var1.data");
+      EXPECT_EQ (columnOffset.index, 0);
+      EXPECT_EQ (columnOffset.type, &typeid (uint32_t));
+      EXPECT_EQ (columnOffset.accessMode, ColumnAccessMode::input);
+      EXPECT_EQ (columnOffset.offsetName, "EventInfo.var1.offset");
+      auto& columnData = columns[2];
+      EXPECT_EQ (columnData.name, "EventInfo.var1.offset");
+      EXPECT_EQ (columnData.index, 0);
+      EXPECT_EQ (columnData.type, &typeid (ColumnarOffsetType));
+      EXPECT_EQ (columnData.accessMode, ColumnAccessMode::input);
+      EXPECT_EQ (columnData.offsetName, numberOfEventsName);
+    }
+    tool.setColumnIndex ("EventInfo.var1.offset", 1);
+    tool.setColumnIndex ("EventInfo.var1.data", 2);
+    std::vector<void*> data (3, nullptr);
+    std::vector<ColumnarOffsetType> var1Offsets = {0, 1, 3, 6, 7};
+    std::vector<uint32_t> var1Data = {0, 1, 2, 3, 4, 5, 6};
+    data[1] = var1Offsets.data();
+    data[2] = var1Data.data();
+    MyId<ContainerId::eventInfo> id1 {data.data(), 1};
+    MyId<ContainerId::eventInfo> id2 {data.data(), 2};
+    EXPECT_EQ (eventAccessor (id1).size(), 2);
+    EXPECT_EQ (eventAccessor (id2).size(), 3);
+    EXPECT_EQ (eventAccessor(id1)[0],1);
+    EXPECT_EQ (eventAccessor(id1)[1],2);
+    EXPECT_EQ (eventAccessor(id2)[0],3);
+    EXPECT_EQ (eventAccessor(id2)[2],5);
+
+    EXPECT_EQ (eventRetypeAccessor (id1).size(), 2);
+    EXPECT_EQ (eventRetypeAccessor (id2).size(), 3);
+    EXPECT_EQ (eventRetypeAccessor(id1)[0],1);
+    EXPECT_EQ (eventRetypeAccessor(id1)[1],2);
+    EXPECT_EQ (eventRetypeAccessor(id2)[0],3);
+    EXPECT_EQ (eventRetypeAccessor(id2)[2],5);
+  }
 }
 
 ATLAS_GOOGLE_TEST_MAIN

PhysicsAnalysis/Columnar/ColumnarExampleTools/ColumnarExampleTools/VectorExampleTool.h

+/*
+  Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
+*/
+
+/// @author Nils Krumnack
+
+
+#ifndef COLUMNAR_EXAMPLE_TOOLS_VECTOR_EXAMPLE_TOOL_H
+#define COLUMNAR_EXAMPLE_TOOLS_VECTOR_EXAMPLE_TOOL_H
+
+#include <AsgTools/AsgTool.h>
+#include <AsgTools/PropertyWrapper.h>
+#include <ColumnarCore/ColumnAccessor.h>
+#include <ColumnarCore/ColumnarTool.h>
+#include <ColumnarCore/ObjectColumn.h>
+#include <ColumnarCore/ParticleDef.h>
+#include <ColumnarCore/VectorColumn.h>
+
+namespace columnar
+{
+  /// @brief an example of a columnar tool that reads a vector column
+  ///
+  /// This illustrates how to read vector columns in a tool.  The actual
+  /// example is a bit arcane, but it hopefully illustrates how to read
+  /// vector columns.
+
+  class VectorExampleTool final
+    : public asg::AsgTool,
+      public ColumnarTool<>
+  {
+  public:
+
+    VectorExampleTool (const std::string& name);
+
+    virtual StatusCode initialize () override;
+
+    virtual void callEvents (EventContextRange events) const override;
+
+
+    /// @brief the pt cut to apply
+    Gaudi::Property<float> m_ptCut {this, "ptCut", 10e3, "pt cut (in MeV)"};
+
+
+    /// @brief the object accessor for the particles
+    ///
+    /// This is equivalent to a `ReadHandleKey` in the xAOD world.  It
+    /// is used to access the particle range/container for a given
+    /// event.
+    ParticleAccessor<ObjectColumn> particlesHandle {*this, "Particles"};
+
+
+    /// @brief the pt accessor for the particle container
+    ///
+    /// This is the equivalent to an `AuxElement::Accessor` in the xAOD
+    /// world.  The main difference is that it registers with the tool,
+    /// as that is needed for column accessors.  Also, it is specific to
+    /// the container, and can't be used with other containers.
+    ParticleAccessor<float> ptAcc {*this, "pt"};
+
+
+    /// @brief a vector column accessor
+    ///
+    /// There aren't a lot of cases in which there is a simple
+    /// POD-vector column in PHYSLITE, so I picked up this rather
+    /// obscure one.
+    ParticleAccessor<std::vector<int>> trknumAcc {*this, "NumTrkPt500"};
+
+
+    /// @brief a vector accessor involving retyping
+    ///
+    /// This mostly illustrates and tests that view can also read
+    /// columns that need to do a conversion of the underlying type.  In
+    /// PHYSLITE this mostly happens for ElementLink columns, and maybe
+    /// enum columns, but for a simple example I'm changing from `float`
+    /// to `double`.
+    ParticleAccessor<std::vector<RetypeColumn<double,float>>> trksumptAcc {*this, "SumPtTrkPt500"};
+
+
+    /// @brief the selection decorator for the particles
+    ///
+    /// This is the equivalent to an `AuxElement::Decorator` in the xAOD
+    /// world.  One thing to note is that the tools generally run on
+    /// many objects, not a a single object.  So the tool doesn't have
+    /// the option to return individual output values.  Instead it needs
+    /// to provide an output value per object, which in the columnar
+    /// world is done by filling a column.
+    ParticleDecorator<char> selectionDec {*this, "selection"};
+  };
+}
+
+#endif

PhysicsAnalysis/Columnar/ColumnarExampleTools/Root/VectorExampleTool.cxx

+/*
+  Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
+*/
+
+/// @author Nils Krumnack
+
+//
+// includes
+//
+
+#include <ColumnarExampleTools/VectorExampleTool.h>
+
+//
+// method implementations
+//
+
+namespace columnar
+{
+  VectorExampleTool ::
+  VectorExampleTool (const std::string& name)
+    : AsgTool (name)
+  {}
+
+
+
+  StatusCode VectorExampleTool ::
+  initialize ()
+  {
+    // give the base class a chance to initialize the column accessor
+    // backends
+    ANA_CHECK (initializeColumns());
+    return StatusCode::SUCCESS;
+  }
+
+
+
+  void VectorExampleTool ::
+  callEvents (EventContextRange events) const
+  {
+    // loop over all events and particles.  note that this is
+    // deliberately looping by value, as the ID classes are very small
+    // and can be copied cheaply.  this could have also been written as
+    // a single loop over all particles in the event range, but I chose
+    // to split it up into two loops as most tools will need to do some
+    // per-event things, e.g. retrieve `EventInfo`.
+    for (columnar::EventContextId event : events)
+    {
+      for (ParticleId particle : particlesHandle(event))
+      {
+        // in pactical terms we should find the index of the primary
+        // vertex, but for purposes of the example we just use the first
+        // vertex instead
+        const std::size_t index = 0;
+
+        // it is actually safe to copy the return value of the
+        // accessors, as those are ranges that are cheap to copy
+        auto trknum = trknumAcc(particle);
+        auto trksumpt = trksumptAcc(particle);
+
+        // this is probably not a meaningful selection, but it hopefully
+        // illustrates how to use the vector accessors
+        selectionDec(particle) = ptAcc(particle) > m_ptCut.value() && trknum.size() > index && trknum[index] > 2 && trksumpt.size() > index && trksumpt[index] > 2e3;
+      }
+    }
+  }
+}