DPL Analysis: allow for per-origin AOD metadata handling

Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx

@@ -11,12 +11,10 @@
 
 #include "AODJAlienReaderHelpers.h"
 #include <algorithm>
-#include <charconv>
 #include <cctype>
 #include <cstdlib>
 #include <exception>
 #include <memory>
-#include <ranges>
 #include <string>
 #include <string_view>
 #include <vector>
@@ -27,6 +25,7 @@
 #include "Framework/AlgorithmSpec.h"
 #include "Framework/ArrowContext.h"
 #include "Framework/ConfigParamRegistry.h"
+#include "Framework/Array2D.h"
 #include "Framework/ControlService.h"
 #include "Framework/CallbackService.h"
 #include "Framework/EndOfStreamContext.h"
@@ -155,24 +154,11 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const
     parentAccessLevel = ctx.options().get<int>("aod-parent-access-level");
   }
   std::vector<std::pair<std::string, int>> originLevelMapping;
-  if (ctx.options().isSet("aod-origin-level-mapping")) {
-    auto originLevelMappingStr = ctx.options().get<std::string>("aod-origin-level-mapping");
-    for (auto pairRange : originLevelMappingStr | std::views::split(',')) {
-      std::string_view pair{pairRange.begin(), pairRange.end()};
-      auto colonPos = pair.find(':');
-      if (colonPos == std::string_view::npos) {
-        LOGP(fatal, "Badly formatted aod-origin-level-mapping entry: \"{}\"", pair);
-        continue;
-      }
-      std::string key(pair.substr(0, colonPos));
-      std::string_view valueStr = pair.substr(colonPos + 1);
-      int value{};
-      auto [ptr, ec] = std::from_chars(valueStr.data(), valueStr.data() + valueStr.size(), value);
-      if (ec == std::errc{}) {
-        originLevelMapping.emplace_back(std::move(key), value);
-      } else {
-        LOGP(fatal, "Unable to parse level in aod-origin-level-mapping entry: \"{}\"", pair);
-      }
+  if (ctx.options().hasOption("aod-origin-level-map")) {
+    auto originLevelMap = ctx.options().get<LabeledArray<int>>("aod-origin-level-map");
+    auto origins = originLevelMap.getLabelsRows();
+    for (auto i = 0U; i < originLevelMap.rows(); ++i) {
+      originLevelMapping.emplace_back(origins[i], originLevelMap.get(i, 0U));
     }
   }
   auto callback = AlgorithmSpec{adaptStateful([parentFileReplacement, parentAccessLevel, originLevelMapping](ConfigParamRegistry const& options,

Framework/AnalysisSupport/src/Plugin.cxx

@@ -23,6 +23,8 @@
 #include <TObjString.h>
 #include <TString.h>
 #include <fmt/format.h>
+#include <algorithm>
+#include <map>
 #include <memory>
 
 O2_DECLARE_DYNAMIC_LOG(analysis_support);
@@ -105,7 +107,7 @@ struct RunSummary : o2::framework::ServicePlugin {
   }
 };
 
-std::vector<std::string> getListOfTables(std::unique_ptr<TFile>& f)
+std::vector<std::string> getListOfTables(TFile* f)
 {
   std::vector<std::string> r;
   TList* keyList = f->GetListOfKeys();
@@ -155,17 +157,17 @@ std::vector<std::string> getListOfTables(std::unique_ptr<TFile>& f)
   return r;
 }
 
-auto readMetadata(std::unique_ptr<TFile>& currentFile) -> std::vector<ConfigParamSpec>
+// Ordered key / value pairs of the metadata, empty if there is none
+auto readMetadata(TFile* currentFile) -> std::vector<std::pair<std::string, std::string>>
 {
   // Get the metadata, if any
   auto m = (TMap*)currentFile->Get("metaData");
   if (!m) {
     return {};
   }
-  std::vector<ConfigParamSpec> results;
+  std::vector<std::pair<std::string, std::string>> results;
   auto it = m->MakeIterator();
 
-  // Serialise metadata into a ; separated string with : separating key and value
   bool first = true;
   while (auto obj = it->Next()) {
     if (first) {
@@ -174,14 +176,90 @@ auto readMetadata(std::unique_ptr<TFile>& currentFile) -> std::vector<ConfigPara
     }
     auto objString = (TObjString*)m->GetValue(obj);
     LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
-    std::string key = "aod-metadata-" + std::string(obj->GetName());
-    char const* value = strdup(objString->String());
-    results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
+    results.emplace_back(obj->GetName(), objString->String().Data());
   }
 
   return results;
 }
 
+auto metadataToSpecs(std::vector<std::pair<std::string, std::string>> const& metadata) -> std::vector<ConfigParamSpec>
+{
+  std::vector<ConfigParamSpec> results;
+  for (auto const& [key, value] : metadata) {
+    results.push_back(ConfigParamSpec{"aod-metadata-" + key, VariantType::String, value, {"Metadata in AOD"}});
+  }
+  return results;
+}
+
+auto metadataKeys(std::vector<std::pair<std::string, std::string>> const& metadata) -> std::vector<std::string>
+{
+  std::vector<std::string> keys;
+  for (auto const& [key, value] : metadata) {
+    keys.push_back(key);
+  }
+  return keys;
+}
+
+struct FileAtLevel {
+  std::unique_ptr<TFile> owned; // empty for level 0, which is the file we started from
+  TFile* file = nullptr;        // nullptr if the chain of parents is shorter than the level
+  std::string name;
+};
+
+// Follow the first parent file `level` times, starting from `file`
+FileAtLevel openAtLevel(TFile* file, std::string const& filename, int level, ConfigParamRegistry& registry)
+{
+  FileAtLevel result{nullptr, file, filename};
+  for (auto l = 0; l < level; ++l) {
+    auto parentFiles = (TMap*)result.file->Get("parentFiles");
+    if (!parentFiles || parentFiles->GetSize() == 0) {
+      LOGP(info, "No parent file found for \"{}\"", result.name);
+      return {nullptr, nullptr, result.name};
+    }
+    std::string parentFilename;
+    for (auto* p : *parentFiles) {
+      parentFilename = ((TPair*)p)->Value()->GetName();
+      break;
+    }
+    // Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be
+    // a workflow option (because the metadata itself is potentially changing the topology).
+    if (registry.isSet("aod-parent-base-path-replacement")) {
+      auto parentFileReplacement = registry.get<std::string>("aod-parent-base-path-replacement");
+      auto pos = parentFileReplacement.find(';');
+      if (pos == std::string::npos) {
+        throw std::runtime_error(fmt::format("Invalid syntax in aod-parent-base-path-replacement: \"{}\"", parentFileReplacement.c_str()));
+      }
+      auto from = parentFileReplacement.substr(0, pos);
+      auto to = parentFileReplacement.substr(pos + 1);
+      pos = parentFilename.find(from);
+      if (pos != std::string::npos) {
+        parentFilename.replace(pos, from.length(), to);
+      }
+    }
+
+    if (parentFilename.starts_with("alien://") && !gGrid) {
+      TGrid::Connect("alien://");
+    }
+
+    std::unique_ptr<TFile> parentFile{TFile::Open(parentFilename.c_str())};
+    if (parentFile.get() == nullptr) {
+      LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename);
+    }
+    result.file = parentFile.get();
+    result.owned = std::move(parentFile);
+    result.name = parentFilename;
+  }
+  return result;
+}
+
+// What was found at a given parent level
+struct LevelMetadata {
+  bool found = false; // the file at this level exists
+  std::string source;
+  std::vector<std::pair<std::string, std::string>> metadata;
+  std::vector<std::string> tables;
+};
+
 struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
   ConfigDiscovery* create() override
   {
@@ -210,73 +288,112 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
         if (currentFile.get() == nullptr) {
           LOGP(fatal, "Couldn't open file \"{}\"!", filename);
         }
-        std::vector<ConfigParamSpec> results = readMetadata(currentFile);
-        const bool metaDataEmpty = results.empty();
-        auto tables = getListOfTables(currentFile);
-        if (tables.empty() == false) {
-          results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
-        }
 
-        // Found metadata already in the main file.
-        if (!metaDataEmpty) {
-          results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
-          return results;
+        // Each level is read only once, even if several origins map to it
+        std::map<int, LevelMetadata> levels;
+        auto readLevel = [&](int level) -> LevelMetadata const& {
+          if (auto it = levels.find(level); it != levels.end()) {
+            return it->second;
+          }
+          LevelMetadata info;
+          auto atLevel = openAtLevel(currentFile.get(), filename, level, registry);
+          if (atLevel.file) {
+            info.found = true;
+            info.source = atLevel.name;
+            info.metadata = readMetadata(atLevel.file);
+            info.tables = getListOfTables(atLevel.file);
+          }
+          return levels.emplace(level, std::move(info)).first->second;
+        };
+        int parentAccessLevel = 0;
+        if (registry.isSet("aod-parent-access-level")) {
+          parentAccessLevel = registry.get<int>("aod-parent-access-level");
         }
 
-        if (!registry.isSet("aod-parent-access-level") || registry.get<int>("aod-parent-access-level") == 0) {
+        // The plain aod-metadata-* options, from the file itself or from its first parent
+        auto const& top = readLevel(0);
+        std::vector<ConfigParamSpec> results = metadataToSpecs(top.metadata);
+        std::vector<std::string> plainKeys;
+        if (!top.tables.empty()) {
+          results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, top.tables, {"Tables in first AOD"}});
+        }
+        if (!top.metadata.empty()) {
+          // Found metadata already in the main file.
+          results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
+          plainKeys = metadataKeys(top.metadata);
+        } else if (parentAccessLevel == 0) {
           LOGP(info, "No metadata found in file \"{}\" and parent level 0 prevents further lookup.", filename);
           results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
-          return results;
+        } else {
+          // Lets try in parent file.
+          LOGP(info, "No metadata found in file \"{}\", checking in its parents.", filename);
+          auto const& parent = readLevel(1);
+          if (!parent.found) {
+            LOGP(info, "No metadata found in file \"{}\"", filename);
+          } else {
+            results = metadataToSpecs(parent.metadata);
+            if (!parent.metadata.empty()) {
+              if (!parent.tables.empty()) {
+                results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, parent.tables, {"Tables in first AOD"}});
+              }
+              results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
+              plainKeys = metadataKeys(parent.metadata);
+            } else {
+              LOGP(info, "No metadata found in file \"{}\" nor in its parent file \"{}\"", filename, parent.source);
+            }
+          }
+          if (plainKeys.empty()) {
+            results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
+          }
         }
 
-        // Lets try in parent file.
-        auto parentFiles = (TMap*)currentFile->Get("parentFiles");
-        if (!parentFiles) {
-          LOGP(info, "No metadata found in file \"{}\"", filename);
-          results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
+        if (!registry.hasOption("aod-origin-level-map")) {
           return results;
         }
-        LOGP(info, "No metadata found in file \"{}\", checking in its parents.", filename);
-        for (auto* p : *parentFiles) {
-          std::string parentFilename = ((TPair*)p)->Value()->GetName();
-          // Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be
-          // a workflow option (because the metadata itself is potentially changing the topology).
-          if (registry.isSet("aod-parent-base-path-replacement")) {
-            auto parentFileReplacement = registry.get<std::string>("aod-parent-base-path-replacement");
-            auto pos = parentFileReplacement.find(';');
-            if (pos == std::string::npos) {
-              throw std::runtime_error(fmt::format("Invalid syntax in aod-parent-base-path-replacement: \"{}\"", parentFileReplacement.c_str()));
-            }
-            auto from = parentFileReplacement.substr(0, pos);
-            auto to = parentFileReplacement.substr(pos + 1);
-            pos = parentFilename.find(from);
-            if (pos != std::string::npos) {
-              parentFilename.replace(pos, from.length(), to);
+
+        // The metadata of the level each origin reads, one row per origin. The columns are
+        // all the keys found, including the plain ones, so that an empty cell always means
+        // that the key is missing for that origin.
+        auto levelMap = registry.get<LabeledArray<int>>("aod-origin-level-map");
+        auto origins = levelMap.getLabelsRows();
+        std::vector<std::string> keys = plainKeys;
+        for (auto i = 0U; i < levelMap.rows(); ++i) {
+          auto level = levelMap.get(i, 0U);
+          if (level > parentAccessLevel) {
+            LOGP(fatal, "Origin {} is mapped to parent level {}, but aod-parent-access-level is {}", origins[i], level, parentAccessLevel);
+          }
+          for (auto const& [key, value] : readLevel(level).metadata) {
+            if (std::ranges::find(keys, key) == keys.end()) {
+              keys.push_back(key);
             }
           }
+        }
+        auto sourceColumn = keys.size();
+        keys.emplace_back("source");
+        auto disableColumn = keys.size();
+        keys.emplace_back("disable");
 
-          if (parentFilename.starts_with("alien://") && !gGrid) {
-            TGrid::Connect("alien://");
+        std::vector<std::string> values(origins.size() * keys.size());
+        for (auto i = 0U; i < origins.size(); ++i) {
+          auto const& info = readLevel(levelMap.get(i, 0U));
+          auto* row = values.data() + i * keys.size();
+          for (auto const& [key, value] : info.metadata) {
+            row[std::ranges::find(keys, key) - keys.begin()] = value;
           }
-
-          std::unique_ptr<TFile> parentFile{TFile::Open(parentFilename.c_str())};
-          if (parentFile.get() == nullptr) {
-            LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename);
+          row[sourceColumn] = info.source;
+          if (info.metadata.empty()) {
+            LOGP(info, "Metadata info missing for origin {}", origins[i]);
+            row[disableColumn] = "1";
           }
-          results = readMetadata(parentFile);
-          // Found metadata already in the main file.
-          if (!results.empty()) {
-            auto tables = getListOfTables(parentFile);
-            if (tables.empty() == false) {
-              results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
-            }
-            results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
-            return results;
+        }
+        LabeledArray<std::string> originMetadata{values.data(), static_cast<uint32_t>(origins.size()), static_cast<uint32_t>(keys.size()), origins, keys};
+        results.push_back(ConfigParamSpec{"aod-metadata-origins", VariantType::LabeledArrayString, originMetadata, {"Metadata in AOD for each origin: one row per origin, one column per key"}});
+        for (auto i = 0U; i < origins.size(); ++i) {
+          auto const& tables = readLevel(levelMap.get(i, 0U)).tables;
+          if (!tables.empty()) {
+            results.push_back(ConfigParamSpec{"aod-metadata-origin-" + origins[i] + "-tables", VariantType::ArrayString, tables, {"Tables in the AOD read for this origin"}});
           }
-          LOGP(info, "No metadata found in file \"{}\" nor in its parent file \"{}\"", filename, parentFilename);
-          break;
         }
-        results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
         return results;
       }};
   }

Framework/Core/include/Framework/Array2D.h

@@ -79,19 +79,29 @@ struct Array2D {
 
   Array2D& operator=(Array2D<T> const& other)
   {
-    this->rows = other.rows;
-    this->cols = other.cols;
-    data = new T[rows * cols];
-    for (auto i = 0U; i < rows; ++i) {
-      for (auto j = 0U; j < cols; ++j) {
-        data[i * cols + j] = *(other.data + (i * cols + j));
+    if (this == &other) {
+      return *this;
+    }
+    // Copy into a new buffer first, so that a throwing copy leaves this array untouched
+    auto* newData = new T[other.rows * other.cols];
+    for (auto i = 0U; i < other.rows; ++i) {
+      for (auto j = 0U; j < other.cols; ++j) {
+        newData[i * other.cols + j] = other.data[i * other.cols + j];
       }
     }
+    delete[] data;
+    data = newData;
+    this->rows = other.rows;
+    this->cols = other.cols;
     return *this;
   }
 
   Array2D& operator=(Array2D<T>&& other)
   {
+    if (this == &other) {
+      return *this;
+    }
+    delete[] data;
     this->rows = other.rows;
     this->cols = other.cols;
     data = other.data;
@@ -231,6 +241,13 @@ class LabeledArray : public LabelMap
     return values[y];
   }
 
+  // nullptr if there is no row with this label
+  T* getRow(const char* y) const
+  {
+    auto it = rowmap.find(y);
+    return it == rowmap.end() ? nullptr : values[it->second];
+  }
+
   T* operator[](uint32_t y) const
   {
     return values[y];