Skip to content

Commit cd319b0

Browse files
aalkinaalkin
committed
working state
1 parent b8e6441 commit cd319b0

9 files changed

Lines changed: 311 additions & 80 deletions

File tree

‎Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx‎

Lines changed: 6 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,10 @@
1111

1212
#include "AODJAlienReaderHelpers.h"
1313
#include <algorithm>
14-
#include <charconv>
1514
#include <cctype>
1615
#include <cstdlib>
1716
#include <exception>
1817
#include <memory>
19-
#include <ranges>
2018
#include <string>
2119
#include <string_view>
2220
#include <vector>
@@ -27,6 +25,7 @@
2725
#include "Framework/AlgorithmSpec.h"
2826
#include "Framework/ArrowContext.h"
2927
#include "Framework/ConfigParamRegistry.h"
28+
#include "Framework/Array2D.h"
3029
#include "Framework/ControlService.h"
3130
#include "Framework/CallbackService.h"
3231
#include "Framework/EndOfStreamContext.h"
@@ -155,24 +154,11 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const
155154
parentAccessLevel = ctx.options().get<int>("aod-parent-access-level");
156155
}
157156
std::vector<std::pair<std::string, int>> originLevelMapping;
158-
if (ctx.options().isSet("aod-origin-level-mapping")) {
159-
auto originLevelMappingStr = ctx.options().get<std::string>("aod-origin-level-mapping");
160-
for (auto pairRange : originLevelMappingStr | std::views::split(',')) {
161-
std::string_view pair{pairRange.begin(), pairRange.end()};
162-
auto colonPos = pair.find(':');
163-
if (colonPos == std::string_view::npos) {
164-
LOGP(fatal, "Badly formatted aod-origin-level-mapping entry: \"{}\"", pair);
165-
continue;
166-
}
167-
std::string key(pair.substr(0, colonPos));
168-
std::string_view valueStr = pair.substr(colonPos + 1);
169-
int value{};
170-
auto [ptr, ec] = std::from_chars(valueStr.data(), valueStr.data() + valueStr.size(), value);
171-
if (ec == std::errc{}) {
172-
originLevelMapping.emplace_back(std::move(key), value);
173-
} else {
174-
LOGP(fatal, "Unable to parse level in aod-origin-level-mapping entry: \"{}\"", pair);
175-
}
157+
if (ctx.options().hasOption("aod-origin-level-map")) {
158+
auto originLevelMap = ctx.options().get<LabeledArray<int>>("aod-origin-level-map");
159+
auto origins = originLevelMap.getLabelsRows();
160+
for (auto i = 0U; i < originLevelMap.rows(); ++i) {
161+
originLevelMapping.emplace_back(origins[i], originLevelMap.get(i, 0U));
176162
}
177163
}
178164
auto callback = AlgorithmSpec{adaptStateful([parentFileReplacement, parentAccessLevel, originLevelMapping](ConfigParamRegistry const& options,

‎Framework/AnalysisSupport/src/Plugin.cxx‎

Lines changed: 175 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
#include <TObjString.h>
2424
#include <TString.h>
2525
#include <fmt/format.h>
26+
#include <algorithm>
27+
#include <map>
2628
#include <memory>
2729

2830
O2_DECLARE_DYNAMIC_LOG(analysis_support);
@@ -105,7 +107,7 @@ struct RunSummary : o2::framework::ServicePlugin {
105107
}
106108
};
107109

108-
std::vector<std::string> getListOfTables(std::unique_ptr<TFile>& f)
110+
std::vector<std::string> getListOfTables(TFile* f)
109111
{
110112
std::vector<std::string> r;
111113
TList* keyList = f->GetListOfKeys();
@@ -155,17 +157,17 @@ std::vector<std::string> getListOfTables(std::unique_ptr<TFile>& f)
155157
return r;
156158
}
157159

158-
auto readMetadata(std::unique_ptr<TFile>& currentFile) -> std::vector<ConfigParamSpec>
160+
// Ordered key / value pairs of the metadata, empty if there is none
161+
auto readMetadata(TFile* currentFile) -> std::vector<std::pair<std::string, std::string>>
159162
{
160163
// Get the metadata, if any
161164
auto m = (TMap*)currentFile->Get("metaData");
162165
if (!m) {
163166
return {};
164167
}
165-
std::vector<ConfigParamSpec> results;
168+
std::vector<std::pair<std::string, std::string>> results;
166169
auto it = m->MakeIterator();
167170

168-
// Serialise metadata into a ; separated string with : separating key and value
169171
bool first = true;
170172
while (auto obj = it->Next()) {
171173
if (first) {
@@ -174,14 +176,90 @@ auto readMetadata(std::unique_ptr<TFile>& currentFile) -> std::vector<ConfigPara
174176
}
175177
auto objString = (TObjString*)m->GetValue(obj);
176178
LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
177-
std::string key = "aod-metadata-" + std::string(obj->GetName());
178-
char const* value = strdup(objString->String());
179-
results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
179+
results.emplace_back(obj->GetName(), objString->String().Data());
180180
}
181181

182182
return results;
183183
}
184184

185+
auto metadataToSpecs(std::vector<std::pair<std::string, std::string>> const& metadata) -> std::vector<ConfigParamSpec>
186+
{
187+
std::vector<ConfigParamSpec> results;
188+
for (auto const& [key, value] : metadata) {
189+
results.push_back(ConfigParamSpec{"aod-metadata-" + key, VariantType::String, value, {"Metadata in AOD"}});
190+
}
191+
return results;
192+
}
193+
194+
auto metadataKeys(std::vector<std::pair<std::string, std::string>> const& metadata) -> std::vector<std::string>
195+
{
196+
std::vector<std::string> keys;
197+
for (auto const& [key, value] : metadata) {
198+
keys.push_back(key);
199+
}
200+
return keys;
201+
}
202+
203+
struct FileAtLevel {
204+
std::unique_ptr<TFile> owned; // empty for level 0, which is the file we started from
205+
TFile* file = nullptr; // nullptr if the chain of parents is shorter than the level
206+
std::string name;
207+
};
208+
209+
// Follow the first parent file `level` times, starting from `file`
210+
FileAtLevel openAtLevel(TFile* file, std::string const& filename, int level, ConfigParamRegistry& registry)
211+
{
212+
FileAtLevel result{nullptr, file, filename};
213+
for (auto l = 0; l < level; ++l) {
214+
auto parentFiles = (TMap*)result.file->Get("parentFiles");
215+
if (!parentFiles || parentFiles->GetSize() == 0) {
216+
LOGP(info, "No parent file found for \"{}\"", result.name);
217+
return {nullptr, nullptr, result.name};
218+
}
219+
std::string parentFilename;
220+
for (auto* p : *parentFiles) {
221+
parentFilename = ((TPair*)p)->Value()->GetName();
222+
break;
223+
}
224+
// Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be
225+
// a workflow option (because the metadata itself is potentially changing the topology).
226+
if (registry.isSet("aod-parent-base-path-replacement")) {
227+
auto parentFileReplacement = registry.get<std::string>("aod-parent-base-path-replacement");
228+
auto pos = parentFileReplacement.find(';');
229+
if (pos == std::string::npos) {
230+
throw std::runtime_error(fmt::format("Invalid syntax in aod-parent-base-path-replacement: \"{}\"", parentFileReplacement.c_str()));
231+
}
232+
auto from = parentFileReplacement.substr(0, pos);
233+
auto to = parentFileReplacement.substr(pos + 1);
234+
pos = parentFilename.find(from);
235+
if (pos != std::string::npos) {
236+
parentFilename.replace(pos, from.length(), to);
237+
}
238+
}
239+
240+
if (parentFilename.starts_with("alien://") && !gGrid) {
241+
TGrid::Connect("alien://");
242+
}
243+
244+
std::unique_ptr<TFile> parentFile{TFile::Open(parentFilename.c_str())};
245+
if (parentFile.get() == nullptr) {
246+
LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename);
247+
}
248+
result.file = parentFile.get();
249+
result.owned = std::move(parentFile);
250+
result.name = parentFilename;
251+
}
252+
return result;
253+
}
254+
255+
// What was found at a given parent level
256+
struct LevelMetadata {
257+
bool found = false; // the file at this level exists
258+
std::string source;
259+
std::vector<std::pair<std::string, std::string>> metadata;
260+
std::vector<std::string> tables;
261+
};
262+
185263
struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
186264
ConfigDiscovery* create() override
187265
{
@@ -210,73 +288,112 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
210288
if (currentFile.get() == nullptr) {
211289
LOGP(fatal, "Couldn't open file \"{}\"!", filename);
212290
}
213-
std::vector<ConfigParamSpec> results = readMetadata(currentFile);
214-
const bool metaDataEmpty = results.empty();
215-
auto tables = getListOfTables(currentFile);
216-
if (tables.empty() == false) {
217-
results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
218-
}
219291

220-
// Found metadata already in the main file.
221-
if (!metaDataEmpty) {
222-
results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
223-
return results;
292+
// Each level is read only once, even if several origins map to it
293+
std::map<int, LevelMetadata> levels;
294+
auto readLevel = [&](int level) -> LevelMetadata const& {
295+
if (auto it = levels.find(level); it != levels.end()) {
296+
return it->second;
297+
}
298+
LevelMetadata info;
299+
auto atLevel = openAtLevel(currentFile.get(), filename, level, registry);
300+
if (atLevel.file) {
301+
info.found = true;
302+
info.source = atLevel.name;
303+
info.metadata = readMetadata(atLevel.file);
304+
info.tables = getListOfTables(atLevel.file);
305+
}
306+
return levels.emplace(level, std::move(info)).first->second;
307+
};
308+
int parentAccessLevel = 0;
309+
if (registry.isSet("aod-parent-access-level")) {
310+
parentAccessLevel = registry.get<int>("aod-parent-access-level");
224311
}
225312

226-
if (!registry.isSet("aod-parent-access-level") || registry.get<int>("aod-parent-access-level") == 0) {
313+
// The plain aod-metadata-* options, from the file itself or from its first parent
314+
auto const& top = readLevel(0);
315+
std::vector<ConfigParamSpec> results = metadataToSpecs(top.metadata);
316+
std::vector<std::string> plainKeys;
317+
if (!top.tables.empty()) {
318+
results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, top.tables, {"Tables in first AOD"}});
319+
}
320+
if (!top.metadata.empty()) {
321+
// Found metadata already in the main file.
322+
results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
323+
plainKeys = metadataKeys(top.metadata);
324+
} else if (parentAccessLevel == 0) {
227325
LOGP(info, "No metadata found in file \"{}\" and parent level 0 prevents further lookup.", filename);
228326
results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
229-
return results;
327+
} else {
328+
// Lets try in parent file.
329+
LOGP(info, "No metadata found in file \"{}\", checking in its parents.", filename);
330+
auto const& parent = readLevel(1);
331+
if (!parent.found) {
332+
LOGP(info, "No metadata found in file \"{}\"", filename);
333+
} else {
334+
results = metadataToSpecs(parent.metadata);
335+
if (!parent.metadata.empty()) {
336+
if (!parent.tables.empty()) {
337+
results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, parent.tables, {"Tables in first AOD"}});
338+
}
339+
results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
340+
plainKeys = metadataKeys(parent.metadata);
341+
} else {
342+
LOGP(info, "No metadata found in file \"{}\" nor in its parent file \"{}\"", filename, parent.source);
343+
}
344+
}
345+
if (plainKeys.empty()) {
346+
results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
347+
}
230348
}
231349

232-
// Lets try in parent file.
233-
auto parentFiles = (TMap*)currentFile->Get("parentFiles");
234-
if (!parentFiles) {
235-
LOGP(info, "No metadata found in file \"{}\"", filename);
236-
results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
350+
if (!registry.hasOption("aod-origin-level-map")) {
237351
return results;
238352
}
239-
LOGP(info, "No metadata found in file \"{}\", checking in its parents.", filename);
240-
for (auto* p : *parentFiles) {
241-
std::string parentFilename = ((TPair*)p)->Value()->GetName();
242-
// Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be
243-
// a workflow option (because the metadata itself is potentially changing the topology).
244-
if (registry.isSet("aod-parent-base-path-replacement")) {
245-
auto parentFileReplacement = registry.get<std::string>("aod-parent-base-path-replacement");
246-
auto pos = parentFileReplacement.find(';');
247-
if (pos == std::string::npos) {
248-
throw std::runtime_error(fmt::format("Invalid syntax in aod-parent-base-path-replacement: \"{}\"", parentFileReplacement.c_str()));
249-
}
250-
auto from = parentFileReplacement.substr(0, pos);
251-
auto to = parentFileReplacement.substr(pos + 1);
252-
pos = parentFilename.find(from);
253-
if (pos != std::string::npos) {
254-
parentFilename.replace(pos, from.length(), to);
353+
354+
// The metadata of the level each origin reads, one row per origin. The columns are
355+
// all the keys found, including the plain ones, so that an empty cell always means
356+
// that the key is missing for that origin.
357+
auto levelMap = registry.get<LabeledArray<int>>("aod-origin-level-map");
358+
auto origins = levelMap.getLabelsRows();
359+
std::vector<std::string> keys = plainKeys;
360+
for (auto i = 0U; i < levelMap.rows(); ++i) {
361+
auto level = levelMap.get(i, 0U);
362+
if (level > parentAccessLevel) {
363+
LOGP(fatal, "Origin {} is mapped to parent level {}, but aod-parent-access-level is {}", origins[i], level, parentAccessLevel);
364+
}
365+
for (auto const& [key, value] : readLevel(level).metadata) {
366+
if (std::ranges::find(keys, key) == keys.end()) {
367+
keys.push_back(key);
255368
}
256369
}
370+
}
371+
auto sourceColumn = keys.size();
372+
keys.emplace_back("source");
373+
auto disableColumn = keys.size();
374+
keys.emplace_back("disable");
257375

258-
if (parentFilename.starts_with("alien://") && !gGrid) {
259-
TGrid::Connect("alien://");
376+
std::vector<std::string> values(origins.size() * keys.size());
377+
for (auto i = 0U; i < origins.size(); ++i) {
378+
auto const& info = readLevel(levelMap.get(i, 0U));
379+
auto* row = values.data() + i * keys.size();
380+
for (auto const& [key, value] : info.metadata) {
381+
row[std::ranges::find(keys, key) - keys.begin()] = value;
260382
}
261-
262-
std::unique_ptr<TFile> parentFile{TFile::Open(parentFilename.c_str())};
263-
if (parentFile.get() == nullptr) {
264-
LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename);
383+
row[sourceColumn] = info.source;
384+
if (info.metadata.empty()) {
385+
LOGP(info, "Metadata info missing for origin {}", origins[i]);
386+
row[disableColumn] = "1";
265387
}
266-
results = readMetadata(parentFile);
267-
// Found metadata already in the main file.
268-
if (!results.empty()) {
269-
auto tables = getListOfTables(parentFile);
270-
if (tables.empty() == false) {
271-
results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
272-
}
273-
results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
274-
return results;
388+
}
389+
LabeledArray<std::string> originMetadata{values.data(), static_cast<uint32_t>(origins.size()), static_cast<uint32_t>(keys.size()), origins, keys};
390+
results.push_back(ConfigParamSpec{"aod-metadata-origins", VariantType::LabeledArrayString, originMetadata, {"Metadata in AOD for each origin: one row per origin, one column per key"}});
391+
for (auto i = 0U; i < origins.size(); ++i) {
392+
auto const& tables = readLevel(levelMap.get(i, 0U)).tables;
393+
if (!tables.empty()) {
394+
results.push_back(ConfigParamSpec{"aod-metadata-origin-" + origins[i] + "-tables", VariantType::ArrayString, tables, {"Tables in the AOD read for this origin"}});
275395
}
276-
LOGP(info, "No metadata found in file \"{}\" nor in its parent file \"{}\"", filename, parentFilename);
277-
break;
278396
}
279-
results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
280397
return results;
281398
}};
282399
}

‎Framework/Core/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,7 @@ add_executable(o2-test-framework-core
217217
test/test_ControlServiceHelpers.cxx
218218
test/test_ConfigParamStore.cxx
219219
test/test_ConfigParamRegistry.cxx
220+
test/test_ConfigParamDiscovery.cxx
220221
test/test_DataDescriptorMatcher.cxx
221222
test/test_DataDescriptorQueryBuilder.cxx
222223
test/test_DataProcessingStats.cxx

‎Framework/Core/include/Framework/Array2D.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,13 @@ class LabeledArray : public LabelMap
241241
return values[y];
242242
}
243243

244+
// nullptr if there is no row with this label
245+
T* getRow(const char* y) const
246+
{
247+
auto it = rowmap.find(y);
248+
return it == rowmap.end() ? nullptr : values[it->second];
249+
}
250+
244251
T* operator[](uint32_t y) const
245252
{
246253
return values[y];

‎Framework/Core/include/Framework/ConfigParamDiscovery.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,26 @@
1313
#define O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_
1414

1515
#include "Framework/ConfigParamRegistry.h"
16+
#include "Framework/Array2D.h"
17+
18+
#include <string_view>
1619

1720
namespace o2::framework
1821
{
1922
// A plugin which can be used to inject extra configuration
2023
// options.
2124
struct ConfigParamDiscovery {
2225
static std::vector<ConfigParamSpec> discover(ConfigParamRegistry&, int, char**);
26+
27+
/// Parse the value of aod-origin-level-mapping, "ORIGIN:LEVEL[,ORIGIN2:LEVEL2,...]",
28+
/// into a LabeledArray with one row per origin and a single "level" column.
29+
/// Throws std::runtime_error if the mapping is malformed.
30+
static LabeledArray<int> parseOriginLevelMapping(std::string_view mapping);
31+
32+
/// Replace the plain aod-metadata-* options with the ones in the row of
33+
/// aod-metadata-origins which corresponds to the origin this process reads
34+
/// (aod-origin-replace, or AOD). Does nothing if there is no such row.
35+
static void applyOriginMetadata(ConfigParamRegistry& registry);
2336
};
2437

2538
struct ConfigDiscovery {

0 commit comments

Comments
 (0)