Add barrel reco configParams to AOD metadata. (#15645)

* Method to extract ConfigurableParam as JSON string and update from such string

* Add all barrel tracking configurables to AOD metadata

Add all barrel tracking configurables to AOD metadata:
write them as ConfigParam_<device>_<configname>.json
files from every device (1st lane only in pipelined)
Also, send them as META/<PROCNAME>/0 sporadic DPL output
only once for the direct collection by the metadata writer
(when it will be supported by the DPL).

* Option to create configParams metadata from json dumps

With the option `--collect-config-files` provided, the AODProducer will collect the json dumps of ConfigParams
made by the upstream processes and will add them as `<configName> : <configParam json>` TObjString pairs
in the AOD metadata.
Once the DPL supports collecting META/<PROCNAME>/0 sporadic inputs, the metadata will be created directly
from these inputs, so this option will not be needed for the data processing.
But in the case of MC, collecting json files via this option is the only way to add configs to the AOD metadata
(as it is done in the sim_challenge.sh).

Author: shahor02

Common/Utils/include/CommonUtils/ConfigurableParam.h

@@ -407,6 +407,8 @@ class ConfigurableParam
   // writes a human readable INI or JSON file depending on the extension
   static void write(std::string const& filename, std::string const& keyOnly = "");
 
+  static std::string asJSON(std::string const& keyOnly = "");
+
   // can be used instead of using API on concrete child classes
   template <typename T>
   static T getValueAs(std::string key)
@@ -494,6 +496,9 @@ class ConfigurableParam
   // be updated, absence of data for any of requested params will lead to fatal
   static void updateFromFile(std::string const&, std::string const& paramsList = "", bool unchangedOnly = false);
 
+  // update from a JSON string with the same filtering semantics as updateFromFile
+  static void updateFromJSONString(std::string const&, std::string const& paramsList = "", bool unchangedOnly = false);
+
   // interface for use from the CCDB API; allows to sync objects read from CCDB with the information
   // stored in the registry; modifies given object as well as registry
   virtual void syncCCDBandRegistry(void* obj) = 0;

Common/Utils/src/ConfigurableParam.cxx

@@ -37,6 +37,7 @@
 #endif
 #include <cassert>
 #include <iostream>
+#include <sstream>
 #include <string>
 #include <fairlogger/Logger.h>
 #include <typeindex>
@@ -754,6 +755,29 @@ void ConfigurableParam::writeJSON(std::string const& filename, std::string const
 
 // ------------------------------------------------------------------
 
+std::string ConfigurableParam::asJSON(std::string const& keyOnly)
+{
+  initPropertyTree(); // update the boost tree before writing
+  std::ostringstream os;
+  if (!keyOnly.empty()) { // write ini for selected key only
+    try {
+      boost::property_tree::ptree kTree;
+      auto keys = o2::utils::Str::tokenize(keyOnly, " ,;", true, true);
+      for (const auto& k : keys) {
+        kTree.add_child(k, sPtree->get_child(k));
+      }
+      boost::property_tree::write_json(os, kTree);
+    } catch (const boost::property_tree::ptree_bad_path& err) {
+      LOG(fatal) << "non-existing key " << keyOnly << " provided to writeJSON";
+    }
+  } else {
+    boost::property_tree::write_json(os, *sPtree);
+  }
+  return os.str();
+}
+
+// ------------------------------------------------------------------
+
 void ConfigurableParam::initPropertyTree()
 {
   sPtree->clear();
@@ -870,26 +894,10 @@ void ConfigurableParam::printAllRegisteredParamNames()
 
 // ------------------------------------------------------------------
 
-// Update the storage map of params from the given configuration file.
-// It can be in JSON or INI format.
-// If nonempty comma-separated paramsList is provided, only those params will
-// be updated, absence of data for any of requested params will lead to fatal
-// If unchangedOnly is true, then only those parameters whose provenance is kCODE will be updated
-// (to allow preference of run-time settings)
-void ConfigurableParam::updateFromFile(std::string const& configFile, std::string const& paramsList, bool unchangedOnly)
+namespace
+{
+void updateFromPropertyTree(boost::property_tree::ptree const& pt, std::string const& source, std::string const& paramsList, bool unchangedOnly)
 {
-  if (!sIsFullyInitialized) {
-    initialize();
-  }
-
-  auto cfgfile = o2::utils::Str::trim_copy(configFile);
-
-  if (cfgfile.length() == 0) {
-    return;
-  }
-
-  boost::property_tree::ptree pt = ConfigurableParamReaders::readConfigFile(cfgfile);
-
   std::vector<std::pair<std::string, std::string>> keyValPairs;
   auto request = o2::utils::Str::tokenize(paramsList, ',', true);
   std::unordered_map<std::string, int> requestMap;
@@ -913,7 +921,7 @@ void ConfigurableParam::updateFromFile(std::string const& configFile, std::strin
         auto name = subKey.first;
         auto value = subKey.second.get_value<std::string>();
         std::string key = mainKey + "." + name;
-        if (!unchangedOnly || getProvenance(key) == kCODE) {
+        if (!unchangedOnly || ConfigurableParam::getProvenance(key) == ConfigurableParam::kCODE) {
           std::pair<std::string, std::string> pair = std::make_pair(key, o2::utils::Str::trim_copy(value));
           keyValPairs.push_back(pair);
         }
@@ -928,16 +936,62 @@ void ConfigurableParam::updateFromFile(std::string const& configFile, std::strin
   // make sure all requested params were retrieved
   for (const auto& req : requestMap) {
     if (req.second == 0) {
-      throw std::runtime_error(fmt::format("Param {:s} was not found in {:s}", req.first, configFile));
+      throw std::runtime_error(fmt::format("Param {:s} was not found in {:s}", req.first, source));
     }
   }
 
   try {
-    setValues(keyValPairs);
+    ConfigurableParam::setValues(keyValPairs);
   } catch (std::exception const& error) {
     LOG(error) << "Error while setting values " << error.what();
   }
 }
+} // namespace
+
+// Update the storage map of params from the given configuration file.
+// It can be in JSON or INI format.
+// If nonempty comma-separated paramsList is provided, only those params will
+// be updated, absence of data for any of requested params will lead to fatal
+// If unchangedOnly is true, then only those parameters whose provenance is kCODE will be updated
+// (to allow preference of run-time settings)
+void ConfigurableParam::updateFromFile(std::string const& configFile, std::string const& paramsList, bool unchangedOnly)
+{
+  if (!sIsFullyInitialized) {
+    initialize();
+  }
+
+  auto cfgfile = o2::utils::Str::trim_copy(configFile);
+
+  if (cfgfile.length() == 0) {
+    return;
+  }
+
+  updateFromPropertyTree(ConfigurableParamReaders::readConfigFile(cfgfile), configFile, paramsList, unchangedOnly);
+}
+
+// ------------------------------------------------------------------
+
+void ConfigurableParam::updateFromJSONString(std::string const& configJSON, std::string const& paramsList, bool unchangedOnly)
+{
+  if (!sIsFullyInitialized) {
+    initialize();
+  }
+
+  auto json = o2::utils::Str::trim_copy(configJSON);
+  if (json.length() == 0) {
+    return;
+  }
+
+  boost::property_tree::ptree pt;
+  std::istringstream input(json);
+  try {
+    boost::property_tree::read_json(input, pt);
+  } catch (const boost::property_tree::ptree_error& e) {
+    LOG(fatal) << "Failed to read JSON config string (" << e.what() << ")";
+  }
+
+  updateFromPropertyTree(pt, "provided JSON string", paramsList, unchangedOnly);
+}
 
 // ------------------------------------------------------------------
 // ------------------------------------------------------------------

Common/Utils/test/testConfigurableParam.cxx

@@ -151,6 +151,42 @@ BOOST_AUTO_TEST_CASE(ConfigurableParam_FileIO_Json)
   std::remove(testFileName.c_str());
 }
 
+BOOST_AUTO_TEST_CASE(ConfigurableParam_JSONString_UnchangedOnly)
+{
+  ConfigurableParam::setValue("TestParam.ulValue", "2");
+  ConfigurableParam::setProvenance("TestParam", "lValue", ConfigurableParam::kCODE);
+  ConfigurableParam::setProvenance("TestParam", "ulValue", ConfigurableParam::kRT);
+  ConfigurableParam::updateFromJSONString(R"json({"TestParam":{"lValue":"77","ulValue":"88"}})json", "TestParam", true);
+
+  BOOST_CHECK_EQUAL(TestParam::Instance().lValue, 77);
+  BOOST_CHECK_EQUAL(TestParam::Instance().ulValue, 2);
+}
+
+BOOST_AUTO_TEST_CASE(ConfigurableParam_JSONString_FromAsJSON)
+{
+  ConfigurableParam::setValue("TestParam.iValue", "321");
+  ConfigurableParam::setValue("TestParam.sValue", "json-source");
+  ConfigurableParam::setValues({{"TestParam.map", "{7:8,9:10}"}});
+  const auto mapBefore = TestParam::Instance().map;
+  const auto json = ConfigurableParam::asJSON("TestParam");
+
+  ConfigurableParam::setValue("TestParam.iValue", "999");
+  ConfigurableParam::setValue("TestParam.sValue", "json-modified");
+  ConfigurableParam::setValues({{"TestParam.map", "{1:2}"}});
+  ConfigurableParam::updateFromJSONString(json, "TestParam");
+
+  BOOST_CHECK_EQUAL(TestParam::Instance().iValue, 321);
+  BOOST_CHECK_EQUAL(TestParam::Instance().sValue, "json-source");
+  BOOST_CHECK_EQUAL(TestParam::Instance().map.size(), mapBefore.size());
+  BOOST_CHECK_EQUAL(TestParam::Instance().map.at(7), mapBefore.at(7));
+  BOOST_CHECK_EQUAL(TestParam::Instance().map.at(9), mapBefore.at(9));
+}
+
+BOOST_AUTO_TEST_CASE(ConfigurableParam_JSONString_ParamsListMissing)
+{
+  BOOST_CHECK_THROW(ConfigurableParam::updateFromJSONString(ConfigurableParam::asJSON("TestParam"), "MissingParam"), std::runtime_error);
+}
+
 BOOST_AUTO_TEST_CASE(ConfigurableParam_FileIO_ROOT)
 {
   // test for root file serialization

Detectors/AOD/include/AODProducerWorkflow/AODProducerWorkflowSpec.h

@@ -246,6 +246,8 @@ class AODProducerWorkflowDPL : public Task
     return std::uint64_t(mStartIR.toLong()) + relativeTime_to_LocalBC(relativeTimeStampInNS);
   }
 
+  bool collectConfigFiles(std::vector<TString>& keys, std::vector<TString>& values, int indent = -1);
+
   bool mThinTracks{false};
   bool mPropTracks{false};
   bool mPropMuons{false};
@@ -280,6 +282,7 @@ class AODProducerWorkflowDPL : public Task
   bool mEnableFITextra = false;
   bool mEnableTRDextra = false;
   bool mFieldON = false;
+  bool mCollectConfigFiles = false;
   const float cSpeed = 0.029979246f; // speed of light in TOF units
 
   GID::mask_t mInputSources;

Detectors/AOD/src/AODProducerWorkflowSpec.cxx

@@ -54,6 +54,8 @@
 #include "Framework/TableBuilder.h"
 #include "Framework/CCDBParamSpec.h"
 #include "CommonUtils/TreeStreamRedirector.h"
+#include "CommonUtils/KeyValParam.h"
+#include "CommonUtils/NameConf.h"
 #include "FT0Base/Geometry.h"
 #include "GlobalTracking/MatchTOF.h"
 #include "ReconstructionDataFormats/Cascade.h"
@@ -88,6 +90,7 @@
 #include "MathUtils/Utils.h"
 #include "Math/SMatrix.h"
 #include "TString.h"
+#include <fnmatch.h>
 #include <limits>
 #include <map>
 #include <numeric>
@@ -1902,6 +1905,8 @@ void AODProducerWorkflowDPL::init(InitContext& ic)
 
   mUseSigFiltMC = ic.options().get<bool>("mc-signal-filt");
 
+  mCollectConfigFiles = ic.options().get<bool>("collect-config-files");
+
   // set no truncation if selected by user
   if (mTruncate != 1) {
     LOG(info) << "Truncation is not used!";
@@ -2670,6 +2675,10 @@ void AODProducerWorkflowDPL::run(ProcessingContext& pc)
   mMetaDataVals = {dataType, "3", O2Version, ROOTVersion, mRecoPass, mAnchorProd, mAnchorPass, mLPMProdTag, mUser};
   add_additional_meta_info(mMetaDataKeys, mMetaDataVals);
 
+  if (mCollectConfigFiles) {
+    collectConfigFiles(mMetaDataKeys, mMetaDataVals);
+  }
+
   pc.outputs().snapshot(Output{"AMD", "AODMetadataKeys", 0}, mMetaDataKeys);
   pc.outputs().snapshot(Output{"AMD", "AODMetadataVals", 0}, mMetaDataVals);
 
@@ -3405,6 +3414,102 @@ std::vector<uint8_t> AODProducerWorkflowDPL::fillBCFlags(const o2::globaltrackin
   return flags;
 }
 
+bool AODProducerWorkflowDPL::collectConfigFiles(std::vector<TString>& keys, std::vector<TString>& values, int indent)
+{
+  // collect JSON-files of ConfigParams dumped by different upstream processors and add to medata
+  static std::string pattern, directory;
+  static size_t cachedNumberOfFiles = 0, cachedTotalFileSize = 0;
+  static bool first = true, discard = false;
+  if (discard) {
+    return false;
+  }
+  std::error_code ec;
+  if (first) {
+    first = false;
+    pattern = o2::base::NameConf::Instance().getConfigOutputFileName("*");
+    auto dir = o2::conf::KeyValParam::Instance().getOutputDir();
+    if (dir == "/dev/null") {
+      LOGP(warn, "ConfigParams output is disabled, abandoning {} files collection for metadata", pattern);
+      discard = true;
+      return false;
+    }
+    directory = (dir.empty() || dir == "none") ? "." : dir;
+    if (!std::filesystem::is_directory(directory, ec)) {
+      LOGP(error, R"(No directory "{}" is found to look for {} configuration files)", directory, pattern);
+      discard = true;
+      return false;
+    }
+  }
+  static std::unordered_map<std::string, std::string> cachedMap;
+  std::vector<std::filesystem::path> files;
+  size_t currentTotalFileSize = 0;
+
+  for (const auto& entry : std::filesystem::directory_iterator(directory)) {
+    if (!entry.is_regular_file()) {
+      continue;
+    }
+    const std::string fileName = entry.path().filename().string();
+    if (fnmatch(pattern.c_str(), fileName.c_str(), 0) != 0) {
+      continue;
+    }
+    const auto fileSize = entry.file_size(ec);
+    if (ec) {
+      LOGP(error, "Cannot determine size of file {}, reason: {}", entry.path().string(), ec.message());
+    }
+    files.push_back(entry.path());
+    currentTotalFileSize += static_cast<size_t>(fileSize);
+  }
+
+  if (files.size() != cachedNumberOfFiles || currentTotalFileSize != cachedTotalFileSize) { // need to create a new map
+    cachedNumberOfFiles = files.size();
+    cachedTotalFileSize = currentTotalFileSize;
+    cachedMap.clear();
+  }
+
+  if (!files.empty() && cachedMap.empty()) {
+    for (const auto& fname : files) {
+      std::ifstream input(fname);
+      if (!input) {
+        LOGP(error, "Cannot open JSON file {}", fname.string());
+        cachedTotalFileSize = 0; // will trigger a new trial next time
+        continue;
+      }
+      nlohmann::json document;
+      try {
+        input >> document;
+      } catch (const nlohmann::json::parse_error& e) {
+        LOGP(error, "Cannot parse JSON file {}, reason: {}", fname.string(), e.what());
+        cachedTotalFileSize = 0; // will trigger a new trial next time
+        continue;
+      }
+
+      if (!document.is_object()) {
+        LOGP(error, "Top-level JSON value is not an object in file: {}", fname.string());
+        cachedTotalFileSize = 0; // will trigger a new trial next time
+        continue;
+      }
+
+      for (auto it = document.begin(); it != document.end(); ++it) {
+        const std::string& key = it.key();
+        if (cachedMap.find(key) != cachedMap.end()) {
+          LOGP(error, "Duplicate top-level key {} in file {}", key, fname.string());
+          continue;
+        }
+        LOGP(info, "Adding json config {} from file {} to AOD metadata", key, fname.string());
+        nlohmann::json valueDocument = nlohmann::json::object();
+        valueDocument[key] = it.value();
+        cachedMap[key] = valueDocument.dump(indent);
+      }
+    }
+  }
+
+  for (const auto& kv : cachedMap) {
+    keys.push_back(kv.first.c_str());
+    values.push_back(kv.second.c_str());
+  }
+  return true;
+}
+
 void AODProducerWorkflowDPL::endOfStream(EndOfStreamContext& /*ec*/)
 {
   LOGF(info, "aod producer dpl total timing: Cpu: %.3e Real: %.3e s in %d slots",
@@ -3565,7 +3670,9 @@ DataProcessorSpec getAODProducerWorkflowSpec(GID::mask_t src, bool enableSV, boo
       ConfigParamSpec{"trackqc-tpc-pt", VariantType::Float, 0.2f, {"Keep TPC standalone track with this pt"}},
       ConfigParamSpec{"with-streamers", VariantType::String, "", {"Bit-mask to steer writing of intermediate streamer files"}},
       ConfigParamSpec{"seed", VariantType::Int, 0, {"Set seed for random generator used for sampling (0 (default) means using a random_device)"}},
-      ConfigParamSpec{"mc-signal-filt", VariantType::Bool, false, {"Enable usage of signal filtering (only for MC with embedding)"}}}};
+      ConfigParamSpec{"mc-signal-filt", VariantType::Bool, false, {"Enable usage of signal filtering (only for MC with embedding)"}},
+      ConfigParamSpec{"collect-config-files", VariantType::Bool, false, {"Collect ConfigParams json files written by upsteam processors"}},
+    }};
 }
 
 } // namespace o2::aodproducer

Detectors/GlobalTrackingWorkflow/include/GlobalTrackingWorkflow/StrangenessTrackingSpec.h

@@ -50,6 +50,7 @@ class StrangenessTrackerSpec : public framework::Task
 
  private:
   void updateTimeDependentParams(framework::ProcessingContext& pc);
+  void storeConfigs(framework::ProcessingContext& pc);
 
   bool mUseMC = false;
   TStopwatch mTimer;