From 52d21bd3f18f5909ed5fde1f677e45e6c01cba70 Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Thu, 18 Nov 2021 15:09:42 +0100 Subject: [PATCH 1/5] [QC-652] Add an executable to update the metadata of an object. --- Framework/CMakeLists.txt | 9 ++- Framework/src/runMetadataUpdater.cxx | 90 ++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 Framework/src/runMetadataUpdater.cxx diff --git a/Framework/CMakeLists.txt b/Framework/CMakeLists.txt index 972ff87fce..e58873987d 100644 --- a/Framework/CMakeLists.txt +++ b/Framework/CMakeLists.txt @@ -137,7 +137,8 @@ set(EXE_SRCS src/runLocationCalculator.cxx src/runMergerCalculator.cxx src/runUploadRootObjects.cxx - src/runFileMerger.cxx) + src/runFileMerger.cxx + src/runMetadataUpdater.cxx) set(EXE_NAMES o2-qc-run-producer @@ -153,7 +154,8 @@ set(EXE_NAMES o2-qc-location-calculator o2-qc-merger-calculator o2-qc-upload-root-objects - o2-qc-file-merger) + o2-qc-file-merger + o2-qc-metadata-updater) # These were the original names before the convention changed. We will get rid # of them but for the time being we want to create symlinks to avoid confusion. @@ -172,7 +174,8 @@ set(EXE_OLD_NAMES o2-qc-location-calculator o2-qc-merger-calculator o2-qc-upload-root-objects - o2-qc-file-merger) + o2-qc-file-merger + o2-qc-metadata-updater) # As per https://stackoverflow.com/questions/35765106/symbolic-links-cmake macro(install_symlink filepath sympath) diff --git a/Framework/src/runMetadataUpdater.cxx b/Framework/src/runMetadataUpdater.cxx new file mode 100644 index 0000000000..1e41a64bf2 --- /dev/null +++ b/Framework/src/runMetadataUpdater.cxx @@ -0,0 +1,90 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file runMetadataUpdater.cxx +/// \author Barthelemy von Haller +/// +/// \brief Easily update the metadata of an object in the QCDB. + +#include +#include +#include +#include +#include +#include +#include + +namespace bpo = boost::program_options; +using namespace std; + +int main(int argc, const char* argv[]) +{ + try { + bpo::options_description desc{ "Options" }; + desc.add_options() + ("help,h", "Help screen") + ("url,u", bpo::value()->required(), "URL to the QCDB") + ("path,p", bpo::value()->required(), "Path to the object to update") + ("timestamp,t", bpo::value()->default_value(o2::ccdb::getCurrentTimestamp()), "Timestamp to select the object") + ("id", bpo::value()->default_value(""), "Id of the object to select") + ("pair", bpo::value< vector >()->required(), "Key-value pair to update the metadata (e.g. --pair \"1,oil\", can be added multiple times)"); + + bpo::variables_map vm; + store(parse_command_line(argc, argv, desc), vm); + notify(vm); + + if (vm.count("help")) { + std::cout << desc << std::endl; + return 0; + } + + const auto url = vm["url"].as(); + const auto path = vm["path"].as(); + const auto timestamp = vm["timestamp"].as(); + const auto id = vm["id"].as(); + const auto pairs = vm["pair"].as>(); + + // prepare the key value map + map metadata; + for(auto p: pairs) { + if(p.find(',') results; + boost::algorithm::split(results, p, boost::is_any_of(",")); + metadata[results[0]] = results[1]; + } + } + + std::cout << "PARAMETERS" << std::endl; + std::cout << "url................" << url << std::endl; + std::cout << "path,.............." << path << std::endl; + std::cout << "timestamp.........." << timestamp << std::endl; + std::cout << "id................." << id << std::endl; + std::cout << "pairs" << std::endl; + for(auto m : metadata) { + std::cout << " |........" << m.first << " -> " << m.second << endl; + } + std::cout << std::endl; + + o2::ccdb::CcdbApi api; + api.init(url); + api.updateMetadata(path, metadata, timestamp, id); + + return 0; + } catch (const bpo::error& ex) { + std::cerr << "Exception caught: " << ex.what() << std::endl; + return 1; + } + + return 0; +} + +// -0.378347x + 2782.30 \ No newline at end of file From df8980ee51f225044bc998f2fa6b79d010f180e6 Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Thu, 18 Nov 2021 15:18:36 +0100 Subject: [PATCH 2/5] Add an example --- Framework/src/CcdbDatabase.cxx | 1 - Framework/src/runMetadataUpdater.cxx | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Framework/src/CcdbDatabase.cxx b/Framework/src/CcdbDatabase.cxx index 65884ecfac..3a12682561 100644 --- a/Framework/src/CcdbDatabase.cxx +++ b/Framework/src/CcdbDatabase.cxx @@ -35,7 +35,6 @@ #include #include // boost -#include #include #include #include diff --git a/Framework/src/runMetadataUpdater.cxx b/Framework/src/runMetadataUpdater.cxx index 1e41a64bf2..a0267c57a7 100644 --- a/Framework/src/runMetadataUpdater.cxx +++ b/Framework/src/runMetadataUpdater.cxx @@ -13,7 +13,9 @@ /// \file runMetadataUpdater.cxx /// \author Barthelemy von Haller /// -/// \brief Easily update the metadata of an object in the QCDB. +/// \brief Easily update the metadata of an object in the QCDB or add new metadata if it does not exist yet. +/// +/// Example: o2-qc-metadata-updater --url ccdb-test.cern.ch:8080 --path Test/pid61065/Test --pair something,else --id 8b9728fe-486b-11ec-afda-2001171b226b --pair key1,value1 #include #include From 5f7f44b9664598f0b9c3cd8f7ebbce2a13b41562 Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Mon, 22 Nov 2021 08:22:15 +0100 Subject: [PATCH 3/5] format --- Framework/src/runMetadataUpdater.cxx | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/Framework/src/runMetadataUpdater.cxx b/Framework/src/runMetadataUpdater.cxx index a0267c57a7..3b694686b1 100644 --- a/Framework/src/runMetadataUpdater.cxx +++ b/Framework/src/runMetadataUpdater.cxx @@ -32,13 +32,7 @@ int main(int argc, const char* argv[]) { try { bpo::options_description desc{ "Options" }; - desc.add_options() - ("help,h", "Help screen") - ("url,u", bpo::value()->required(), "URL to the QCDB") - ("path,p", bpo::value()->required(), "Path to the object to update") - ("timestamp,t", bpo::value()->default_value(o2::ccdb::getCurrentTimestamp()), "Timestamp to select the object") - ("id", bpo::value()->default_value(""), "Id of the object to select") - ("pair", bpo::value< vector >()->required(), "Key-value pair to update the metadata (e.g. --pair \"1,oil\", can be added multiple times)"); + desc.add_options()("help,h", "Help screen")("url,u", bpo::value()->required(), "URL to the QCDB")("path,p", bpo::value()->required(), "Path to the object to update")("timestamp,t", bpo::value()->default_value(o2::ccdb::getCurrentTimestamp()), "Timestamp to select the object")("id", bpo::value()->default_value(""), "Id of the object to select")("pair", bpo::value>()->required(), "Key-value pair to update the metadata (e.g. --pair \"1,oil\", can be added multiple times)"); bpo::variables_map vm; store(parse_command_line(argc, argv, desc), vm); @@ -57,8 +51,8 @@ int main(int argc, const char* argv[]) // prepare the key value map map metadata; - for(auto p: pairs) { - if(p.find(',') results; boost::algorithm::split(results, p, boost::is_any_of(",")); metadata[results[0]] = results[1]; @@ -70,8 +64,8 @@ int main(int argc, const char* argv[]) std::cout << "path,.............." << path << std::endl; std::cout << "timestamp.........." << timestamp << std::endl; std::cout << "id................." << id << std::endl; - std::cout << "pairs" << std::endl; - for(auto m : metadata) { + std::cout << "pairs" << std::endl; + for (auto m : metadata) { std::cout << " |........" << m.first << " -> " << m.second << endl; } std::cout << std::endl; From 0d6e1ef3cc735f42087a0ace2a39b953dfa7a26c Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Mon, 22 Nov 2021 11:41:18 +0100 Subject: [PATCH 4/5] Add support for escaped commas --- Framework/src/runMetadataUpdater.cxx | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/Framework/src/runMetadataUpdater.cxx b/Framework/src/runMetadataUpdater.cxx index 3b694686b1..2e6ef8599c 100644 --- a/Framework/src/runMetadataUpdater.cxx +++ b/Framework/src/runMetadataUpdater.cxx @@ -16,6 +16,9 @@ /// \brief Easily update the metadata of an object in the QCDB or add new metadata if it does not exist yet. /// /// Example: o2-qc-metadata-updater --url ccdb-test.cern.ch:8080 --path Test/pid61065/Test --pair something,else --id 8b9728fe-486b-11ec-afda-2001171b226b --pair key1,value1 +/// +/// Note: commas can be escaped if they must be part of the key: "my,key" --> "my\\,key". Note that it needs double escaping. +/// commas don't have to escaped in the value. #include #include @@ -49,14 +52,21 @@ int main(int argc, const char* argv[]) const auto id = vm["id"].as(); const auto pairs = vm["pair"].as>(); - // prepare the key value map + // prepare the key value map, take into account escaped commas map metadata; for (auto p : pairs) { - if (p.find(',') < p.length()) { - std::vector results; - boost::algorithm::split(results, p, boost::is_any_of(",")); - metadata[results[0]] = results[1]; + size_t hit = -1; // on purpose ... don't worry + do { // make sure we ignore the escaped commas + hit = p.find(',', hit+1); + } while(hit != string::npos && hit > 0 && p.at(hit-1) == '\\'); + if (hit == string::npos) { + continue; } + metadata[p.substr(0,hit)] = p.substr(hit+1); + } + if (metadata.empty()) { + cout << "No proper pairs found, aborting." << endl; + return -1; } std::cout << "PARAMETERS" << std::endl; From 9ec2af8ab752c1de03897d4f53d33b23e1e4f810 Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Mon, 22 Nov 2021 13:28:53 +0100 Subject: [PATCH 5/5] format --- Framework/src/runMetadataUpdater.cxx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Framework/src/runMetadataUpdater.cxx b/Framework/src/runMetadataUpdater.cxx index 2e6ef8599c..4804cd3595 100644 --- a/Framework/src/runMetadataUpdater.cxx +++ b/Framework/src/runMetadataUpdater.cxx @@ -56,13 +56,13 @@ int main(int argc, const char* argv[]) map metadata; for (auto p : pairs) { size_t hit = -1; // on purpose ... don't worry - do { // make sure we ignore the escaped commas - hit = p.find(',', hit+1); - } while(hit != string::npos && hit > 0 && p.at(hit-1) == '\\'); + do { // make sure we ignore the escaped commas + hit = p.find(',', hit + 1); + } while (hit != string::npos && hit > 0 && p.at(hit - 1) == '\\'); if (hit == string::npos) { continue; } - metadata[p.substr(0,hit)] = p.substr(hit+1); + metadata[p.substr(0, hit)] = p.substr(hit + 1); } if (metadata.empty()) { cout << "No proper pairs found, aborting." << endl;