From 49d79c51191770bf1f32e4766fab1370282cc117 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Mon, 22 Aug 2022 10:56:04 +0200 Subject: [PATCH 001/126] 78 - edit smake file and add CtaStorageClass.spp --- cmdline/CMakeLists.txt | 8 +++ cmdline/CtaChangeStorageClass.cpp | 85 +++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 cmdline/CtaChangeStorageClass.cpp diff --git a/cmdline/CMakeLists.txt b/cmdline/CMakeLists.txt index 91d0a453fe..f2c312aad2 100644 --- a/cmdline/CMakeLists.txt +++ b/cmdline/CMakeLists.txt @@ -36,6 +36,14 @@ add_executable(cta-admin CtaAdminCmd.cpp CtaAdminCmdParse.cpp CtaAdminTextFormat target_link_libraries(cta-admin XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon) set_property (TARGET cta-admin APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) +# +# cta-change-storage-class +# +# +add_executable(cta-change-storage-class CtaChangeStorageClass.cpp) +target_link_libraries(cta-change-storage-class) +set_property(TARGET cta-change-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) + # # cta-send-event CLOSEW|PREPARE|ABORT_PREPARE # injects a workflow event into the CTA Frontend diff --git a/cmdline/CtaChangeStorageClass.cpp b/cmdline/CtaChangeStorageClass.cpp new file mode 100644 index 0000000000..c1460ffca8 --- /dev/null +++ b/cmdline/CtaChangeStorageClass.cpp @@ -0,0 +1,85 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranPty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#include <stdexcept> +#include <iostream> +#include <map> + +#include "cta_admin.pb.h" +#include "CtaCmdOptions.hpp" + +namespace cta { +namespace admin { + + +/* + * Type aliases + */ +using cmd_val_t = std::vector<Option>; + +// Usage exception +const std::runtime_error Usage("Usage: ..."); + +StringOption files_to_be_changed {"--files", "-f", false}; + +/*! + * Map string list options to Protocol Buffer enum values + */ +const std::map<std::string, OptionStrList::Key> strListOptions = { + { "--fxidfile", OptionStrList::FILE_ID }, + { "--vidfile", OptionStrList::VID } +}; + +void parseOptions(int start, int argc, const char *const *const argv, const cmd_val_t &options) +{ + for(int i = start; i < argc; ++i) + { + int opt_num = i-start; + + cmd_val_t::const_iterator opt_it; + + // Scan options for a match + + for(opt_it = options.begin(); opt_it != options.end(); ++opt_it) { + // Special case of OPT_CMD type has an implicit key + if(opt_num-- == 0 && opt_it->get_type() == Option::OPT_CMD) break; + + if(*opt_it == argv[i]) break; + } + if(opt_it == options.end()) { + throwUsage(std::string("Invalid option: ") + argv[i]); + } + if((i += opt_it->num_params()) == argc) { + throw std::runtime_error(std::string(argv[i-1]) + " expects a parameter: " + opt_it->help()); + } + + std::cout << *opt_it + } +} + + +/* + * Start here + */ +int main(int argc, const char **argv) +{ + parse_cmd(1, argc, argv, strListOptions); + return 1; +} + +} // namespace admin +} // namespace cta \ No newline at end of file -- GitLab From f5ddaa77375785c15525d498920cc180ef9f687c Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Mon, 22 Aug 2022 17:41:19 +0200 Subject: [PATCH 002/126] 78 - Add simpler string list option --- cmdline/CtaChangeStorageClass.cpp | 90 +++++++++++++++---------------- cmdline/CtaCmdOptions.hpp | 77 ++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 45 deletions(-) diff --git a/cmdline/CtaChangeStorageClass.cpp b/cmdline/CtaChangeStorageClass.cpp index c1460ffca8..ac0e810420 100644 --- a/cmdline/CtaChangeStorageClass.cpp +++ b/cmdline/CtaChangeStorageClass.cpp @@ -19,67 +19,67 @@ #include <iostream> #include <map> -#include "cta_admin.pb.h" #include "CtaCmdOptions.hpp" -namespace cta { -namespace admin { - - -/* - * Type aliases - */ -using cmd_val_t = std::vector<Option>; // Usage exception const std::runtime_error Usage("Usage: ..."); -StringOption files_to_be_changed {"--files", "-f", false}; +StringListOption option_fxid {"--fxid", "-fxid", true}; +StringListOption option_fid {"--fid", "-fid", true}; -/*! - * Map string list options to Protocol Buffer enum values - */ -const std::map<std::string, OptionStrList::Key> strListOptions = { - { "--fxidfile", OptionStrList::FILE_ID }, - { "--vidfile", OptionStrList::VID } -}; - -void parseOptions(int start, int argc, const char *const *const argv, const cmd_val_t &options) -{ - for(int i = start; i < argc; ++i) - { - int opt_num = i-start; - cmd_val_t::const_iterator opt_it; - - // Scan options for a match - - for(opt_it = options.begin(); opt_it != options.end(); ++opt_it) { - // Special case of OPT_CMD type has an implicit key - if(opt_num-- == 0 && opt_it->get_type() == Option::OPT_CMD) break; +std::map<std::string, StringListOption*> option_map = { + {"-fxid", &option_fxid}, + {"-fid", &option_fid}, + {"--fxid", &option_fxid}, + {"--fid", &option_fid} +}; - if(*opt_it == argv[i]) break; - } - if(opt_it == options.end()) { - throwUsage(std::string("Invalid option: ") + argv[i]); - } - if((i += opt_it->num_params()) == argc) { - throw std::runtime_error(std::string(argv[i-1]) + " expects a parameter: " + opt_it->help()); - } +void validate_cmd() { + for (auto &it: option_map) { + auto option = it.second; + if (!option->is_optional() && !option->is_present()) { + std::cout << "Error: Option " << option->get_name() << " is mandatory but not present." << std::endl; + throw Usage; + } + } +} - std::cout << *opt_it - } +void parse_cmd(const int argc, const char *const *const argv) { + unsigned int num_values_for_option = 1; + for (int i = 1; i < argc; i += num_values_for_option) { + auto search = option_map.find(argv[i]); + if (search == option_map.end()) { + std::cout << "Error: Unknown option: " << argv[i] << std::endl; + throw Usage; + } + + auto option = search->second; + option->set_present(); + + for (int j = 1; j < argc - i + 1; j += 1) { + option->add_value(argv[i + j]); + bool end_of_option = option_map.find(argv[i + j]) == option_map.end(); + std::cout << argc << std::endl; + if (end_of_option) + { + num_values_for_option = j + 1; + break; + } + } + } } - /* * Start here */ int main(int argc, const char **argv) { - parse_cmd(1, argc, argv, strListOptions); + parse_cmd(argc, argv); + validate_cmd(); + + std::cout << option_fid.get_value()[0]; + return 1; } - -} // namespace admin -} // namespace cta \ No newline at end of file diff --git a/cmdline/CtaCmdOptions.hpp b/cmdline/CtaCmdOptions.hpp index 8a3e5ed164..ae4e934f69 100644 --- a/cmdline/CtaCmdOptions.hpp +++ b/cmdline/CtaCmdOptions.hpp @@ -17,6 +17,9 @@ #pragma once +#include <string> +#include <vector> + /*! * Simple command line option class for simpler commands like cta-verify-file or cta-send-event */ @@ -92,3 +95,77 @@ private: }; +/*! +* Simple command line option class for simpler commands like cta-verify-file or cta-send-event +*/ +class StringListOption { +public: + /*! + * Constructor + */ + StringListOption(const std::string &long_opt, const std::string &short_opt, const bool is_optional) : + m_long_opt(long_opt), + m_short_opt(short_opt), + m_is_optional(is_optional), + m_is_present(false) {} + + /*! + * Check if the supplied key matches the option + */ + bool operator==(const std::string &option) const { + return option == m_short_opt || option == m_long_opt; + } + + /*! + * Return whether the option is optional + */ + bool is_optional() const { + return m_is_optional; + } + + /*! + * Return whether the option is present on the command line + */ + bool is_present() const { + return m_is_present; + } + + /*! + * Sets the option as present on the command line + */ + void set_present() { + m_is_present = true; + } + + /*! + * Return the value of the option given from the command line + */ + const std::vector<std::string> &get_value() const { + return m_value_list; + } + + /*! + * Return the name of the option (it's long command option) + */ + const std::string &get_name() const { + return m_long_opt; + } + + /*! + * Sets the value of the option from the command line + */ + void add_value(const std::string &mValue) { + m_value_list.push_back(mValue); + } + +private: + + //member variables + const std::string m_long_opt; //!< Long command option + const std::string m_short_opt; //!< Short command option + const bool m_is_optional; //!< Option is optional or compulsory + bool m_is_present; //!< Option is present on the command line + std::vector<std::string> m_value_list; //!< Option value +}; + + -- GitLab From 8ebf6daa12c44f1f8bec8759e62f2d9b663eca0f Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 23 Aug 2022 16:52:16 +0200 Subject: [PATCH 003/126] 78 - add parsing of arguments --- cmdline/CtaChangeStorageClass.cpp | 32 ++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/cmdline/CtaChangeStorageClass.cpp b/cmdline/CtaChangeStorageClass.cpp index ac0e810420..0ef3e901b8 100644 --- a/cmdline/CtaChangeStorageClass.cpp +++ b/cmdline/CtaChangeStorageClass.cpp @@ -25,15 +25,12 @@ // Usage exception const std::runtime_error Usage("Usage: ..."); -StringListOption option_fxid {"--fxid", "-fxid", true}; -StringListOption option_fid {"--fid", "-fid", true}; +StringListOption option_vid {"--vid", "-v", true}; std::map<std::string, StringListOption*> option_map = { - {"-fxid", &option_fxid}, - {"-fid", &option_fid}, - {"--fxid", &option_fxid}, - {"--fid", &option_fid} + {"--vid", &option_vid}, + {"-v", &option_vid}, }; void validate_cmd() { @@ -49,6 +46,7 @@ void validate_cmd() { void parse_cmd(const int argc, const char *const *const argv) { unsigned int num_values_for_option = 1; for (int i = 1; i < argc; i += num_values_for_option) { + num_values_for_option = 1; auto search = option_map.find(argv[i]); if (search == option_map.end()) { std::cout << "Error: Unknown option: " << argv[i] << std::endl; @@ -58,19 +56,23 @@ void parse_cmd(const int argc, const char *const *const argv) { auto option = search->second; option->set_present(); - for (int j = 1; j < argc - i + 1; j += 1) { - option->add_value(argv[i + j]); - bool end_of_option = option_map.find(argv[i + j]) == option_map.end(); - std::cout << argc << std::endl; - if (end_of_option) - { - num_values_for_option = j + 1; - break; + if (option->get_name() == "--vid") + { + int num_arg_left = argc - i; + for (int j = 1; j < num_arg_left; j += 1) { + bool end_of_option = (option_map.find(argv[i + j]) != option_map.end()); + option->add_value(argv[i + j]); + if (end_of_option || j == (num_arg_left - 1)) + { + num_values_for_option = j + 1; + break; + } } } } } + /* * Start here */ @@ -79,7 +81,7 @@ int main(int argc, const char **argv) parse_cmd(argc, argv); validate_cmd(); - std::cout << option_fid.get_value()[0]; + std::cout << option_vid.get_value().size() << std::endl; return 1; } -- GitLab From 8aae23d8afa4186b3e05067ef9bbb05a9fe38773 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 23 Aug 2022 17:11:23 +0200 Subject: [PATCH 004/126] 78 - edit formatiing --- cmdline/CtaChangeStorageClass.cpp | 80 +++++++++++++++---------------- 1 file changed, 38 insertions(+), 42 deletions(-) diff --git a/cmdline/CtaChangeStorageClass.cpp b/cmdline/CtaChangeStorageClass.cpp index 0ef3e901b8..d2ae7c3b73 100644 --- a/cmdline/CtaChangeStorageClass.cpp +++ b/cmdline/CtaChangeStorageClass.cpp @@ -21,67 +21,63 @@ #include "CtaCmdOptions.hpp" - // Usage exception const std::runtime_error Usage("Usage: ..."); -StringListOption option_vid {"--vid", "-v", true}; - +StringListOption option_vid{"--vid", "-v", true}; -std::map<std::string, StringListOption*> option_map = { - {"--vid", &option_vid}, - {"-v", &option_vid}, +std::map<std::string, StringListOption *> option_map = { + {"--vid", &option_vid}, + {"-v", &option_vid}, }; void validate_cmd() { - for (auto &it: option_map) { - auto option = it.second; - if (!option->is_optional() && !option->is_present()) { - std::cout << "Error: Option " << option->get_name() << " is mandatory but not present." << std::endl; - throw Usage; - } - } + for (auto &it : option_map) { + auto option = it.second; + if (!option->is_optional() && !option->is_present()) { + std::cout << "Error: Option " << option->get_name() << " is mandatory but not present." << std::endl; + throw Usage; + } + } } void parse_cmd(const int argc, const char *const *const argv) { - unsigned int num_values_for_option = 1; - for (int i = 1; i < argc; i += num_values_for_option) { - num_values_for_option = 1; - auto search = option_map.find(argv[i]); - if (search == option_map.end()) { - std::cout << "Error: Unknown option: " << argv[i] << std::endl; - throw Usage; - } + unsigned int num_values_for_option = 1; + for (int i = 1; i < argc; i += num_values_for_option){ + num_values_for_option = 1; + auto search = option_map.find(argv[i]); + if (search == option_map.end()) { + std::cout << "Error: Unknown option: " << argv[i] << std::endl; + throw Usage; + } + + auto option = search->second; + option->set_present(); - auto option = search->second; - option->set_present(); - - if (option->get_name() == "--vid") - { - int num_arg_left = argc - i; - for (int j = 1; j < num_arg_left; j += 1) { - bool end_of_option = (option_map.find(argv[i + j]) != option_map.end()); - option->add_value(argv[i + j]); - if (end_of_option || j == (num_arg_left - 1)) - { - num_values_for_option = j + 1; - break; - } - } - } + if (option->get_name() == "--vid") { + int num_arg_left = argc - i; + for (int j = 1; j < num_arg_left; j += 1) { + bool end_of_option = (option_map.find(argv[i + j]) != option_map.end()); + option->add_value(argv[i + j]); + if (end_of_option || j == (num_arg_left - 1)) + { + num_values_for_option = j + 1; + break; + } + } } + } } - /* * Start here */ int main(int argc, const char **argv) { - parse_cmd(argc, argv); - validate_cmd(); + parse_cmd(argc, argv); + validate_cmd(); - std::cout << option_vid.get_value().size() << std::endl; + std::cout << option_vid.get_value().size() << std::endl; - return 1; + return 1; } -- GitLab From 23e820c87c9c3894f1e89e12ee8814a75892bb8c Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 23 Aug 2022 17:20:07 +0200 Subject: [PATCH 005/126] #78 - update file name --- ...taChangeStorageClass.cpp => CtaUpdateStorageClass.cpp} | 8 ++++++++ 1 file changed, 8 insertions(+) rename cmdline/{CtaChangeStorageClass.cpp => CtaUpdateStorageClass.cpp} (98%) diff --git a/cmdline/CtaChangeStorageClass.cpp b/cmdline/CtaUpdateStorageClass.cpp similarity index 98% rename from cmdline/CtaChangeStorageClass.cpp rename to cmdline/CtaUpdateStorageClass.cpp index d2ae7c3b73..684eed95e8 100644 --- a/cmdline/CtaChangeStorageClass.cpp +++ b/cmdline/CtaUpdateStorageClass.cpp @@ -31,6 +31,7 @@ std::map<std::string, StringListOption *> option_map = { {"-v", &option_vid}, }; + void validate_cmd() { for (auto &it : option_map) { auto option = it.second; @@ -41,6 +42,7 @@ void validate_cmd() { } } + void parse_cmd(const int argc, const char *const *const argv) { unsigned int num_values_for_option = 1; for (int i = 1; i < argc; i += num_values_for_option){ @@ -69,6 +71,12 @@ void parse_cmd(const int argc, const char *const *const argv) { } } + +void updateExtAttr() { + +} + + /* * Start here */ -- GitLab From 35d0d184de0ae240a4655c47463aba55ff1ce3e8 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 23 Aug 2022 17:24:45 +0200 Subject: [PATCH 006/126] #78 - tabs to spaces --- cmdline/CtaUpdateStorageClass.cpp | 47 +++++++++++++++---------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/cmdline/CtaUpdateStorageClass.cpp b/cmdline/CtaUpdateStorageClass.cpp index 684eed95e8..4cc27d3229 100644 --- a/cmdline/CtaUpdateStorageClass.cpp +++ b/cmdline/CtaUpdateStorageClass.cpp @@ -27,19 +27,19 @@ const std::runtime_error Usage("Usage: ..."); StringListOption option_vid{"--vid", "-v", true}; std::map<std::string, StringListOption *> option_map = { - {"--vid", &option_vid}, - {"-v", &option_vid}, + {"--vid", &option_vid}, + {"-v", &option_vid}, }; void validate_cmd() { - for (auto &it : option_map) { - auto option = it.second; - if (!option->is_optional() && !option->is_present()) { - std::cout << "Error: Option " << option->get_name() << " is mandatory but not present." << std::endl; - throw Usage; - } - } + for (auto &it : option_map) { + auto option = it.second; + if (!option->is_optional() && !option->is_present()) { + std::cout << "Error: Option " << option->get_name() << " is mandatory but not present." << std::endl; + throw Usage; + } + } } @@ -57,23 +57,22 @@ void parse_cmd(const int argc, const char *const *const argv) { option->set_present(); if (option->get_name() == "--vid") { - int num_arg_left = argc - i; - for (int j = 1; j < num_arg_left; j += 1) { - bool end_of_option = (option_map.find(argv[i + j]) != option_map.end()); - option->add_value(argv[i + j]); - if (end_of_option || j == (num_arg_left - 1)) - { - num_values_for_option = j + 1; - break; - } - } + int num_arg_left = argc - i; + for (int j = 1; j < num_arg_left; j += 1) { + bool end_of_option = (option_map.find(argv[i + j]) != option_map.end()); + option->add_value(argv[i + j]); + if (end_of_option || j == (num_arg_left - 1)) { + num_values_for_option = j + 1; + break; + } + } } } } void updateExtAttr() { - + } @@ -82,10 +81,10 @@ void updateExtAttr() { */ int main(int argc, const char **argv) { - parse_cmd(argc, argv); - validate_cmd(); + parse_cmd(argc, argv); + validate_cmd(); - std::cout << option_vid.get_value().size() << std::endl; + std::cout << option_vid.get_value().size() << std::endl; - return 1; + return 1; } -- GitLab From 21bdb24f90cad4d642ea673043374f44b857ed8c Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 25 Aug 2022 10:37:01 +0200 Subject: [PATCH 007/126] 78 - add validation of args --- cmdline/CMakeLists.txt | 8 ++++---- cmdline/CtaUpdateStorageClass.cpp | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cmdline/CMakeLists.txt b/cmdline/CMakeLists.txt index f2c312aad2..a88d54682b 100644 --- a/cmdline/CMakeLists.txt +++ b/cmdline/CMakeLists.txt @@ -37,12 +37,12 @@ target_link_libraries(cta-admin XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon) set_property (TARGET cta-admin APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) # -# cta-change-storage-class +# cta-update-storage-class # # -add_executable(cta-change-storage-class CtaChangeStorageClass.cpp) -target_link_libraries(cta-change-storage-class) -set_property(TARGET cta-change-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) +add_executable(cta-update-storage-class CtaUpdateStorageClass.cpp) +target_link_libraries(cta-update-storage-class) +set_property(TARGET cta-update-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) # # cta-send-event CLOSEW|PREPARE|ABORT_PREPARE diff --git a/cmdline/CtaUpdateStorageClass.cpp b/cmdline/CtaUpdateStorageClass.cpp index 4cc27d3229..292f5401e9 100644 --- a/cmdline/CtaUpdateStorageClass.cpp +++ b/cmdline/CtaUpdateStorageClass.cpp @@ -32,7 +32,10 @@ std::map<std::string, StringListOption *> option_map = { }; -void validate_cmd() { +void validate_cmd(int argc) { + if (argc < 3) { + throw Usage; + } for (auto &it : option_map) { auto option = it.second; if (!option->is_optional() && !option->is_present()) { @@ -82,7 +85,7 @@ void updateExtAttr() { int main(int argc, const char **argv) { parse_cmd(argc, argv); - validate_cmd(); + validate_cmd(argc); std::cout << option_vid.get_value().size() << std::endl; -- GitLab From 2fa369ed501da641c7c7047c0c3ab5ab2ba5b073 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 25 Aug 2022 14:03:21 +0200 Subject: [PATCH 008/126] 78 - add xRootd server communication --- cmdline/CMakeLists.txt | 5 +- cmdline/CtaUpdateStorageClass.cpp | 149 ++++++++++++++++++++++++++++-- 2 files changed, 142 insertions(+), 12 deletions(-) diff --git a/cmdline/CMakeLists.txt b/cmdline/CMakeLists.txt index a88d54682b..d4904121fd 100644 --- a/cmdline/CMakeLists.txt +++ b/cmdline/CMakeLists.txt @@ -38,10 +38,9 @@ set_property (TARGET cta-admin APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) # # cta-update-storage-class -# # -add_executable(cta-update-storage-class CtaUpdateStorageClass.cpp) -target_link_libraries(cta-update-storage-class) +add_executable(cta-update-storage-class CtaUpdateStorageClass.cpp CtaAdminCmdParse.cpp CtaAdminTextFormatter.cpp) +target_link_libraries(cta-update-storage-class ctacommon XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon) set_property(TARGET cta-update-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) # diff --git a/cmdline/CtaUpdateStorageClass.cpp b/cmdline/CtaUpdateStorageClass.cpp index 292f5401e9..21cf5b7691 100644 --- a/cmdline/CtaUpdateStorageClass.cpp +++ b/cmdline/CtaUpdateStorageClass.cpp @@ -15,23 +15,75 @@ * submit itself to any jurisdiction. */ -#include <stdexcept> +#include <sstream> #include <iostream> -#include <map> -#include "CtaCmdOptions.hpp" +#include <XrdSsiPbLog.hpp> +#include <XrdSsiPbIStreamBuffer.hpp> +#include <XrdSsiPbLog.hpp> + +#include "cmdline/CtaAdminCmdParse.hpp" +#include "version.h" +#include "cmdline/CtaCmdOptions.hpp" + +#include "cta_frontend.pb.h" + + +namespace XrdSsiPb { + + +/*! +* User error exception +*/ +class UserException : public std::runtime_error +{ +public: + UserException(const std::string &err_msg) : std::runtime_error(err_msg) {} +}; + + +/*! + * Alert callback. + * + * Defines how Alert messages should be logged + */ +template<> +void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) +{ + std::cout << "AlertCallback():" << std::endl; + Log::DumpProtobuf(Log::PROTOBUF, &alert); +} + +} // namespace Xrdcp + +namespace cta{ +namespace updateStorageClass { + +using namespace cta::admin; + +std::atomic<bool> is_json(false); +std::atomic<bool> is_first_record(true); +std::atomic<bool> isHeaderSent(false); + +const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses +const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by + + +std::string jsonCloseDelim() { + return is_first_record ? "[]" : "]"; +} // Usage exception const std::runtime_error Usage("Usage: ..."); -StringListOption option_vid{"--vid", "-v", true}; +StringListOption option_vid{"--fid", "-f", true}; std::map<std::string, StringListOption *> option_map = { - {"--vid", &option_vid}, - {"-v", &option_vid}, + {"--fid", &option_vid}, + {"-f", &option_vid}, }; - + void validate_cmd(int argc) { if (argc < 3) { throw Usage; @@ -59,7 +111,7 @@ void parse_cmd(const int argc, const char *const *const argv) { auto option = search->second; option->set_present(); - if (option->get_name() == "--vid") { + if (option->get_name() == "--fid") { int num_arg_left = argc - i; for (int j = 1; j < num_arg_left; j += 1) { bool end_of_option = (option_map.find(argv[i + j]) != option_map.end()); @@ -74,19 +126,98 @@ void parse_cmd(const int argc, const char *const *const argv) { } -void updateExtAttr() { +void sendTapeFile(cta::xrd::Request m_request) { + // Set configuration options + const std::string config_file = "/etc/cta/cta-cli.conf"; + XrdSsiPb::Config config(config_file, "cta"); + config.set("resource", "/ctafrontend"); + config.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes + config.set("request_timeout", DefaultRequestTimeout); // default value = 10s + + // Allow environment variables to override config file + config.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); + + // If XRDDEBUG=1, switch on all logging + if(getenv("XRDDEBUG")) { + config.set("log", "all"); + } + // If fine-grained control over log level is required, use XrdSsiPbLogLevel + config.getEnv("log", "XrdSsiPbLogLevel"); + + // Validate that endpoint was specified in the config file + if(!config.getOptionValueStr("endpoint").first) { + throw std::runtime_error("Configuration error: cta.endpoint missing from " + config_file); + } + // If the server is down, we want an immediate failure. Set client retry to a single attempt. + XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); + + // Obtain a Service Provider + XrdSsiPbServiceType cta_service(config); + + // Send the Request to the Service and get a Response + cta::xrd::Response response; + auto stream_future = cta_service.SendAsync(m_request, response); + + // Handle responses + switch(response.type()) + { + using namespace cta::xrd; + using namespace cta::admin; + + case Response::RSP_SUCCESS: + // Print message text + std::cout << response.message_txt(); + // Print streaming response header + if(true) switch(response.show_header()) { + case HeaderType::TAPEFILE_LS: std::cout << "hello"; break; + default: break; + } + // Allow stream processing to commence + isHeaderSent = true; + break; + case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); + case Response::RSP_ERR_USER: throw XrdSsiPb::UserException(response.message_txt()); + case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); + default: throw XrdSsiPb::PbException("Invalid response type."); + } + + // If there is a Data/Stream payload, wait until it has been processed before exiting + stream_future.wait(); + + // JSON output is an array of structs, close bracket + if(is_json) { std::cout << jsonCloseDelim(); } } +void getTapeFile(cta::xrd::Request m_request) { + auto &admincmd = *(m_request.mutable_admincmd()); + + m_request.set_client_cta_version(CTA_VERSION); + m_request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); + + admincmd.set_cmd(AdminCmd::CMD_TAPEFILE); + admincmd.set_subcmd(AdminCmd::SUBCMD_LS); + + sendTapeFile(m_request); +} +} // namespace updateStorageClass +} // namespace cta + + /* * Start here */ int main(int argc, const char **argv) { + using namespace cta::updateStorageClass; + parse_cmd(argc, argv); validate_cmd(argc); + cta::xrd::Request m_request; + getTapeFile(m_request); + std::cout << option_vid.get_value().size() << std::endl; return 1; -- GitLab From e53b1717c96dd2764a70259ea46b6da6345e89a2 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 26 Aug 2022 16:25:49 +0200 Subject: [PATCH 009/126] #78 - add class structure for update_storage_class tool --- cmdline/CMakeLists.txt | 7 +- cmdline/CtaUpdateStorageClass.cpp | 224 -------- cmdline/update_storage_class/CMakeLists.txt | 36 ++ .../CtaUpdateStorageClass.cpp | 529 ++++++++++++++++++ .../CtaUpdateStorageClass.hpp | 211 +++++++ .../CtaUpdateStorageClassCmdLineArgs.cpp | 193 +++++++ .../CtaUpdateStorageClassCmdLineArgs.hpp | 95 ++++ .../CtaUpdateStorageClassMain.cpp | 48 ++ 8 files changed, 1113 insertions(+), 230 deletions(-) delete mode 100644 cmdline/CtaUpdateStorageClass.cpp create mode 100644 cmdline/update_storage_class/CMakeLists.txt create mode 100644 cmdline/update_storage_class/CtaUpdateStorageClass.cpp create mode 100644 cmdline/update_storage_class/CtaUpdateStorageClass.hpp create mode 100644 cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp create mode 100644 cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp create mode 100644 cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp diff --git a/cmdline/CMakeLists.txt b/cmdline/CMakeLists.txt index d4904121fd..09b8c112e6 100644 --- a/cmdline/CMakeLists.txt +++ b/cmdline/CMakeLists.txt @@ -16,6 +16,7 @@ cmake_minimum_required (VERSION 3.17) add_subdirectory (restore_files) +add_subdirectory (update_storage_class) find_package(xrootdclient REQUIRED) find_package(Protobuf3 REQUIRED) @@ -36,12 +37,6 @@ add_executable(cta-admin CtaAdminCmd.cpp CtaAdminCmdParse.cpp CtaAdminTextFormat target_link_libraries(cta-admin XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon) set_property (TARGET cta-admin APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) -# -# cta-update-storage-class -# -add_executable(cta-update-storage-class CtaUpdateStorageClass.cpp CtaAdminCmdParse.cpp CtaAdminTextFormatter.cpp) -target_link_libraries(cta-update-storage-class ctacommon XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon) -set_property(TARGET cta-update-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) # # cta-send-event CLOSEW|PREPARE|ABORT_PREPARE diff --git a/cmdline/CtaUpdateStorageClass.cpp b/cmdline/CtaUpdateStorageClass.cpp deleted file mode 100644 index 21cf5b7691..0000000000 --- a/cmdline/CtaUpdateStorageClass.cpp +++ /dev/null @@ -1,224 +0,0 @@ -/* - * @project The CERN Tape Archive (CTA) - * @copyright Copyright © 2021-2022 CERN - * @license This program is free software, distributed under the terms of the GNU General Public - * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can - * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranPty of MERCHANTABILITY or FITNESS FOR A - * PARTICULAR PURPOSE. See the GNU General Public License for more details. - * - * In applying this licence, CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization or - * submit itself to any jurisdiction. - */ - -#include <sstream> -#include <iostream> - -#include <XrdSsiPbLog.hpp> -#include <XrdSsiPbIStreamBuffer.hpp> -#include <XrdSsiPbLog.hpp> - -#include "cmdline/CtaAdminCmdParse.hpp" -#include "version.h" -#include "cmdline/CtaCmdOptions.hpp" - -#include "cta_frontend.pb.h" - - -namespace XrdSsiPb { - - -/*! -* User error exception -*/ -class UserException : public std::runtime_error -{ -public: - UserException(const std::string &err_msg) : std::runtime_error(err_msg) {} -}; - - -/*! - * Alert callback. - * - * Defines how Alert messages should be logged - */ -template<> -void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) -{ - std::cout << "AlertCallback():" << std::endl; - Log::DumpProtobuf(Log::PROTOBUF, &alert); -} - -} // namespace Xrdcp - -namespace cta{ -namespace updateStorageClass { - -using namespace cta::admin; - -std::atomic<bool> is_json(false); -std::atomic<bool> is_first_record(true); -std::atomic<bool> isHeaderSent(false); - -const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses -const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by - - -std::string jsonCloseDelim() { - return is_first_record ? "[]" : "]"; -} - -// Usage exception -const std::runtime_error Usage("Usage: ..."); - -StringListOption option_vid{"--fid", "-f", true}; - -std::map<std::string, StringListOption *> option_map = { - {"--fid", &option_vid}, - {"-f", &option_vid}, -}; - - -void validate_cmd(int argc) { - if (argc < 3) { - throw Usage; - } - for (auto &it : option_map) { - auto option = it.second; - if (!option->is_optional() && !option->is_present()) { - std::cout << "Error: Option " << option->get_name() << " is mandatory but not present." << std::endl; - throw Usage; - } - } -} - - -void parse_cmd(const int argc, const char *const *const argv) { - unsigned int num_values_for_option = 1; - for (int i = 1; i < argc; i += num_values_for_option){ - num_values_for_option = 1; - auto search = option_map.find(argv[i]); - if (search == option_map.end()) { - std::cout << "Error: Unknown option: " << argv[i] << std::endl; - throw Usage; - } - - auto option = search->second; - option->set_present(); - - if (option->get_name() == "--fid") { - int num_arg_left = argc - i; - for (int j = 1; j < num_arg_left; j += 1) { - bool end_of_option = (option_map.find(argv[i + j]) != option_map.end()); - option->add_value(argv[i + j]); - if (end_of_option || j == (num_arg_left - 1)) { - num_values_for_option = j + 1; - break; - } - } - } - } -} - - -void sendTapeFile(cta::xrd::Request m_request) { - // Set configuration options - const std::string config_file = "/etc/cta/cta-cli.conf"; - XrdSsiPb::Config config(config_file, "cta"); - config.set("resource", "/ctafrontend"); - config.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes - config.set("request_timeout", DefaultRequestTimeout); // default value = 10s - - // Allow environment variables to override config file - config.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); - - // If XRDDEBUG=1, switch on all logging - if(getenv("XRDDEBUG")) { - config.set("log", "all"); - } - // If fine-grained control over log level is required, use XrdSsiPbLogLevel - config.getEnv("log", "XrdSsiPbLogLevel"); - - // Validate that endpoint was specified in the config file - if(!config.getOptionValueStr("endpoint").first) { - throw std::runtime_error("Configuration error: cta.endpoint missing from " + config_file); - } - - // If the server is down, we want an immediate failure. Set client retry to a single attempt. - XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); - - // Obtain a Service Provider - XrdSsiPbServiceType cta_service(config); - - // Send the Request to the Service and get a Response - cta::xrd::Response response; - auto stream_future = cta_service.SendAsync(m_request, response); - - // Handle responses - switch(response.type()) - { - using namespace cta::xrd; - using namespace cta::admin; - - case Response::RSP_SUCCESS: - // Print message text - std::cout << response.message_txt(); - // Print streaming response header - if(true) switch(response.show_header()) { - case HeaderType::TAPEFILE_LS: std::cout << "hello"; break; - default: break; - } - // Allow stream processing to commence - isHeaderSent = true; - break; - case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); - case Response::RSP_ERR_USER: throw XrdSsiPb::UserException(response.message_txt()); - case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); - default: throw XrdSsiPb::PbException("Invalid response type."); - } - - // If there is a Data/Stream payload, wait until it has been processed before exiting - stream_future.wait(); - - // JSON output is an array of structs, close bracket - if(is_json) { std::cout << jsonCloseDelim(); } -} - - -void getTapeFile(cta::xrd::Request m_request) { - auto &admincmd = *(m_request.mutable_admincmd()); - - m_request.set_client_cta_version(CTA_VERSION); - m_request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); - - admincmd.set_cmd(AdminCmd::CMD_TAPEFILE); - admincmd.set_subcmd(AdminCmd::SUBCMD_LS); - - sendTapeFile(m_request); -} -} // namespace updateStorageClass -} // namespace cta - - -/* - * Start here - */ -int main(int argc, const char **argv) -{ - using namespace cta::updateStorageClass; - - parse_cmd(argc, argv); - validate_cmd(argc); - - cta::xrd::Request m_request; - getTapeFile(m_request); - - std::cout << option_vid.get_value().size() << std::endl; - - return 1; -} diff --git a/cmdline/update_storage_class/CMakeLists.txt b/cmdline/update_storage_class/CMakeLists.txt new file mode 100644 index 0000000000..45521ae5ad --- /dev/null +++ b/cmdline/update_storage_class/CMakeLists.txt @@ -0,0 +1,36 @@ +# @project The CERN Tape Archive (CTA) +# @copyright Copyright © 2015-2022 CERN +# @license This program is free software, distributed under the terms of the GNU General Public +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can +# redistribute it and/or modify it under the terms of the GPL Version 3, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# In applying this licence, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +cmake_minimum_required (VERSION 3.17) + +find_package(xrootdclient REQUIRED) +find_package(Protobuf3 REQUIRED) + +# XRootD SSI +include_directories(${XROOTD_INCLUDE_DIR} ${XROOTD_INCLUDE_DIR}/private ) + +# XRootD SSI Protocol Buffer bindings +include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include) + +# Compiled protocol buffers +include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS}) + +add_executable(cta-update-storage-class CtaUpdateStorageClass.cpp CtaUpdateStorageClassCmdLineArgs.cpp CtaUpdateStorageClassMain.cpp ../restore_files/CmdLineTool.cpp) +target_link_libraries(cta-update-storage-class ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient) +set_property (TARGET cta-update-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) + + +install(TARGETS cta-update-storage-class DESTINATION usr/bin) +install(FILES cta-update-storage-class.1cta DESTINATION usr/share/man/man1) diff --git a/cmdline/update_storage_class/CtaUpdateStorageClass.cpp b/cmdline/update_storage_class/CtaUpdateStorageClass.cpp new file mode 100644 index 0000000000..aa9f09c59b --- /dev/null +++ b/cmdline/update_storage_class/CtaUpdateStorageClass.cpp @@ -0,0 +1,529 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#include "cmdline/update_storage_class/CtaUpdateStorageClass.hpp" +#include "cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp" +#include "common/utils/utils.hpp" +#include "common/checksum/ChecksumBlob.hpp" +#include "CtaFrontendApi.hpp" + +#include <XrdSsiPbLog.hpp> +#include <XrdSsiPbIStreamBuffer.hpp> + +#include <grpc++/grpc++.h> +#include "Rpc.grpc.pb.h" + +#include <sys/stat.h> +#include <iostream> +#include <memory> + +// GLOBAL VARIABLES : used to pass information between main thread and stream handler thread + +// global synchronisation flag +std::atomic<bool> isHeaderSent(false); + +std::list<cta::admin::RecycleTapeFileLsItem> deletedTapeFiles; +std::list<std::pair<std::string,std::string>> listedTapeFiles; + +namespace XrdSsiPb { + +/*! + * User error exception + */ +class UserException : public std::runtime_error +{ +public: + UserException(const std::string &err_msg) : std::runtime_error(err_msg) {} +}; // class UserException + +/*! + * Alert callback. + * + * Defines how Alert messages should be logged + */ +template<> +void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) +{ + std::cout << "AlertCallback():" << std::endl; + Log::DumpProtobuf(Log::PROTOBUF, &alert); +} + +/*! + * Data/Stream callback. + * + * Defines how incoming records from the stream should be handled + */ +template<> +void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const +{ + using namespace cta::xrd; + using namespace cta::admin; + + // Wait for primary response to be handled before allowing stream response + while(!isHeaderSent) { std::this_thread::yield(); } + + switch(record.data_case()) { + case Data::kRtflsItem: + { + auto item = record.rtfls_item(); + deletedTapeFiles.push_back(item); + } + break; + case Data::kTflsItem: + { + auto item = record.tfls_item(); + auto instanceAndFid = std::make_pair(item.df().disk_instance(), item.df().disk_id()); + listedTapeFiles.push_back(instanceAndFid); + } + break; + default: + throw std::runtime_error("Received invalid stream data from CTA Frontend for the cta-restore-deleted-files command."); + } +} + +} // namespace XrdSsiPb + + +namespace cta{ +namespace admin { + +/*! + * UpdateFilesCmdException + */ +class UpdateFilesCmdException : public std::runtime_error +{ +public: + UpdateFilesCmdException(const std::string &err_msg) : std::runtime_error(err_msg) {} +}; // class UserException + + + +//------------------------------------------------------------------------------ +// constructor +//------------------------------------------------------------------------------ +CtaUpdateStorageClass::CtaUpdateStorageClass(std::istream &inStream, std::ostream &outStream, + std::ostream &errStream, cta::log::StdoutLogger &log): + CmdLineTool(inStream, outStream, errStream), + m_log(log) { + + // Default layout: see EOS common/LayoutId.hh for definitions of constants + const int kAdler = 0x2; + const int kReplica = (0x1 << 4); + const int kStripeSize = (0x0 << 8); // 1 stripe + const int kStripeWidth = (0x0 << 16); // 4K blocks + const int kBlockChecksum = (0x1 << 20); + + // Default single replica layout id should be 00100012 + m_defaultFileLayout = kReplica | kAdler | kStripeSize | kStripeWidth | kBlockChecksum; + +} + +//------------------------------------------------------------------------------ +// exceptionThrowingMain +//------------------------------------------------------------------------------ +int CtaUpdateStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { + CtaUpdateStorageClassCmdLineArgs cmdLineArgs(argc, argv); + if (cmdLineArgs.m_help) { + printUsage(m_out); + return 0; + } + return 0; +} + +//------------------------------------------------------------------------------ +// readAndSetConfiguration +//------------------------------------------------------------------------------ +void CtaUpdateStorageClass::readAndSetConfiguration( + const std::string &userName, + const CtaUpdateStorageClassCmdLineArgs &cmdLineArgs) { + + m_vid = cmdLineArgs.m_vid; + m_diskInstance = cmdLineArgs.m_diskInstance; + m_eosFids = cmdLineArgs.m_eosFids; + m_copyNumber = cmdLineArgs.m_copyNumber; + m_archiveFileId = cmdLineArgs.m_archiveFileId; + + if (cmdLineArgs.m_debug) { + m_log.setLogMask("DEBUG"); + } else { + m_log.setLogMask("INFO"); + } + + // Set CTA frontend configuration options + const std::string cli_config_file = "/etc/cta/cta-cli.conf"; + XrdSsiPb::Config cliConfig(cli_config_file, "cta"); + cliConfig.set("resource", "/ctafrontend"); + cliConfig.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes + cliConfig.set("request_timeout", DefaultRequestTimeout); // default value = 10s + + // Allow environment variables to override config file + cliConfig.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); + + // If XRDDEBUG=1, switch on all logging + if(getenv("XRDDEBUG")) { + cliConfig.set("log", "all"); + } + // If fine-grained control over log level is required, use XrdSsiPbLogLevel + cliConfig.getEnv("log", "XrdSsiPbLogLevel"); + + // Validate that endpoint was specified in the config file + if(!cliConfig.getOptionValueStr("endpoint").first) { + throw std::runtime_error("Configuration error: cta.endpoint missing from " + cli_config_file); + } + + // If the server is down, we want an immediate failure. Set client retry to a single attempt. + XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); + + m_serviceProviderPtr.reset(new XrdSsiPbServiceType(cliConfig)); + + // Set CTA frontend configuration options to connect to eos + const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf"; + XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); + + // Get the endpoint for namespace queries + auto nsConf = frontendXrootdConfig.getOptionValueStr("ns.config"); + if(nsConf.first) { + setNamespaceMap(nsConf.second); + } else { + throw std::runtime_error("Configuration error: cta.ns.config missing from " + frontend_xrootd_config_file); + } +} + +void CtaUpdateStorageClass::setNamespaceMap(const std::string &keytab_file) { + // Open the keytab file for reading + std::ifstream file(keytab_file); + if(!file) { + throw cta::exception::UserError("Failed to open namespace keytab configuration file " + keytab_file); + } + ::eos::client::NamespaceMap_t namespaceMap; + // Parse the keytab line by line + std::string line; + for(int lineno = 0; std::getline(file, line); ++lineno) { + // Strip out comments + auto pos = line.find('#'); + if(pos != std::string::npos) { + line.resize(pos); + } + + // Parse one line + std::stringstream ss(line); + std::string diskInstance; + std::string endpoint; + std::string token; + std::string eol; + ss >> diskInstance >> endpoint >> token >> eol; + + // Ignore blank lines, all other lines must have exactly 3 elements + if(token.empty() || !eol.empty()) { + if(diskInstance.empty() && endpoint.empty() && token.empty()) continue; + throw cta::exception::UserError("Could not parse namespace keytab configuration file line " + std::to_string(lineno) + ": " + line); + } + namespaceMap.insert(std::make_pair(diskInstance, ::eos::client::Namespace(endpoint, token))); + } + m_endpointMapPtr = std::make_unique<::eos::client::EndpointMap>(namespaceMap); +} + + +//------------------------------------------------------------------------------ +// addContainerEos +//------------------------------------------------------------------------------ +uint64_t CtaUpdateStorageClass::addContainerEos(const std::string &diskInstance, const std::string &path, const std::string &sc) const { + auto c_id = containerExistsEos(diskInstance, path); + if (c_id) { + return c_id; + } + auto enclosingPath = cta::utils::getEnclosingPath(path); + auto parent_id = containerExistsEos(diskInstance, enclosingPath); + if (!parent_id) { + //parent does not exist, need to add it as well + parent_id = addContainerEos(diskInstance, enclosingPath, sc); + } + + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", getUsername())); + params.push_back(cta::log::Param("diskInstance", diskInstance)); + params.push_back(cta::log::Param("path", path)); + m_log(cta::log::DEBUG, "Inserting container in EOS namespace", params); + + ::eos::rpc::ContainerMdProto dir; + dir.set_path(path); + dir.set_name(cta::utils::getEnclosedName(path)); + + // Filemode: filter out S_ISUID, S_ISGID and S_ISVTX because EOS does not follow POSIX semantics for these bits + uint64_t filemode = (S_IRWXU | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); // 0755 permissions by default + filemode &= ~(S_ISUID | S_ISGID | S_ISVTX); + dir.set_mode(filemode); + + auto time = ::time(nullptr); + // Timestamps + dir.mutable_ctime()->set_sec(time); + dir.mutable_mtime()->set_sec(time); + // we don't care about dir.stime (sync time, used for CERNBox) + + dir.mutable_xattrs()->insert(google::protobuf::MapPair<std::string,std::string>("sys.archive.storage_class", sc)); + + + auto reply = m_endpointMapPtr->containerInsert(diskInstance, dir); + + m_log(cta::log::DEBUG, "Inserted container in EOS namespace successfully, querying again for its id", params); + + auto cont_id = containerExistsEos(diskInstance, path); + if (!cont_id) { + throw UpdateFilesCmdException(std::string("Container ") + path + " does not exist after being inserted in EOS."); + } + return cont_id; +} + +//------------------------------------------------------------------------------ +// containerExistsEos +//------------------------------------------------------------------------------ +uint64_t CtaUpdateStorageClass::containerExistsEos(const std::string &diskInstance, const std::string &path) const { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", getUsername())); + params.push_back(cta::log::Param("diskInstance", diskInstance)); + params.push_back(cta::log::Param("path", path)); + + m_log(cta::log::DEBUG, "Verifying if the container exists in the EOS namespace", params); + + auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::CONTAINER, 0, path, false); + auto cid = md_response.cmd().id(); + params.push_back(cta::log::Param("containerId", cid)); + if (cid != 0) { + m_log(cta::log::DEBUG, "Container exists in the EOS namespace", params); + } else { + m_log(cta::log::DEBUG, "Container does not exist in the EOS namespace", params); + } + return cid; +} + + +//------------------------------------------------------------------------------ +// archiveFileExistsCTA +//------------------------------------------------------------------------------ +bool CtaUpdateStorageClass::archiveFileExistsCTA(const uint64_t &archiveFileId) const { + + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", getUsername())); + params.push_back(cta::log::Param("archiveFileId", archiveFileId)); + + m_log(cta::log::DEBUG, "Looking for archive file in the CTA catalogue", params); + + cta::xrd::Request request; + + auto &admincmd = *(request.mutable_admincmd()); + + request.set_client_cta_version(CTA_VERSION); + request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); + admincmd.set_cmd(AdminCmd::CMD_TAPEFILE); + admincmd.set_subcmd(AdminCmd::SUBCMD_LS); + + auto key = OptionUInt64::ARCHIVE_FILE_ID; + auto new_opt = admincmd.add_option_uint64(); + new_opt->set_key(key); + new_opt->set_value(archiveFileId); + + // Send the Request to the Service and get a Response + cta::xrd::Response response; + auto stream_future = m_serviceProviderPtr->SendAsync(request, response); + + bool ret; + // Handle responses + switch(response.type()) + { + using namespace cta::xrd; + using namespace cta::admin; + case Response::RSP_SUCCESS: ret = true; break; //success sent if archive file does not exist + case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); + case Response::RSP_ERR_USER: ret = false; break; //user error sent if archive file does not exist + case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); + default: throw XrdSsiPb::PbException("Invalid response type."); + } + + // wait until the data stream has been processed before exiting + if (ret) { + stream_future.wait(); + } + if (ret) { + m_log(cta::log::DEBUG, "Archive file is present in the CTA catalogue", params); + } else { + m_log(cta::log::DEBUG, "Archive file is missing in the CTA catalogue", params); + } + return ret; +} + +//------------------------------------------------------------------------------ +// fileExistsEos +//------------------------------------------------------------------------------ +bool CtaUpdateStorageClass::fileExistsEos(const std::string &diskInstance, const std::string &diskFileId) const { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", getUsername())); + params.push_back(cta::log::Param("diskInstance", diskInstance)); + params.push_back(cta::log::Param("diskFileId", diskFileId)); + + m_log(cta::log::DEBUG, "Verifying if EOS fid exists in the EOS namespace", params); + try { + auto path = m_endpointMapPtr->getPath(diskInstance, diskFileId); + params.push_back(cta::log::Param("diskFilePath", path)); + m_log(cta::log::DEBUG, "EOS fid exists in the EOS namespace"); + return true; + } catch(cta::exception::Exception&) { + m_log(cta::log::DEBUG, "EOS fid does not exist in the EOS namespace"); + return false; + } +} + +//------------------------------------------------------------------------------ +// getFileIdEos +//------------------------------------------------------------------------------ +uint64_t CtaUpdateStorageClass::getFileIdEos(const std::string &diskInstance, const std::string &path) const { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", getUsername())); + params.push_back(cta::log::Param("diskInstance", diskInstance)); + params.push_back(cta::log::Param("path", path)); + + m_log(cta::log::DEBUG, "Querying for file metadata in the EOS namespace", params); + auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::FILE, 0, path, false); + auto fid = md_response.fmd().id(); + params.push_back(cta::log::Param("diskFileId", fid)); + if (fid != 0) { + m_log(cta::log::DEBUG, "File path exists in the EOS namespace", params); + } else { + m_log(cta::log::DEBUG, "File path does not exist in the EOS namespace", params); + } + return fid; +} + +//------------------------------------------------------------------------------ +// getCurrentEosIds +//------------------------------------------------------------------------------ +void CtaUpdateStorageClass::getCurrentEosIds(const std::string &diskInstance) const { + uint64_t cid; + uint64_t fid; + m_endpointMapPtr->getCurrentIds(diskInstance, cid, fid); + + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("diskInstance", diskInstance)); + params.push_back(cta::log::Param("ContainerId", cid)); + params.push_back(cta::log::Param("FileId", fid)); + m_log(cta::log::DEBUG, "Obtained current EOS container and file id", params); +} + + +//------------------------------------------------------------------------------ +// printUsage +//------------------------------------------------------------------------------ +void CtaUpdateStorageClass::printUsage(std::ostream &os) { + CtaUpdateStorageClassCmdLineArgs::printUsage(os); +} + + +//------------------------------------------------------------------------------ +// getFxidFromCTA +//------------------------------------------------------------------------------ +std::pair<std::string,std::string> CtaUpdateStorageClass::getInstanceAndFidFromCTA(const RecycleTapeFileLsItem& file) { + { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("archiveFileId", file.archive_file_id())); + m_log(cta::log::DEBUG, "cta-admin tapefile ls", params); + } + + cta::xrd::Request request; + auto &admincmd = *(request.mutable_admincmd()); + + request.set_client_cta_version(CTA_VERSION); + request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); + admincmd.set_cmd(AdminCmd::CMD_TAPEFILE); + admincmd.set_subcmd(AdminCmd::SUBCMD_LS); + auto new_opt = admincmd.add_option_uint64(); + new_opt->set_key(OptionUInt64::ARCHIVE_FILE_ID); + new_opt->set_value(file.archive_file_id()); + + // Send the Request to the Service and get a Response + cta::xrd::Response response; + auto stream_future = m_serviceProviderPtr->SendAsync(request, response); + + // Handle responses + switch(response.type()) + { + using namespace cta::xrd; + using namespace cta::admin; + case Response::RSP_SUCCESS: + // Print message text + std::cout << response.message_txt(); + // Allow stream processing to commence + isHeaderSent = true; + break; + case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); + case Response::RSP_ERR_USER: throw XrdSsiPb::UserException(response.message_txt()); + case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); + default: throw XrdSsiPb::PbException("Invalid response type."); + } + + // wait until the data stream has been processed before exiting + stream_future.wait(); + if(listedTapeFiles.size() != 1) { + throw std::runtime_error("Unexpected result set: listedTapeFiles size expected=1 received=" + std::to_string(listedTapeFiles.size())); + } + auto listedTapeFile = listedTapeFiles.back(); + listedTapeFiles.clear(); + { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("diskInstance", listedTapeFile.first)); + params.push_back(cta::log::Param("diskFileId", listedTapeFile.second)); + m_log(cta::log::DEBUG, "Obtained file metadata from CTA", params); + } + return listedTapeFile; +} + +//------------------------------------------------------------------------------ +// getArchiveFileIdFromEOS +//------------------------------------------------------------------------------ +std::pair<uint64_t, std::string> CtaUpdateStorageClass::getArchiveFileIdAndChecksumFromEOS( + const std::string& diskInstance, const std::string& fidStr) { + auto fid = strtoul(fidStr.c_str(), nullptr, 10); + if(fid < 1) { + throw std::runtime_error(fid + " (base 10) is not a valid disk file ID"); + } + { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("diskInstance", diskInstance)); + params.push_back(cta::log::Param("fid", fid)); + std::stringstream ss; + ss << std::hex << fid; + params.push_back(cta::log::Param("fxid", ss.str())); + m_log(cta::log::DEBUG, "Querying EOS namespace for archiveFileId and checksum", params); + } + auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::FILE, fid, "", false); + auto archiveFileIdItor = md_response.fmd().xattrs().find("sys.archive.file_id"); + if(md_response.fmd().xattrs().end() == archiveFileIdItor) { + throw std::runtime_error("archiveFileId extended attribute not found."); + } + auto archiveFileId = strtoul(archiveFileIdItor->second.c_str(), nullptr, 10); + auto byteArray = md_response.fmd().checksum().value(); + auto checksumValue = checksum::ChecksumBlob::ByteArrayToHex(std::string(byteArray.rbegin(), byteArray.rend())); + { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("archiveFileId", archiveFileId)); + params.push_back(cta::log::Param("checksumValue", checksumValue)); + m_log(cta::log::DEBUG, "Response from EOS nameserver", params); + } + + return std::make_pair(archiveFileId,checksumValue); +} + +} // namespace admin +} // namespace cta \ No newline at end of file diff --git a/cmdline/update_storage_class/CtaUpdateStorageClass.hpp b/cmdline/update_storage_class/CtaUpdateStorageClass.hpp new file mode 100644 index 0000000000..47ead6bee6 --- /dev/null +++ b/cmdline/update_storage_class/CtaUpdateStorageClass.hpp @@ -0,0 +1,211 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#pragma once + +#include "cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp" +#include "cmdline/restore_files/CmdLineTool.hpp" +#include "eos_grpc_client/GrpcEndpoint.hpp" +#include "catalogue/Catalogue.hpp" +#include "common/log/StdoutLogger.hpp" + +#include <optional> +#include <memory> + +#include "CtaFrontendApi.hpp" + +namespace cta { +namespace admin { + +class CtaUpdateStorageClass: public CmdLineTool { +public: + /** + * Constructor. + * + * @param inStream Standard input stream. + * @param outStream Standard output stream. + * @param errStream Standard error stream. + * @param log The object representing the API of the CTA logging system. + */ + CtaUpdateStorageClass(std::istream &inStream, std::ostream &outStream, + std::ostream &errStream, cta::log::StdoutLogger &log); + + /** + * An exception throwing version of main(). + * + * @param argc The number of command-line arguments including the program name. + * @param argv The command-line arguments. + * @return The exit value of the program. + */ + int exceptionThrowingMain(const int argc, char *const *const argv) override; + + /** + * Sets internal configuration parameters to be used for reading. + * It reads cta frontend parameters from /etc/cta/cta-cli.conf + * + * @param username The name of the user running the command-line tool. + * @param cmdLineArgs The arguments parsed from the command line. + */ + void readAndSetConfiguration(const std::string &userName, const CtaUpdateStorageClassCmdLineArgs &cmdLineArgs); + + /** + * Populate the namespace endpoint configuration from a keytab file + */ + void setNamespaceMap(const std::string &keytab_file); + + + /** + * Queries the eos mgm for the current eos file and container id + * Must be called before any other call to EOS, to initialize the grpc + * client cid and fid + * @param diskInstance the disk instance of the eos instance + */ + void getCurrentEosIds(const std::string &diskInstance) const; + + + /** + * Adds a container in the eos namespace, if it does not exist + * @param diskInstance eos disk instance + * @param path the path of the container + * @param sc the storage class of the container + * @returns the container id of the container identified by path + */ + uint64_t addContainerEos(const std::string &diskInstance, const std::string &path, const std::string &sc) const; + + /** + * Returns the container id of the container in the EOS namespace + * + * @param diskInstance eos disk instance + * @param path the path of the container + * + * @returns the container id of the container in the EOS namespace, or zero if it does not exist + */ + uint64_t containerExistsEos(const std::string &diskInstance, const std::string &path) const; + + /** + * Returns true (i.e. not zero) if a file with given id exists in the eos namespace + * @param diskInstance eos disk instance + * @param diskFileId the eos file id to check + */ + bool fileExistsEos(const std::string &diskInstance, const std::string &diskFileId) const; + + + /** + * Returns true (i.e. not zero) if an archive file with given id exists in the cta catalogue + * @param archiveFileId the archive file id to check + */ + bool archiveFileExistsCTA(const uint64_t &archiveFileId) const; + + /** + * Returns the id of a given file in eos or zero if the files does not exist + * @param diskInstance eos disk instance + * @param path the path to check + */ + uint64_t getFileIdEos(const std::string &diskInstance, const std::string &path) const; + + + /** + * Prints the usage message of the command-line tool. + * + * @param os The output stream to which the usage message is to be printed. + */ + void printUsage(std::ostream &os) override; + +private: + /** + * Query CTA for the disk instance and fid of the restored file + * + * @param file The restored tape file in CTA + * @return Tuple of EOS disk instance and file ID (fid) as a decimal string + */ + std::pair<std::string,std::string> getInstanceAndFidFromCTA(const RecycleTapeFileLsItem& file); + + /** + * Query EOS for the archiveFileId and checksum of the restored file + * + * @param diskInstance Which EOS disk instance to query + * @param fxid fid of the file in EOS as a decimal string + * @return tuple containing archiveFileId and checksum of the file + */ + std::pair<uint64_t, std::string> getArchiveFileIdAndChecksumFromEOS(const std::string& diskInstance, + const std::string& fidStr); + + /** + * The object representing the API of the CTA logging system. + */ + cta::log::StdoutLogger &m_log; + + /** + * True if the command should just restore deleted tape file copies + */ + bool m_restoreCopies; + + /** + * True if the command should just restore deleted archive files + */ + bool m_restoreFiles; + + /** + * Archive file id of the files to restore + */ + std::optional<uint64_t> m_archiveFileId; + + /** + * Disk instance of the files to restore + */ + std::optional<std::string> m_diskInstance; + + /** + * Fids of the files to restore + */ + std::optional<std::list<uint64_t>> m_eosFids; + + /** + * Vid of the tape of the files to restore + */ + std::optional<std::string> m_vid; + + /** + *Copy number of the files to restore + */ + std::optional<uint64_t> m_copyNumber; + + /** + *Default file layout for restored files in EOS + */ + int m_defaultFileLayout; + + /** + * CTA Frontend service provider + */ + std::unique_ptr<XrdSsiPbServiceType> m_serviceProviderPtr; + + std::unique_ptr<::eos::client::EndpointMap> m_endpointMapPtr; + + const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses + const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by + //!< XRD_REQUESTTIMEOUT environment variable. + + const std::string m_ctaVersion = CTA_VERSION; //!< The version of CTA + const std::string m_xrootdSsiProtobufInterfaceVersion = //!< The xrootd-ssi-protobuf-interface version (=tag) + XROOTD_SSI_PROTOBUF_INTERFACE_VERSION; + + +} ; // class CtaUpdateStorageClass + +} // namespace admin +} // namespace cta diff --git a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp new file mode 100644 index 0000000000..28a1e630e6 --- /dev/null +++ b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp @@ -0,0 +1,193 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#include "cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp" +#include "common/exception/CommandLineNotParsed.hpp" + +#include <getopt.h> +#include <ostream> +#include <sstream> +#include <iostream> +#include <string> +#include <limits.h> +#include <fstream> + + +namespace cta { +namespace admin{ + +//------------------------------------------------------------------------------ +// constructor +//------------------------------------------------------------------------------ +CtaUpdateStorageClassCmdLineArgs::CtaUpdateStorageClassCmdLineArgs(const int argc, char *const *const argv): +m_help(false), m_debug(false) { + + static struct option longopts[] = { + {"id", required_argument, nullptr, 'I'}, + {"instance", required_argument, nullptr, 'i'}, + {"fxid", required_argument, nullptr, 'f'}, + {"fxidfile", required_argument, nullptr, 'F'}, + {"vid", required_argument, nullptr, 'v'}, + {"copynb", required_argument, nullptr, 'c'}, + {"help", no_argument, nullptr, 'h'}, + {"debug", no_argument, nullptr, 'd'}, + {nullptr, 0, nullptr, 0} + }; + + opterr = 0; + int opt = 0; + int opt_index = 3; + + while ((opt = getopt_long(argc, argv, "I:i:f:F:v:c:hd", longopts, &opt_index)) != -1) { + switch(opt) { + case 'I': + { + int64_t archiveId = std::stol(std::string(optarg)); + if(archiveId < 0) throw std::out_of_range("archive id value cannot be negative"); + m_archiveFileId = archiveId; + break; + } + case 'i': + { + m_diskInstance = std::string(optarg); + break; + } + case 'f': + { + if (! m_eosFids) { + m_eosFids = std::list<uint64_t>(); + } + auto fid = strtoul(optarg, nullptr, 16); + if(fid < 1) { + throw std::runtime_error(std::string(optarg) + " is not a valid file ID"); + } + m_eosFids->push_back(fid); + break; + } + case 'F': + { + if (! m_eosFids) { + m_eosFids = std::list<uint64_t>(); + } + readFidListFromFile(std::string(optarg), m_eosFids.value()); + break; + } + case 'v': + { + m_vid = std::string(optarg); + break; + } + case 'c': + { + int64_t copyNumber = std::stol(std::string(optarg)); + if(copyNumber < 0) throw std::out_of_range("copy number value cannot be negative"); + m_copyNumber = copyNumber; + break; + } + case 'h': + { + m_help = true; + break; + } + case 'd': + { + m_debug = true; + break; + } + case ':': // Missing parameter + { + exception::CommandLineNotParsed ex; + ex.getMessage() << "The -" << (char)optopt << " option requires a parameter"; + throw ex; + } + case '?': // Unknown option + { + exception::CommandLineNotParsed ex; + if(0 == optopt) { + ex.getMessage() << "Unknown command-line option"; + } else { + ex.getMessage() << "Unknown command-line option: -" << (char)optopt; + } + throw ex; + } + default: + { + exception::CommandLineNotParsed ex; + ex.getMessage() << + "getopt_long returned the following unknown value: 0x" << + std::hex << (int)opt; + throw ex; + } + } + } +} + +//------------------------------------------------------------------------------ +// readFidListFromFile +//------------------------------------------------------------------------------ +void CtaUpdateStorageClassCmdLineArgs::readFidListFromFile(const std::string &filename, std::list<std::uint64_t> &fidList) { + std::ifstream file(filename); + if (file.fail()) { + throw std::runtime_error("Unable to open file " + filename); + } + + std::string line; + + while(std::getline(file, line)) { + // Strip out comments + auto pos = line.find('#'); + if(pos != std::string::npos) { + line.resize(pos); + } + + // Extract the list items + std::stringstream ss(line); + while(!ss.eof()) { + std::string item; + ss >> item; + // skip blank lines or lines consisting only of whitespace + if(item.empty()) continue; + + // Special handling for file id lists. The output from "eos find --fid <fid> /path" is: + // path=/path fid=<fid> + // We discard everything except the list of fids. <fid> is a zero-padded hexadecimal number, + // but in the CTA catalogue we store disk IDs as a decimal string, so we need to convert it. + if(item.substr(0, 4) == "fid=") { + auto fid = strtol(item.substr(4).c_str(), nullptr, 16); + if(fid < 1 || fid == LONG_MAX) { + throw std::runtime_error(item + " is not a valid file ID"); + } + fidList.push_back(fid); + } else { + continue; + } + } + } +} + +//------------------------------------------------------------------------------ +// printUsage +//------------------------------------------------------------------------------ +void CtaUpdateStorageClassCmdLineArgs::printUsage(std::ostream &os) { + os << "Usage:" << std::endl << + " cta-update-storage-class [--id/-I <archive_file_id>] [--instance/-i <disk_instance>]" << std::endl << + " [--fxid/-f <eos_fxid>] [--fxidfile/-F <filename>]" << std::endl << + " [--vid/-v <vid>] [--copynb/-c <copy_number>] [--debug/-d]" << std::endl; +} + +} // namespace admin +} // namespace cta diff --git a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp new file mode 100644 index 0000000000..1af45fd2f2 --- /dev/null +++ b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp @@ -0,0 +1,95 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#pragma once + +#include "version.h" + +#include <list> +#include <optional> + +namespace cta { +namespace admin { + +/** + * Structure to store the command-line arguments of the command-line tool + * named cta-update-deleted-archive. + */ +struct CtaUpdateStorageClassCmdLineArgs { + /** + * True if the usage message should be printed. + */ + bool m_help; + + /** + * True if debug messages should be printed + */ + bool m_debug; + + /** + * Archive file id of the files to update + */ + std::optional<uint64_t> m_archiveFileId; + + /** + * Disk instance of the files to update + */ + std::optional<std::string> m_diskInstance; + + /** + * Fids of the files to update + */ + std::optional<std::list<uint64_t>> m_eosFids; + + /** + * Vid of the tape of the files to update + */ + std::optional<std::string> m_vid; + + /** + *Copy number of the files to update + */ + std::optional<uint64_t> m_copyNumber; + + /** + * Constructor that parses the specified command-line arguments. + * + * @param argc The number of command-line arguments including the name of the + * executable. + * @param argv The vector of command-line arguments. + */ + CtaUpdateStorageClassCmdLineArgs(const int argc, char *const *const argv); + + /** + * Read a list of eos file ids from a file and write the options to a list + * + * @param filename The name of the file to read + * @param fidList The list of file IDs + */ + void readFidListFromFile(const std::string &filename, std::list<uint64_t> &fidList); + + /** + * Prints the usage message of the command-line tool. + * + * @param os The output stream to which the usage message is to be printed. + */ + static void printUsage(std::ostream &os); + +}; // class CtaUpdateStorageClassCmdLineArgs + +} // namespace admin +} // namespace cta diff --git a/cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp b/cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp new file mode 100644 index 0000000000..e07a11fcc6 --- /dev/null +++ b/cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp @@ -0,0 +1,48 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + + +#include <sstream> +#include <iostream> + +#include <XrdSsiPbLog.hpp> +#include <XrdSsiPbIStreamBuffer.hpp> + +#include "cmdline/update_storage_class/CtaUpdateStorageClass.hpp" +#include "common/utils/utils.hpp" +#include "common/log/StdoutLogger.hpp" + +//------------------------------------------------------------------------------ +// main +//------------------------------------------------------------------------------ +int main(const int argc, char *const *const argv) { + char buf[256]; + std::string hostName; + if(gethostname(buf, sizeof(buf))) { + hostName = "UNKNOWN"; + } else { + buf[sizeof(buf) - 1] = '\0'; + hostName = buf; + } + cta::log::StdoutLogger log(hostName, "cta-restore-deleted-files"); + + cta::admin::CtaUpdateStorageClass cmd(std::cin, std::cout, std::cerr, log); + int ret = cmd.main(argc, argv); + // Delete all global objects allocated by libprotobuf + google::protobuf::ShutdownProtobufLibrary(); + return ret; +} \ No newline at end of file -- GitLab From 51d94ca4438644ac304ff6ca1cadfae4d5e0ee48 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Mon, 29 Aug 2022 11:48:31 +0200 Subject: [PATCH 010/126] #78 add storage class name as input option to the tool --- cmdline/CtaUpdateStorageClass.cpp | 71 +++++++++++++++++++------------ 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/cmdline/CtaUpdateStorageClass.cpp b/cmdline/CtaUpdateStorageClass.cpp index 21cf5b7691..4ebe3502ad 100644 --- a/cmdline/CtaUpdateStorageClass.cpp +++ b/cmdline/CtaUpdateStorageClass.cpp @@ -76,11 +76,17 @@ std::string jsonCloseDelim() { // Usage exception const std::runtime_error Usage("Usage: ..."); -StringListOption option_vid{"--fid", "-f", true}; +StringListOption option_fid{"--fid", "-f", true}; +StringOption option_storage_class_name{"--storage_class_name", "-scn", true}; -std::map<std::string, StringListOption *> option_map = { - {"--fid", &option_vid}, - {"-f", &option_vid}, +std::map<std::string, StringListOption *> option_with_multiple_values_map = { + {"--fid", &option_fid}, + {"-f", &option_fid}, +}; + +std::map<std::string, StringOption *> option_map = { + {"--storage_class_name", &option_storage_class_name}, + {"-scn", &option_storage_class_name}, }; @@ -88,7 +94,7 @@ void validate_cmd(int argc) { if (argc < 3) { throw Usage; } - for (auto &it : option_map) { + for (auto &it : option_with_multiple_values_map) { auto option = it.second; if (!option->is_optional() && !option->is_present()) { std::cout << "Error: Option " << option->get_name() << " is mandatory but not present." << std::endl; @@ -99,29 +105,41 @@ void validate_cmd(int argc) { void parse_cmd(const int argc, const char *const *const argv) { - unsigned int num_values_for_option = 1; - for (int i = 1; i < argc; i += num_values_for_option){ - num_values_for_option = 1; - auto search = option_map.find(argv[i]); - if (search == option_map.end()) { - std::cout << "Error: Unknown option: " << argv[i] << std::endl; - throw Usage; + unsigned int num_cmd_line_enrties_for_option = 2; + for (int i = 1; i < argc; i += num_cmd_line_enrties_for_option){ + num_cmd_line_enrties_for_option = 2; + auto search_option_with_multiple_values_map = option_with_multiple_values_map.find(argv[i]); + if (search_option_with_multiple_values_map != option_with_multiple_values_map.end()) { + auto stringListOption = search_option_with_multiple_values_map->second; + stringListOption->set_present(); + + if (stringListOption->get_name() == "--fid") { + int num_arg_left = argc - i; + for (int j = 1; j < num_arg_left; j += 1) { + bool end_of_option = (option_with_multiple_values_map.find(argv[i + j]) != option_with_multiple_values_map.end()); + stringListOption->add_value(argv[i + j]); + if (end_of_option || j == (num_arg_left - 1)) { + num_cmd_line_enrties_for_option = j + 1; + break; + } + } + } } - auto option = search->second; - option->set_present(); - - if (option->get_name() == "--fid") { - int num_arg_left = argc - i; - for (int j = 1; j < num_arg_left; j += 1) { - bool end_of_option = (option_map.find(argv[i + j]) != option_map.end()); - option->add_value(argv[i + j]); - if (end_of_option || j == (num_arg_left - 1)) { - num_values_for_option = j + 1; - break; - } + auto search_option = option_map.find(argv[i]); + if (search_option != option_map.end()) { + auto stringOption = search_option->second; + stringOption->set_present(); + + if (stringOption->get_name() == "--storage_class_name") { + stringOption->set_value(argv[i + 1]); } } + + if (search_option_with_multiple_values_map == option_with_multiple_values_map.end() && search_option == option_map.end()) { + std::cout << "Error: Unknown option: " << argv[i] << std::endl; + throw Usage; + } } } @@ -215,10 +233,7 @@ int main(int argc, const char **argv) parse_cmd(argc, argv); validate_cmd(argc); - cta::xrd::Request m_request; - getTapeFile(m_request); - - std::cout << option_vid.get_value().size() << std::endl; + std::cout << option_storage_class_name.get_value() << std::endl; return 1; } -- GitLab From 0763c60c4d22aed9587decf96edaef17e0522a3e Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Mon, 29 Aug 2022 14:42:09 +0200 Subject: [PATCH 011/126] #78 - fix parse_cmd bug --- cmdline/CtaUpdateStorageClass.cpp | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/cmdline/CtaUpdateStorageClass.cpp b/cmdline/CtaUpdateStorageClass.cpp index 955983d9c5..0638118c06 100644 --- a/cmdline/CtaUpdateStorageClass.cpp +++ b/cmdline/CtaUpdateStorageClass.cpp @@ -104,6 +104,14 @@ void validate_cmd(int argc) { } +bool isEndOfOption(int nextOptionOrValueIndex, const char *const *const argv) { + return ( + option_with_multiple_values_map.find(argv[nextOptionOrValueIndex]) != option_with_multiple_values_map.end() || + option_map.find(argv[nextOptionOrValueIndex]) != option_map.end() + ); +} + + void parse_cmd(const int argc, const char *const *const argv) { unsigned int num_cmd_line_enrties_for_option = 2; for (int i = 1; i < argc; i += num_cmd_line_enrties_for_option){ @@ -114,13 +122,13 @@ void parse_cmd(const int argc, const char *const *const argv) { stringListOption->set_present(); if (stringListOption->get_name() == "--fid") { - int num_arg_left = argc - i; - for (int j = 1; j < num_arg_left; j += 1) { - bool end_of_option = (option_with_multiple_values_map.find(argv[i + j]) != option_with_multiple_values_map.end()); + for (int j = 1; j < argc; j += 1) { + int nextOptionOrValueIndex = i + j + 1; + bool end_of_option = isEndOfOption(nextOptionOrValueIndex, argv); stringListOption->add_value(argv[i + j]); - if (end_of_option || j == (num_arg_left - 1)) { - num_cmd_line_enrties_for_option = j + 1; - break; + if (end_of_option || (j + i) == (argc - 1)) { + num_cmd_line_enrties_for_option = j + 1; + break; } } } @@ -130,7 +138,6 @@ void parse_cmd(const int argc, const char *const *const argv) { if (search_option != option_map.end()) { auto stringOption = search_option->second; stringOption->set_present(); - if (stringOption->get_name() == "--storage_class_name") { stringOption->set_value(argv[i + 1]); } @@ -235,6 +242,8 @@ int main(int argc, const char **argv) validate_cmd(argc); std::cout << option_storage_class_name.get_value() << std::endl; + std::cout << option_fid.get_value()[0] << std::endl; + std::cout << option_fid.get_value()[1] << std::endl; return 1; } -- GitLab From 8d994e161e580de4c843adc7ff0dbd74cbc89c3e Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 30 Aug 2022 09:31:33 +0200 Subject: [PATCH 012/126] #78 - commit small changes before reset --- cmdline/CtaUpdateStorageClass.cpp | 109 ++++++++++++------------------ 1 file changed, 44 insertions(+), 65 deletions(-) diff --git a/cmdline/CtaUpdateStorageClass.cpp b/cmdline/CtaUpdateStorageClass.cpp index 0638118c06..b826757b8b 100644 --- a/cmdline/CtaUpdateStorageClass.cpp +++ b/cmdline/CtaUpdateStorageClass.cpp @@ -65,9 +65,7 @@ std::atomic<bool> is_json(false); std::atomic<bool> is_first_record(true); std::atomic<bool> isHeaderSent(false); -const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses -const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by - +const std::string config_file = "/etc/cta/cta-cli.conf"; std::string jsonCloseDelim() { return is_first_record ? "[]" : "]"; @@ -90,7 +88,7 @@ std::map<std::string, StringOption *> option_map = { }; -void validate_cmd(int argc) { +void validateCmd(int argc) { if (argc < 3) { throw Usage; } @@ -112,7 +110,7 @@ bool isEndOfOption(int nextOptionOrValueIndex, const char *const *const argv) { } -void parse_cmd(const int argc, const char *const *const argv) { +void parseCmd(const int argc, const char *const *const argv) { unsigned int num_cmd_line_enrties_for_option = 2; for (int i = 1; i < argc; i += num_cmd_line_enrties_for_option){ num_cmd_line_enrties_for_option = 2; @@ -151,80 +149,61 @@ void parse_cmd(const int argc, const char *const *const argv) { } -void sendTapeFile(cta::xrd::Request m_request) { - // Set configuration options - const std::string config_file = "/etc/cta/cta-cli.conf"; - XrdSsiPb::Config config(config_file, "cta"); - config.set("resource", "/ctafrontend"); - config.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes - config.set("request_timeout", DefaultRequestTimeout); // default value = 10s +void setStorageClassNameInEOSNamespace(std::string storageClassName, ) { + const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses + const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by + + + XrdSsiPb::Config config(config_file, "eos"); + for (const auto &conf_option : std::vector<std::string>({ "instance", "requester.user", "requester.group" })) { + if (!config.getOptionValueStr(conf_option).first) { + throw std::runtime_error(conf_option + " must be specified in " + config_file); + } + } + std::string instance = config.getOptionValueStr("instance").second; + std::string user = config.getOptionValueStr("requester.user").second; + std::string group = config.getOptionValueStr("requester.group").second; + + + + // Set CTA frontend configuration options + const std::string cli_config_file = "/etc/cta/cta-cli.conf"; + XrdSsiPb::Config cliConfig(cli_config_file, "cta"); + cliConfig.set("resource", "/ctafrontend"); + cliConfig.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes + cliConfig.set("request_timeout", DefaultRequestTimeout); // default value = 10s // Allow environment variables to override config file - config.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); + cliConfig.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); // If XRDDEBUG=1, switch on all logging if(getenv("XRDDEBUG")) { - config.set("log", "all"); + cliConfig.set("log", "all"); } // If fine-grained control over log level is required, use XrdSsiPbLogLevel - config.getEnv("log", "XrdSsiPbLogLevel"); + cliConfig.getEnv("log", "XrdSsiPbLogLevel"); // Validate that endpoint was specified in the config file - if(!config.getOptionValueStr("endpoint").first) { - throw std::runtime_error("Configuration error: cta.endpoint missing from " + config_file); + if(!cliConfig.getOptionValueStr("endpoint").first) { + throw std::runtime_error("Configuration error: cta.endpoint missing from " + cli_config_file); } // If the server is down, we want an immediate failure. Set client retry to a single attempt. XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); - // Obtain a Service Provider - XrdSsiPbServiceType cta_service(config); - - // Send the Request to the Service and get a Response - cta::xrd::Response response; - auto stream_future = cta_service.SendAsync(m_request, response); - - // Handle responses - switch(response.type()) - { - using namespace cta::xrd; - using namespace cta::admin; - - case Response::RSP_SUCCESS: - // Print message text - std::cout << response.message_txt(); - // Print streaming response header - if(true) switch(response.show_header()) { - case HeaderType::TAPEFILE_LS: std::cout << "hello"; break; - default: break; - } - // Allow stream processing to commence - isHeaderSent = true; - break; - case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); - case Response::RSP_ERR_USER: throw XrdSsiPb::UserException(response.message_txt()); - case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); - default: throw XrdSsiPb::PbException("Invalid response type."); - } - - // If there is a Data/Stream payload, wait until it has been processed before exiting - stream_future.wait(); - - // JSON output is an array of structs, close bracket - if(is_json) { std::cout << jsonCloseDelim(); } -} - + m_serviceProviderPtr.reset(new XrdSsiPbServiceType(cliConfig)); -void getTapeFile(cta::xrd::Request m_request) { - auto &admincmd = *(m_request.mutable_admincmd()); - - m_request.set_client_cta_version(CTA_VERSION); - m_request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); + // Set CTA frontend configuration options to connect to eos + const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf"; + XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); - admincmd.set_cmd(AdminCmd::CMD_TAPEFILE); - admincmd.set_subcmd(AdminCmd::SUBCMD_LS); - - sendTapeFile(m_request); + // Get the endpoint for namespace queries + auto nsConf = frontendXrootdConfig.getOptionValueStr("ns.config"); + if(nsConf.first) { + setNamespaceMap(nsConf.second); + } else { + throw std::runtime_error("Configuration error: cta.ns.config missing from " + frontend_xrootd_config_file); + } } } // namespace updateStorageClass } // namespace cta @@ -238,8 +217,8 @@ int main(int argc, const char **argv) using namespace cta::updateStorageClass; // testing - parse_cmd(argc, argv); - validate_cmd(argc); + parseCmd(argc, argv); + validateCmd(argc); std::cout << option_storage_class_name.get_value() << std::endl; std::cout << option_fid.get_value()[0] << std::endl; -- GitLab From 1016fd13f3f7a46764370ecdbb11602ff08bdf45 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 30 Aug 2022 09:35:33 +0200 Subject: [PATCH 013/126] #78 - add class structure --- cmdline/CMakeLists.txt | 7 +- cmdline/CtaUpdateStorageClass.cpp | 228 ++++++++++++++++++++++++++++++ 2 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 cmdline/CtaUpdateStorageClass.cpp diff --git a/cmdline/CMakeLists.txt b/cmdline/CMakeLists.txt index 09b8c112e6..d4904121fd 100644 --- a/cmdline/CMakeLists.txt +++ b/cmdline/CMakeLists.txt @@ -16,7 +16,6 @@ cmake_minimum_required (VERSION 3.17) add_subdirectory (restore_files) -add_subdirectory (update_storage_class) find_package(xrootdclient REQUIRED) find_package(Protobuf3 REQUIRED) @@ -37,6 +36,12 @@ add_executable(cta-admin CtaAdminCmd.cpp CtaAdminCmdParse.cpp CtaAdminTextFormat target_link_libraries(cta-admin XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon) set_property (TARGET cta-admin APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) +# +# cta-update-storage-class +# +add_executable(cta-update-storage-class CtaUpdateStorageClass.cpp CtaAdminCmdParse.cpp CtaAdminTextFormatter.cpp) +target_link_libraries(cta-update-storage-class ctacommon XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon) +set_property(TARGET cta-update-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) # # cta-send-event CLOSEW|PREPARE|ABORT_PREPARE diff --git a/cmdline/CtaUpdateStorageClass.cpp b/cmdline/CtaUpdateStorageClass.cpp new file mode 100644 index 0000000000..b826757b8b --- /dev/null +++ b/cmdline/CtaUpdateStorageClass.cpp @@ -0,0 +1,228 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranPty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#include <sstream> +#include <iostream> + +#include <XrdSsiPbLog.hpp> +#include <XrdSsiPbIStreamBuffer.hpp> +#include <XrdSsiPbLog.hpp> + +#include "cmdline/CtaAdminCmdParse.hpp" +#include "version.h" +#include "cmdline/CtaCmdOptions.hpp" + +#include "cta_frontend.pb.h" + + +namespace XrdSsiPb { + + +/*! +* User error exception +*/ +class UserException : public std::runtime_error +{ +public: + UserException(const std::string &err_msg) : std::runtime_error(err_msg) {} +}; + + +/*! + * Alert callback. + * + * Defines how Alert messages should be logged + */ +template<> +void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) +{ + std::cout << "AlertCallback():" << std::endl; + Log::DumpProtobuf(Log::PROTOBUF, &alert); +} + +} // namespace Xrdcp + +namespace cta{ +namespace updateStorageClass { + +using namespace cta::admin; + +std::atomic<bool> is_json(false); +std::atomic<bool> is_first_record(true); +std::atomic<bool> isHeaderSent(false); + +const std::string config_file = "/etc/cta/cta-cli.conf"; + +std::string jsonCloseDelim() { + return is_first_record ? "[]" : "]"; +} + +// Usage exception +const std::runtime_error Usage("Usage: ..."); + +StringListOption option_fid{"--fid", "-f", true}; +StringOption option_storage_class_name{"--storage_class_name", "-scn", true}; + +std::map<std::string, StringListOption *> option_with_multiple_values_map = { + {"--fid", &option_fid}, + {"-f", &option_fid}, +}; + +std::map<std::string, StringOption *> option_map = { + {"--storage_class_name", &option_storage_class_name}, + {"-scn", &option_storage_class_name}, +}; + + +void validateCmd(int argc) { + if (argc < 3) { + throw Usage; + } + for (auto &it : option_with_multiple_values_map) { + auto option = it.second; + if (!option->is_optional() && !option->is_present()) { + std::cout << "Error: Option " << option->get_name() << " is mandatory but not present." << std::endl; + throw Usage; + } + } +} + + +bool isEndOfOption(int nextOptionOrValueIndex, const char *const *const argv) { + return ( + option_with_multiple_values_map.find(argv[nextOptionOrValueIndex]) != option_with_multiple_values_map.end() || + option_map.find(argv[nextOptionOrValueIndex]) != option_map.end() + ); +} + + +void parseCmd(const int argc, const char *const *const argv) { + unsigned int num_cmd_line_enrties_for_option = 2; + for (int i = 1; i < argc; i += num_cmd_line_enrties_for_option){ + num_cmd_line_enrties_for_option = 2; + auto search_option_with_multiple_values_map = option_with_multiple_values_map.find(argv[i]); + if (search_option_with_multiple_values_map != option_with_multiple_values_map.end()) { + auto stringListOption = search_option_with_multiple_values_map->second; + stringListOption->set_present(); + + if (stringListOption->get_name() == "--fid") { + for (int j = 1; j < argc; j += 1) { + int nextOptionOrValueIndex = i + j + 1; + bool end_of_option = isEndOfOption(nextOptionOrValueIndex, argv); + stringListOption->add_value(argv[i + j]); + if (end_of_option || (j + i) == (argc - 1)) { + num_cmd_line_enrties_for_option = j + 1; + break; + } + } + } + } + + auto search_option = option_map.find(argv[i]); + if (search_option != option_map.end()) { + auto stringOption = search_option->second; + stringOption->set_present(); + if (stringOption->get_name() == "--storage_class_name") { + stringOption->set_value(argv[i + 1]); + } + } + + if (search_option_with_multiple_values_map == option_with_multiple_values_map.end() && search_option == option_map.end()) { + std::cout << "Error: Unknown option: " << argv[i] << std::endl; + throw Usage; + } + } +} + + +void setStorageClassNameInEOSNamespace(std::string storageClassName, ) { + const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses + const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by + + + XrdSsiPb::Config config(config_file, "eos"); + for (const auto &conf_option : std::vector<std::string>({ "instance", "requester.user", "requester.group" })) { + if (!config.getOptionValueStr(conf_option).first) { + throw std::runtime_error(conf_option + " must be specified in " + config_file); + } + } + std::string instance = config.getOptionValueStr("instance").second; + std::string user = config.getOptionValueStr("requester.user").second; + std::string group = config.getOptionValueStr("requester.group").second; + + + + // Set CTA frontend configuration options + const std::string cli_config_file = "/etc/cta/cta-cli.conf"; + XrdSsiPb::Config cliConfig(cli_config_file, "cta"); + cliConfig.set("resource", "/ctafrontend"); + cliConfig.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes + cliConfig.set("request_timeout", DefaultRequestTimeout); // default value = 10s + + // Allow environment variables to override config file + cliConfig.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); + + // If XRDDEBUG=1, switch on all logging + if(getenv("XRDDEBUG")) { + cliConfig.set("log", "all"); + } + // If fine-grained control over log level is required, use XrdSsiPbLogLevel + cliConfig.getEnv("log", "XrdSsiPbLogLevel"); + + // Validate that endpoint was specified in the config file + if(!cliConfig.getOptionValueStr("endpoint").first) { + throw std::runtime_error("Configuration error: cta.endpoint missing from " + cli_config_file); + } + + // If the server is down, we want an immediate failure. Set client retry to a single attempt. + XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); + + m_serviceProviderPtr.reset(new XrdSsiPbServiceType(cliConfig)); + + // Set CTA frontend configuration options to connect to eos + const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf"; + XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); + + // Get the endpoint for namespace queries + auto nsConf = frontendXrootdConfig.getOptionValueStr("ns.config"); + if(nsConf.first) { + setNamespaceMap(nsConf.second); + } else { + throw std::runtime_error("Configuration error: cta.ns.config missing from " + frontend_xrootd_config_file); + } +} +} // namespace updateStorageClass +} // namespace cta + + +/* + * Start here + */ +int main(int argc, const char **argv) +{ + using namespace cta::updateStorageClass; + + // testing + parseCmd(argc, argv); + validateCmd(argc); + + std::cout << option_storage_class_name.get_value() << std::endl; + std::cout << option_fid.get_value()[0] << std::endl; + std::cout << option_fid.get_value()[1] << std::endl; + + return 1; +} -- GitLab From 9ee31343095b4cf8e07997c956118368d13ae2dc Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 30 Aug 2022 10:37:32 +0200 Subject: [PATCH 014/126] #78 - add class structure --- cmdline/CMakeLists.txt | 7 +- cmdline/CtaUpdateStorageClass.cpp | 228 ++++++++++++++++++++++++++++++ 2 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 cmdline/CtaUpdateStorageClass.cpp diff --git a/cmdline/CMakeLists.txt b/cmdline/CMakeLists.txt index 09b8c112e6..d4904121fd 100644 --- a/cmdline/CMakeLists.txt +++ b/cmdline/CMakeLists.txt @@ -16,7 +16,6 @@ cmake_minimum_required (VERSION 3.17) add_subdirectory (restore_files) -add_subdirectory (update_storage_class) find_package(xrootdclient REQUIRED) find_package(Protobuf3 REQUIRED) @@ -37,6 +36,12 @@ add_executable(cta-admin CtaAdminCmd.cpp CtaAdminCmdParse.cpp CtaAdminTextFormat target_link_libraries(cta-admin XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon) set_property (TARGET cta-admin APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) +# +# cta-update-storage-class +# +add_executable(cta-update-storage-class CtaUpdateStorageClass.cpp CtaAdminCmdParse.cpp CtaAdminTextFormatter.cpp) +target_link_libraries(cta-update-storage-class ctacommon XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon) +set_property(TARGET cta-update-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) # # cta-send-event CLOSEW|PREPARE|ABORT_PREPARE diff --git a/cmdline/CtaUpdateStorageClass.cpp b/cmdline/CtaUpdateStorageClass.cpp new file mode 100644 index 0000000000..b826757b8b --- /dev/null +++ b/cmdline/CtaUpdateStorageClass.cpp @@ -0,0 +1,228 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranPty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#include <sstream> +#include <iostream> + +#include <XrdSsiPbLog.hpp> +#include <XrdSsiPbIStreamBuffer.hpp> +#include <XrdSsiPbLog.hpp> + +#include "cmdline/CtaAdminCmdParse.hpp" +#include "version.h" +#include "cmdline/CtaCmdOptions.hpp" + +#include "cta_frontend.pb.h" + + +namespace XrdSsiPb { + + +/*! +* User error exception +*/ +class UserException : public std::runtime_error +{ +public: + UserException(const std::string &err_msg) : std::runtime_error(err_msg) {} +}; + + +/*! + * Alert callback. + * + * Defines how Alert messages should be logged + */ +template<> +void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) +{ + std::cout << "AlertCallback():" << std::endl; + Log::DumpProtobuf(Log::PROTOBUF, &alert); +} + +} // namespace Xrdcp + +namespace cta{ +namespace updateStorageClass { + +using namespace cta::admin; + +std::atomic<bool> is_json(false); +std::atomic<bool> is_first_record(true); +std::atomic<bool> isHeaderSent(false); + +const std::string config_file = "/etc/cta/cta-cli.conf"; + +std::string jsonCloseDelim() { + return is_first_record ? "[]" : "]"; +} + +// Usage exception +const std::runtime_error Usage("Usage: ..."); + +StringListOption option_fid{"--fid", "-f", true}; +StringOption option_storage_class_name{"--storage_class_name", "-scn", true}; + +std::map<std::string, StringListOption *> option_with_multiple_values_map = { + {"--fid", &option_fid}, + {"-f", &option_fid}, +}; + +std::map<std::string, StringOption *> option_map = { + {"--storage_class_name", &option_storage_class_name}, + {"-scn", &option_storage_class_name}, +}; + + +void validateCmd(int argc) { + if (argc < 3) { + throw Usage; + } + for (auto &it : option_with_multiple_values_map) { + auto option = it.second; + if (!option->is_optional() && !option->is_present()) { + std::cout << "Error: Option " << option->get_name() << " is mandatory but not present." << std::endl; + throw Usage; + } + } +} + + +bool isEndOfOption(int nextOptionOrValueIndex, const char *const *const argv) { + return ( + option_with_multiple_values_map.find(argv[nextOptionOrValueIndex]) != option_with_multiple_values_map.end() || + option_map.find(argv[nextOptionOrValueIndex]) != option_map.end() + ); +} + + +void parseCmd(const int argc, const char *const *const argv) { + unsigned int num_cmd_line_enrties_for_option = 2; + for (int i = 1; i < argc; i += num_cmd_line_enrties_for_option){ + num_cmd_line_enrties_for_option = 2; + auto search_option_with_multiple_values_map = option_with_multiple_values_map.find(argv[i]); + if (search_option_with_multiple_values_map != option_with_multiple_values_map.end()) { + auto stringListOption = search_option_with_multiple_values_map->second; + stringListOption->set_present(); + + if (stringListOption->get_name() == "--fid") { + for (int j = 1; j < argc; j += 1) { + int nextOptionOrValueIndex = i + j + 1; + bool end_of_option = isEndOfOption(nextOptionOrValueIndex, argv); + stringListOption->add_value(argv[i + j]); + if (end_of_option || (j + i) == (argc - 1)) { + num_cmd_line_enrties_for_option = j + 1; + break; + } + } + } + } + + auto search_option = option_map.find(argv[i]); + if (search_option != option_map.end()) { + auto stringOption = search_option->second; + stringOption->set_present(); + if (stringOption->get_name() == "--storage_class_name") { + stringOption->set_value(argv[i + 1]); + } + } + + if (search_option_with_multiple_values_map == option_with_multiple_values_map.end() && search_option == option_map.end()) { + std::cout << "Error: Unknown option: " << argv[i] << std::endl; + throw Usage; + } + } +} + + +void setStorageClassNameInEOSNamespace(std::string storageClassName, ) { + const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses + const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by + + + XrdSsiPb::Config config(config_file, "eos"); + for (const auto &conf_option : std::vector<std::string>({ "instance", "requester.user", "requester.group" })) { + if (!config.getOptionValueStr(conf_option).first) { + throw std::runtime_error(conf_option + " must be specified in " + config_file); + } + } + std::string instance = config.getOptionValueStr("instance").second; + std::string user = config.getOptionValueStr("requester.user").second; + std::string group = config.getOptionValueStr("requester.group").second; + + + + // Set CTA frontend configuration options + const std::string cli_config_file = "/etc/cta/cta-cli.conf"; + XrdSsiPb::Config cliConfig(cli_config_file, "cta"); + cliConfig.set("resource", "/ctafrontend"); + cliConfig.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes + cliConfig.set("request_timeout", DefaultRequestTimeout); // default value = 10s + + // Allow environment variables to override config file + cliConfig.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); + + // If XRDDEBUG=1, switch on all logging + if(getenv("XRDDEBUG")) { + cliConfig.set("log", "all"); + } + // If fine-grained control over log level is required, use XrdSsiPbLogLevel + cliConfig.getEnv("log", "XrdSsiPbLogLevel"); + + // Validate that endpoint was specified in the config file + if(!cliConfig.getOptionValueStr("endpoint").first) { + throw std::runtime_error("Configuration error: cta.endpoint missing from " + cli_config_file); + } + + // If the server is down, we want an immediate failure. Set client retry to a single attempt. + XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); + + m_serviceProviderPtr.reset(new XrdSsiPbServiceType(cliConfig)); + + // Set CTA frontend configuration options to connect to eos + const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf"; + XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); + + // Get the endpoint for namespace queries + auto nsConf = frontendXrootdConfig.getOptionValueStr("ns.config"); + if(nsConf.first) { + setNamespaceMap(nsConf.second); + } else { + throw std::runtime_error("Configuration error: cta.ns.config missing from " + frontend_xrootd_config_file); + } +} +} // namespace updateStorageClass +} // namespace cta + + +/* + * Start here + */ +int main(int argc, const char **argv) +{ + using namespace cta::updateStorageClass; + + // testing + parseCmd(argc, argv); + validateCmd(argc); + + std::cout << option_storage_class_name.get_value() << std::endl; + std::cout << option_fid.get_value()[0] << std::endl; + std::cout << option_fid.get_value()[1] << std::endl; + + return 1; +} -- GitLab From 0ba9a39ba5afe95174b616f3a65078e1befaaadf Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 30 Aug 2022 10:48:48 +0200 Subject: [PATCH 015/126] #78 - add to class structure --- cmdline/CMakeLists.txt | 9 +- cmdline/update_storage_class/CMakeLists.txt | 36 ++ .../CtaUpdateStorageClass.cpp | 529 ++++++++++++++++++ .../CtaUpdateStorageClass.hpp | 211 +++++++ .../CtaUpdateStorageClassCmdLineArgs.cpp | 193 +++++++ .../CtaUpdateStorageClassCmdLineArgs.hpp | 95 ++++ .../CtaUpdateStorageClassMain.cpp | 48 ++ 7 files changed, 1114 insertions(+), 7 deletions(-) create mode 100644 cmdline/update_storage_class/CMakeLists.txt create mode 100644 cmdline/update_storage_class/CtaUpdateStorageClass.cpp create mode 100644 cmdline/update_storage_class/CtaUpdateStorageClass.hpp create mode 100644 cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp create mode 100644 cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp create mode 100644 cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp diff --git a/cmdline/CMakeLists.txt b/cmdline/CMakeLists.txt index d4904121fd..75929acf80 100644 --- a/cmdline/CMakeLists.txt +++ b/cmdline/CMakeLists.txt @@ -16,6 +16,7 @@ cmake_minimum_required (VERSION 3.17) add_subdirectory (restore_files) +add_subdirectory (update_storage_class) find_package(xrootdclient REQUIRED) find_package(Protobuf3 REQUIRED) @@ -36,12 +37,6 @@ add_executable(cta-admin CtaAdminCmd.cpp CtaAdminCmdParse.cpp CtaAdminTextFormat target_link_libraries(cta-admin XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon) set_property (TARGET cta-admin APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) -# -# cta-update-storage-class -# -add_executable(cta-update-storage-class CtaUpdateStorageClass.cpp CtaAdminCmdParse.cpp CtaAdminTextFormatter.cpp) -target_link_libraries(cta-update-storage-class ctacommon XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon) -set_property(TARGET cta-update-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) # # cta-send-event CLOSEW|PREPARE|ABORT_PREPARE @@ -75,4 +70,4 @@ install(FILES cta-admin.1cta DESTINATION usr/share/man/man1) install(TARGETS cta-send-event DESTINATION usr/bin) install(FILES cta-send-closew.sh DESTINATION usr/bin) install(TARGETS cta-verify-file DESTINATION usr/bin) -install(FILES cta-cli.conf.example DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/cta) +install(FILES cta-cli.conf.example DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}/cta) \ No newline at end of file diff --git a/cmdline/update_storage_class/CMakeLists.txt b/cmdline/update_storage_class/CMakeLists.txt new file mode 100644 index 0000000000..45521ae5ad --- /dev/null +++ b/cmdline/update_storage_class/CMakeLists.txt @@ -0,0 +1,36 @@ +# @project The CERN Tape Archive (CTA) +# @copyright Copyright © 2015-2022 CERN +# @license This program is free software, distributed under the terms of the GNU General Public +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can +# redistribute it and/or modify it under the terms of the GPL Version 3, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# In applying this licence, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +cmake_minimum_required (VERSION 3.17) + +find_package(xrootdclient REQUIRED) +find_package(Protobuf3 REQUIRED) + +# XRootD SSI +include_directories(${XROOTD_INCLUDE_DIR} ${XROOTD_INCLUDE_DIR}/private ) + +# XRootD SSI Protocol Buffer bindings +include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include) + +# Compiled protocol buffers +include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS}) + +add_executable(cta-update-storage-class CtaUpdateStorageClass.cpp CtaUpdateStorageClassCmdLineArgs.cpp CtaUpdateStorageClassMain.cpp ../restore_files/CmdLineTool.cpp) +target_link_libraries(cta-update-storage-class ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient) +set_property (TARGET cta-update-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) + + +install(TARGETS cta-update-storage-class DESTINATION usr/bin) +install(FILES cta-update-storage-class.1cta DESTINATION usr/share/man/man1) diff --git a/cmdline/update_storage_class/CtaUpdateStorageClass.cpp b/cmdline/update_storage_class/CtaUpdateStorageClass.cpp new file mode 100644 index 0000000000..75cfdcbe6f --- /dev/null +++ b/cmdline/update_storage_class/CtaUpdateStorageClass.cpp @@ -0,0 +1,529 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#include "cmdline/update_storage_class/CtaUpdateStorageClass.hpp" +#include "cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp" +#include "common/utils/utils.hpp" +#include "common/checksum/ChecksumBlob.hpp" +#include "CtaFrontendApi.hpp" + +#include <XrdSsiPbLog.hpp> +#include <XrdSsiPbIStreamBuffer.hpp> + +#include <grpc++/grpc++.h> +#include "Rpc.grpc.pb.h" + +#include <sys/stat.h> +#include <iostream> +#include <memory> + +// GLOBAL VARIABLES : used to pass information between main thread and stream handler thread + +// global synchronisation flag +std::atomic<bool> isHeaderSent(false); + +std::list<cta::admin::RecycleTapeFileLsItem> deletedTapeFiles; +std::list<std::pair<std::string,std::string>> listedTapeFiles; + +namespace XrdSsiPb { + +/*! + * User error exception + */ +class UserException : public std::runtime_error +{ +public: + UserException(const std::string &err_msg) : std::runtime_error(err_msg) {} +}; // class UserException + +/*! + * Alert callback. + * + * Defines how Alert messages should be logged + */ +template<> +void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) +{ + std::cout << "AlertCallback():" << std::endl; + Log::DumpProtobuf(Log::PROTOBUF, &alert); +} + +/*! + * Data/Stream callback. + * + * Defines how incoming records from the stream should be handled + */ +template<> +void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const +{ + using namespace cta::xrd; + using namespace cta::admin; + + // Wait for primary response to be handled before allowing stream response + while(!isHeaderSent) { std::this_thread::yield(); } + + switch(record.data_case()) { + case Data::kRtflsItem: + { + auto item = record.rtfls_item(); + deletedTapeFiles.push_back(item); + } + break; + case Data::kTflsItem: + { + auto item = record.tfls_item(); + auto instanceAndFid = std::make_pair(item.df().disk_instance(), item.df().disk_id()); + listedTapeFiles.push_back(instanceAndFid); + } + break; + default: + throw std::runtime_error("Received invalid stream data from CTA Frontend for the cta-restore-deleted-files command."); + } +} + +} // namespace XrdSsiPb + + +namespace cta{ +namespace admin { + +/*! + * UpdateFilesCmdException + */ +class UpdateFilesCmdException : public std::runtime_error +{ +public: + UpdateFilesCmdException(const std::string &err_msg) : std::runtime_error(err_msg) {} +}; // class UserException + + + +//------------------------------------------------------------------------------ +// constructor +//------------------------------------------------------------------------------ +CtaUpdateStorageClass::CtaUpdateStorageClass(std::istream &inStream, std::ostream &outStream, + std::ostream &errStream, cta::log::StdoutLogger &log): + CmdLineTool(inStream, outStream, errStream), + m_log(log) { + + // Default layout: see EOS common/LayoutId.hh for definitions of constants + const int kAdler = 0x2; + const int kReplica = (0x1 << 4); + const int kStripeSize = (0x0 << 8); // 1 stripe + const int kStripeWidth = (0x0 << 16); // 4K blocks + const int kBlockChecksum = (0x1 << 20); + + // Default single replica layout id should be 00100012 + m_defaultFileLayout = kReplica | kAdler | kStripeSize | kStripeWidth | kBlockChecksum; + +} + +//------------------------------------------------------------------------------ +// exceptionThrowingMain +//------------------------------------------------------------------------------ +int CtaUpdateStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { + CtaUpdateStorageClassCmdLineArgs cmdLineArgs(argc, argv); + if (cmdLineArgs.m_help) { + printUsage(m_out); + return 0; + } + return 0; +} + +//------------------------------------------------------------------------------ +// readAndSetConfiguration +//------------------------------------------------------------------------------ +void CtaUpdateStorageClass::readAndSetConfiguration( + const std::string &userName, + const CtaUpdateStorageClassCmdLineArgs &cmdLineArgs) { + + m_vid = cmdLineArgs.m_vid; + m_diskInstance = cmdLineArgs.m_diskInstance; + m_eosFids = cmdLineArgs.m_eosFids; + m_copyNumber = cmdLineArgs.m_copyNumber; + m_archiveFileId = cmdLineArgs.m_archiveFileId; + + if (cmdLineArgs.m_debug) { + m_log.setLogMask("DEBUG"); + } else { + m_log.setLogMask("INFO"); + } + + // Set CTA frontend configuration options + const std::string cli_config_file = "/etc/cta/cta-cli.conf"; + XrdSsiPb::Config cliConfig(cli_config_file, "cta"); + cliConfig.set("resource", "/ctafrontend"); + cliConfig.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes + cliConfig.set("request_timeout", DefaultRequestTimeout); // default value = 10s + + // Allow environment variables to override config file + cliConfig.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); + + // If XRDDEBUG=1, switch on all logging + if(getenv("XRDDEBUG")) { + cliConfig.set("log", "all"); + } + // If fine-grained control over log level is required, use XrdSsiPbLogLevel + cliConfig.getEnv("log", "XrdSsiPbLogLevel"); + + // Validate that endpoint was specified in the config file + if(!cliConfig.getOptionValueStr("endpoint").first) { + throw std::runtime_error("Configuration error: cta.endpoint missing from " + cli_config_file); + } + + // If the server is down, we want an immediate failure. Set client retry to a single attempt. + XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); + + m_serviceProviderPtr.reset(new XrdSsiPbServiceType(cliConfig)); + + // Set CTA frontend configuration options to connect to eos + const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf"; + XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); + + // Get the endpoint for namespace queries + auto nsConf = frontendXrootdConfig.getOptionValueStr("ns.config"); + if(nsConf.first) { + setNamespaceMap(nsConf.second); + } else { + throw std::runtime_error("Configuration error: cta.ns.config missing from " + frontend_xrootd_config_file); + } +} + +void CtaUpdateStorageClass::setNamespaceMap(const std::string &keytab_file) { + // Open the keytab file for reading + std::ifstream file(keytab_file); + if(!file) { + throw cta::exception::UserError("Failed to open namespace keytab configuration file " + keytab_file); + } + ::eos::client::NamespaceMap_t namespaceMap; + // Parse the keytab line by line + std::string line; + for(int lineno = 0; std::getline(file, line); ++lineno) { + // Strip out comments + auto pos = line.find('#'); + if(pos != std::string::npos) { + line.resize(pos); + } + + // Parse one line + std::stringstream ss(line); + std::string diskInstance; + std::string endpoint; + std::string token; + std::string eol; + ss >> diskInstance >> endpoint >> token >> eol; + + // Ignore blank lines, all other lines must have exactly 3 elements + if(token.empty() || !eol.empty()) { + if(diskInstance.empty() && endpoint.empty() && token.empty()) continue; + throw cta::exception::UserError("Could not parse namespace keytab configuration file line " + std::to_string(lineno) + ": " + line); + } + namespaceMap.insert(std::make_pair(diskInstance, ::eos::client::Namespace(endpoint, token))); + } + m_endpointMapPtr = std::make_unique<::eos::client::EndpointMap>(namespaceMap); +} + + +//------------------------------------------------------------------------------ +// addContainerEos +//------------------------------------------------------------------------------ +uint64_t CtaUpdateStorageClass::addContainerEos(const std::string &diskInstance, const std::string &path, const std::string &sc) const { + auto c_id = containerExistsEos(diskInstance, path); + if (c_id) { + return c_id; + } + auto enclosingPath = cta::utils::getEnclosingPath(path); + auto parent_id = containerExistsEos(diskInstance, enclosingPath); + if (!parent_id) { + //parent does not exist, need to add it as well + parent_id = addContainerEos(diskInstance, enclosingPath, sc); + } + + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", getUsername())); + params.push_back(cta::log::Param("diskInstance", diskInstance)); + params.push_back(cta::log::Param("path", path)); + m_log(cta::log::DEBUG, "Inserting container in EOS namespace", params); + + ::eos::rpc::ContainerMdProto dir; + dir.set_path(path); + dir.set_name(cta::utils::getEnclosedName(path)); + + // Filemode: filter out S_ISUID, S_ISGID and S_ISVTX because EOS does not follow POSIX semantics for these bits + uint64_t filemode = (S_IRWXU | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); // 0755 permissions by default + filemode &= ~(S_ISUID | S_ISGID | S_ISVTX); + dir.set_mode(filemode); + + auto time = ::time(nullptr); + // Timestamps + dir.mutable_ctime()->set_sec(time); + dir.mutable_mtime()->set_sec(time); + // we don't care about dir.stime (sync time, used for CERNBox) + + dir.mutable_xattrs()->insert(google::protobuf::MapPair<std::string,std::string>("sys.archive.storage_class", sc)); + + + auto reply = m_endpointMapPtr->containerInsert(diskInstance, dir); + + m_log(cta::log::DEBUG, "Inserted container in EOS namespace successfully, querying again for its id", params); + + auto cont_id = containerExistsEos(diskInstance, path); + if (!cont_id) { + throw UpdateFilesCmdException(std::string("Container ") + path + " does not exist after being inserted in EOS."); + } + return cont_id; +} + +//------------------------------------------------------------------------------ +// containerExistsEos +//------------------------------------------------------------------------------ +uint64_t CtaUpdateStorageClass::containerExistsEos(const std::string &diskInstance, const std::string &path) const { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", getUsername())); + params.push_back(cta::log::Param("diskInstance", diskInstance)); + params.push_back(cta::log::Param("path", path)); + + m_log(cta::log::DEBUG, "Verifying if the container exists in the EOS namespace", params); + + auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::CONTAINER, 0, path, false); + auto cid = md_response.cmd().id(); + params.push_back(cta::log::Param("containerId", cid)); + if (cid != 0) { + m_log(cta::log::DEBUG, "Container exists in the EOS namespace", params); + } else { + m_log(cta::log::DEBUG, "Container does not exist in the EOS namespace", params); + } + return cid; +} + + +//------------------------------------------------------------------------------ +// archiveFileExistsCTA +//------------------------------------------------------------------------------ +bool CtaUpdateStorageClass::archiveFileExistsCTA(const uint64_t &archiveFileId) const { + + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", getUsername())); + params.push_back(cta::log::Param("archiveFileId", archiveFileId)); + + m_log(cta::log::DEBUG, "Looking for archive file in the CTA catalogue", params); + + cta::xrd::Request request; + + auto &admincmd = *(request.mutable_admincmd()); + + request.set_client_cta_version(CTA_VERSION); + request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); + admincmd.set_cmd(AdminCmd::CMD_TAPEFILE); + admincmd.set_subcmd(AdminCmd::SUBCMD_LS); + + auto key = OptionUInt64::ARCHIVE_FILE_ID; + auto new_opt = admincmd.add_option_uint64(); + new_opt->set_key(key); + new_opt->set_value(archiveFileId); + + // Send the Request to the Service and get a Response + cta::xrd::Response response; + auto stream_future = m_serviceProviderPtr->SendAsync(request, response); + + bool ret; + // Handle responses + switch(response.type()) + { + using namespace cta::xrd; + using namespace cta::admin; + case Response::RSP_SUCCESS: ret = true; break; //success sent if archive file does not exist + case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); + case Response::RSP_ERR_USER: ret = false; break; //user error sent if archive file does not exist + case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); + default: throw XrdSsiPb::PbException("Invalid response type."); + } + + // wait until the data stream has been processed before exiting + if (ret) { + stream_future.wait(); + } + if (ret) { + m_log(cta::log::DEBUG, "Archive file is present in the CTA catalogue", params); + } else { + m_log(cta::log::DEBUG, "Archive file is missing in the CTA catalogue", params); + } + return ret; +} + +//------------------------------------------------------------------------------ +// fileExistsEos +//------------------------------------------------------------------------------ +bool CtaUpdateStorageClass::fileExistsEos(const std::string &diskInstance, const std::string &diskFileId) const { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", getUsername())); + params.push_back(cta::log::Param("diskInstance", diskInstance)); + params.push_back(cta::log::Param("diskFileId", diskFileId)); + + m_log(cta::log::DEBUG, "Verifying if EOS fid exists in the EOS namespace", params); + try { + auto path = m_endpointMapPtr->getPath(diskInstance, diskFileId); + params.push_back(cta::log::Param("diskFilePath", path)); + m_log(cta::log::DEBUG, "EOS fid exists in the EOS namespace"); + return true; + } catch(cta::exception::Exception&) { + m_log(cta::log::DEBUG, "EOS fid does not exist in the EOS namespace"); + return false; + } +} + +//------------------------------------------------------------------------------ +// getFileIdEos +//------------------------------------------------------------------------------ +uint64_t CtaUpdateStorageClass::getFileIdEos(const std::string &diskInstance, const std::string &path) const { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("userName", getUsername())); + params.push_back(cta::log::Param("diskInstance", diskInstance)); + params.push_back(cta::log::Param("path", path)); + + m_log(cta::log::DEBUG, "Querying for file metadata in the EOS namespace", params); + auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::FILE, 0, path, false); + auto fid = md_response.fmd().id(); + params.push_back(cta::log::Param("diskFileId", fid)); + if (fid != 0) { + m_log(cta::log::DEBUG, "File path exists in the EOS namespace", params); + } else { + m_log(cta::log::DEBUG, "File path does not exist in the EOS namespace", params); + } + return fid; +} + +//------------------------------------------------------------------------------ +// getCurrentEosIds +//------------------------------------------------------------------------------ +void CtaUpdateStorageClass::getCurrentEosIds(const std::string &diskInstance) const { + uint64_t cid; + uint64_t fid; + m_endpointMapPtr->getCurrentIds(diskInstance, cid, fid); + + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("diskInstance", diskInstance)); + params.push_back(cta::log::Param("ContainerId", cid)); + params.push_back(cta::log::Param("FileId", fid)); + m_log(cta::log::DEBUG, "Obtained current EOS container and file id", params); +} + + +//------------------------------------------------------------------------------ +// printUsage +//------------------------------------------------------------------------------ +void CtaUpdateStorageClass::printUsage(std::ostream &os) { + CtaUpdateStorageClassCmdLineArgs::printUsage(os); +} + + +//------------------------------------------------------------------------------ +// getFxidFromCTA +//------------------------------------------------------------------------------ +std::pair<std::string,std::string> CtaUpdateStorageClass::getInstanceAndFidFromCTA(const RecycleTapeFileLsItem& file) { + { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("archiveFileId", file.archive_file_id())); + m_log(cta::log::DEBUG, "cta-admin tapefile ls", params); + } + + cta::xrd::Request request; + auto &admincmd = *(request.mutable_admincmd()); + + request.set_client_cta_version(CTA_VERSION); + request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); + admincmd.set_cmd(AdminCmd::CMD_TAPEFILE); + admincmd.set_subcmd(AdminCmd::SUBCMD_LS); + auto new_opt = admincmd.add_option_uint64(); + new_opt->set_key(OptionUInt64::ARCHIVE_FILE_ID); + new_opt->set_value(file.archive_file_id()); + + // Send the Request to the Service and get a Response + cta::xrd::Response response; + auto stream_future = m_serviceProviderPtr->SendAsync(request, response); + + // Handle responses + switch(response.type()) + { + using namespace cta::xrd; + using namespace cta::admin; + case Response::RSP_SUCCESS: + // Print message text + std::cout << response.message_txt(); + // Allow stream processing to commence + isHeaderSent = true; + break; + case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); + case Response::RSP_ERR_USER: throw XrdSsiPb::UserException(response.message_txt()); + case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); + default: throw XrdSsiPb::PbException("Invalid response type."); + } + + // wait until the data stream has been processed before exiting + stream_future.wait(); + if(listedTapeFiles.size() != 1) { + throw std::runtime_error("Unexpected result set: listedTapeFiles size expected=1 received=" + std::to_string(listedTapeFiles.size())); + } + auto listedTapeFile = listedTapeFiles.back(); + listedTapeFiles.clear(); + { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("diskInstance", listedTapeFile.first)); + params.push_back(cta::log::Param("diskFileId", listedTapeFile.second)); + m_log(cta::log::DEBUG, "Obtained file metadata from CTA", params); + } + return listedTapeFile; +} + +//------------------------------------------------------------------------------ +// getArchiveFileIdFromEOS +//------------------------------------------------------------------------------ +std::pair<uint64_t, std::string> CtaUpdateStorageClass::getArchiveFileIdAndChecksumFromEOS( + const std::string& diskInstance, const std::string& fidStr) { + auto fid = strtoul(fidStr.c_str(), nullptr, 10); + if(fid < 1) { + throw std::runtime_error(fid + " (base 10) is not a valid disk file ID"); + } + { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("diskInstance", diskInstance)); + params.push_back(cta::log::Param("fid", fid)); + std::stringstream ss; + ss << std::hex << fid; + params.push_back(cta::log::Param("fxid", ss.str())); + m_log(cta::log::DEBUG, "Querying EOS namespace for archiveFileId and checksum", params); + } + auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::FILE, fid, "", false); + auto archiveFileIdItor = md_response.fmd().xattrs().find("sys.archive.file_id"); + if(md_response.fmd().xattrs().end() == archiveFileIdItor) { + throw std::runtime_error("archiveFileId extended attribute not found."); + } + auto archiveFileId = strtoul(archiveFileIdItor->second.c_str(), nullptr, 10); + auto byteArray = md_response.fmd().checksum().value(); + auto checksumValue = checksum::ChecksumBlob::ByteArrayToHex(std::string(byteArray.rbegin(), byteArray.rend())); + { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("archiveFileId", archiveFileId)); + params.push_back(cta::log::Param("checksumValue", checksumValue)); + m_log(cta::log::DEBUG, "Response from EOS nameserver", params); + } + + return std::make_pair(archiveFileId,checksumValue); +} + +} // namespace admin +} // namespace cta diff --git a/cmdline/update_storage_class/CtaUpdateStorageClass.hpp b/cmdline/update_storage_class/CtaUpdateStorageClass.hpp new file mode 100644 index 0000000000..47ead6bee6 --- /dev/null +++ b/cmdline/update_storage_class/CtaUpdateStorageClass.hpp @@ -0,0 +1,211 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#pragma once + +#include "cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp" +#include "cmdline/restore_files/CmdLineTool.hpp" +#include "eos_grpc_client/GrpcEndpoint.hpp" +#include "catalogue/Catalogue.hpp" +#include "common/log/StdoutLogger.hpp" + +#include <optional> +#include <memory> + +#include "CtaFrontendApi.hpp" + +namespace cta { +namespace admin { + +class CtaUpdateStorageClass: public CmdLineTool { +public: + /** + * Constructor. + * + * @param inStream Standard input stream. + * @param outStream Standard output stream. + * @param errStream Standard error stream. + * @param log The object representing the API of the CTA logging system. + */ + CtaUpdateStorageClass(std::istream &inStream, std::ostream &outStream, + std::ostream &errStream, cta::log::StdoutLogger &log); + + /** + * An exception throwing version of main(). + * + * @param argc The number of command-line arguments including the program name. + * @param argv The command-line arguments. + * @return The exit value of the program. + */ + int exceptionThrowingMain(const int argc, char *const *const argv) override; + + /** + * Sets internal configuration parameters to be used for reading. + * It reads cta frontend parameters from /etc/cta/cta-cli.conf + * + * @param username The name of the user running the command-line tool. + * @param cmdLineArgs The arguments parsed from the command line. + */ + void readAndSetConfiguration(const std::string &userName, const CtaUpdateStorageClassCmdLineArgs &cmdLineArgs); + + /** + * Populate the namespace endpoint configuration from a keytab file + */ + void setNamespaceMap(const std::string &keytab_file); + + + /** + * Queries the eos mgm for the current eos file and container id + * Must be called before any other call to EOS, to initialize the grpc + * client cid and fid + * @param diskInstance the disk instance of the eos instance + */ + void getCurrentEosIds(const std::string &diskInstance) const; + + + /** + * Adds a container in the eos namespace, if it does not exist + * @param diskInstance eos disk instance + * @param path the path of the container + * @param sc the storage class of the container + * @returns the container id of the container identified by path + */ + uint64_t addContainerEos(const std::string &diskInstance, const std::string &path, const std::string &sc) const; + + /** + * Returns the container id of the container in the EOS namespace + * + * @param diskInstance eos disk instance + * @param path the path of the container + * + * @returns the container id of the container in the EOS namespace, or zero if it does not exist + */ + uint64_t containerExistsEos(const std::string &diskInstance, const std::string &path) const; + + /** + * Returns true (i.e. not zero) if a file with given id exists in the eos namespace + * @param diskInstance eos disk instance + * @param diskFileId the eos file id to check + */ + bool fileExistsEos(const std::string &diskInstance, const std::string &diskFileId) const; + + + /** + * Returns true (i.e. not zero) if an archive file with given id exists in the cta catalogue + * @param archiveFileId the archive file id to check + */ + bool archiveFileExistsCTA(const uint64_t &archiveFileId) const; + + /** + * Returns the id of a given file in eos or zero if the files does not exist + * @param diskInstance eos disk instance + * @param path the path to check + */ + uint64_t getFileIdEos(const std::string &diskInstance, const std::string &path) const; + + + /** + * Prints the usage message of the command-line tool. + * + * @param os The output stream to which the usage message is to be printed. + */ + void printUsage(std::ostream &os) override; + +private: + /** + * Query CTA for the disk instance and fid of the restored file + * + * @param file The restored tape file in CTA + * @return Tuple of EOS disk instance and file ID (fid) as a decimal string + */ + std::pair<std::string,std::string> getInstanceAndFidFromCTA(const RecycleTapeFileLsItem& file); + + /** + * Query EOS for the archiveFileId and checksum of the restored file + * + * @param diskInstance Which EOS disk instance to query + * @param fxid fid of the file in EOS as a decimal string + * @return tuple containing archiveFileId and checksum of the file + */ + std::pair<uint64_t, std::string> getArchiveFileIdAndChecksumFromEOS(const std::string& diskInstance, + const std::string& fidStr); + + /** + * The object representing the API of the CTA logging system. + */ + cta::log::StdoutLogger &m_log; + + /** + * True if the command should just restore deleted tape file copies + */ + bool m_restoreCopies; + + /** + * True if the command should just restore deleted archive files + */ + bool m_restoreFiles; + + /** + * Archive file id of the files to restore + */ + std::optional<uint64_t> m_archiveFileId; + + /** + * Disk instance of the files to restore + */ + std::optional<std::string> m_diskInstance; + + /** + * Fids of the files to restore + */ + std::optional<std::list<uint64_t>> m_eosFids; + + /** + * Vid of the tape of the files to restore + */ + std::optional<std::string> m_vid; + + /** + *Copy number of the files to restore + */ + std::optional<uint64_t> m_copyNumber; + + /** + *Default file layout for restored files in EOS + */ + int m_defaultFileLayout; + + /** + * CTA Frontend service provider + */ + std::unique_ptr<XrdSsiPbServiceType> m_serviceProviderPtr; + + std::unique_ptr<::eos::client::EndpointMap> m_endpointMapPtr; + + const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses + const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by + //!< XRD_REQUESTTIMEOUT environment variable. + + const std::string m_ctaVersion = CTA_VERSION; //!< The version of CTA + const std::string m_xrootdSsiProtobufInterfaceVersion = //!< The xrootd-ssi-protobuf-interface version (=tag) + XROOTD_SSI_PROTOBUF_INTERFACE_VERSION; + + +} ; // class CtaUpdateStorageClass + +} // namespace admin +} // namespace cta diff --git a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp new file mode 100644 index 0000000000..28a1e630e6 --- /dev/null +++ b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp @@ -0,0 +1,193 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#include "cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp" +#include "common/exception/CommandLineNotParsed.hpp" + +#include <getopt.h> +#include <ostream> +#include <sstream> +#include <iostream> +#include <string> +#include <limits.h> +#include <fstream> + + +namespace cta { +namespace admin{ + +//------------------------------------------------------------------------------ +// constructor +//------------------------------------------------------------------------------ +CtaUpdateStorageClassCmdLineArgs::CtaUpdateStorageClassCmdLineArgs(const int argc, char *const *const argv): +m_help(false), m_debug(false) { + + static struct option longopts[] = { + {"id", required_argument, nullptr, 'I'}, + {"instance", required_argument, nullptr, 'i'}, + {"fxid", required_argument, nullptr, 'f'}, + {"fxidfile", required_argument, nullptr, 'F'}, + {"vid", required_argument, nullptr, 'v'}, + {"copynb", required_argument, nullptr, 'c'}, + {"help", no_argument, nullptr, 'h'}, + {"debug", no_argument, nullptr, 'd'}, + {nullptr, 0, nullptr, 0} + }; + + opterr = 0; + int opt = 0; + int opt_index = 3; + + while ((opt = getopt_long(argc, argv, "I:i:f:F:v:c:hd", longopts, &opt_index)) != -1) { + switch(opt) { + case 'I': + { + int64_t archiveId = std::stol(std::string(optarg)); + if(archiveId < 0) throw std::out_of_range("archive id value cannot be negative"); + m_archiveFileId = archiveId; + break; + } + case 'i': + { + m_diskInstance = std::string(optarg); + break; + } + case 'f': + { + if (! m_eosFids) { + m_eosFids = std::list<uint64_t>(); + } + auto fid = strtoul(optarg, nullptr, 16); + if(fid < 1) { + throw std::runtime_error(std::string(optarg) + " is not a valid file ID"); + } + m_eosFids->push_back(fid); + break; + } + case 'F': + { + if (! m_eosFids) { + m_eosFids = std::list<uint64_t>(); + } + readFidListFromFile(std::string(optarg), m_eosFids.value()); + break; + } + case 'v': + { + m_vid = std::string(optarg); + break; + } + case 'c': + { + int64_t copyNumber = std::stol(std::string(optarg)); + if(copyNumber < 0) throw std::out_of_range("copy number value cannot be negative"); + m_copyNumber = copyNumber; + break; + } + case 'h': + { + m_help = true; + break; + } + case 'd': + { + m_debug = true; + break; + } + case ':': // Missing parameter + { + exception::CommandLineNotParsed ex; + ex.getMessage() << "The -" << (char)optopt << " option requires a parameter"; + throw ex; + } + case '?': // Unknown option + { + exception::CommandLineNotParsed ex; + if(0 == optopt) { + ex.getMessage() << "Unknown command-line option"; + } else { + ex.getMessage() << "Unknown command-line option: -" << (char)optopt; + } + throw ex; + } + default: + { + exception::CommandLineNotParsed ex; + ex.getMessage() << + "getopt_long returned the following unknown value: 0x" << + std::hex << (int)opt; + throw ex; + } + } + } +} + +//------------------------------------------------------------------------------ +// readFidListFromFile +//------------------------------------------------------------------------------ +void CtaUpdateStorageClassCmdLineArgs::readFidListFromFile(const std::string &filename, std::list<std::uint64_t> &fidList) { + std::ifstream file(filename); + if (file.fail()) { + throw std::runtime_error("Unable to open file " + filename); + } + + std::string line; + + while(std::getline(file, line)) { + // Strip out comments + auto pos = line.find('#'); + if(pos != std::string::npos) { + line.resize(pos); + } + + // Extract the list items + std::stringstream ss(line); + while(!ss.eof()) { + std::string item; + ss >> item; + // skip blank lines or lines consisting only of whitespace + if(item.empty()) continue; + + // Special handling for file id lists. The output from "eos find --fid <fid> /path" is: + // path=/path fid=<fid> + // We discard everything except the list of fids. <fid> is a zero-padded hexadecimal number, + // but in the CTA catalogue we store disk IDs as a decimal string, so we need to convert it. + if(item.substr(0, 4) == "fid=") { + auto fid = strtol(item.substr(4).c_str(), nullptr, 16); + if(fid < 1 || fid == LONG_MAX) { + throw std::runtime_error(item + " is not a valid file ID"); + } + fidList.push_back(fid); + } else { + continue; + } + } + } +} + +//------------------------------------------------------------------------------ +// printUsage +//------------------------------------------------------------------------------ +void CtaUpdateStorageClassCmdLineArgs::printUsage(std::ostream &os) { + os << "Usage:" << std::endl << + " cta-update-storage-class [--id/-I <archive_file_id>] [--instance/-i <disk_instance>]" << std::endl << + " [--fxid/-f <eos_fxid>] [--fxidfile/-F <filename>]" << std::endl << + " [--vid/-v <vid>] [--copynb/-c <copy_number>] [--debug/-d]" << std::endl; +} + +} // namespace admin +} // namespace cta diff --git a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp new file mode 100644 index 0000000000..1af45fd2f2 --- /dev/null +++ b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp @@ -0,0 +1,95 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#pragma once + +#include "version.h" + +#include <list> +#include <optional> + +namespace cta { +namespace admin { + +/** + * Structure to store the command-line arguments of the command-line tool + * named cta-update-deleted-archive. + */ +struct CtaUpdateStorageClassCmdLineArgs { + /** + * True if the usage message should be printed. + */ + bool m_help; + + /** + * True if debug messages should be printed + */ + bool m_debug; + + /** + * Archive file id of the files to update + */ + std::optional<uint64_t> m_archiveFileId; + + /** + * Disk instance of the files to update + */ + std::optional<std::string> m_diskInstance; + + /** + * Fids of the files to update + */ + std::optional<std::list<uint64_t>> m_eosFids; + + /** + * Vid of the tape of the files to update + */ + std::optional<std::string> m_vid; + + /** + *Copy number of the files to update + */ + std::optional<uint64_t> m_copyNumber; + + /** + * Constructor that parses the specified command-line arguments. + * + * @param argc The number of command-line arguments including the name of the + * executable. + * @param argv The vector of command-line arguments. + */ + CtaUpdateStorageClassCmdLineArgs(const int argc, char *const *const argv); + + /** + * Read a list of eos file ids from a file and write the options to a list + * + * @param filename The name of the file to read + * @param fidList The list of file IDs + */ + void readFidListFromFile(const std::string &filename, std::list<uint64_t> &fidList); + + /** + * Prints the usage message of the command-line tool. + * + * @param os The output stream to which the usage message is to be printed. + */ + static void printUsage(std::ostream &os); + +}; // class CtaUpdateStorageClassCmdLineArgs + +} // namespace admin +} // namespace cta diff --git a/cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp b/cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp new file mode 100644 index 0000000000..70e2cc464b --- /dev/null +++ b/cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp @@ -0,0 +1,48 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + + +#include <sstream> +#include <iostream> + +#include <XrdSsiPbLog.hpp> +#include <XrdSsiPbIStreamBuffer.hpp> + +#include "cmdline/update_storage_class/CtaUpdateStorageClass.hpp" +#include "common/utils/utils.hpp" +#include "common/log/StdoutLogger.hpp" + +//------------------------------------------------------------------------------ +// main +//------------------------------------------------------------------------------ +int main(const int argc, char *const *const argv) { + char buf[256]; + std::string hostName; + if(gethostname(buf, sizeof(buf))) { + hostName = "UNKNOWN"; + } else { + buf[sizeof(buf) - 1] = '\0'; + hostName = buf; + } + cta::log::StdoutLogger log(hostName, "cta-restore-deleted-files"); + + cta::admin::CtaUpdateStorageClass cmd(std::cin, std::cout, std::cerr, log); + int ret = cmd.main(argc, argv); + // Delete all global objects allocated by libprotobuf + google::protobuf::ShutdownProtobufLibrary(); + return ret; +} -- GitLab From 747b500cfca571a86ec3e718d1025884f2b4c6d6 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 30 Aug 2022 15:30:04 +0200 Subject: [PATCH 016/126] #78 - fix inlude errors --- cmdline/CtaUpdateStorageClass.cpp | 228 ------------------ cmdline/update_storage_class/CMakeLists.txt | 2 +- .../update_storage_class/CtaCmdOptions.hpp | 171 +++++++++++++ .../CtaUpdateStorageClass.cpp | 99 +------- .../CtaUpdateStorageClass.hpp | 2 - .../CtaUpdateStorageClassCmdLineArgs.cpp | 222 ++++++----------- .../CtaUpdateStorageClassCmdLineArgs.hpp | 61 ++--- .../CtaUpdateStorageClassMain.cpp | 2 +- 8 files changed, 294 insertions(+), 493 deletions(-) delete mode 100644 cmdline/CtaUpdateStorageClass.cpp create mode 100644 cmdline/update_storage_class/CtaCmdOptions.hpp diff --git a/cmdline/CtaUpdateStorageClass.cpp b/cmdline/CtaUpdateStorageClass.cpp deleted file mode 100644 index b826757b8b..0000000000 --- a/cmdline/CtaUpdateStorageClass.cpp +++ /dev/null @@ -1,228 +0,0 @@ -/* - * @project The CERN Tape Archive (CTA) - * @copyright Copyright © 2021-2022 CERN - * @license This program is free software, distributed under the terms of the GNU General Public - * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can - * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranPty of MERCHANTABILITY or FITNESS FOR A - * PARTICULAR PURPOSE. See the GNU General Public License for more details. - * - * In applying this licence, CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization or - * submit itself to any jurisdiction. - */ - -#include <sstream> -#include <iostream> - -#include <XrdSsiPbLog.hpp> -#include <XrdSsiPbIStreamBuffer.hpp> -#include <XrdSsiPbLog.hpp> - -#include "cmdline/CtaAdminCmdParse.hpp" -#include "version.h" -#include "cmdline/CtaCmdOptions.hpp" - -#include "cta_frontend.pb.h" - - -namespace XrdSsiPb { - - -/*! -* User error exception -*/ -class UserException : public std::runtime_error -{ -public: - UserException(const std::string &err_msg) : std::runtime_error(err_msg) {} -}; - - -/*! - * Alert callback. - * - * Defines how Alert messages should be logged - */ -template<> -void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) -{ - std::cout << "AlertCallback():" << std::endl; - Log::DumpProtobuf(Log::PROTOBUF, &alert); -} - -} // namespace Xrdcp - -namespace cta{ -namespace updateStorageClass { - -using namespace cta::admin; - -std::atomic<bool> is_json(false); -std::atomic<bool> is_first_record(true); -std::atomic<bool> isHeaderSent(false); - -const std::string config_file = "/etc/cta/cta-cli.conf"; - -std::string jsonCloseDelim() { - return is_first_record ? "[]" : "]"; -} - -// Usage exception -const std::runtime_error Usage("Usage: ..."); - -StringListOption option_fid{"--fid", "-f", true}; -StringOption option_storage_class_name{"--storage_class_name", "-scn", true}; - -std::map<std::string, StringListOption *> option_with_multiple_values_map = { - {"--fid", &option_fid}, - {"-f", &option_fid}, -}; - -std::map<std::string, StringOption *> option_map = { - {"--storage_class_name", &option_storage_class_name}, - {"-scn", &option_storage_class_name}, -}; - - -void validateCmd(int argc) { - if (argc < 3) { - throw Usage; - } - for (auto &it : option_with_multiple_values_map) { - auto option = it.second; - if (!option->is_optional() && !option->is_present()) { - std::cout << "Error: Option " << option->get_name() << " is mandatory but not present." << std::endl; - throw Usage; - } - } -} - - -bool isEndOfOption(int nextOptionOrValueIndex, const char *const *const argv) { - return ( - option_with_multiple_values_map.find(argv[nextOptionOrValueIndex]) != option_with_multiple_values_map.end() || - option_map.find(argv[nextOptionOrValueIndex]) != option_map.end() - ); -} - - -void parseCmd(const int argc, const char *const *const argv) { - unsigned int num_cmd_line_enrties_for_option = 2; - for (int i = 1; i < argc; i += num_cmd_line_enrties_for_option){ - num_cmd_line_enrties_for_option = 2; - auto search_option_with_multiple_values_map = option_with_multiple_values_map.find(argv[i]); - if (search_option_with_multiple_values_map != option_with_multiple_values_map.end()) { - auto stringListOption = search_option_with_multiple_values_map->second; - stringListOption->set_present(); - - if (stringListOption->get_name() == "--fid") { - for (int j = 1; j < argc; j += 1) { - int nextOptionOrValueIndex = i + j + 1; - bool end_of_option = isEndOfOption(nextOptionOrValueIndex, argv); - stringListOption->add_value(argv[i + j]); - if (end_of_option || (j + i) == (argc - 1)) { - num_cmd_line_enrties_for_option = j + 1; - break; - } - } - } - } - - auto search_option = option_map.find(argv[i]); - if (search_option != option_map.end()) { - auto stringOption = search_option->second; - stringOption->set_present(); - if (stringOption->get_name() == "--storage_class_name") { - stringOption->set_value(argv[i + 1]); - } - } - - if (search_option_with_multiple_values_map == option_with_multiple_values_map.end() && search_option == option_map.end()) { - std::cout << "Error: Unknown option: " << argv[i] << std::endl; - throw Usage; - } - } -} - - -void setStorageClassNameInEOSNamespace(std::string storageClassName, ) { - const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses - const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by - - - XrdSsiPb::Config config(config_file, "eos"); - for (const auto &conf_option : std::vector<std::string>({ "instance", "requester.user", "requester.group" })) { - if (!config.getOptionValueStr(conf_option).first) { - throw std::runtime_error(conf_option + " must be specified in " + config_file); - } - } - std::string instance = config.getOptionValueStr("instance").second; - std::string user = config.getOptionValueStr("requester.user").second; - std::string group = config.getOptionValueStr("requester.group").second; - - - - // Set CTA frontend configuration options - const std::string cli_config_file = "/etc/cta/cta-cli.conf"; - XrdSsiPb::Config cliConfig(cli_config_file, "cta"); - cliConfig.set("resource", "/ctafrontend"); - cliConfig.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes - cliConfig.set("request_timeout", DefaultRequestTimeout); // default value = 10s - - // Allow environment variables to override config file - cliConfig.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); - - // If XRDDEBUG=1, switch on all logging - if(getenv("XRDDEBUG")) { - cliConfig.set("log", "all"); - } - // If fine-grained control over log level is required, use XrdSsiPbLogLevel - cliConfig.getEnv("log", "XrdSsiPbLogLevel"); - - // Validate that endpoint was specified in the config file - if(!cliConfig.getOptionValueStr("endpoint").first) { - throw std::runtime_error("Configuration error: cta.endpoint missing from " + cli_config_file); - } - - // If the server is down, we want an immediate failure. Set client retry to a single attempt. - XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); - - m_serviceProviderPtr.reset(new XrdSsiPbServiceType(cliConfig)); - - // Set CTA frontend configuration options to connect to eos - const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf"; - XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); - - // Get the endpoint for namespace queries - auto nsConf = frontendXrootdConfig.getOptionValueStr("ns.config"); - if(nsConf.first) { - setNamespaceMap(nsConf.second); - } else { - throw std::runtime_error("Configuration error: cta.ns.config missing from " + frontend_xrootd_config_file); - } -} -} // namespace updateStorageClass -} // namespace cta - - -/* - * Start here - */ -int main(int argc, const char **argv) -{ - using namespace cta::updateStorageClass; - - // testing - parseCmd(argc, argv); - validateCmd(argc); - - std::cout << option_storage_class_name.get_value() << std::endl; - std::cout << option_fid.get_value()[0] << std::endl; - std::cout << option_fid.get_value()[1] << std::endl; - - return 1; -} diff --git a/cmdline/update_storage_class/CMakeLists.txt b/cmdline/update_storage_class/CMakeLists.txt index 45521ae5ad..ac446247d0 100644 --- a/cmdline/update_storage_class/CMakeLists.txt +++ b/cmdline/update_storage_class/CMakeLists.txt @@ -27,7 +27,7 @@ include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include) # Compiled protocol buffers include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS}) -add_executable(cta-update-storage-class CtaUpdateStorageClass.cpp CtaUpdateStorageClassCmdLineArgs.cpp CtaUpdateStorageClassMain.cpp ../restore_files/CmdLineTool.cpp) +add_executable(cta-update-storage-class CtaUpdateStorageClass.cpp CtaUpdateStorageClassCmdLineArgs.cpp CtaUpdateStorageClassMain.cpp ../restore_files/CmdLineTool.cpp CtaCmdOptions.hpp) target_link_libraries(cta-update-storage-class ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient) set_property (TARGET cta-update-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) diff --git a/cmdline/update_storage_class/CtaCmdOptions.hpp b/cmdline/update_storage_class/CtaCmdOptions.hpp new file mode 100644 index 0000000000..ae4e934f69 --- /dev/null +++ b/cmdline/update_storage_class/CtaCmdOptions.hpp @@ -0,0 +1,171 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#pragma once + +#include <string> +#include <vector> + +/*! +* Simple command line option class for simpler commands like cta-verify-file or cta-send-event +*/ +class StringOption { +public: + /*! + * Constructor + */ + StringOption(const std::string &long_opt, const std::string &short_opt, const bool is_optional) : + m_long_opt(long_opt), + m_short_opt(short_opt), + m_is_optional(is_optional), + m_is_present(false), + m_value("") {} + + /*! + * Check if the supplied key matches the option + */ + bool operator==(const std::string &option) const { + return option == m_short_opt || option == m_long_opt; + } + + /*! + * Return whether the option is optional + */ + bool is_optional() const { + return m_is_optional; + } + + /*! + * Return whether the option is present on the command line + */ + bool is_present() const { + return m_is_present; + } + + /*! + * Sets the option as present on the command line + */ + void set_present() { + m_is_present = true; + } + + /*! + * Return the value of the option given from the command line + */ + const std::string &get_value() const { + return m_value; + } + + /*! + * Return the name of the option (it's long command option) + */ + const std::string &get_name() const { + return m_long_opt; + } + + /*! + * Sets the value of the option from the command line + */ + void set_value(const std::string &mValue) { + m_value = mValue; + } + +private: + + //member variables + const std::string m_long_opt; //!< Long command option + const std::string m_short_opt; //!< Short command option + const bool m_is_optional; //!< Option is optional or compulsory + bool m_is_present; //!< Option is present on the command line + std::string m_value; //!< Option value +}; + + +/*! +* Simple command line option class for simpler commands like cta-verify-file or cta-send-event +*/ +class StringListOption { +public: + /*! + * Constructor + */ + StringListOption(const std::string &long_opt, const std::string &short_opt, const bool is_optional) : + m_long_opt(long_opt), + m_short_opt(short_opt), + m_is_optional(is_optional), + m_is_present(false) {} + + /*! + * Check if the supplied key matches the option + */ + bool operator==(const std::string &option) const { + return option == m_short_opt || option == m_long_opt; + } + + /*! + * Return whether the option is optional + */ + bool is_optional() const { + return m_is_optional; + } + + /*! + * Return whether the option is present on the command line + */ + bool is_present() const { + return m_is_present; + } + + /*! + * Sets the option as present on the command line + */ + void set_present() { + m_is_present = true; + } + + /*! + * Return the value of the option given from the command line + */ + const std::vector<std::string> &get_value() const { + return m_value_list; + } + + /*! + * Return the name of the option (it's long command option) + */ + const std::string &get_name() const { + return m_long_opt; + } + + /*! + * Sets the value of the option from the command line + */ + void add_value(const std::string &mValue) { + m_value_list.push_back(mValue); + } + +private: + + //member variables + const std::string m_long_opt; //!< Long command option + const std::string m_short_opt; //!< Short command option + const bool m_is_optional; //!< Option is optional or compulsory + bool m_is_present; //!< Option is present on the command line + std::vector<std::string> m_value_list; //!< Option value +}; + + diff --git a/cmdline/update_storage_class/CtaUpdateStorageClass.cpp b/cmdline/update_storage_class/CtaUpdateStorageClass.cpp index 75cfdcbe6f..e0e331b261 100644 --- a/cmdline/update_storage_class/CtaUpdateStorageClass.cpp +++ b/cmdline/update_storage_class/CtaUpdateStorageClass.cpp @@ -132,14 +132,20 @@ CtaUpdateStorageClass::CtaUpdateStorageClass(std::istream &inStream, std::ostrea } +void CtaUpdateStorageClass::printUsage(std::ostream &os) { + os << "Usage:" << std::endl << + " cta-update-storage-class [--id/-I <archive_file_id>] [--instance/-i <disk_instance>]" << std::endl << + " [--fxid/-f <eos_fxid>] [--fxidfile/-F <filename>]" << std::endl << + " [--vid/-v <vid>] [--copynb/-c <copy_number>] [--debug/-d]" << std::endl; +} + //------------------------------------------------------------------------------ // exceptionThrowingMain //------------------------------------------------------------------------------ int CtaUpdateStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { CtaUpdateStorageClassCmdLineArgs cmdLineArgs(argc, argv); if (cmdLineArgs.m_help) { - printUsage(m_out); - return 0; + cmdLineArgs.printUsage(); } return 0; } @@ -151,18 +157,10 @@ void CtaUpdateStorageClass::readAndSetConfiguration( const std::string &userName, const CtaUpdateStorageClassCmdLineArgs &cmdLineArgs) { - m_vid = cmdLineArgs.m_vid; m_diskInstance = cmdLineArgs.m_diskInstance; m_eosFids = cmdLineArgs.m_eosFids; - m_copyNumber = cmdLineArgs.m_copyNumber; m_archiveFileId = cmdLineArgs.m_archiveFileId; - if (cmdLineArgs.m_debug) { - m_log.setLogMask("DEBUG"); - } else { - m_log.setLogMask("INFO"); - } - // Set CTA frontend configuration options const std::string cli_config_file = "/etc/cta/cta-cli.conf"; XrdSsiPb::Config cliConfig(cli_config_file, "cta"); @@ -238,79 +236,6 @@ void CtaUpdateStorageClass::setNamespaceMap(const std::string &keytab_file) { } -//------------------------------------------------------------------------------ -// addContainerEos -//------------------------------------------------------------------------------ -uint64_t CtaUpdateStorageClass::addContainerEos(const std::string &diskInstance, const std::string &path, const std::string &sc) const { - auto c_id = containerExistsEos(diskInstance, path); - if (c_id) { - return c_id; - } - auto enclosingPath = cta::utils::getEnclosingPath(path); - auto parent_id = containerExistsEos(diskInstance, enclosingPath); - if (!parent_id) { - //parent does not exist, need to add it as well - parent_id = addContainerEos(diskInstance, enclosingPath, sc); - } - - std::list<cta::log::Param> params; - params.push_back(cta::log::Param("userName", getUsername())); - params.push_back(cta::log::Param("diskInstance", diskInstance)); - params.push_back(cta::log::Param("path", path)); - m_log(cta::log::DEBUG, "Inserting container in EOS namespace", params); - - ::eos::rpc::ContainerMdProto dir; - dir.set_path(path); - dir.set_name(cta::utils::getEnclosedName(path)); - - // Filemode: filter out S_ISUID, S_ISGID and S_ISVTX because EOS does not follow POSIX semantics for these bits - uint64_t filemode = (S_IRWXU | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); // 0755 permissions by default - filemode &= ~(S_ISUID | S_ISGID | S_ISVTX); - dir.set_mode(filemode); - - auto time = ::time(nullptr); - // Timestamps - dir.mutable_ctime()->set_sec(time); - dir.mutable_mtime()->set_sec(time); - // we don't care about dir.stime (sync time, used for CERNBox) - - dir.mutable_xattrs()->insert(google::protobuf::MapPair<std::string,std::string>("sys.archive.storage_class", sc)); - - - auto reply = m_endpointMapPtr->containerInsert(diskInstance, dir); - - m_log(cta::log::DEBUG, "Inserted container in EOS namespace successfully, querying again for its id", params); - - auto cont_id = containerExistsEos(diskInstance, path); - if (!cont_id) { - throw UpdateFilesCmdException(std::string("Container ") + path + " does not exist after being inserted in EOS."); - } - return cont_id; -} - -//------------------------------------------------------------------------------ -// containerExistsEos -//------------------------------------------------------------------------------ -uint64_t CtaUpdateStorageClass::containerExistsEos(const std::string &diskInstance, const std::string &path) const { - std::list<cta::log::Param> params; - params.push_back(cta::log::Param("userName", getUsername())); - params.push_back(cta::log::Param("diskInstance", diskInstance)); - params.push_back(cta::log::Param("path", path)); - - m_log(cta::log::DEBUG, "Verifying if the container exists in the EOS namespace", params); - - auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::CONTAINER, 0, path, false); - auto cid = md_response.cmd().id(); - params.push_back(cta::log::Param("containerId", cid)); - if (cid != 0) { - m_log(cta::log::DEBUG, "Container exists in the EOS namespace", params); - } else { - m_log(cta::log::DEBUG, "Container does not exist in the EOS namespace", params); - } - return cid; -} - - //------------------------------------------------------------------------------ // archiveFileExistsCTA //------------------------------------------------------------------------------ @@ -423,14 +348,6 @@ void CtaUpdateStorageClass::getCurrentEosIds(const std::string &diskInstance) co } -//------------------------------------------------------------------------------ -// printUsage -//------------------------------------------------------------------------------ -void CtaUpdateStorageClass::printUsage(std::ostream &os) { - CtaUpdateStorageClassCmdLineArgs::printUsage(os); -} - - //------------------------------------------------------------------------------ // getFxidFromCTA //------------------------------------------------------------------------------ diff --git a/cmdline/update_storage_class/CtaUpdateStorageClass.hpp b/cmdline/update_storage_class/CtaUpdateStorageClass.hpp index 47ead6bee6..46715b6dd6 100644 --- a/cmdline/update_storage_class/CtaUpdateStorageClass.hpp +++ b/cmdline/update_storage_class/CtaUpdateStorageClass.hpp @@ -120,8 +120,6 @@ public: /** * Prints the usage message of the command-line tool. - * - * @param os The output stream to which the usage message is to be printed. */ void printUsage(std::ostream &os) override; diff --git a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp index 28a1e630e6..607e356cf4 100644 --- a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp +++ b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp @@ -26,7 +26,6 @@ #include <limits.h> #include <fstream> - namespace cta { namespace admin{ @@ -34,159 +33,98 @@ namespace admin{ // constructor //------------------------------------------------------------------------------ CtaUpdateStorageClassCmdLineArgs::CtaUpdateStorageClassCmdLineArgs(const int argc, char *const *const argv): -m_help(false), m_debug(false) { - - static struct option longopts[] = { - {"id", required_argument, nullptr, 'I'}, - {"instance", required_argument, nullptr, 'i'}, - {"fxid", required_argument, nullptr, 'f'}, - {"fxidfile", required_argument, nullptr, 'F'}, - {"vid", required_argument, nullptr, 'v'}, - {"copynb", required_argument, nullptr, 'c'}, - {"help", no_argument, nullptr, 'h'}, - {"debug", no_argument, nullptr, 'd'}, - {nullptr, 0, nullptr, 0} - }; - - opterr = 0; - int opt = 0; - int opt_index = 3; - - while ((opt = getopt_long(argc, argv, "I:i:f:F:v:c:hd", longopts, &opt_index)) != -1) { - switch(opt) { - case 'I': - { - int64_t archiveId = std::stol(std::string(optarg)); - if(archiveId < 0) throw std::out_of_range("archive id value cannot be negative"); - m_archiveFileId = archiveId; - break; - } - case 'i': - { - m_diskInstance = std::string(optarg); - break; - } - case 'f': - { - if (! m_eosFids) { - m_eosFids = std::list<uint64_t>(); - } - auto fid = strtoul(optarg, nullptr, 16); - if(fid < 1) { - throw std::runtime_error(std::string(optarg) + " is not a valid file ID"); - } - m_eosFids->push_back(fid); - break; - } - case 'F': - { - if (! m_eosFids) { - m_eosFids = std::list<uint64_t>(); - } - readFidListFromFile(std::string(optarg), m_eosFids.value()); - break; - } - case 'v': - { - m_vid = std::string(optarg); - break; - } - case 'c': - { - int64_t copyNumber = std::stol(std::string(optarg)); - if(copyNumber < 0) throw std::out_of_range("copy number value cannot be negative"); - m_copyNumber = copyNumber; - break; - } - case 'h': - { - m_help = true; - break; - } - case 'd': - { - m_debug = true; - break; - } - case ':': // Missing parameter - { - exception::CommandLineNotParsed ex; - ex.getMessage() << "The -" << (char)optopt << " option requires a parameter"; - throw ex; - } - case '?': // Unknown option - { - exception::CommandLineNotParsed ex; - if(0 == optopt) { - ex.getMessage() << "Unknown command-line option"; - } else { - ex.getMessage() << "Unknown command-line option: -" << (char)optopt; - } - throw ex; - } - default: - { - exception::CommandLineNotParsed ex; - ex.getMessage() << - "getopt_long returned the following unknown value: 0x" << - std::hex << (int)opt; - throw ex; - } - } - } + m_help(false), + m_option_fids("--fid", "-f", true), + m_option_storage_class_name("--storage_class_name", "-scn", true) { + + option_with_multiple_values_map["--fid"] = &m_option_fids; + option_with_multiple_values_map["-f"] = &m_option_fids; + + option_map["--storage_class_name"] = &m_option_storage_class_name; + option_map["-scn"] = &m_option_storage_class_name; + + parseCmd(argc, argv); } + //------------------------------------------------------------------------------ -// readFidListFromFile +// printUsage //------------------------------------------------------------------------------ -void CtaUpdateStorageClassCmdLineArgs::readFidListFromFile(const std::string &filename, std::list<std::uint64_t> &fidList) { - std::ifstream file(filename); - if (file.fail()) { - throw std::runtime_error("Unable to open file " + filename); +void CtaUpdateStorageClassCmdLineArgs::printUsage() { + std::cout << "Usage:" << std::endl << + " cta-update-storage-class [--id/-I <archive_file_id>] [--instance/-i <disk_instance>]" << std::endl << + " [--fxid/-f <eos_fxid>] [--fxidfile/-F <filename>]" << std::endl << + " [--vid/-v <vid>] [--copynb/-c <copy_number>] [--debug/-d]" << std::endl; +} + +//------------------------------------------------------------------------------ +// validateCmd +//------------------------------------------------------------------------------ +void CtaUpdateStorageClassCmdLineArgs::validateCmd(int argc) { + if (argc < 3) { + printUsage(); + throw std::runtime_error(""); } + for (auto &it : option_with_multiple_values_map) { + auto option = it.second; + if (!option->is_optional() && !option->is_present()) { + std::cout << "Error: Option " << option->get_name() << " is mandatory but not present." << std::endl; + printUsage(); + throw std::runtime_error(""); + } + } +} - std::string line; +//------------------------------------------------------------------------------ +// isEndOfOption +//------------------------------------------------------------------------------ +bool CtaUpdateStorageClassCmdLineArgs::isEndOfOption(int nextOptionOrValueIndex, const char *const *const argv) { + return ( + option_with_multiple_values_map.find(argv[nextOptionOrValueIndex]) != option_with_multiple_values_map.end() || + option_map.find(argv[nextOptionOrValueIndex]) != option_map.end() + ); +} - while(std::getline(file, line)) { - // Strip out comments - auto pos = line.find('#'); - if(pos != std::string::npos) { - line.resize(pos); - } +//------------------------------------------------------------------------------ +// parseCmd +//------------------------------------------------------------------------------ +void CtaUpdateStorageClassCmdLineArgs::parseCmd(const int argc, const char *const *const argv) { + unsigned int num_cmd_line_enrties_for_option = 2; + for (int i = 1; i < argc; i += num_cmd_line_enrties_for_option){ + num_cmd_line_enrties_for_option = 2; + auto search_option_with_multiple_values_map = option_with_multiple_values_map.find(argv[i]); + if (search_option_with_multiple_values_map != option_with_multiple_values_map.end()) { + auto stringListOption = search_option_with_multiple_values_map->second; + stringListOption->set_present(); - // Extract the list items - std::stringstream ss(line); - while(!ss.eof()) { - std::string item; - ss >> item; - // skip blank lines or lines consisting only of whitespace - if(item.empty()) continue; - - // Special handling for file id lists. The output from "eos find --fid <fid> /path" is: - // path=/path fid=<fid> - // We discard everything except the list of fids. <fid> is a zero-padded hexadecimal number, - // but in the CTA catalogue we store disk IDs as a decimal string, so we need to convert it. - if(item.substr(0, 4) == "fid=") { - auto fid = strtol(item.substr(4).c_str(), nullptr, 16); - if(fid < 1 || fid == LONG_MAX) { - throw std::runtime_error(item + " is not a valid file ID"); + if (stringListOption->get_name() == "--fid") { + for (int j = 1; j < argc; j += 1) { + int nextOptionOrValueIndex = i + j + 1; + bool end_of_option = isEndOfOption(nextOptionOrValueIndex, argv); + stringListOption->add_value(argv[i + j]); + if (end_of_option || (j + i) == (argc - 1)) { + num_cmd_line_enrties_for_option = j + 1; + break; + } } - fidList.push_back(fid); - } else { - continue; } } - } -} -//------------------------------------------------------------------------------ -// printUsage -//------------------------------------------------------------------------------ -void CtaUpdateStorageClassCmdLineArgs::printUsage(std::ostream &os) { - os << "Usage:" << std::endl << - " cta-update-storage-class [--id/-I <archive_file_id>] [--instance/-i <disk_instance>]" << std::endl << - " [--fxid/-f <eos_fxid>] [--fxidfile/-F <filename>]" << std::endl << - " [--vid/-v <vid>] [--copynb/-c <copy_number>] [--debug/-d]" << std::endl; + auto search_option = option_map.find(argv[i]); + if (search_option != option_map.end()) { + auto stringOption = search_option->second; + stringOption->set_present(); + if (stringOption->get_name() == "--storage_class_name") { + stringOption->set_value(argv[i + 1]); + } + } + + if (search_option_with_multiple_values_map == option_with_multiple_values_map.end() && search_option == option_map.end()) { + std::cout << "Error: Unknown option: " << argv[i] << std::endl; + printUsage(); + throw std::runtime_error(""); + } + } } } // namespace admin diff --git a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp index 1af45fd2f2..dc3d49266d 100644 --- a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp +++ b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp @@ -17,29 +17,31 @@ #pragma once -#include "version.h" - #include <list> #include <optional> +#include <map> +#include <iostream> +#include <sstream> + +#include "version.h" +#include "cmdline/CtaCmdOptions.hpp" + namespace cta { namespace admin { /** - * Structure to store the command-line arguments of the command-line tool - * named cta-update-deleted-archive. + * class to store the command-line arguments of the command-line tool + * named cta-update-storage-class. */ -struct CtaUpdateStorageClassCmdLineArgs { +class CtaUpdateStorageClassCmdLineArgs { + +public: /** * True if the usage message should be printed. */ bool m_help; - /** - * True if debug messages should be printed - */ - bool m_debug; - /** * Archive file id of the files to update */ @@ -55,15 +57,18 @@ struct CtaUpdateStorageClassCmdLineArgs { */ std::optional<std::list<uint64_t>> m_eosFids; - /** - * Vid of the tape of the files to update - */ - std::optional<std::string> m_vid; /** - *Copy number of the files to update - */ - std::optional<uint64_t> m_copyNumber; + * List of archive file ids + */ + StringListOption m_option_fids; + + + /** + * The storage name used to replace the names for the files with file id found in m_fids + */ + StringOption m_option_storage_class_name; + /** * Constructor that parses the specified command-line arguments. @@ -74,20 +79,20 @@ struct CtaUpdateStorageClassCmdLineArgs { */ CtaUpdateStorageClassCmdLineArgs(const int argc, char *const *const argv); - /** - * Read a list of eos file ids from a file and write the options to a list - * - * @param filename The name of the file to read - * @param fidList The list of file IDs - */ - void readFidListFromFile(const std::string &filename, std::list<uint64_t> &fidList); - /** - * Prints the usage message of the command-line tool. + /** + * Print the correct usage of the tool * - * @param os The output stream to which the usage message is to be printed. */ - static void printUsage(std::ostream &os); + void printUsage(); + +private: + void validateCmd(int argc); + bool isEndOfOption(int nextOptionOrValueIndex, const char *const *const argv); + void parseCmd(const int argc, const char *const *const argv); + + std::map<std::string, StringListOption *> option_with_multiple_values_map; + std::map<std::string, StringOption *> option_map; }; // class CtaUpdateStorageClassCmdLineArgs diff --git a/cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp b/cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp index 70e2cc464b..41acbf1e2c 100644 --- a/cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp +++ b/cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp @@ -38,7 +38,7 @@ int main(const int argc, char *const *const argv) { buf[sizeof(buf) - 1] = '\0'; hostName = buf; } - cta::log::StdoutLogger log(hostName, "cta-restore-deleted-files"); + cta::log::StdoutLogger log(hostName, "cta-update-storage-class"); cta::admin::CtaUpdateStorageClass cmd(std::cin, std::cout, std::cerr, log); int ret = cmd.main(argc, argv); -- GitLab From d4369e1ada482ca27c24600dbac5e9ab2e957b10 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 30 Aug 2022 15:40:15 +0200 Subject: [PATCH 017/126] #78 - add code clean up --- .../CtaUpdateStorageClass.cpp | 228 ------------------ .../CtaUpdateStorageClass.hpp | 74 ------ .../CtaUpdateStorageClassCmdLineArgs.cpp | 2 - 3 files changed, 304 deletions(-) diff --git a/cmdline/update_storage_class/CtaUpdateStorageClass.cpp b/cmdline/update_storage_class/CtaUpdateStorageClass.cpp index e0e331b261..971429d651 100644 --- a/cmdline/update_storage_class/CtaUpdateStorageClass.cpp +++ b/cmdline/update_storage_class/CtaUpdateStorageClass.cpp @@ -191,124 +191,6 @@ void CtaUpdateStorageClass::readAndSetConfiguration( // Set CTA frontend configuration options to connect to eos const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf"; XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); - - // Get the endpoint for namespace queries - auto nsConf = frontendXrootdConfig.getOptionValueStr("ns.config"); - if(nsConf.first) { - setNamespaceMap(nsConf.second); - } else { - throw std::runtime_error("Configuration error: cta.ns.config missing from " + frontend_xrootd_config_file); - } -} - -void CtaUpdateStorageClass::setNamespaceMap(const std::string &keytab_file) { - // Open the keytab file for reading - std::ifstream file(keytab_file); - if(!file) { - throw cta::exception::UserError("Failed to open namespace keytab configuration file " + keytab_file); - } - ::eos::client::NamespaceMap_t namespaceMap; - // Parse the keytab line by line - std::string line; - for(int lineno = 0; std::getline(file, line); ++lineno) { - // Strip out comments - auto pos = line.find('#'); - if(pos != std::string::npos) { - line.resize(pos); - } - - // Parse one line - std::stringstream ss(line); - std::string diskInstance; - std::string endpoint; - std::string token; - std::string eol; - ss >> diskInstance >> endpoint >> token >> eol; - - // Ignore blank lines, all other lines must have exactly 3 elements - if(token.empty() || !eol.empty()) { - if(diskInstance.empty() && endpoint.empty() && token.empty()) continue; - throw cta::exception::UserError("Could not parse namespace keytab configuration file line " + std::to_string(lineno) + ": " + line); - } - namespaceMap.insert(std::make_pair(diskInstance, ::eos::client::Namespace(endpoint, token))); - } - m_endpointMapPtr = std::make_unique<::eos::client::EndpointMap>(namespaceMap); -} - - -//------------------------------------------------------------------------------ -// archiveFileExistsCTA -//------------------------------------------------------------------------------ -bool CtaUpdateStorageClass::archiveFileExistsCTA(const uint64_t &archiveFileId) const { - - std::list<cta::log::Param> params; - params.push_back(cta::log::Param("userName", getUsername())); - params.push_back(cta::log::Param("archiveFileId", archiveFileId)); - - m_log(cta::log::DEBUG, "Looking for archive file in the CTA catalogue", params); - - cta::xrd::Request request; - - auto &admincmd = *(request.mutable_admincmd()); - - request.set_client_cta_version(CTA_VERSION); - request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); - admincmd.set_cmd(AdminCmd::CMD_TAPEFILE); - admincmd.set_subcmd(AdminCmd::SUBCMD_LS); - - auto key = OptionUInt64::ARCHIVE_FILE_ID; - auto new_opt = admincmd.add_option_uint64(); - new_opt->set_key(key); - new_opt->set_value(archiveFileId); - - // Send the Request to the Service and get a Response - cta::xrd::Response response; - auto stream_future = m_serviceProviderPtr->SendAsync(request, response); - - bool ret; - // Handle responses - switch(response.type()) - { - using namespace cta::xrd; - using namespace cta::admin; - case Response::RSP_SUCCESS: ret = true; break; //success sent if archive file does not exist - case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); - case Response::RSP_ERR_USER: ret = false; break; //user error sent if archive file does not exist - case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); - default: throw XrdSsiPb::PbException("Invalid response type."); - } - - // wait until the data stream has been processed before exiting - if (ret) { - stream_future.wait(); - } - if (ret) { - m_log(cta::log::DEBUG, "Archive file is present in the CTA catalogue", params); - } else { - m_log(cta::log::DEBUG, "Archive file is missing in the CTA catalogue", params); - } - return ret; -} - -//------------------------------------------------------------------------------ -// fileExistsEos -//------------------------------------------------------------------------------ -bool CtaUpdateStorageClass::fileExistsEos(const std::string &diskInstance, const std::string &diskFileId) const { - std::list<cta::log::Param> params; - params.push_back(cta::log::Param("userName", getUsername())); - params.push_back(cta::log::Param("diskInstance", diskInstance)); - params.push_back(cta::log::Param("diskFileId", diskFileId)); - - m_log(cta::log::DEBUG, "Verifying if EOS fid exists in the EOS namespace", params); - try { - auto path = m_endpointMapPtr->getPath(diskInstance, diskFileId); - params.push_back(cta::log::Param("diskFilePath", path)); - m_log(cta::log::DEBUG, "EOS fid exists in the EOS namespace"); - return true; - } catch(cta::exception::Exception&) { - m_log(cta::log::DEBUG, "EOS fid does not exist in the EOS namespace"); - return false; - } } //------------------------------------------------------------------------------ @@ -332,115 +214,5 @@ uint64_t CtaUpdateStorageClass::getFileIdEos(const std::string &diskInstance, co return fid; } -//------------------------------------------------------------------------------ -// getCurrentEosIds -//------------------------------------------------------------------------------ -void CtaUpdateStorageClass::getCurrentEosIds(const std::string &diskInstance) const { - uint64_t cid; - uint64_t fid; - m_endpointMapPtr->getCurrentIds(diskInstance, cid, fid); - - std::list<cta::log::Param> params; - params.push_back(cta::log::Param("diskInstance", diskInstance)); - params.push_back(cta::log::Param("ContainerId", cid)); - params.push_back(cta::log::Param("FileId", fid)); - m_log(cta::log::DEBUG, "Obtained current EOS container and file id", params); -} - - -//------------------------------------------------------------------------------ -// getFxidFromCTA -//------------------------------------------------------------------------------ -std::pair<std::string,std::string> CtaUpdateStorageClass::getInstanceAndFidFromCTA(const RecycleTapeFileLsItem& file) { - { - std::list<cta::log::Param> params; - params.push_back(cta::log::Param("archiveFileId", file.archive_file_id())); - m_log(cta::log::DEBUG, "cta-admin tapefile ls", params); - } - - cta::xrd::Request request; - auto &admincmd = *(request.mutable_admincmd()); - - request.set_client_cta_version(CTA_VERSION); - request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); - admincmd.set_cmd(AdminCmd::CMD_TAPEFILE); - admincmd.set_subcmd(AdminCmd::SUBCMD_LS); - auto new_opt = admincmd.add_option_uint64(); - new_opt->set_key(OptionUInt64::ARCHIVE_FILE_ID); - new_opt->set_value(file.archive_file_id()); - - // Send the Request to the Service and get a Response - cta::xrd::Response response; - auto stream_future = m_serviceProviderPtr->SendAsync(request, response); - - // Handle responses - switch(response.type()) - { - using namespace cta::xrd; - using namespace cta::admin; - case Response::RSP_SUCCESS: - // Print message text - std::cout << response.message_txt(); - // Allow stream processing to commence - isHeaderSent = true; - break; - case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); - case Response::RSP_ERR_USER: throw XrdSsiPb::UserException(response.message_txt()); - case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); - default: throw XrdSsiPb::PbException("Invalid response type."); - } - - // wait until the data stream has been processed before exiting - stream_future.wait(); - if(listedTapeFiles.size() != 1) { - throw std::runtime_error("Unexpected result set: listedTapeFiles size expected=1 received=" + std::to_string(listedTapeFiles.size())); - } - auto listedTapeFile = listedTapeFiles.back(); - listedTapeFiles.clear(); - { - std::list<cta::log::Param> params; - params.push_back(cta::log::Param("diskInstance", listedTapeFile.first)); - params.push_back(cta::log::Param("diskFileId", listedTapeFile.second)); - m_log(cta::log::DEBUG, "Obtained file metadata from CTA", params); - } - return listedTapeFile; -} - -//------------------------------------------------------------------------------ -// getArchiveFileIdFromEOS -//------------------------------------------------------------------------------ -std::pair<uint64_t, std::string> CtaUpdateStorageClass::getArchiveFileIdAndChecksumFromEOS( - const std::string& diskInstance, const std::string& fidStr) { - auto fid = strtoul(fidStr.c_str(), nullptr, 10); - if(fid < 1) { - throw std::runtime_error(fid + " (base 10) is not a valid disk file ID"); - } - { - std::list<cta::log::Param> params; - params.push_back(cta::log::Param("diskInstance", diskInstance)); - params.push_back(cta::log::Param("fid", fid)); - std::stringstream ss; - ss << std::hex << fid; - params.push_back(cta::log::Param("fxid", ss.str())); - m_log(cta::log::DEBUG, "Querying EOS namespace for archiveFileId and checksum", params); - } - auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::FILE, fid, "", false); - auto archiveFileIdItor = md_response.fmd().xattrs().find("sys.archive.file_id"); - if(md_response.fmd().xattrs().end() == archiveFileIdItor) { - throw std::runtime_error("archiveFileId extended attribute not found."); - } - auto archiveFileId = strtoul(archiveFileIdItor->second.c_str(), nullptr, 10); - auto byteArray = md_response.fmd().checksum().value(); - auto checksumValue = checksum::ChecksumBlob::ByteArrayToHex(std::string(byteArray.rbegin(), byteArray.rend())); - { - std::list<cta::log::Param> params; - params.push_back(cta::log::Param("archiveFileId", archiveFileId)); - params.push_back(cta::log::Param("checksumValue", checksumValue)); - m_log(cta::log::DEBUG, "Response from EOS nameserver", params); - } - - return std::make_pair(archiveFileId,checksumValue); -} - } // namespace admin } // namespace cta diff --git a/cmdline/update_storage_class/CtaUpdateStorageClass.hpp b/cmdline/update_storage_class/CtaUpdateStorageClass.hpp index 46715b6dd6..3d45ececf0 100644 --- a/cmdline/update_storage_class/CtaUpdateStorageClass.hpp +++ b/cmdline/update_storage_class/CtaUpdateStorageClass.hpp @@ -62,11 +62,6 @@ public: */ void readAndSetConfiguration(const std::string &userName, const CtaUpdateStorageClassCmdLineArgs &cmdLineArgs); - /** - * Populate the namespace endpoint configuration from a keytab file - */ - void setNamespaceMap(const std::string &keytab_file); - /** * Queries the eos mgm for the current eos file and container id @@ -76,40 +71,6 @@ public: */ void getCurrentEosIds(const std::string &diskInstance) const; - - /** - * Adds a container in the eos namespace, if it does not exist - * @param diskInstance eos disk instance - * @param path the path of the container - * @param sc the storage class of the container - * @returns the container id of the container identified by path - */ - uint64_t addContainerEos(const std::string &diskInstance, const std::string &path, const std::string &sc) const; - - /** - * Returns the container id of the container in the EOS namespace - * - * @param diskInstance eos disk instance - * @param path the path of the container - * - * @returns the container id of the container in the EOS namespace, or zero if it does not exist - */ - uint64_t containerExistsEos(const std::string &diskInstance, const std::string &path) const; - - /** - * Returns true (i.e. not zero) if a file with given id exists in the eos namespace - * @param diskInstance eos disk instance - * @param diskFileId the eos file id to check - */ - bool fileExistsEos(const std::string &diskInstance, const std::string &diskFileId) const; - - - /** - * Returns true (i.e. not zero) if an archive file with given id exists in the cta catalogue - * @param archiveFileId the archive file id to check - */ - bool archiveFileExistsCTA(const uint64_t &archiveFileId) const; - /** * Returns the id of a given file in eos or zero if the files does not exist * @param diskInstance eos disk instance @@ -124,38 +85,12 @@ public: void printUsage(std::ostream &os) override; private: - /** - * Query CTA for the disk instance and fid of the restored file - * - * @param file The restored tape file in CTA - * @return Tuple of EOS disk instance and file ID (fid) as a decimal string - */ - std::pair<std::string,std::string> getInstanceAndFidFromCTA(const RecycleTapeFileLsItem& file); - - /** - * Query EOS for the archiveFileId and checksum of the restored file - * - * @param diskInstance Which EOS disk instance to query - * @param fxid fid of the file in EOS as a decimal string - * @return tuple containing archiveFileId and checksum of the file - */ - std::pair<uint64_t, std::string> getArchiveFileIdAndChecksumFromEOS(const std::string& diskInstance, - const std::string& fidStr); /** * The object representing the API of the CTA logging system. */ cta::log::StdoutLogger &m_log; - /** - * True if the command should just restore deleted tape file copies - */ - bool m_restoreCopies; - - /** - * True if the command should just restore deleted archive files - */ - bool m_restoreFiles; /** * Archive file id of the files to restore @@ -172,15 +107,6 @@ private: */ std::optional<std::list<uint64_t>> m_eosFids; - /** - * Vid of the tape of the files to restore - */ - std::optional<std::string> m_vid; - - /** - *Copy number of the files to restore - */ - std::optional<uint64_t> m_copyNumber; /** *Default file layout for restored files in EOS diff --git a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp index 607e356cf4..375bc77f27 100644 --- a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp +++ b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp @@ -20,8 +20,6 @@ #include <getopt.h> #include <ostream> -#include <sstream> -#include <iostream> #include <string> #include <limits.h> #include <fstream> -- GitLab From 1aa600aeb9a422edbcc471cc2358b1c4bfaaa9cd Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 30 Aug 2022 16:38:41 +0200 Subject: [PATCH 018/126] #78 - add casting of command line arguments --- .../CtaUpdateStorageClass.cpp | 55 ++++++------------- .../CtaUpdateStorageClass.hpp | 39 +++++++------ .../CtaUpdateStorageClassCmdLineArgs.cpp | 6 +- .../CtaUpdateStorageClassCmdLineArgs.hpp | 19 +------ .../CtaUpdateStorageClassMain.cpp | 2 +- 5 files changed, 40 insertions(+), 81 deletions(-) diff --git a/cmdline/update_storage_class/CtaUpdateStorageClass.cpp b/cmdline/update_storage_class/CtaUpdateStorageClass.cpp index 971429d651..3ffccc93a2 100644 --- a/cmdline/update_storage_class/CtaUpdateStorageClass.cpp +++ b/cmdline/update_storage_class/CtaUpdateStorageClass.cpp @@ -115,20 +115,21 @@ public: //------------------------------------------------------------------------------ // constructor //------------------------------------------------------------------------------ -CtaUpdateStorageClass::CtaUpdateStorageClass(std::istream &inStream, std::ostream &outStream, - std::ostream &errStream, cta::log::StdoutLogger &log): +CtaUpdateStorageClass::CtaUpdateStorageClass( + std::istream &inStream, + std::ostream &outStream, + std::ostream &errStream, + cta::log::StdoutLogger &log, + const int& argc, + char *const *const argv): CmdLineTool(inStream, outStream, errStream), - m_log(log) { - - // Default layout: see EOS common/LayoutId.hh for definitions of constants - const int kAdler = 0x2; - const int kReplica = (0x1 << 4); - const int kStripeSize = (0x0 << 8); // 1 stripe - const int kStripeWidth = (0x0 << 16); // 4K blocks - const int kBlockChecksum = (0x1 << 20); - - // Default single replica layout id should be 00100012 - m_defaultFileLayout = kReplica | kAdler | kStripeSize | kStripeWidth | kBlockChecksum; + m_log(log), + m_cmdLineArgs(argc, argv) { + + std::vector<std::string> fids = m_cmdLineArgs.m_optionFids.get_value(); + for(const auto& fid: fids) { + m_archiveFileIds.push_back(static_cast<uint32_t>(std::stoul(fid))); + } } @@ -154,13 +155,8 @@ int CtaUpdateStorageClass::exceptionThrowingMain(const int argc, char *const *co // readAndSetConfiguration //------------------------------------------------------------------------------ void CtaUpdateStorageClass::readAndSetConfiguration( - const std::string &userName, - const CtaUpdateStorageClassCmdLineArgs &cmdLineArgs) { + const std::string &userName) { - m_diskInstance = cmdLineArgs.m_diskInstance; - m_eosFids = cmdLineArgs.m_eosFids; - m_archiveFileId = cmdLineArgs.m_archiveFileId; - // Set CTA frontend configuration options const std::string cli_config_file = "/etc/cta/cta-cli.conf"; XrdSsiPb::Config cliConfig(cli_config_file, "cta"); @@ -193,26 +189,7 @@ void CtaUpdateStorageClass::readAndSetConfiguration( XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); } -//------------------------------------------------------------------------------ -// getFileIdEos -//------------------------------------------------------------------------------ -uint64_t CtaUpdateStorageClass::getFileIdEos(const std::string &diskInstance, const std::string &path) const { - std::list<cta::log::Param> params; - params.push_back(cta::log::Param("userName", getUsername())); - params.push_back(cta::log::Param("diskInstance", diskInstance)); - params.push_back(cta::log::Param("path", path)); - - m_log(cta::log::DEBUG, "Querying for file metadata in the EOS namespace", params); - auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::FILE, 0, path, false); - auto fid = md_response.fmd().id(); - params.push_back(cta::log::Param("diskFileId", fid)); - if (fid != 0) { - m_log(cta::log::DEBUG, "File path exists in the EOS namespace", params); - } else { - m_log(cta::log::DEBUG, "File path does not exist in the EOS namespace", params); - } - return fid; -} + } // namespace admin } // namespace cta diff --git a/cmdline/update_storage_class/CtaUpdateStorageClass.hpp b/cmdline/update_storage_class/CtaUpdateStorageClass.hpp index 3d45ececf0..c37672a3f8 100644 --- a/cmdline/update_storage_class/CtaUpdateStorageClass.hpp +++ b/cmdline/update_storage_class/CtaUpdateStorageClass.hpp @@ -40,9 +40,16 @@ public: * @param outStream Standard output stream. * @param errStream Standard error stream. * @param log The object representing the API of the CTA logging system. + * @param argc + * @param argv */ - CtaUpdateStorageClass(std::istream &inStream, std::ostream &outStream, - std::ostream &errStream, cta::log::StdoutLogger &log); + CtaUpdateStorageClass( + std::istream &inStream, + std::ostream &outStream, + std::ostream &errStream, + cta::log::StdoutLogger &log, + const int& argc, + char *const *const argv); /** * An exception throwing version of main(). @@ -60,23 +67,14 @@ public: * @param username The name of the user running the command-line tool. * @param cmdLineArgs The arguments parsed from the command line. */ - void readAndSetConfiguration(const std::string &userName, const CtaUpdateStorageClassCmdLineArgs &cmdLineArgs); + void readAndSetConfiguration(const std::string &userName); /** - * Queries the eos mgm for the current eos file and container id - * Must be called before any other call to EOS, to initialize the grpc - * client cid and fid - * @param diskInstance the disk instance of the eos instance - */ - void getCurrentEosIds(const std::string &diskInstance) const; - - /** - * Returns the id of a given file in eos or zero if the files does not exist + * Returns the id of a storage class * @param diskInstance eos disk instance - * @param path the path to check */ - uint64_t getFileIdEos(const std::string &diskInstance, const std::string &path) const; + uint64_t getStorageClassId() const; /** @@ -93,25 +91,26 @@ private: /** - * Archive file id of the files to restore + * Archive file id of the files to update */ - std::optional<uint64_t> m_archiveFileId; + std::vector<uint64_t> m_archiveFileIds; /** - * Disk instance of the files to restore + * Disk instance of the files to update */ std::optional<std::string> m_diskInstance; /** - * Fids of the files to restore + * Fids of the files to update */ std::optional<std::list<uint64_t>> m_eosFids; /** - *Default file layout for restored files in EOS + *Comman line arguments */ - int m_defaultFileLayout; + CtaUpdateStorageClassCmdLineArgs m_cmdLineArgs; + /** * CTA Frontend service provider diff --git a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp index 375bc77f27..786b94ec40 100644 --- a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp +++ b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp @@ -32,11 +32,11 @@ namespace admin{ //------------------------------------------------------------------------------ CtaUpdateStorageClassCmdLineArgs::CtaUpdateStorageClassCmdLineArgs(const int argc, char *const *const argv): m_help(false), - m_option_fids("--fid", "-f", true), + m_optionFids("--fid", "-f", true), m_option_storage_class_name("--storage_class_name", "-scn", true) { - option_with_multiple_values_map["--fid"] = &m_option_fids; - option_with_multiple_values_map["-f"] = &m_option_fids; + option_with_multiple_values_map["--fid"] = &m_optionFids; + option_with_multiple_values_map["-f"] = &m_optionFids; option_map["--storage_class_name"] = &m_option_storage_class_name; option_map["-scn"] = &m_option_storage_class_name; diff --git a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp index dc3d49266d..5cf7968e6b 100644 --- a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp +++ b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp @@ -42,27 +42,10 @@ public: */ bool m_help; - /** - * Archive file id of the files to update - */ - std::optional<uint64_t> m_archiveFileId; - - /** - * Disk instance of the files to update - */ - std::optional<std::string> m_diskInstance; - - /** - * Fids of the files to update - */ - std::optional<std::list<uint64_t>> m_eosFids; - - /** * List of archive file ids */ - StringListOption m_option_fids; - + StringListOption m_optionFids; /** * The storage name used to replace the names for the files with file id found in m_fids diff --git a/cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp b/cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp index 41acbf1e2c..f15c119f3c 100644 --- a/cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp +++ b/cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp @@ -40,7 +40,7 @@ int main(const int argc, char *const *const argv) { } cta::log::StdoutLogger log(hostName, "cta-update-storage-class"); - cta::admin::CtaUpdateStorageClass cmd(std::cin, std::cout, std::cerr, log); + cta::admin::CtaUpdateStorageClass cmd(std::cin, std::cout, std::cerr, log, argc, argv); int ret = cmd.main(argc, argv); // Delete all global objects allocated by libprotobuf google::protobuf::ShutdownProtobufLibrary(); -- GitLab From 852e5dd31bbfbdf86c7d06dfc26297bcfce86347 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 30 Aug 2022 17:15:43 +0200 Subject: [PATCH 019/126] #78 - add class member values --- .../CtaUpdateStorageClass.cpp | 37 ++++++++++++++----- .../CtaUpdateStorageClass.hpp | 10 ++++- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/cmdline/update_storage_class/CtaUpdateStorageClass.cpp b/cmdline/update_storage_class/CtaUpdateStorageClass.cpp index 3ffccc93a2..7f461c7af1 100644 --- a/cmdline/update_storage_class/CtaUpdateStorageClass.cpp +++ b/cmdline/update_storage_class/CtaUpdateStorageClass.cpp @@ -131,8 +131,22 @@ CtaUpdateStorageClass::CtaUpdateStorageClass( m_archiveFileIds.push_back(static_cast<uint32_t>(std::stoul(fid))); } + m_storageClassName = m_cmdLineArgs.m_option_storage_class_name.get_value(); + +} + + +//------------------------------------------------------------------------------ +// getStorageClassId +//------------------------------------------------------------------------------ +uint64_t getStorageClassId() { + return 0; } + +//------------------------------------------------------------------------------ +// printUsage +//------------------------------------------------------------------------------ void CtaUpdateStorageClass::printUsage(std::ostream &os) { os << "Usage:" << std::endl << " cta-update-storage-class [--id/-I <archive_file_id>] [--instance/-i <disk_instance>]" << std::endl << @@ -140,16 +154,6 @@ void CtaUpdateStorageClass::printUsage(std::ostream &os) { " [--vid/-v <vid>] [--copynb/-c <copy_number>] [--debug/-d]" << std::endl; } -//------------------------------------------------------------------------------ -// exceptionThrowingMain -//------------------------------------------------------------------------------ -int CtaUpdateStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { - CtaUpdateStorageClassCmdLineArgs cmdLineArgs(argc, argv); - if (cmdLineArgs.m_help) { - cmdLineArgs.printUsage(); - } - return 0; -} //------------------------------------------------------------------------------ // readAndSetConfiguration @@ -189,6 +193,19 @@ void CtaUpdateStorageClass::readAndSetConfiguration( XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); } +//------------------------------------------------------------------------------ +// exceptionThrowingMain +//------------------------------------------------------------------------------ +int CtaUpdateStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { + CtaUpdateStorageClassCmdLineArgs cmdLineArgs(argc, argv); + if (cmdLineArgs.m_help) { + cmdLineArgs.printUsage(); + } + + readAndSetConfiguration(getUsername()); + + return 0; +} } // namespace admin diff --git a/cmdline/update_storage_class/CtaUpdateStorageClass.hpp b/cmdline/update_storage_class/CtaUpdateStorageClass.hpp index c37672a3f8..24e042c86e 100644 --- a/cmdline/update_storage_class/CtaUpdateStorageClass.hpp +++ b/cmdline/update_storage_class/CtaUpdateStorageClass.hpp @@ -72,7 +72,6 @@ public: /** * Returns the id of a storage class - * @param diskInstance eos disk instance */ uint64_t getStorageClassId() const; @@ -95,6 +94,13 @@ private: */ std::vector<uint64_t> m_archiveFileIds; + + /** + * Archive file id of the files to update + */ + std::string m_storageClassName; + + /** * Disk instance of the files to update */ @@ -107,7 +113,7 @@ private: /** - *Comman line arguments + *Command line arguments */ CtaUpdateStorageClassCmdLineArgs m_cmdLineArgs; -- GitLab From 6af19e340ccd48fa4b2a0a5c8e94ac50e816a429 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 31 Aug 2022 14:29:17 +0200 Subject: [PATCH 020/126] #78 - change naming to match issue specification --- cmdline/CMakeLists.txt | 2 +- cmdline/change_storage_class/CMakeLists.txt | 36 +++ .../CtaChangeStorageClass.cpp | 207 ++++++++++++++++++ .../CtaChangeStorageClass.hpp | 138 ++++++++++++ .../CtaChangeStorageClassCmdLineArgs.cpp | 127 +++++++++++ .../CtaChangeStorageClassCmdLineArgs.hpp | 83 +++++++ .../CtaChangeStorageClassMain.cpp | 48 ++++ .../change_storage_class/CtaCmdOptions.hpp | 171 +++++++++++++++ .../CtaUpdateStorageClass.cpp | 49 ++++- .../CtaUpdateStorageClass.hpp | 8 +- 10 files changed, 852 insertions(+), 17 deletions(-) create mode 100644 cmdline/change_storage_class/CMakeLists.txt create mode 100644 cmdline/change_storage_class/CtaChangeStorageClass.cpp create mode 100644 cmdline/change_storage_class/CtaChangeStorageClass.hpp create mode 100644 cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp create mode 100644 cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp create mode 100644 cmdline/change_storage_class/CtaChangeStorageClassMain.cpp create mode 100644 cmdline/change_storage_class/CtaCmdOptions.hpp diff --git a/cmdline/CMakeLists.txt b/cmdline/CMakeLists.txt index 75929acf80..35fcbed6b7 100644 --- a/cmdline/CMakeLists.txt +++ b/cmdline/CMakeLists.txt @@ -16,7 +16,7 @@ cmake_minimum_required (VERSION 3.17) add_subdirectory (restore_files) -add_subdirectory (update_storage_class) +add_subdirectory (change_storage_class) find_package(xrootdclient REQUIRED) find_package(Protobuf3 REQUIRED) diff --git a/cmdline/change_storage_class/CMakeLists.txt b/cmdline/change_storage_class/CMakeLists.txt new file mode 100644 index 0000000000..c08c2b472b --- /dev/null +++ b/cmdline/change_storage_class/CMakeLists.txt @@ -0,0 +1,36 @@ +# @project The CERN Tape Archive (CTA) +# @copyright Copyright © 2015-2022 CERN +# @license This program is free software, distributed under the terms of the GNU General Public +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can +# redistribute it and/or modify it under the terms of the GPL Version 3, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# In applying this licence, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +cmake_minimum_required (VERSION 3.17) + +find_package(xrootdclient REQUIRED) +find_package(Protobuf3 REQUIRED) + +# XRootD SSI +include_directories(${XROOTD_INCLUDE_DIR} ${XROOTD_INCLUDE_DIR}/private ) + +# XRootD SSI Protocol Buffer bindings +include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include) + +# Compiled protocol buffers +include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS}) + +add_executable(cta-change-storage-class CtaChangeStorageClass.cpp CtaChangeStorageClassCmdLineArgs.cpp CtaChangeStorageClassMain.cpp ../restore_files/CmdLineTool.cpp CtaCmdOptions.hpp) +target_link_libraries(cta-change-storage-class ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient) +set_property (TARGET cta-change-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) + + +install(TARGETS cta-change-storage-class DESTINATION usr/bin) +install(FILES cta-change-storage-class.1cta DESTINATION usr/share/man/man1) diff --git a/cmdline/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/change_storage_class/CtaChangeStorageClass.cpp new file mode 100644 index 0000000000..a39700b325 --- /dev/null +++ b/cmdline/change_storage_class/CtaChangeStorageClass.cpp @@ -0,0 +1,207 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#include "cmdline/change_storage_class/CtaChangeStorageClass.hpp" +#include "cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp" +#include "common/utils/utils.hpp" +#include "common/checksum/ChecksumBlob.hpp" +#include "CtaFrontendApi.hpp" + +#include <XrdSsiPbLog.hpp> +#include <XrdSsiPbIStreamBuffer.hpp> + +#include <grpc++/grpc++.h> +#include "Rpc.grpc.pb.h" + +#include <sys/stat.h> +#include <iostream> +#include <memory> + +// GLOBAL VARIABLES : used to pass information between main thread and stream handler thread + +// global synchronisation flag +std::atomic<bool> isHeaderSent(false); + +std::list<cta::admin::RecycleTapeFileLsItem> deletedTapeFiles; +std::list<std::pair<std::string,std::string>> listedTapeFiles; + +namespace XrdSsiPb { + +/*! + * User error exception + */ +class UserException : public std::runtime_error +{ +public: + UserException(const std::string &err_msg) : std::runtime_error(err_msg) {} +}; // class UserException + +/*! + * Alert callback. + * + * Defines how Alert messages should be logged + */ +template<> +void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) +{ + std::cout << "AlertCallback():" << std::endl; + Log::DumpProtobuf(Log::PROTOBUF, &alert); +} + + +} // namespace XrdSsiPb + + +namespace cta { +namespace admin { + +/*! + * ChangeFilesCmdException + */ +class ChangeFilesCmdException : public std::runtime_error +{ +public: + ChangeFilesCmdException(const std::string &err_msg) : std::runtime_error(err_msg) {} +}; // class UserException + + + +//------------------------------------------------------------------------------ +// constructor +//------------------------------------------------------------------------------ +CtaChangeStorageClass::CtaChangeStorageClass( + std::istream &inStream, + std::ostream &outStream, + std::ostream &errStream, + cta::log::StdoutLogger &log, + const int& argc, + char *const *const argv): + CmdLineTool(inStream, outStream, errStream), + m_log(log), + m_cmdLineArgs(argc, argv) { + + std::vector<std::string> fids = m_cmdLineArgs.m_optionFids.get_value(); + for(const auto& fid: fids) { + m_archiveFileIds.push_back(static_cast<uint32_t>(std::stoul(fid))); + } + + m_storageClassName = m_cmdLineArgs.m_option_storage_class_name.get_value(); + +} + + +//------------------------------------------------------------------------------ +// getStorageClassId +//------------------------------------------------------------------------------ +uint64_t CtaChangeStorageClass::changeCatalogue() { + cta::xrd::Request request; + + auto &admincmd = *(request.mutable_admincmd()); + + request.set_client_cta_version(CTA_VERSION); + request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); + admincmd.set_cmd(AdminCmd::CMD_STORAGECLASS); // TODO: define new admin request + admincmd.set_subcmd(AdminCmd::SUBCMD_LS); + + // Send the Request to the Service and get a Response + cta::xrd::Response response; + m_serviceProviderPtr->Send(request, response); + + // Handle responses + switch(response.type()) + { + using namespace cta::xrd; + using namespace cta::admin; + case Response::RSP_SUCCESS: + // Print message text + std::cout << response.message_txt(); + break; + case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); + case Response::RSP_ERR_USER: throw XrdSsiPb::UserException(response.message_txt()); + case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); + default: throw XrdSsiPb::PbException("Invalid response type."); + } + return 0; +} + + +//------------------------------------------------------------------------------ +// printUsage +//------------------------------------------------------------------------------ +void CtaChangeStorageClass::printUsage(std::ostream &os) { + os << "Usage:" << std::endl << + " cta-change-storage-class [--fid/-f <archive_file_id>] [--storage_class_name/-scn <storage_class_name>]" << std::endl; +} + + +//------------------------------------------------------------------------------ +// readAndSetConfiguration +//------------------------------------------------------------------------------ +void CtaChangeStorageClass::readAndSetConfiguration( + const std::string &userName) { + + // Set CTA frontend configuration options + const std::string cli_config_file = "/etc/cta/cta-cli.conf"; + XrdSsiPb::Config cliConfig(cli_config_file, "cta"); + cliConfig.set("resource", "/ctafrontend"); + cliConfig.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes + cliConfig.set("request_timeout", DefaultRequestTimeout); // default value = 10s + + // Allow environment variables to override config file + cliConfig.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); + + // If XRDDEBUG=1, switch on all logging + if(getenv("XRDDEBUG")) { + cliConfig.set("log", "all"); + } + // If fine-grained control over log level is required, use XrdSsiPbLogLevel + cliConfig.getEnv("log", "XrdSsiPbLogLevel"); + + // Validate that endpoint was specified in the config file + if(!cliConfig.getOptionValueStr("endpoint").first) { + throw std::runtime_error("Configuration error: cta.endpoint missing from " + cli_config_file); + } + + // If the server is down, we want an immediate failure. Set client retry to a single attempt. + XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); + + m_serviceProviderPtr.reset(new XrdSsiPbServiceType(cliConfig)); + + // Set CTA frontend configuration options to connect to eos + const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf"; + XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); +} + +//------------------------------------------------------------------------------ +// exceptionThrowingMain +//------------------------------------------------------------------------------ +int CtaChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { + CtaChangeStorageClassCmdLineArgs cmdLineArgs(argc, argv); + if (cmdLineArgs.m_help) { + cmdLineArgs.printUsage(); + } + + readAndSetConfiguration(getUsername()); + + changeCatalogue(); + + return 0; +} + + +} // namespace admin +} // namespace cta diff --git a/cmdline/change_storage_class/CtaChangeStorageClass.hpp b/cmdline/change_storage_class/CtaChangeStorageClass.hpp new file mode 100644 index 0000000000..0c627ac392 --- /dev/null +++ b/cmdline/change_storage_class/CtaChangeStorageClass.hpp @@ -0,0 +1,138 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#pragma once + +#include "cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp" +#include "cmdline/restore_files/CmdLineTool.hpp" +#include "eos_grpc_client/GrpcEndpoint.hpp" +#include "catalogue/Catalogue.hpp" +#include "common/log/StdoutLogger.hpp" + +#include <optional> +#include <memory> + +#include "CtaFrontendApi.hpp" + +namespace cta { +namespace admin { + +class CtaChangeStorageClass: public CmdLineTool { +public: + /** + * Constructor. + * + * @param inStream Standard input stream. + * @param outStream Standard output stream. + * @param errStream Standard error stream. + * @param log The object representing the API of the CTA logging system. + * @param argc + * @param argv + */ + CtaChangeStorageClass( + std::istream &inStream, + std::ostream &outStream, + std::ostream &errStream, + cta::log::StdoutLogger &log, + const int& argc, + char *const *const argv); + + /** + * An exception throwing version of main(). + * + * @param argc The number of command-line arguments including the program name. + * @param argv The command-line arguments. + * @return The exit value of the program. + */ + int exceptionThrowingMain(const int argc, char *const *const argv) override; + + /** + * Sets internal configuration parameters to be used for reading. + * It reads cta frontend parameters from /etc/cta/cta-cli.conf + * + * @param username The name of the user running the command-line tool. + * @param cmdLineArgs The arguments parsed from the command line. + */ + void readAndSetConfiguration(const std::string &userName); + + + /** + * Prints the usage message of the command-line tool. + */ + void printUsage(std::ostream &os) override; + +private: + + /** + * The object representing the API of the CTA logging system. + */ + cta::log::StdoutLogger &m_log; + + + /** + * Archive file id of the files to change + */ + std::vector<uint64_t> m_archiveFileIds; + + + /** + * Archive file id of the files to change + */ + std::string m_storageClassName; + + + /** + * Disk instance of the files to change + */ + std::optional<std::string> m_diskInstance; + + /** + * Fids of the files to change + */ + std::optional<std::list<uint64_t>> m_eosFids; + + + /** + *Command line arguments + */ + CtaChangeStorageClassCmdLineArgs m_cmdLineArgs; + + + /** + * CTA Frontend service provider + */ + std::unique_ptr<XrdSsiPbServiceType> m_serviceProviderPtr; + + std::unique_ptr<::eos::client::EndpointMap> m_endpointMapPtr; + + const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses + const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by + //!< XRD_REQUESTTIMEOUT environment variable. + + const std::string m_ctaVersion = CTA_VERSION; //!< The version of CTA + const std::string m_xrootdSsiProtobufInterfaceVersion = //!< The xrootd-ssi-protobuf-interface version (=tag) + XROOTD_SSI_PROTOBUF_INTERFACE_VERSION; + + /** + * Changes the storage class in the catalogue + */ + uint64_t changeCatalogue(); + +} ; // class CtaChangeStorageClass + +} // namespace admin +} // namespace cta diff --git a/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp b/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp new file mode 100644 index 0000000000..420464833b --- /dev/null +++ b/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp @@ -0,0 +1,127 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#include "cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp" +#include "common/exception/CommandLineNotParsed.hpp" + +#include <getopt.h> +#include <ostream> +#include <string> +#include <limits.h> +#include <fstream> + +namespace cta { +namespace admin{ + +//------------------------------------------------------------------------------ +// constructor +//------------------------------------------------------------------------------ +CtaChangeStorageClassCmdLineArgs::CtaChangeStorageClassCmdLineArgs(const int argc, char *const *const argv): + m_help(false), + m_optionFids("--fid", "-f", true), + m_option_storage_class_name("--storage_class_name", "-scn", true) { + + option_with_multiple_values_map["--fid"] = &m_optionFids; + option_with_multiple_values_map["-f"] = &m_optionFids; + + option_map["--storage_class_name"] = &m_option_storage_class_name; + option_map["-scn"] = &m_option_storage_class_name; + + parseCmd(argc, argv); +} + + +//------------------------------------------------------------------------------ +// printUsage +//------------------------------------------------------------------------------ +void CtaChangeStorageClassCmdLineArgs::printUsage() { + std::cout << "Usage:" << std::endl << + " cta-update-storage-class [--fid/-f <archive_file_id>] [--storage_class_name/-scn <storage_class_name>]" << std::endl; +} + +//------------------------------------------------------------------------------ +// validateCmd +//------------------------------------------------------------------------------ +void CtaChangeStorageClassCmdLineArgs::validateCmd(int argc) { + if (argc < 3) { + printUsage(); + throw std::runtime_error(""); + } + for (auto &it : option_with_multiple_values_map) { + auto option = it.second; + if (!option->is_optional() && !option->is_present()) { + std::cout << "Error: Option " << option->get_name() << " is mandatory but not present." << std::endl; + printUsage(); + throw std::runtime_error(""); + } + } +} + +//------------------------------------------------------------------------------ +// isEndOfOption +//------------------------------------------------------------------------------ +bool CtaChangeStorageClassCmdLineArgs::isEndOfOption(int nextOptionOrValueIndex, const char *const *const argv) { + return ( + option_with_multiple_values_map.find(argv[nextOptionOrValueIndex]) != option_with_multiple_values_map.end() || + option_map.find(argv[nextOptionOrValueIndex]) != option_map.end() + ); +} + +//------------------------------------------------------------------------------ +// parseCmd +//------------------------------------------------------------------------------ +void CtaChangeStorageClassCmdLineArgs::parseCmd(const int argc, const char *const *const argv) { + unsigned int num_cmd_line_enrties_for_option = 2; + for (int i = 1; i < argc; i += num_cmd_line_enrties_for_option){ + num_cmd_line_enrties_for_option = 2; + auto search_option_with_multiple_values_map = option_with_multiple_values_map.find(argv[i]); + if (search_option_with_multiple_values_map != option_with_multiple_values_map.end()) { + auto stringListOption = search_option_with_multiple_values_map->second; + stringListOption->set_present(); + + if (stringListOption->get_name() == "--fid") { + for (int j = 1; j < argc; j += 1) { + int nextOptionOrValueIndex = i + j + 1; + bool end_of_option = isEndOfOption(nextOptionOrValueIndex, argv); + stringListOption->add_value(argv[i + j]); + if (end_of_option || (j + i) == (argc - 1)) { + num_cmd_line_enrties_for_option = j + 1; + break; + } + } + } + } + + auto search_option = option_map.find(argv[i]); + if (search_option != option_map.end()) { + auto stringOption = search_option->second; + stringOption->set_present(); + if (stringOption->get_name() == "--storage_class_name") { + stringOption->set_value(argv[i + 1]); + } + } + + if (search_option_with_multiple_values_map == option_with_multiple_values_map.end() && search_option == option_map.end()) { + std::cout << "Error: Unknown option: " << argv[i] << std::endl; + printUsage(); + throw std::runtime_error(""); + } + } +} + +} // namespace admin +} // namespace cta diff --git a/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp b/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp new file mode 100644 index 0000000000..27ca41d154 --- /dev/null +++ b/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp @@ -0,0 +1,83 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#pragma once + +#include <list> +#include <optional> +#include <map> +#include <iostream> +#include <sstream> + +#include "version.h" +#include "cmdline/CtaCmdOptions.hpp" + + +namespace cta { +namespace admin { + +/** + * class to store the command-line arguments of the command-line tool + * named cta-change-storage-class. + */ +class CtaChangeStorageClassCmdLineArgs { + +public: + /** + * True if the usage message should be printed. + */ + bool m_help; + + /** + * List of archive file ids + */ + StringListOption m_optionFids; + + /** + * The storage name used to replace the names for the files with file id found in m_fids + */ + StringOption m_option_storage_class_name; + + + /** + * Constructor that parses the specified command-line arguments. + * + * @param argc The number of command-line arguments including the name of the + * executable. + * @param argv The vector of command-line arguments. + */ + CtaChangeStorageClassCmdLineArgs(const int argc, char *const *const argv); + + + /** + * Print the correct usage of the tool + * + */ + void printUsage(); + +private: + void validateCmd(int argc); + bool isEndOfOption(int nextOptionOrValueIndex, const char *const *const argv); + void parseCmd(const int argc, const char *const *const argv); + + std::map<std::string, StringListOption *> option_with_multiple_values_map; + std::map<std::string, StringOption *> option_map; + +}; // class CtaChangeStorageClassCmdLineArgs + +} // namespace admin +} // namespace cta diff --git a/cmdline/change_storage_class/CtaChangeStorageClassMain.cpp b/cmdline/change_storage_class/CtaChangeStorageClassMain.cpp new file mode 100644 index 0000000000..60a6125379 --- /dev/null +++ b/cmdline/change_storage_class/CtaChangeStorageClassMain.cpp @@ -0,0 +1,48 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + + +#include <sstream> +#include <iostream> + +#include <XrdSsiPbLog.hpp> +#include <XrdSsiPbIStreamBuffer.hpp> + +#include "cmdline/change_storage_class/CtaChangeStorageClass.hpp" +#include "common/utils/utils.hpp" +#include "common/log/StdoutLogger.hpp" + +//------------------------------------------------------------------------------ +// main +//------------------------------------------------------------------------------ +int main(const int argc, char *const *const argv) { + char buf[256]; + std::string hostName; + if(gethostname(buf, sizeof(buf))) { + hostName = "UNKNOWN"; + } else { + buf[sizeof(buf) - 1] = '\0'; + hostName = buf; + } + cta::log::StdoutLogger log(hostName, "cta-change-storage-class"); + + cta::admin::CtaChangeStorageClass cmd(std::cin, std::cout, std::cerr, log, argc, argv); + int ret = cmd.main(argc, argv); + // Delete all global objects allocated by libprotobuf + google::protobuf::ShutdownProtobufLibrary(); + return ret; +} diff --git a/cmdline/change_storage_class/CtaCmdOptions.hpp b/cmdline/change_storage_class/CtaCmdOptions.hpp new file mode 100644 index 0000000000..ae4e934f69 --- /dev/null +++ b/cmdline/change_storage_class/CtaCmdOptions.hpp @@ -0,0 +1,171 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#pragma once + +#include <string> +#include <vector> + +/*! +* Simple command line option class for simpler commands like cta-verify-file or cta-send-event +*/ +class StringOption { +public: + /*! + * Constructor + */ + StringOption(const std::string &long_opt, const std::string &short_opt, const bool is_optional) : + m_long_opt(long_opt), + m_short_opt(short_opt), + m_is_optional(is_optional), + m_is_present(false), + m_value("") {} + + /*! + * Check if the supplied key matches the option + */ + bool operator==(const std::string &option) const { + return option == m_short_opt || option == m_long_opt; + } + + /*! + * Return whether the option is optional + */ + bool is_optional() const { + return m_is_optional; + } + + /*! + * Return whether the option is present on the command line + */ + bool is_present() const { + return m_is_present; + } + + /*! + * Sets the option as present on the command line + */ + void set_present() { + m_is_present = true; + } + + /*! + * Return the value of the option given from the command line + */ + const std::string &get_value() const { + return m_value; + } + + /*! + * Return the name of the option (it's long command option) + */ + const std::string &get_name() const { + return m_long_opt; + } + + /*! + * Sets the value of the option from the command line + */ + void set_value(const std::string &mValue) { + m_value = mValue; + } + +private: + + //member variables + const std::string m_long_opt; //!< Long command option + const std::string m_short_opt; //!< Short command option + const bool m_is_optional; //!< Option is optional or compulsory + bool m_is_present; //!< Option is present on the command line + std::string m_value; //!< Option value +}; + + +/*! +* Simple command line option class for simpler commands like cta-verify-file or cta-send-event +*/ +class StringListOption { +public: + /*! + * Constructor + */ + StringListOption(const std::string &long_opt, const std::string &short_opt, const bool is_optional) : + m_long_opt(long_opt), + m_short_opt(short_opt), + m_is_optional(is_optional), + m_is_present(false) {} + + /*! + * Check if the supplied key matches the option + */ + bool operator==(const std::string &option) const { + return option == m_short_opt || option == m_long_opt; + } + + /*! + * Return whether the option is optional + */ + bool is_optional() const { + return m_is_optional; + } + + /*! + * Return whether the option is present on the command line + */ + bool is_present() const { + return m_is_present; + } + + /*! + * Sets the option as present on the command line + */ + void set_present() { + m_is_present = true; + } + + /*! + * Return the value of the option given from the command line + */ + const std::vector<std::string> &get_value() const { + return m_value_list; + } + + /*! + * Return the name of the option (it's long command option) + */ + const std::string &get_name() const { + return m_long_opt; + } + + /*! + * Sets the value of the option from the command line + */ + void add_value(const std::string &mValue) { + m_value_list.push_back(mValue); + } + +private: + + //member variables + const std::string m_long_opt; //!< Long command option + const std::string m_short_opt; //!< Short command option + const bool m_is_optional; //!< Option is optional or compulsory + bool m_is_present; //!< Option is present on the command line + std::vector<std::string> m_value_list; //!< Option value +}; + + diff --git a/cmdline/update_storage_class/CtaUpdateStorageClass.cpp b/cmdline/update_storage_class/CtaUpdateStorageClass.cpp index 7f461c7af1..b07f98b6dc 100644 --- a/cmdline/update_storage_class/CtaUpdateStorageClass.cpp +++ b/cmdline/update_storage_class/CtaUpdateStorageClass.cpp @@ -83,13 +83,6 @@ void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const deletedTapeFiles.push_back(item); } break; - case Data::kTflsItem: - { - auto item = record.tfls_item(); - auto instanceAndFid = std::make_pair(item.df().disk_instance(), item.df().disk_id()); - listedTapeFiles.push_back(instanceAndFid); - } - break; default: throw std::runtime_error("Received invalid stream data from CTA Frontend for the cta-restore-deleted-files command."); } @@ -139,8 +132,40 @@ CtaUpdateStorageClass::CtaUpdateStorageClass( //------------------------------------------------------------------------------ // getStorageClassId //------------------------------------------------------------------------------ -uint64_t getStorageClassId() { - return 0; +uint64_t CtaUpdateStorageClass::updateCatalogue() { + cta::xrd::Request request; + + auto &admincmd = *(request.mutable_admincmd()); + + request.set_client_cta_version(CTA_VERSION); + request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); + admincmd.set_cmd(AdminCmd::CMD_STORAGECLASS); // TODO: define new admin request + admincmd.set_subcmd(AdminCmd::STORAGECLASS_LS); + + m_log(cta::log::DEBUG, "Getting Storage Classes", params); + + // Send the Request to the Service and get a Response + cta::xrd::Response response; + m_serviceProviderPtr->Send(request, response); + + // Handle responses + switch(response.type()) + { + using namespace cta::xrd; + using namespace cta::admin; + case Response::RSP_SUCCESS: + // Print message text + std::cout << response.message_txt(); + m_log(cta::log::INFO, "Recieved Storage Class from Cta Catalogue", params); + break; + case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); + case Response::RSP_ERR_USER: throw XrdSsiPb::UserException(response.message_txt()); + case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); + default: throw XrdSsiPb::PbException("Invalid response type."); + } + + // wait until the data stream has been processed before exiting + stream_future.wait(); } @@ -149,9 +174,7 @@ uint64_t getStorageClassId() { //------------------------------------------------------------------------------ void CtaUpdateStorageClass::printUsage(std::ostream &os) { os << "Usage:" << std::endl << - " cta-update-storage-class [--id/-I <archive_file_id>] [--instance/-i <disk_instance>]" << std::endl << - " [--fxid/-f <eos_fxid>] [--fxidfile/-F <filename>]" << std::endl << - " [--vid/-v <vid>] [--copynb/-c <copy_number>] [--debug/-d]" << std::endl; + " cta-update-storage-class [--fid/-f <archive_file_id>] [--storage_class_name/-scn <storage_class_name>]" << std::endl; } @@ -204,6 +227,8 @@ int CtaUpdateStorageClass::exceptionThrowingMain(const int argc, char *const *co readAndSetConfiguration(getUsername()); + getStorageClassId(); + return 0; } diff --git a/cmdline/update_storage_class/CtaUpdateStorageClass.hpp b/cmdline/update_storage_class/CtaUpdateStorageClass.hpp index 24e042c86e..fced2ff14f 100644 --- a/cmdline/update_storage_class/CtaUpdateStorageClass.hpp +++ b/cmdline/update_storage_class/CtaUpdateStorageClass.hpp @@ -70,10 +70,6 @@ public: void readAndSetConfiguration(const std::string &userName); - /** - * Returns the id of a storage class - */ - uint64_t getStorageClassId() const; /** @@ -133,6 +129,10 @@ private: const std::string m_xrootdSsiProtobufInterfaceVersion = //!< The xrootd-ssi-protobuf-interface version (=tag) XROOTD_SSI_PROTOBUF_INTERFACE_VERSION; + /** + * Returns the id of a storage class + */ + uint64_t updateCatalogue() const; } ; // class CtaUpdateStorageClass -- GitLab From 8e742dd23cef5f1a624b616ae02be0c57ec9436c Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 1 Sep 2022 10:17:46 +0200 Subject: [PATCH 021/126] #78 - Add new frontend cmds --- cmdline/CtaAdminCmdParse.hpp | 6 + .../CtaChangeStorageClass.cpp | 42 +++- .../CtaChangeStorageClass.hpp | 2 +- .../CtaChangeStorageClassCmdLineArgs.cpp | 18 -- .../CtaChangeStorageClassCmdLineArgs.hpp | 1 - cmdline/update_storage_class/CMakeLists.txt | 36 --- .../update_storage_class/CtaCmdOptions.hpp | 171 ------------- .../CtaUpdateStorageClass.cpp | 237 ------------------ .../CtaUpdateStorageClass.hpp | 140 ----------- .../CtaUpdateStorageClassCmdLineArgs.cpp | 129 ---------- .../CtaUpdateStorageClassCmdLineArgs.hpp | 83 ------ .../CtaUpdateStorageClassMain.cpp | 48 ---- xroot_plugins/XrdSsiCtaRequestMessage.cpp | 12 + xroot_plugins/XrdSsiCtaRequestMessage.hpp | 1 + 14 files changed, 49 insertions(+), 877 deletions(-) delete mode 100644 cmdline/update_storage_class/CMakeLists.txt delete mode 100644 cmdline/update_storage_class/CtaCmdOptions.hpp delete mode 100644 cmdline/update_storage_class/CtaUpdateStorageClass.cpp delete mode 100644 cmdline/update_storage_class/CtaUpdateStorageClass.hpp delete mode 100644 cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp delete mode 100644 cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp delete mode 100644 cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp diff --git a/cmdline/CtaAdminCmdParse.hpp b/cmdline/CtaAdminCmdParse.hpp index 4d1c799593..2a40167f69 100644 --- a/cmdline/CtaAdminCmdParse.hpp +++ b/cmdline/CtaAdminCmdParse.hpp @@ -557,6 +557,9 @@ const Option opt_diskinstancespace_alias { Option::OPT_STR, "--name", const Option opt_verificationstatus { Option::OPT_STR, "--verificationstatus", "--vs", " <verification_status>" }; const Option opt_disabledreason { Option::OPT_STR, "--disabledreason", "--dr", " <disabled_reason>" }; +const Option opt_storage_class_name { Option::OPT_STR, "--storage_class_name", "-scn", " <storage_class_name>" }; +const Option opt_archive_file_ids { Option::OPT_STR_LIST, "--fid", "-f", " <archive_file_id>" }; + /*! * Map valid options to commands */ @@ -707,6 +710,9 @@ const std::map<cmd_key_t, cmd_val_t> cmdOptions = { { opt_instance, opt_username_alias, opt_activityregex, opt_mountpolicy.optional(), opt_comment.optional() }}, {{ AdminCmd::CMD_ACTIVITYMOUNTRULE, AdminCmd::SUBCMD_RM }, { opt_instance, opt_username_alias, opt_activityregex }}, {{ AdminCmd::CMD_ACTIVITYMOUNTRULE, AdminCmd::SUBCMD_LS }, { }}, + /*----------------------------------------------------------------------------------------------------*/ + {{ AdminCmd::CMD_CHANGESTORAGECLASS, AdminCmd::SUBCMD_CH }, + { opt_storage_class_name, opt_archive_file_ids }}, }; diff --git a/cmdline/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/change_storage_class/CtaChangeStorageClass.cpp index a39700b325..7742453d7b 100644 --- a/cmdline/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/change_storage_class/CtaChangeStorageClass.cpp @@ -50,6 +50,7 @@ public: UserException(const std::string &err_msg) : std::runtime_error(err_msg) {} }; // class UserException + /*! * Alert callback. * @@ -62,7 +63,6 @@ void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) Log::DumpProtobuf(Log::PROTOBUF, &alert); } - } // namespace XrdSsiPb @@ -94,18 +94,13 @@ CtaChangeStorageClass::CtaChangeStorageClass( m_log(log), m_cmdLineArgs(argc, argv) { - std::vector<std::string> fids = m_cmdLineArgs.m_optionFids.get_value(); - for(const auto& fid: fids) { - m_archiveFileIds.push_back(static_cast<uint32_t>(std::stoul(fid))); - } - + m_archiveFileIds = m_cmdLineArgs.m_optionFids.get_value(); m_storageClassName = m_cmdLineArgs.m_option_storage_class_name.get_value(); - } //------------------------------------------------------------------------------ -// getStorageClassId +// changeCatalogue //------------------------------------------------------------------------------ uint64_t CtaChangeStorageClass::changeCatalogue() { cta::xrd::Request request; @@ -114,8 +109,24 @@ uint64_t CtaChangeStorageClass::changeCatalogue() { request.set_client_cta_version(CTA_VERSION); request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); - admincmd.set_cmd(AdminCmd::CMD_STORAGECLASS); // TODO: define new admin request - admincmd.set_subcmd(AdminCmd::SUBCMD_LS); + admincmd.set_cmd(AdminCmd::CMD_CHANGESTORAGECLASS); + admincmd.set_subcmd(AdminCmd::SUBCMD_CH); + + { + auto key = OptionString::STORAGE_CLASS_NAME; + auto new_opt = admincmd.add_option_str(); + new_opt->set_key(key); + new_opt->set_value(m_storageClassName); + } + + { + auto key = OptionStrList::FILE_ID; + auto new_opt = admincmd.add_option_str_list(); + new_opt->set_key(key); + for (auto &archiveFileId : m_archiveFileIds) { + new_opt->add_item(archiveFileId); + } + } // Send the Request to the Service and get a Response cta::xrd::Response response; @@ -135,6 +146,7 @@ uint64_t CtaChangeStorageClass::changeCatalogue() { case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); default: throw XrdSsiPb::PbException("Invalid response type."); } + return 0; } @@ -190,13 +202,17 @@ void CtaChangeStorageClass::readAndSetConfiguration( // exceptionThrowingMain //------------------------------------------------------------------------------ int CtaChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { - CtaChangeStorageClassCmdLineArgs cmdLineArgs(argc, argv); - if (cmdLineArgs.m_help) { - cmdLineArgs.printUsage(); + if (m_cmdLineArgs.m_help) { + m_cmdLineArgs.printUsage(); } readAndSetConfiguration(getUsername()); + std::cout << getUsername() << std::endl; + std::cout << getHostname() << std::endl; + + std::cout << typeid(m_storageClassName).name() << std::endl; + changeCatalogue(); return 0; diff --git a/cmdline/change_storage_class/CtaChangeStorageClass.hpp b/cmdline/change_storage_class/CtaChangeStorageClass.hpp index 0c627ac392..2fddf2b016 100644 --- a/cmdline/change_storage_class/CtaChangeStorageClass.hpp +++ b/cmdline/change_storage_class/CtaChangeStorageClass.hpp @@ -86,7 +86,7 @@ private: /** * Archive file id of the files to change */ - std::vector<uint64_t> m_archiveFileIds; + std::vector<std::string> m_archiveFileIds; /** diff --git a/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp b/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp index 420464833b..d4fef3de86 100644 --- a/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp +++ b/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp @@ -53,24 +53,6 @@ void CtaChangeStorageClassCmdLineArgs::printUsage() { " cta-update-storage-class [--fid/-f <archive_file_id>] [--storage_class_name/-scn <storage_class_name>]" << std::endl; } -//------------------------------------------------------------------------------ -// validateCmd -//------------------------------------------------------------------------------ -void CtaChangeStorageClassCmdLineArgs::validateCmd(int argc) { - if (argc < 3) { - printUsage(); - throw std::runtime_error(""); - } - for (auto &it : option_with_multiple_values_map) { - auto option = it.second; - if (!option->is_optional() && !option->is_present()) { - std::cout << "Error: Option " << option->get_name() << " is mandatory but not present." << std::endl; - printUsage(); - throw std::runtime_error(""); - } - } -} - //------------------------------------------------------------------------------ // isEndOfOption //------------------------------------------------------------------------------ diff --git a/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp b/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp index 27ca41d154..0e7679b576 100644 --- a/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp +++ b/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp @@ -70,7 +70,6 @@ public: void printUsage(); private: - void validateCmd(int argc); bool isEndOfOption(int nextOptionOrValueIndex, const char *const *const argv); void parseCmd(const int argc, const char *const *const argv); diff --git a/cmdline/update_storage_class/CMakeLists.txt b/cmdline/update_storage_class/CMakeLists.txt deleted file mode 100644 index ac446247d0..0000000000 --- a/cmdline/update_storage_class/CMakeLists.txt +++ /dev/null @@ -1,36 +0,0 @@ -# @project The CERN Tape Archive (CTA) -# @copyright Copyright © 2015-2022 CERN -# @license This program is free software, distributed under the terms of the GNU General Public -# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can -# redistribute it and/or modify it under the terms of the GPL Version 3, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -# PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# In applying this licence, CERN does not waive the privileges and immunities -# granted to it by virtue of its status as an Intergovernmental Organization or -# submit itself to any jurisdiction. - -cmake_minimum_required (VERSION 3.17) - -find_package(xrootdclient REQUIRED) -find_package(Protobuf3 REQUIRED) - -# XRootD SSI -include_directories(${XROOTD_INCLUDE_DIR} ${XROOTD_INCLUDE_DIR}/private ) - -# XRootD SSI Protocol Buffer bindings -include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include) - -# Compiled protocol buffers -include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS}) - -add_executable(cta-update-storage-class CtaUpdateStorageClass.cpp CtaUpdateStorageClassCmdLineArgs.cpp CtaUpdateStorageClassMain.cpp ../restore_files/CmdLineTool.cpp CtaCmdOptions.hpp) -target_link_libraries(cta-update-storage-class ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient) -set_property (TARGET cta-update-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) - - -install(TARGETS cta-update-storage-class DESTINATION usr/bin) -install(FILES cta-update-storage-class.1cta DESTINATION usr/share/man/man1) diff --git a/cmdline/update_storage_class/CtaCmdOptions.hpp b/cmdline/update_storage_class/CtaCmdOptions.hpp deleted file mode 100644 index ae4e934f69..0000000000 --- a/cmdline/update_storage_class/CtaCmdOptions.hpp +++ /dev/null @@ -1,171 +0,0 @@ -/* - * @project The CERN Tape Archive (CTA) - * @copyright Copyright © 2021-2022 CERN - * @license This program is free software, distributed under the terms of the GNU General Public - * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can - * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A - * PARTICULAR PURPOSE. See the GNU General Public License for more details. - * - * In applying this licence, CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization or - * submit itself to any jurisdiction. - */ - -#pragma once - -#include <string> -#include <vector> - -/*! -* Simple command line option class for simpler commands like cta-verify-file or cta-send-event -*/ -class StringOption { -public: - /*! - * Constructor - */ - StringOption(const std::string &long_opt, const std::string &short_opt, const bool is_optional) : - m_long_opt(long_opt), - m_short_opt(short_opt), - m_is_optional(is_optional), - m_is_present(false), - m_value("") {} - - /*! - * Check if the supplied key matches the option - */ - bool operator==(const std::string &option) const { - return option == m_short_opt || option == m_long_opt; - } - - /*! - * Return whether the option is optional - */ - bool is_optional() const { - return m_is_optional; - } - - /*! - * Return whether the option is present on the command line - */ - bool is_present() const { - return m_is_present; - } - - /*! - * Sets the option as present on the command line - */ - void set_present() { - m_is_present = true; - } - - /*! - * Return the value of the option given from the command line - */ - const std::string &get_value() const { - return m_value; - } - - /*! - * Return the name of the option (it's long command option) - */ - const std::string &get_name() const { - return m_long_opt; - } - - /*! - * Sets the value of the option from the command line - */ - void set_value(const std::string &mValue) { - m_value = mValue; - } - -private: - - //member variables - const std::string m_long_opt; //!< Long command option - const std::string m_short_opt; //!< Short command option - const bool m_is_optional; //!< Option is optional or compulsory - bool m_is_present; //!< Option is present on the command line - std::string m_value; //!< Option value -}; - - -/*! -* Simple command line option class for simpler commands like cta-verify-file or cta-send-event -*/ -class StringListOption { -public: - /*! - * Constructor - */ - StringListOption(const std::string &long_opt, const std::string &short_opt, const bool is_optional) : - m_long_opt(long_opt), - m_short_opt(short_opt), - m_is_optional(is_optional), - m_is_present(false) {} - - /*! - * Check if the supplied key matches the option - */ - bool operator==(const std::string &option) const { - return option == m_short_opt || option == m_long_opt; - } - - /*! - * Return whether the option is optional - */ - bool is_optional() const { - return m_is_optional; - } - - /*! - * Return whether the option is present on the command line - */ - bool is_present() const { - return m_is_present; - } - - /*! - * Sets the option as present on the command line - */ - void set_present() { - m_is_present = true; - } - - /*! - * Return the value of the option given from the command line - */ - const std::vector<std::string> &get_value() const { - return m_value_list; - } - - /*! - * Return the name of the option (it's long command option) - */ - const std::string &get_name() const { - return m_long_opt; - } - - /*! - * Sets the value of the option from the command line - */ - void add_value(const std::string &mValue) { - m_value_list.push_back(mValue); - } - -private: - - //member variables - const std::string m_long_opt; //!< Long command option - const std::string m_short_opt; //!< Short command option - const bool m_is_optional; //!< Option is optional or compulsory - bool m_is_present; //!< Option is present on the command line - std::vector<std::string> m_value_list; //!< Option value -}; - - diff --git a/cmdline/update_storage_class/CtaUpdateStorageClass.cpp b/cmdline/update_storage_class/CtaUpdateStorageClass.cpp deleted file mode 100644 index b07f98b6dc..0000000000 --- a/cmdline/update_storage_class/CtaUpdateStorageClass.cpp +++ /dev/null @@ -1,237 +0,0 @@ -/* - * @project The CERN Tape Archive (CTA) - * @copyright Copyright © 2021-2022 CERN - * @license This program is free software, distributed under the terms of the GNU General Public - * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can - * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A - * PARTICULAR PURPOSE. See the GNU General Public License for more details. - * - * In applying this licence, CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization or - * submit itself to any jurisdiction. - */ - -#include "cmdline/update_storage_class/CtaUpdateStorageClass.hpp" -#include "cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp" -#include "common/utils/utils.hpp" -#include "common/checksum/ChecksumBlob.hpp" -#include "CtaFrontendApi.hpp" - -#include <XrdSsiPbLog.hpp> -#include <XrdSsiPbIStreamBuffer.hpp> - -#include <grpc++/grpc++.h> -#include "Rpc.grpc.pb.h" - -#include <sys/stat.h> -#include <iostream> -#include <memory> - -// GLOBAL VARIABLES : used to pass information between main thread and stream handler thread - -// global synchronisation flag -std::atomic<bool> isHeaderSent(false); - -std::list<cta::admin::RecycleTapeFileLsItem> deletedTapeFiles; -std::list<std::pair<std::string,std::string>> listedTapeFiles; - -namespace XrdSsiPb { - -/*! - * User error exception - */ -class UserException : public std::runtime_error -{ -public: - UserException(const std::string &err_msg) : std::runtime_error(err_msg) {} -}; // class UserException - -/*! - * Alert callback. - * - * Defines how Alert messages should be logged - */ -template<> -void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) -{ - std::cout << "AlertCallback():" << std::endl; - Log::DumpProtobuf(Log::PROTOBUF, &alert); -} - -/*! - * Data/Stream callback. - * - * Defines how incoming records from the stream should be handled - */ -template<> -void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const -{ - using namespace cta::xrd; - using namespace cta::admin; - - // Wait for primary response to be handled before allowing stream response - while(!isHeaderSent) { std::this_thread::yield(); } - - switch(record.data_case()) { - case Data::kRtflsItem: - { - auto item = record.rtfls_item(); - deletedTapeFiles.push_back(item); - } - break; - default: - throw std::runtime_error("Received invalid stream data from CTA Frontend for the cta-restore-deleted-files command."); - } -} - -} // namespace XrdSsiPb - - -namespace cta{ -namespace admin { - -/*! - * UpdateFilesCmdException - */ -class UpdateFilesCmdException : public std::runtime_error -{ -public: - UpdateFilesCmdException(const std::string &err_msg) : std::runtime_error(err_msg) {} -}; // class UserException - - - -//------------------------------------------------------------------------------ -// constructor -//------------------------------------------------------------------------------ -CtaUpdateStorageClass::CtaUpdateStorageClass( - std::istream &inStream, - std::ostream &outStream, - std::ostream &errStream, - cta::log::StdoutLogger &log, - const int& argc, - char *const *const argv): - CmdLineTool(inStream, outStream, errStream), - m_log(log), - m_cmdLineArgs(argc, argv) { - - std::vector<std::string> fids = m_cmdLineArgs.m_optionFids.get_value(); - for(const auto& fid: fids) { - m_archiveFileIds.push_back(static_cast<uint32_t>(std::stoul(fid))); - } - - m_storageClassName = m_cmdLineArgs.m_option_storage_class_name.get_value(); - -} - - -//------------------------------------------------------------------------------ -// getStorageClassId -//------------------------------------------------------------------------------ -uint64_t CtaUpdateStorageClass::updateCatalogue() { - cta::xrd::Request request; - - auto &admincmd = *(request.mutable_admincmd()); - - request.set_client_cta_version(CTA_VERSION); - request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); - admincmd.set_cmd(AdminCmd::CMD_STORAGECLASS); // TODO: define new admin request - admincmd.set_subcmd(AdminCmd::STORAGECLASS_LS); - - m_log(cta::log::DEBUG, "Getting Storage Classes", params); - - // Send the Request to the Service and get a Response - cta::xrd::Response response; - m_serviceProviderPtr->Send(request, response); - - // Handle responses - switch(response.type()) - { - using namespace cta::xrd; - using namespace cta::admin; - case Response::RSP_SUCCESS: - // Print message text - std::cout << response.message_txt(); - m_log(cta::log::INFO, "Recieved Storage Class from Cta Catalogue", params); - break; - case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); - case Response::RSP_ERR_USER: throw XrdSsiPb::UserException(response.message_txt()); - case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); - default: throw XrdSsiPb::PbException("Invalid response type."); - } - - // wait until the data stream has been processed before exiting - stream_future.wait(); -} - - -//------------------------------------------------------------------------------ -// printUsage -//------------------------------------------------------------------------------ -void CtaUpdateStorageClass::printUsage(std::ostream &os) { - os << "Usage:" << std::endl << - " cta-update-storage-class [--fid/-f <archive_file_id>] [--storage_class_name/-scn <storage_class_name>]" << std::endl; -} - - -//------------------------------------------------------------------------------ -// readAndSetConfiguration -//------------------------------------------------------------------------------ -void CtaUpdateStorageClass::readAndSetConfiguration( - const std::string &userName) { - - // Set CTA frontend configuration options - const std::string cli_config_file = "/etc/cta/cta-cli.conf"; - XrdSsiPb::Config cliConfig(cli_config_file, "cta"); - cliConfig.set("resource", "/ctafrontend"); - cliConfig.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes - cliConfig.set("request_timeout", DefaultRequestTimeout); // default value = 10s - - // Allow environment variables to override config file - cliConfig.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); - - // If XRDDEBUG=1, switch on all logging - if(getenv("XRDDEBUG")) { - cliConfig.set("log", "all"); - } - // If fine-grained control over log level is required, use XrdSsiPbLogLevel - cliConfig.getEnv("log", "XrdSsiPbLogLevel"); - - // Validate that endpoint was specified in the config file - if(!cliConfig.getOptionValueStr("endpoint").first) { - throw std::runtime_error("Configuration error: cta.endpoint missing from " + cli_config_file); - } - - // If the server is down, we want an immediate failure. Set client retry to a single attempt. - XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); - - m_serviceProviderPtr.reset(new XrdSsiPbServiceType(cliConfig)); - - // Set CTA frontend configuration options to connect to eos - const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf"; - XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); -} - -//------------------------------------------------------------------------------ -// exceptionThrowingMain -//------------------------------------------------------------------------------ -int CtaUpdateStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { - CtaUpdateStorageClassCmdLineArgs cmdLineArgs(argc, argv); - if (cmdLineArgs.m_help) { - cmdLineArgs.printUsage(); - } - - readAndSetConfiguration(getUsername()); - - getStorageClassId(); - - return 0; -} - - -} // namespace admin -} // namespace cta diff --git a/cmdline/update_storage_class/CtaUpdateStorageClass.hpp b/cmdline/update_storage_class/CtaUpdateStorageClass.hpp deleted file mode 100644 index fced2ff14f..0000000000 --- a/cmdline/update_storage_class/CtaUpdateStorageClass.hpp +++ /dev/null @@ -1,140 +0,0 @@ -/* - * @project The CERN Tape Archive (CTA) - * @copyright Copyright © 2021-2022 CERN - * @license This program is free software, distributed under the terms of the GNU General Public - * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can - * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A - * PARTICULAR PURPOSE. See the GNU General Public License for more details. - * - * In applying this licence, CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization or - * submit itself to any jurisdiction. - */ - -#pragma once - -#include "cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp" -#include "cmdline/restore_files/CmdLineTool.hpp" -#include "eos_grpc_client/GrpcEndpoint.hpp" -#include "catalogue/Catalogue.hpp" -#include "common/log/StdoutLogger.hpp" - -#include <optional> -#include <memory> - -#include "CtaFrontendApi.hpp" - -namespace cta { -namespace admin { - -class CtaUpdateStorageClass: public CmdLineTool { -public: - /** - * Constructor. - * - * @param inStream Standard input stream. - * @param outStream Standard output stream. - * @param errStream Standard error stream. - * @param log The object representing the API of the CTA logging system. - * @param argc - * @param argv - */ - CtaUpdateStorageClass( - std::istream &inStream, - std::ostream &outStream, - std::ostream &errStream, - cta::log::StdoutLogger &log, - const int& argc, - char *const *const argv); - - /** - * An exception throwing version of main(). - * - * @param argc The number of command-line arguments including the program name. - * @param argv The command-line arguments. - * @return The exit value of the program. - */ - int exceptionThrowingMain(const int argc, char *const *const argv) override; - - /** - * Sets internal configuration parameters to be used for reading. - * It reads cta frontend parameters from /etc/cta/cta-cli.conf - * - * @param username The name of the user running the command-line tool. - * @param cmdLineArgs The arguments parsed from the command line. - */ - void readAndSetConfiguration(const std::string &userName); - - - - - /** - * Prints the usage message of the command-line tool. - */ - void printUsage(std::ostream &os) override; - -private: - - /** - * The object representing the API of the CTA logging system. - */ - cta::log::StdoutLogger &m_log; - - - /** - * Archive file id of the files to update - */ - std::vector<uint64_t> m_archiveFileIds; - - - /** - * Archive file id of the files to update - */ - std::string m_storageClassName; - - - /** - * Disk instance of the files to update - */ - std::optional<std::string> m_diskInstance; - - /** - * Fids of the files to update - */ - std::optional<std::list<uint64_t>> m_eosFids; - - - /** - *Command line arguments - */ - CtaUpdateStorageClassCmdLineArgs m_cmdLineArgs; - - - /** - * CTA Frontend service provider - */ - std::unique_ptr<XrdSsiPbServiceType> m_serviceProviderPtr; - - std::unique_ptr<::eos::client::EndpointMap> m_endpointMapPtr; - - const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses - const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by - //!< XRD_REQUESTTIMEOUT environment variable. - - const std::string m_ctaVersion = CTA_VERSION; //!< The version of CTA - const std::string m_xrootdSsiProtobufInterfaceVersion = //!< The xrootd-ssi-protobuf-interface version (=tag) - XROOTD_SSI_PROTOBUF_INTERFACE_VERSION; - - /** - * Returns the id of a storage class - */ - uint64_t updateCatalogue() const; - -} ; // class CtaUpdateStorageClass - -} // namespace admin -} // namespace cta diff --git a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp deleted file mode 100644 index 786b94ec40..0000000000 --- a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - * @project The CERN Tape Archive (CTA) - * @copyright Copyright © 2021-2022 CERN - * @license This program is free software, distributed under the terms of the GNU General Public - * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can - * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A - * PARTICULAR PURPOSE. See the GNU General Public License for more details. - * - * In applying this licence, CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization or - * submit itself to any jurisdiction. - */ - -#include "cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp" -#include "common/exception/CommandLineNotParsed.hpp" - -#include <getopt.h> -#include <ostream> -#include <string> -#include <limits.h> -#include <fstream> - -namespace cta { -namespace admin{ - -//------------------------------------------------------------------------------ -// constructor -//------------------------------------------------------------------------------ -CtaUpdateStorageClassCmdLineArgs::CtaUpdateStorageClassCmdLineArgs(const int argc, char *const *const argv): - m_help(false), - m_optionFids("--fid", "-f", true), - m_option_storage_class_name("--storage_class_name", "-scn", true) { - - option_with_multiple_values_map["--fid"] = &m_optionFids; - option_with_multiple_values_map["-f"] = &m_optionFids; - - option_map["--storage_class_name"] = &m_option_storage_class_name; - option_map["-scn"] = &m_option_storage_class_name; - - parseCmd(argc, argv); -} - - -//------------------------------------------------------------------------------ -// printUsage -//------------------------------------------------------------------------------ -void CtaUpdateStorageClassCmdLineArgs::printUsage() { - std::cout << "Usage:" << std::endl << - " cta-update-storage-class [--id/-I <archive_file_id>] [--instance/-i <disk_instance>]" << std::endl << - " [--fxid/-f <eos_fxid>] [--fxidfile/-F <filename>]" << std::endl << - " [--vid/-v <vid>] [--copynb/-c <copy_number>] [--debug/-d]" << std::endl; -} - -//------------------------------------------------------------------------------ -// validateCmd -//------------------------------------------------------------------------------ -void CtaUpdateStorageClassCmdLineArgs::validateCmd(int argc) { - if (argc < 3) { - printUsage(); - throw std::runtime_error(""); - } - for (auto &it : option_with_multiple_values_map) { - auto option = it.second; - if (!option->is_optional() && !option->is_present()) { - std::cout << "Error: Option " << option->get_name() << " is mandatory but not present." << std::endl; - printUsage(); - throw std::runtime_error(""); - } - } -} - -//------------------------------------------------------------------------------ -// isEndOfOption -//------------------------------------------------------------------------------ -bool CtaUpdateStorageClassCmdLineArgs::isEndOfOption(int nextOptionOrValueIndex, const char *const *const argv) { - return ( - option_with_multiple_values_map.find(argv[nextOptionOrValueIndex]) != option_with_multiple_values_map.end() || - option_map.find(argv[nextOptionOrValueIndex]) != option_map.end() - ); -} - -//------------------------------------------------------------------------------ -// parseCmd -//------------------------------------------------------------------------------ -void CtaUpdateStorageClassCmdLineArgs::parseCmd(const int argc, const char *const *const argv) { - unsigned int num_cmd_line_enrties_for_option = 2; - for (int i = 1; i < argc; i += num_cmd_line_enrties_for_option){ - num_cmd_line_enrties_for_option = 2; - auto search_option_with_multiple_values_map = option_with_multiple_values_map.find(argv[i]); - if (search_option_with_multiple_values_map != option_with_multiple_values_map.end()) { - auto stringListOption = search_option_with_multiple_values_map->second; - stringListOption->set_present(); - - if (stringListOption->get_name() == "--fid") { - for (int j = 1; j < argc; j += 1) { - int nextOptionOrValueIndex = i + j + 1; - bool end_of_option = isEndOfOption(nextOptionOrValueIndex, argv); - stringListOption->add_value(argv[i + j]); - if (end_of_option || (j + i) == (argc - 1)) { - num_cmd_line_enrties_for_option = j + 1; - break; - } - } - } - } - - auto search_option = option_map.find(argv[i]); - if (search_option != option_map.end()) { - auto stringOption = search_option->second; - stringOption->set_present(); - if (stringOption->get_name() == "--storage_class_name") { - stringOption->set_value(argv[i + 1]); - } - } - - if (search_option_with_multiple_values_map == option_with_multiple_values_map.end() && search_option == option_map.end()) { - std::cout << "Error: Unknown option: " << argv[i] << std::endl; - printUsage(); - throw std::runtime_error(""); - } - } -} - -} // namespace admin -} // namespace cta diff --git a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp b/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp deleted file mode 100644 index 5cf7968e6b..0000000000 --- a/cmdline/update_storage_class/CtaUpdateStorageClassCmdLineArgs.hpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * @project The CERN Tape Archive (CTA) - * @copyright Copyright © 2021-2022 CERN - * @license This program is free software, distributed under the terms of the GNU General Public - * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can - * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A - * PARTICULAR PURPOSE. See the GNU General Public License for more details. - * - * In applying this licence, CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization or - * submit itself to any jurisdiction. - */ - -#pragma once - -#include <list> -#include <optional> -#include <map> -#include <iostream> -#include <sstream> - -#include "version.h" -#include "cmdline/CtaCmdOptions.hpp" - - -namespace cta { -namespace admin { - -/** - * class to store the command-line arguments of the command-line tool - * named cta-update-storage-class. - */ -class CtaUpdateStorageClassCmdLineArgs { - -public: - /** - * True if the usage message should be printed. - */ - bool m_help; - - /** - * List of archive file ids - */ - StringListOption m_optionFids; - - /** - * The storage name used to replace the names for the files with file id found in m_fids - */ - StringOption m_option_storage_class_name; - - - /** - * Constructor that parses the specified command-line arguments. - * - * @param argc The number of command-line arguments including the name of the - * executable. - * @param argv The vector of command-line arguments. - */ - CtaUpdateStorageClassCmdLineArgs(const int argc, char *const *const argv); - - - /** - * Print the correct usage of the tool - * - */ - void printUsage(); - -private: - void validateCmd(int argc); - bool isEndOfOption(int nextOptionOrValueIndex, const char *const *const argv); - void parseCmd(const int argc, const char *const *const argv); - - std::map<std::string, StringListOption *> option_with_multiple_values_map; - std::map<std::string, StringOption *> option_map; - -}; // class CtaUpdateStorageClassCmdLineArgs - -} // namespace admin -} // namespace cta diff --git a/cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp b/cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp deleted file mode 100644 index f15c119f3c..0000000000 --- a/cmdline/update_storage_class/CtaUpdateStorageClassMain.cpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * @project The CERN Tape Archive (CTA) - * @copyright Copyright © 2021-2022 CERN - * @license This program is free software, distributed under the terms of the GNU General Public - * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can - * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A - * PARTICULAR PURPOSE. See the GNU General Public License for more details. - * - * In applying this licence, CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization or - * submit itself to any jurisdiction. - */ - - -#include <sstream> -#include <iostream> - -#include <XrdSsiPbLog.hpp> -#include <XrdSsiPbIStreamBuffer.hpp> - -#include "cmdline/update_storage_class/CtaUpdateStorageClass.hpp" -#include "common/utils/utils.hpp" -#include "common/log/StdoutLogger.hpp" - -//------------------------------------------------------------------------------ -// main -//------------------------------------------------------------------------------ -int main(const int argc, char *const *const argv) { - char buf[256]; - std::string hostName; - if(gethostname(buf, sizeof(buf))) { - hostName = "UNKNOWN"; - } else { - buf[sizeof(buf) - 1] = '\0'; - hostName = buf; - } - cta::log::StdoutLogger log(hostName, "cta-update-storage-class"); - - cta::admin::CtaUpdateStorageClass cmd(std::cin, std::cout, std::cerr, log, argc, argv); - int ret = cmd.main(argc, argv); - // Delete all global objects allocated by libprotobuf - google::protobuf::ShutdownProtobufLibrary(); - return ret; -} diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp index 8113c8e2b5..c6c7d34c23 100644 --- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp +++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp @@ -323,6 +323,9 @@ void RequestMessage::process(const cta::xrd::Request &request, cta::xrd::Respons case cmd_pair(AdminCmd::CMD_RECYCLETAPEFILE, AdminCmd::SUBCMD_RESTORE): processRecycleTapeFile_Restore(response); break; + case cmd_pair(AdminCmd::CMD_CHANGESTORAGECLASS, AdminCmd::SUBCMD_CH): + processChangeStorageClass(response); + break; default: throw PbException("Admin command pair <" + @@ -2541,4 +2544,13 @@ void RequestMessage::processRecycleTapeFile_Restore(cta::xrd::Response& response response.set_type(cta::xrd::Response::RSP_SUCCESS); } +void RequestMessage::processChangeStorageClass(cta::xrd::Response& response) { + using namespace cta::admin; + + getRequired(OptionString::STORAGE_CLASS_NAME); + getRequired(OptionStrList::FILE_ID); + + response.set_type(cta::xrd::Response::RSP_SUCCESS); +} + }} // namespace cta::xrd diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.hpp b/xroot_plugins/XrdSsiCtaRequestMessage.hpp index 4e180e4c81..334a0b4383 100644 --- a/xroot_plugins/XrdSsiCtaRequestMessage.hpp +++ b/xroot_plugins/XrdSsiCtaRequestMessage.hpp @@ -223,6 +223,7 @@ private: void processVirtualOrganization_Ch(cta::xrd::Response &response); void processVirtualOrganization_Rm(cta::xrd::Response &response); void processRecycleTapeFile_Restore(cta::xrd::Response &response); + void processChangeStorageClass (cta::xrd::Response &response); /*! * Process AdminCmd events which can return a stream response -- GitLab From 9e20034289baa55398f9eb7f58551ee30112e3ee Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 2 Sep 2022 13:34:28 +0200 Subject: [PATCH 022/126] #78 - add new request to frontend --- cmdline/CtaAdminCmdParse.hpp | 4 ++- cmdline/change_storage_class/CMakeLists.txt | 2 +- .../CtaChangeStorageClass.cpp | 26 ++++++++++++++++--- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/cmdline/CtaAdminCmdParse.hpp b/cmdline/CtaAdminCmdParse.hpp index 2a40167f69..a01c6e3743 100644 --- a/cmdline/CtaAdminCmdParse.hpp +++ b/cmdline/CtaAdminCmdParse.hpp @@ -346,6 +346,7 @@ const std::map<std::string, OptionString::Key> strOptions = { { "--diskinstancespace", OptionString::DISK_INSTANCE_SPACE }, { "--verificationstatus", OptionString::VERIFICATION_STATUS }, { "--disabledreason", OptionString::DISABLED_REASON }, + { "--storage_class_name", OptionString::STORAGE_CLASS_NAME } }; @@ -355,7 +356,8 @@ const std::map<std::string, OptionString::Key> strOptions = { */ const std::map<std::string, OptionStrList::Key> strListOptions = { { "--fxidfile", OptionStrList::FILE_ID }, - { "--vidfile", OptionStrList::VID } + { "--vidfile", OptionStrList::VID }, + { "--fid", OptionStrList::FILE_ID } }; diff --git a/cmdline/change_storage_class/CMakeLists.txt b/cmdline/change_storage_class/CMakeLists.txt index c08c2b472b..00dc829982 100644 --- a/cmdline/change_storage_class/CMakeLists.txt +++ b/cmdline/change_storage_class/CMakeLists.txt @@ -27,7 +27,7 @@ include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include) # Compiled protocol buffers include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS}) -add_executable(cta-change-storage-class CtaChangeStorageClass.cpp CtaChangeStorageClassCmdLineArgs.cpp CtaChangeStorageClassMain.cpp ../restore_files/CmdLineTool.cpp CtaCmdOptions.hpp) +add_executable(cta-change-storage-class CtaChangeStorageClass.cpp CtaChangeStorageClassCmdLineArgs.cpp CtaChangeStorageClassMain.cpp ../restore_files/CmdLineTool.cpp ../CtaAdminCmdParse.cpp CtaCmdOptions.hpp) target_link_libraries(cta-change-storage-class ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient) set_property (TARGET cta-change-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) diff --git a/cmdline/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/change_storage_class/CtaChangeStorageClass.cpp index 7742453d7b..5fd20abf89 100644 --- a/cmdline/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/change_storage_class/CtaChangeStorageClass.cpp @@ -20,6 +20,8 @@ #include "common/utils/utils.hpp" #include "common/checksum/ChecksumBlob.hpp" #include "CtaFrontendApi.hpp" +#include <cmdline/CtaAdminCmdParse.hpp> + #include <XrdSsiPbLog.hpp> #include <XrdSsiPbIStreamBuffer.hpp> @@ -128,6 +130,15 @@ uint64_t CtaChangeStorageClass::changeCatalogue() { } } + + // Validate the Protocol Buffer + try { + validateCmd(request.admincmd()); + } catch(std::runtime_error &ex) { + throw std::runtime_error("Error: Protocol Buffer validation"); + } + + // Send the Request to the Service and get a Response cta::xrd::Response response; m_serviceProviderPtr->Send(request, response); @@ -165,7 +176,7 @@ void CtaChangeStorageClass::printUsage(std::ostream &os) { //------------------------------------------------------------------------------ void CtaChangeStorageClass::readAndSetConfiguration( const std::string &userName) { - + // Set CTA frontend configuration options const std::string cli_config_file = "/etc/cta/cta-cli.conf"; XrdSsiPb::Config cliConfig(cli_config_file, "cta"); @@ -208,10 +219,17 @@ int CtaChangeStorageClass::exceptionThrowingMain(const int argc, char *const *co readAndSetConfiguration(getUsername()); - std::cout << getUsername() << std::endl; - std::cout << getHostname() << std::endl; + // ------------ For debugging ------------ // + + std::cout << "Username: " << getUsername() << std::endl; + std::cout << "Hostname: " << getHostname() << std::endl; + + std::cout << "m_storageClassName: " << m_storageClassName << " -- Type: " << typeid(m_storageClassName).name() << std::endl; + for (auto &archiveFileId : m_archiveFileIds) { + std::cout << "archiveFileId: " << archiveFileId << " -- Type: " << typeid(archiveFileId).name() << std::endl; + } - std::cout << typeid(m_storageClassName).name() << std::endl; + // --------------------------------------- // changeCatalogue(); -- GitLab From 05af53cf99a9e790b6df2cb9268f13c344109127 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 2 Sep 2022 13:55:46 +0200 Subject: [PATCH 023/126] #78 - add comment describing where the validation function is used --- cmdline/CtaAdminCmdParse.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cmdline/CtaAdminCmdParse.hpp b/cmdline/CtaAdminCmdParse.hpp index a01c6e3743..bbb10c43ca 100644 --- a/cmdline/CtaAdminCmdParse.hpp +++ b/cmdline/CtaAdminCmdParse.hpp @@ -723,6 +723,7 @@ const std::map<cmd_key_t, cmd_val_t> cmdOptions = { * Validate that all required command line options are present * * Throws a std::runtime_error if the command is invalid + * This function is used on the server side */ void validateCmd(const cta::admin::AdminCmd &admincmd); }} // namespace cta::admin -- GitLab From 023960b533865982bf300b65dad723f7ac0c6f4f Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Mon, 5 Sep 2022 15:26:51 +0200 Subject: [PATCH 024/126] 78 - add sql query setting the storage class name in ARCHIVE_FILE --- catalogue/RdbmsCatalogue.cpp | 34 ++++++++++++++++++++++++++++++++++ catalogue/RdbmsCatalogue.hpp | 5 +++++ 2 files changed, 39 insertions(+) diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index ae4a5f8514..56e34d6578 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -2201,6 +2201,40 @@ bool RdbmsCatalogue::archiveFileIdExists(rdbms::Conn &conn, const uint64_t archi } } +// ----------------------------------------------------------------------------- +// changeStorageClassNameInArchiveFile +// ----------------------------------------------------------------------------- + +void changeClassNameInArchiveFileName(rdms::Conn &conn, std::string storageClassName, const uint64_t archiveFileId) { + try { + const char *const sql = + "UPDATE " + "ARCHIVE_FILE " + "SET " + "ARCHIVE_FILE.STORAGE_CLASS_ID = ( + "SELECT " + "STORAGE_CLASS_ID " + "FROM " + "STORAGE_CLASS " + "WHERE " + "STORAGE_CLASS.STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME" + ) + "WHERE " + "ARCHIVE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID"; + auto stmt = conn.createStmt(sql); + stmt.bindString(":STORAGE_CLASS_NAME", storageClassName); + stmt.bindString(":ARCHIVE_FILE_ID", archiveFileId); + auto rset = stmt.executeQuery(); + } catch(exception::UserError &) { + throw; + } catch(excetpion::Exception &ex) { + ex.getMessage().str(std::string(__FUNCTION__) + ": " ex.getMessage().str()); + throw; + } + +} + + //------------------------------------------------------------------------------ // diskFileIdExists //------------------------------------------------------------------------------ diff --git a/catalogue/RdbmsCatalogue.hpp b/catalogue/RdbmsCatalogue.hpp index 562bb76445..78d1d51322 100644 --- a/catalogue/RdbmsCatalogue.hpp +++ b/catalogue/RdbmsCatalogue.hpp @@ -1279,6 +1279,11 @@ protected: */ bool archiveFileIdExists(rdbms::Conn &conn, const uint64_t archiveFileId) const; + /** + * @param storageClassName The name of the storage class + */ + void changeStorageClassNameInArchiveFile(rdbms::Conn &conn, std::string storageClassName, uint64_t archiveFileId) const; + /** * Returns true if the specified disk file identifier exists. * -- GitLab From edd3870353cb636ad256c3f013e51858a35fcaff Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Mon, 5 Sep 2022 15:26:51 +0200 Subject: [PATCH 025/126] 78 - add sql query setting the storage class name in ARCHIVE_FILE --- catalogue/RdbmsCatalogue.cpp | 34 ++++++++++++++++++++++++++++++++++ catalogue/RdbmsCatalogue.hpp | 5 +++++ 2 files changed, 39 insertions(+) diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index ae4a5f8514..755aef1ab2 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -2201,6 +2201,40 @@ bool RdbmsCatalogue::archiveFileIdExists(rdbms::Conn &conn, const uint64_t archi } } +// ----------------------------------------------------------------------------- +// changeStorageClassNameInArchiveFile +// ----------------------------------------------------------------------------- + +void changeClassNameInArchiveFileName(rdbms::Conn &conn, const std::string storageClassName, const uint64_t archiveFileId) { + try { + const char *const sql = + "UPDATE " + "ARCHIVE_FILE " + "SET " + "ARCHIVE_FILE.STORAGE_CLASS_ID = (" + "SELECT " + "STORAGE_CLASS_ID " + "FROM " + "STORAGE_CLASS " + "WHERE " + "STORAGE_CLASS.STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME" + ")" + "WHERE " + "ARCHIVE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID"; + auto stmt = conn.createStmt(sql); + stmt.bindString(":STORAGE_CLASS_NAME", storageClassName); + stmt.bindUint64(":ARCHIVE_FILE_ID", archiveFileId); + auto rset = stmt.executeQuery(); + } catch(exception::UserError &) { + throw; + } catch(exception::Exception &ex) { + ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str()); + throw; + } + +} + + //------------------------------------------------------------------------------ // diskFileIdExists //------------------------------------------------------------------------------ diff --git a/catalogue/RdbmsCatalogue.hpp b/catalogue/RdbmsCatalogue.hpp index 562bb76445..9ac6e02531 100644 --- a/catalogue/RdbmsCatalogue.hpp +++ b/catalogue/RdbmsCatalogue.hpp @@ -1279,6 +1279,11 @@ protected: */ bool archiveFileIdExists(rdbms::Conn &conn, const uint64_t archiveFileId) const; + /** + * @param storageClassName The name of the storage class + */ + void changeStorageClassNameInArchiveFile(rdbms::Conn &conn, const std::string storageClassName, const uint64_t archiveFileId) const; + /** * Returns true if the specified disk file identifier exists. * -- GitLab From 11ef8ba70fdecb2e42bf50c33be86ec8106fd240 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 6 Sep 2022 11:12:47 +0200 Subject: [PATCH 026/126] 78 - add to catalogue and add usage error handeling --- catalogue/Catalogue.hpp | 6 ++ catalogue/CatalogueRetryWrapper.hpp | 4 ++ catalogue/DummyCatalogue.hpp | 1 + catalogue/RdbmsCatalogue.cpp | 61 ++++++++++--------- catalogue/RdbmsCatalogue.hpp | 3 +- .../CtaChangeStorageClass.cpp | 4 +- .../CtaChangeStorageClassCmdLineArgs.cpp | 8 +-- .../CtaChangeStorageClassCmdLineArgs.hpp | 6 +- xroot_plugins/XrdCtaChangeStorageClass.hpp | 43 +++++++++++++ xroot_plugins/XrdSsiCtaRequestMessage.cpp | 8 ++- 10 files changed, 103 insertions(+), 41 deletions(-) create mode 100644 xroot_plugins/XrdCtaChangeStorageClass.hpp diff --git a/catalogue/Catalogue.hpp b/catalogue/Catalogue.hpp index 2d960dbed7..d1f815ef51 100644 --- a/catalogue/Catalogue.hpp +++ b/catalogue/Catalogue.hpp @@ -1418,6 +1418,12 @@ public: */ virtual void releaseDiskSpace(const std::string& driveName, const uint64_t mountId, const DiskSpaceReservationRequest& diskSpaceReservation, log::LogContext & lc) = 0; + /** + * Changes the name of hte storage class + * @param storageClassName The name of the storage class + * @param archiveFileId Id for file found in ARCHIVE_FILE + */ + virtual void changeStorageClassNameInArchiveFile(const std::string &storageClassName, const uint64_t &archiveFileId) const = 0; }; // class Catalogue diff --git a/catalogue/CatalogueRetryWrapper.hpp b/catalogue/CatalogueRetryWrapper.hpp index 8bd0e67608..ed0067b657 100644 --- a/catalogue/CatalogueRetryWrapper.hpp +++ b/catalogue/CatalogueRetryWrapper.hpp @@ -778,6 +778,10 @@ public: return retryOnLostConnection(m_log, [&]{return m_catalogue->releaseDiskSpace(driveName, mountId, diskSpaceReservation, lc);}, m_maxTriesToConnect); } + void changeStorageClassNameInArchiveFile(const std::string &storageClassName, const uint64_t &archiveFileId) const override { + return retryOnLostConnection(m_log, [&]{return m_catalogue->changeStorageClassNameInArchiveFile(storageClassName, archiveFileId);}, m_maxTriesToConnect); + } + protected: /** diff --git a/catalogue/DummyCatalogue.hpp b/catalogue/DummyCatalogue.hpp index 83417a9828..64a7b7ac63 100644 --- a/catalogue/DummyCatalogue.hpp +++ b/catalogue/DummyCatalogue.hpp @@ -192,6 +192,7 @@ public: void tapeMountedForRetrieve(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } bool tapePoolExists(const std::string& tapePoolName) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } void updateDiskFileId(uint64_t archiveFileId, const std::string &diskInstance, const std::string &diskFileId) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } + void changeStorageClassNameInArchiveFile(const std::string &storageClassName, const uint64_t &archiveFileId) const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } void moveArchiveFileToRecycleLog(const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");} // Special functions for unit tests. diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index 5f094fece3..0e82e07d19 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -1414,6 +1414,37 @@ void RdbmsCatalogue::modifyStorageClassName(const common::dataStructures::Securi } } +// ----------------------------------------------------------------------------- +// changeStorageClassNameInArchiveFile +// ----------------------------------------------------------------------------- +void RdbmsCatalogue::changeStorageClassNameInArchiveFile(const std::string &storageClassName, const uint64_t &archiveFileId) const { + try { + const char *const sql = + "UPDATE " + "ARCHIVE_FILE " + "SET " + "ARCHIVE_FILE.STORAGE_CLASS_ID = (" + "SELECT " + "STORAGE_CLASS_ID " + "FROM " + "STORAGE_CLASS " + "WHERE " + "STORAGE_CLASS.STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME" + ")" + "WHERE " + "ARCHIVE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID"; + auto conn = m_connPool.getConn(); + auto stmt = conn.createStmt(sql); + stmt.bindString(":STORAGE_CLASS_NAME", storageClassName); + stmt.bindUint64(":ARCHIVE_FILE_ID", archiveFileId); + auto rset = stmt.executeQuery(); + } catch(exception::UserError &) { + throw; + } catch(exception::Exception &ex) { + ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str()); + } +} + //------------------------------------------------------------------------------ // createMediaType //------------------------------------------------------------------------------ @@ -2151,6 +2182,7 @@ bool RdbmsCatalogue::tapePoolExists(rdbms::Conn &conn, const std::string &tapePo } } + //------------------------------------------------------------------------------ // tapePoolUsedInAnArchiveRoute //------------------------------------------------------------------------------ @@ -2201,35 +2233,6 @@ bool RdbmsCatalogue::archiveFileIdExists(rdbms::Conn &conn, const uint64_t archi } } -// ----------------------------------------------------------------------------- -// changeStorageClassNameInArchiveFile -// ----------------------------------------------------------------------------- -void changeClassNameInArchiveFileName(rdbms::Conn &conn, const std::string storageClassName, const uint64_t archiveFileId) { - try { - const char *const sql = - "UPDATE " - "ARCHIVE_FILE " - "SET " - "ARCHIVE_FILE.STORAGE_CLASS_ID = (" - "SELECT " - "STORAGE_CLASS_ID " - "FROM " - "STORAGE_CLASS " - "WHERE " - "STORAGE_CLASS.STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME" - ")" - "WHERE " - "ARCHIVE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID"; - auto stmt = conn.createStmt(sql); - stmt.bindString(":STORAGE_CLASS_NAME", storageClassName); - stmt.bindUint64(":ARCHIVE_FILE_ID", archiveFileId); - auto rset = stmt.executeQuery(); - } catch(exception::UserError &) { - throw; - } catch(exception::Exception &ex) { - ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str()); - } -} //------------------------------------------------------------------------------ diff --git a/catalogue/RdbmsCatalogue.hpp b/catalogue/RdbmsCatalogue.hpp index 0081d77fd9..58ef6682af 100644 --- a/catalogue/RdbmsCatalogue.hpp +++ b/catalogue/RdbmsCatalogue.hpp @@ -1281,8 +1281,9 @@ protected: /** * @param storageClassName The name of the storage class + * @param archiveFileId Id for file found in ARCHIVE_FILE */ - void changeStorageClassNameInArchiveFile(rdbms::Conn &conn, const std::string storageClassName, const uint64_t archiveFileId) const; + void changeStorageClassNameInArchiveFile(const std::string &storageClassName, const uint64_t &archiveFileId) const override; /** * Returns true if the specified disk file identifier exists. diff --git a/cmdline/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/change_storage_class/CtaChangeStorageClass.cpp index 5fd20abf89..04592bc9f8 100644 --- a/cmdline/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/change_storage_class/CtaChangeStorageClass.cpp @@ -96,6 +96,8 @@ CtaChangeStorageClass::CtaChangeStorageClass( m_log(log), m_cmdLineArgs(argc, argv) { + if (argc < 2) {throw std::runtime_error(m_cmdLineArgs.getUsageText()); } + m_archiveFileIds = m_cmdLineArgs.m_optionFids.get_value(); m_storageClassName = m_cmdLineArgs.m_option_storage_class_name.get_value(); } @@ -214,7 +216,7 @@ void CtaChangeStorageClass::readAndSetConfiguration( //------------------------------------------------------------------------------ int CtaChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { if (m_cmdLineArgs.m_help) { - m_cmdLineArgs.printUsage(); + throw std::runtime_error(m_cmdLineArgs.getUsageText()); } readAndSetConfiguration(getUsername()); diff --git a/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp b/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp index d4fef3de86..df162686cc 100644 --- a/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp +++ b/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp @@ -48,9 +48,8 @@ CtaChangeStorageClassCmdLineArgs::CtaChangeStorageClassCmdLineArgs(const int arg //------------------------------------------------------------------------------ // printUsage //------------------------------------------------------------------------------ -void CtaChangeStorageClassCmdLineArgs::printUsage() { - std::cout << "Usage:" << std::endl << - " cta-update-storage-class [--fid/-f <archive_file_id>] [--storage_class_name/-scn <storage_class_name>]" << std::endl; +const std::string CtaChangeStorageClassCmdLineArgs::getUsageText() const { + return "Usage: cta-update-storage-class [--fid/-f <archive_file_id>] [--storage_class_name/-scn <storage_class_name>]"; } //------------------------------------------------------------------------------ @@ -99,8 +98,7 @@ void CtaChangeStorageClassCmdLineArgs::parseCmd(const int argc, const char *cons if (search_option_with_multiple_values_map == option_with_multiple_values_map.end() && search_option == option_map.end()) { std::cout << "Error: Unknown option: " << argv[i] << std::endl; - printUsage(); - throw std::runtime_error(""); + throw std::runtime_error(getUsageText()); } } } diff --git a/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp b/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp index 0e7679b576..3b81f4d5ec 100644 --- a/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp +++ b/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp @@ -63,11 +63,11 @@ public: CtaChangeStorageClassCmdLineArgs(const int argc, char *const *const argv); - /** - * Print the correct usage of the tool + /** + * Get the text of correct usage of the tool * */ - void printUsage(); + const std::string getUsageText() const; private: bool isEndOfOption(int nextOptionOrValueIndex, const char *const *const argv); diff --git a/xroot_plugins/XrdCtaChangeStorageClass.hpp b/xroot_plugins/XrdCtaChangeStorageClass.hpp new file mode 100644 index 0000000000..c4568013b7 --- /dev/null +++ b/xroot_plugins/XrdCtaChangeStorageClass.hpp @@ -0,0 +1,43 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#pragma once + +#include <xroot_plugins/XrdCtaStream.hpp> +#include <xroot_plugins/XrdSsiCtaRequestMessage.hpp> + + +namespace cta { namespace xrd { + +class XrdCtaChangeStorageClass { +private: + const cta::catalogue::Catalogue &m_catalogue; +public: + XrdCtaChangeStorageClass(cta::catalogue::Catalogue &catalogue); + void updateCatalogue(std::string storageClassName, const std::vector<std::basic_string<char>> archiveFileIds) const; +}; + +XrdCtaChangeStorageClass::XrdCtaChangeStorageClass(cta::catalogue::Catalogue &catalogue): m_catalogue(catalogue) {} + +void XrdCtaChangeStorageClass::updateCatalogue(const std::string storageClassName, const std::vector<std::basic_string<char>> archiveFileIds) const { + for (auto & id : archiveFileIds) { + const uint64_t archiveFileId = std::stoul(id); + m_catalogue.changeStorageClassNameInArchiveFile(storageClassName, archiveFileId); + } +} +} // namespace xrd +} // namespace cta \ No newline at end of file diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp index c6c7d34c23..187a5aaa6b 100644 --- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp +++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp @@ -44,6 +44,7 @@ using XrdSsiPb::PbException; #include "XrdCtaVersion.hpp" #include "XrdCtaSchedulingInfosLs.hpp" #include "XrdCtaRecycleTapeFileLs.hpp" +#include "XrdCtaChangeStorageClass.hpp" #include <limits> #include <sstream> @@ -2547,8 +2548,11 @@ void RequestMessage::processRecycleTapeFile_Restore(cta::xrd::Response& response void RequestMessage::processChangeStorageClass(cta::xrd::Response& response) { using namespace cta::admin; - getRequired(OptionString::STORAGE_CLASS_NAME); - getRequired(OptionStrList::FILE_ID); + std::string storageClassName = getRequired(OptionString::STORAGE_CLASS_NAME); + auto archiveFileIds = getRequired(OptionStrList::FILE_ID); + + XrdCtaChangeStorageClass xrdCtaChangeStorageClass(m_catalogue); + xrdCtaChangeStorageClass.updateCatalogue(storageClassName, archiveFileIds); response.set_type(cta::xrd::Response::RSP_SUCCESS); } -- GitLab From a94e30743fbc01692205e8e54d0e234f4f13aeaf Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 7 Sep 2022 09:24:12 +0200 Subject: [PATCH 027/126] 78 - add user exceptions --- .../CtaChangeStorageClass.cpp | 26 +++---------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/cmdline/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/change_storage_class/CtaChangeStorageClass.cpp index 04592bc9f8..c1d901d003 100644 --- a/cmdline/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/change_storage_class/CtaChangeStorageClass.cpp @@ -43,16 +43,6 @@ std::list<std::pair<std::string,std::string>> listedTapeFiles; namespace XrdSsiPb { -/*! - * User error exception - */ -class UserException : public std::runtime_error -{ -public: - UserException(const std::string &err_msg) : std::runtime_error(err_msg) {} -}; // class UserException - - /*! * Alert callback. * @@ -71,17 +61,6 @@ void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) namespace cta { namespace admin { -/*! - * ChangeFilesCmdException - */ -class ChangeFilesCmdException : public std::runtime_error -{ -public: - ChangeFilesCmdException(const std::string &err_msg) : std::runtime_error(err_msg) {} -}; // class UserException - - - //------------------------------------------------------------------------------ // constructor //------------------------------------------------------------------------------ @@ -95,11 +74,12 @@ CtaChangeStorageClass::CtaChangeStorageClass( CmdLineTool(inStream, outStream, errStream), m_log(log), m_cmdLineArgs(argc, argv) { - - if (argc < 2) {throw std::runtime_error(m_cmdLineArgs.getUsageText()); } + + if (!m_storageClassName || m_archiveFileIds.size() == 0) {throw std::runtime_error(m_cmdLineArgs.getUsageText()); } m_archiveFileIds = m_cmdLineArgs.m_optionFids.get_value(); m_storageClassName = m_cmdLineArgs.m_option_storage_class_name.get_value(); + } -- GitLab From 95b7516729f61a013ce74802d30329ddf2b335e3 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 7 Sep 2022 09:56:19 +0200 Subject: [PATCH 028/126] 78 - add logging for frontend --- .../CtaChangeStorageClass.cpp | 4 ++-- .../CtaChangeStorageClassCmdLineArgs.cpp | 1 + xroot_plugins/XrdCtaChangeStorageClass.hpp | 15 +++++++++++---- xroot_plugins/XrdSsiCtaRequestMessage.cpp | 2 +- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/cmdline/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/change_storage_class/CtaChangeStorageClass.cpp index c1d901d003..f635dbb583 100644 --- a/cmdline/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/change_storage_class/CtaChangeStorageClass.cpp @@ -75,7 +75,7 @@ CtaChangeStorageClass::CtaChangeStorageClass( m_log(log), m_cmdLineArgs(argc, argv) { - if (!m_storageClassName || m_archiveFileIds.size() == 0) {throw std::runtime_error(m_cmdLineArgs.getUsageText()); } + if (m_storageClassName != "" || m_archiveFileIds.size() == 0) {throw std::runtime_error(m_cmdLineArgs.getUsageText()); } m_archiveFileIds = m_cmdLineArgs.m_optionFids.get_value(); m_storageClassName = m_cmdLineArgs.m_option_storage_class_name.get_value(); @@ -135,7 +135,7 @@ uint64_t CtaChangeStorageClass::changeCatalogue() { std::cout << response.message_txt(); break; case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); - case Response::RSP_ERR_USER: throw XrdSsiPb::UserException(response.message_txt()); + case Response::RSP_ERR_USER: throw std::runtime_error(response.message_txt()); case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); default: throw XrdSsiPb::PbException("Invalid response type."); } diff --git a/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp b/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp index df162686cc..8a86a0fa66 100644 --- a/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp +++ b/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp @@ -66,6 +66,7 @@ bool CtaChangeStorageClassCmdLineArgs::isEndOfOption(int nextOptionOrValueIndex, // parseCmd //------------------------------------------------------------------------------ void CtaChangeStorageClassCmdLineArgs::parseCmd(const int argc, const char *const *const argv) { + if (argc < 4) { throw std::runtime_error(getUsageText()); } unsigned int num_cmd_line_enrties_for_option = 2; for (int i = 1; i < argc; i += num_cmd_line_enrties_for_option){ num_cmd_line_enrties_for_option = 2; diff --git a/xroot_plugins/XrdCtaChangeStorageClass.hpp b/xroot_plugins/XrdCtaChangeStorageClass.hpp index c4568013b7..8cb03c6e3e 100644 --- a/xroot_plugins/XrdCtaChangeStorageClass.hpp +++ b/xroot_plugins/XrdCtaChangeStorageClass.hpp @@ -26,16 +26,23 @@ namespace cta { namespace xrd { class XrdCtaChangeStorageClass { private: const cta::catalogue::Catalogue &m_catalogue; + cta::log::LogContext &m_lc; public: - XrdCtaChangeStorageClass(cta::catalogue::Catalogue &catalogue); - void updateCatalogue(std::string storageClassName, const std::vector<std::basic_string<char>> archiveFileIds) const; + XrdCtaChangeStorageClass(cta::catalogue::Catalogue &catalogue, cta::log::LogContext &lc); + void updateCatalogue(std::string storageClassName, const std::vector<std::basic_string<char>> archiveFileIds); }; -XrdCtaChangeStorageClass::XrdCtaChangeStorageClass(cta::catalogue::Catalogue &catalogue): m_catalogue(catalogue) {} +XrdCtaChangeStorageClass::XrdCtaChangeStorageClass(cta::catalogue::Catalogue &catalogue, cta::log::LogContext &lc): m_catalogue(catalogue), m_lc(lc) {} -void XrdCtaChangeStorageClass::updateCatalogue(const std::string storageClassName, const std::vector<std::basic_string<char>> archiveFileIds) const { +void XrdCtaChangeStorageClass::updateCatalogue(const std::string storageClassName, const std::vector<std::basic_string<char>> archiveFileIds) { for (auto & id : archiveFileIds) { const uint64_t archiveFileId = std::stoul(id); + + m_lc.log(cta::log::INFO, std::to_string(archiveFileId)); + m_lc.log(cta::log::INFO, typeid(archiveFileId).name()); + m_lc.log(cta::log::INFO, storageClassName); + m_lc.log(cta::log::INFO, typeid(storageClassName).name()); + m_catalogue.changeStorageClassNameInArchiveFile(storageClassName, archiveFileId); } } diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp index 187a5aaa6b..b83ec53c0b 100644 --- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp +++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp @@ -2551,7 +2551,7 @@ void RequestMessage::processChangeStorageClass(cta::xrd::Response& response) { std::string storageClassName = getRequired(OptionString::STORAGE_CLASS_NAME); auto archiveFileIds = getRequired(OptionStrList::FILE_ID); - XrdCtaChangeStorageClass xrdCtaChangeStorageClass(m_catalogue); + XrdCtaChangeStorageClass xrdCtaChangeStorageClass(m_catalogue, m_lc); xrdCtaChangeStorageClass.updateCatalogue(storageClassName, archiveFileIds); response.set_type(cta::xrd::Response::RSP_SUCCESS); -- GitLab From fbee762ba3be2afa4b7bf5f93af62f4b48c85405 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 7 Sep 2022 12:06:24 +0200 Subject: [PATCH 029/126] 78 - change sql query --- catalogue/RdbmsCatalogue.cpp | 20 +++++++------------ .../CtaChangeStorageClass.cpp | 2 +- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index 0e82e07d19..a888967602 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -1420,19 +1420,13 @@ void RdbmsCatalogue::modifyStorageClassName(const common::dataStructures::Securi void RdbmsCatalogue::changeStorageClassNameInArchiveFile(const std::string &storageClassName, const uint64_t &archiveFileId) const { try { const char *const sql = - "UPDATE " - "ARCHIVE_FILE " - "SET " - "ARCHIVE_FILE.STORAGE_CLASS_ID = (" - "SELECT " - "STORAGE_CLASS_ID " - "FROM " - "STORAGE_CLASS " - "WHERE " - "STORAGE_CLASS.STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME" - ")" - "WHERE " - "ARCHIVE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID"; + "UPDATE ARCHIVE_FILE " + "SET ARCHIVE_FILE.STORAGE_CLASS_ID = STORAGE_CLASS.STORAGE_CLASS_ID " + "JOIN STORAGE_CLASS ON" + "STORAGE_CLASS.STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME " + "WHERE " + "ARCHIVE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID"; + auto conn = m_connPool.getConn(); auto stmt = conn.createStmt(sql); stmt.bindString(":STORAGE_CLASS_NAME", storageClassName); diff --git a/cmdline/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/change_storage_class/CtaChangeStorageClass.cpp index f635dbb583..11bc076e58 100644 --- a/cmdline/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/change_storage_class/CtaChangeStorageClass.cpp @@ -75,11 +75,11 @@ CtaChangeStorageClass::CtaChangeStorageClass( m_log(log), m_cmdLineArgs(argc, argv) { - if (m_storageClassName != "" || m_archiveFileIds.size() == 0) {throw std::runtime_error(m_cmdLineArgs.getUsageText()); } m_archiveFileIds = m_cmdLineArgs.m_optionFids.get_value(); m_storageClassName = m_cmdLineArgs.m_option_storage_class_name.get_value(); + if (m_storageClassName == "" || m_archiveFileIds.size() == 0) {throw std::runtime_error(m_cmdLineArgs.getUsageText()); } } -- GitLab From 182090dd0a4070a8d7021308662fa94ed82f9423 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 13 Sep 2022 09:11:24 +0200 Subject: [PATCH 030/126] 78 - change name of query function to modifyArchiveFileStorageClassId --- catalogue/Catalogue.hpp | 2 +- catalogue/CatalogueRetryWrapper.hpp | 4 ++-- catalogue/DummyCatalogue.hpp | 2 +- catalogue/RdbmsCatalogue.cpp | 4 ++-- catalogue/RdbmsCatalogue.hpp | 2 +- xroot_plugins/XrdCtaChangeStorageClass.hpp | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/catalogue/Catalogue.hpp b/catalogue/Catalogue.hpp index d1f815ef51..5187cca4b5 100644 --- a/catalogue/Catalogue.hpp +++ b/catalogue/Catalogue.hpp @@ -1423,7 +1423,7 @@ public: * @param storageClassName The name of the storage class * @param archiveFileId Id for file found in ARCHIVE_FILE */ - virtual void changeStorageClassNameInArchiveFile(const std::string &storageClassName, const uint64_t &archiveFileId) const = 0; + virtual void modifyArchiveFileStorageClassId(const std::string &storageClassName, const uint64_t &archiveFileId) const = 0; }; // class Catalogue diff --git a/catalogue/CatalogueRetryWrapper.hpp b/catalogue/CatalogueRetryWrapper.hpp index ed0067b657..85e141951e 100644 --- a/catalogue/CatalogueRetryWrapper.hpp +++ b/catalogue/CatalogueRetryWrapper.hpp @@ -778,8 +778,8 @@ public: return retryOnLostConnection(m_log, [&]{return m_catalogue->releaseDiskSpace(driveName, mountId, diskSpaceReservation, lc);}, m_maxTriesToConnect); } - void changeStorageClassNameInArchiveFile(const std::string &storageClassName, const uint64_t &archiveFileId) const override { - return retryOnLostConnection(m_log, [&]{return m_catalogue->changeStorageClassNameInArchiveFile(storageClassName, archiveFileId);}, m_maxTriesToConnect); + void modifyArchiveFileStorageClassId(const std::string &storageClassName, const uint64_t &archiveFileId) const override { + return retryOnLostConnection(m_log, [&]{return m_catalogue->modifyArchiveFileStorageClassId(storageClassName, archiveFileId);}, m_maxTriesToConnect); } protected: diff --git a/catalogue/DummyCatalogue.hpp b/catalogue/DummyCatalogue.hpp index 64a7b7ac63..c052e9eb33 100644 --- a/catalogue/DummyCatalogue.hpp +++ b/catalogue/DummyCatalogue.hpp @@ -192,7 +192,7 @@ public: void tapeMountedForRetrieve(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } bool tapePoolExists(const std::string& tapePoolName) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } void updateDiskFileId(uint64_t archiveFileId, const std::string &diskInstance, const std::string &diskFileId) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } - void changeStorageClassNameInArchiveFile(const std::string &storageClassName, const uint64_t &archiveFileId) const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } + void modifyArchiveFileStorageClassId(const std::string &storageClassName, const uint64_t &archiveFileId) const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } void moveArchiveFileToRecycleLog(const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");} // Special functions for unit tests. diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index a888967602..5f55249d6a 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -1415,9 +1415,9 @@ void RdbmsCatalogue::modifyStorageClassName(const common::dataStructures::Securi } // ----------------------------------------------------------------------------- -// changeStorageClassNameInArchiveFile +// modifyArchiveFileStorageClassId // ----------------------------------------------------------------------------- -void RdbmsCatalogue::changeStorageClassNameInArchiveFile(const std::string &storageClassName, const uint64_t &archiveFileId) const { +void RdbmsCatalogue::modifyArchiveFileStorageClassId(const std::string &storageClassName, const uint64_t &archiveFileId) const { try { const char *const sql = "UPDATE ARCHIVE_FILE " diff --git a/catalogue/RdbmsCatalogue.hpp b/catalogue/RdbmsCatalogue.hpp index 58ef6682af..8b121c153a 100644 --- a/catalogue/RdbmsCatalogue.hpp +++ b/catalogue/RdbmsCatalogue.hpp @@ -1283,7 +1283,7 @@ protected: * @param storageClassName The name of the storage class * @param archiveFileId Id for file found in ARCHIVE_FILE */ - void changeStorageClassNameInArchiveFile(const std::string &storageClassName, const uint64_t &archiveFileId) const override; + void modifyArchiveFileStorageClassId(const std::string &storageClassName, const uint64_t &archiveFileId) const override; /** * Returns true if the specified disk file identifier exists. diff --git a/xroot_plugins/XrdCtaChangeStorageClass.hpp b/xroot_plugins/XrdCtaChangeStorageClass.hpp index 8cb03c6e3e..244ed94e24 100644 --- a/xroot_plugins/XrdCtaChangeStorageClass.hpp +++ b/xroot_plugins/XrdCtaChangeStorageClass.hpp @@ -43,7 +43,7 @@ void XrdCtaChangeStorageClass::updateCatalogue(const std::string storageClassNam m_lc.log(cta::log::INFO, storageClassName); m_lc.log(cta::log::INFO, typeid(storageClassName).name()); - m_catalogue.changeStorageClassNameInArchiveFile(storageClassName, archiveFileId); + m_catalogue.modifyArchiveFileStorageClassId(storageClassName, archiveFileId); } } } // namespace xrd -- GitLab From e907f19127e9f530b1f0271db938d8f5df3d44b9 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 13 Sep 2022 11:10:57 +0200 Subject: [PATCH 031/126] 78 - change variable names and add code style consistency --- catalogue/Catalogue.hpp | 4 ++-- catalogue/CatalogueRetryWrapper.hpp | 4 ++-- catalogue/DummyCatalogue.hpp | 2 +- catalogue/RdbmsCatalogue.cpp | 4 ++-- catalogue/RdbmsCatalogue.hpp | 4 ++-- xroot_plugins/XrdCtaChangeStorageClass.hpp | 10 +++++----- xroot_plugins/XrdSsiCtaRequestMessage.cpp | 4 ++-- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/catalogue/Catalogue.hpp b/catalogue/Catalogue.hpp index 5187cca4b5..3751ed4332 100644 --- a/catalogue/Catalogue.hpp +++ b/catalogue/Catalogue.hpp @@ -1420,10 +1420,10 @@ public: /** * Changes the name of hte storage class - * @param storageClassName The name of the storage class * @param archiveFileId Id for file found in ARCHIVE_FILE + * @param newStorageClassName The name of the storage class */ - virtual void modifyArchiveFileStorageClassId(const std::string &storageClassName, const uint64_t &archiveFileId) const = 0; + virtual void modifyArchiveFileStorageClassId(const uint64_t& archiveFileId, const std::string& newStorageClassName) const = 0; }; // class Catalogue diff --git a/catalogue/CatalogueRetryWrapper.hpp b/catalogue/CatalogueRetryWrapper.hpp index 85e141951e..8e619efd66 100644 --- a/catalogue/CatalogueRetryWrapper.hpp +++ b/catalogue/CatalogueRetryWrapper.hpp @@ -778,8 +778,8 @@ public: return retryOnLostConnection(m_log, [&]{return m_catalogue->releaseDiskSpace(driveName, mountId, diskSpaceReservation, lc);}, m_maxTriesToConnect); } - void modifyArchiveFileStorageClassId(const std::string &storageClassName, const uint64_t &archiveFileId) const override { - return retryOnLostConnection(m_log, [&]{return m_catalogue->modifyArchiveFileStorageClassId(storageClassName, archiveFileId);}, m_maxTriesToConnect); + void modifyArchiveFileStorageClassId(const uint64_t& archiveFileId, const std::string& newStorageClassName) const override { + return retryOnLostConnection(m_log, [&]{return m_catalogue->modifyArchiveFileStorageClassId(archiveFileId, newStorageClassName);}, m_maxTriesToConnect); } protected: diff --git a/catalogue/DummyCatalogue.hpp b/catalogue/DummyCatalogue.hpp index c052e9eb33..dc611d1734 100644 --- a/catalogue/DummyCatalogue.hpp +++ b/catalogue/DummyCatalogue.hpp @@ -192,7 +192,7 @@ public: void tapeMountedForRetrieve(const std::string& vid, const std::string& drive) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } bool tapePoolExists(const std::string& tapePoolName) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } void updateDiskFileId(uint64_t archiveFileId, const std::string &diskInstance, const std::string &diskFileId) override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } - void modifyArchiveFileStorageClassId(const std::string &storageClassName, const uint64_t &archiveFileId) const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } + void modifyArchiveFileStorageClassId(const uint64_t &archiveFileId, const std::string &newStorageClassName) const override { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } void moveArchiveFileToRecycleLog(const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");} // Special functions for unit tests. diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index 5f55249d6a..d8ca036dde 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -1417,7 +1417,7 @@ void RdbmsCatalogue::modifyStorageClassName(const common::dataStructures::Securi // ----------------------------------------------------------------------------- // modifyArchiveFileStorageClassId // ----------------------------------------------------------------------------- -void RdbmsCatalogue::modifyArchiveFileStorageClassId(const std::string &storageClassName, const uint64_t &archiveFileId) const { +void RdbmsCatalogue::modifyArchiveFileStorageClassId(const uint64_t& archiveFileId, const std::string& newStorageClassName) const { try { const char *const sql = "UPDATE ARCHIVE_FILE " @@ -1429,7 +1429,7 @@ void RdbmsCatalogue::modifyArchiveFileStorageClassId(const std::string &storageC auto conn = m_connPool.getConn(); auto stmt = conn.createStmt(sql); - stmt.bindString(":STORAGE_CLASS_NAME", storageClassName); + stmt.bindString(":STORAGE_CLASS_NAME", newStorageClassName); stmt.bindUint64(":ARCHIVE_FILE_ID", archiveFileId); auto rset = stmt.executeQuery(); } catch(exception::UserError &) { diff --git a/catalogue/RdbmsCatalogue.hpp b/catalogue/RdbmsCatalogue.hpp index 8b121c153a..8458b0e92c 100644 --- a/catalogue/RdbmsCatalogue.hpp +++ b/catalogue/RdbmsCatalogue.hpp @@ -1280,10 +1280,10 @@ protected: bool archiveFileIdExists(rdbms::Conn &conn, const uint64_t archiveFileId) const; /** - * @param storageClassName The name of the storage class + * @param newStorageClassName The name of the storage class * @param archiveFileId Id for file found in ARCHIVE_FILE */ - void modifyArchiveFileStorageClassId(const std::string &storageClassName, const uint64_t &archiveFileId) const override; + void modifyArchiveFileStorageClassId(const uint64_t& archiveFileId, const std::string& newStorageClassName) const override; /** * Returns true if the specified disk file identifier exists. diff --git a/xroot_plugins/XrdCtaChangeStorageClass.hpp b/xroot_plugins/XrdCtaChangeStorageClass.hpp index 244ed94e24..887d9b6c59 100644 --- a/xroot_plugins/XrdCtaChangeStorageClass.hpp +++ b/xroot_plugins/XrdCtaChangeStorageClass.hpp @@ -29,21 +29,21 @@ private: cta::log::LogContext &m_lc; public: XrdCtaChangeStorageClass(cta::catalogue::Catalogue &catalogue, cta::log::LogContext &lc); - void updateCatalogue(std::string storageClassName, const std::vector<std::basic_string<char>> archiveFileIds); + void updateCatalogue(const std::vector<std::basic_string<char>> archiveFileIds, std::string newStorageClassName); }; XrdCtaChangeStorageClass::XrdCtaChangeStorageClass(cta::catalogue::Catalogue &catalogue, cta::log::LogContext &lc): m_catalogue(catalogue), m_lc(lc) {} -void XrdCtaChangeStorageClass::updateCatalogue(const std::string storageClassName, const std::vector<std::basic_string<char>> archiveFileIds) { +void XrdCtaChangeStorageClass::updateCatalogue(const std::vector<std::basic_string<char>> archiveFileIds, const std::string newStorageClassName) { for (auto & id : archiveFileIds) { const uint64_t archiveFileId = std::stoul(id); m_lc.log(cta::log::INFO, std::to_string(archiveFileId)); m_lc.log(cta::log::INFO, typeid(archiveFileId).name()); - m_lc.log(cta::log::INFO, storageClassName); - m_lc.log(cta::log::INFO, typeid(storageClassName).name()); + m_lc.log(cta::log::INFO, newStorageClassName); + m_lc.log(cta::log::INFO, typeid(newStorageClassName).name()); - m_catalogue.modifyArchiveFileStorageClassId(storageClassName, archiveFileId); + m_catalogue.modifyArchiveFileStorageClassId(archiveFileId, newStorageClassName); } } } // namespace xrd diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp index b83ec53c0b..2fc5b0cab0 100644 --- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp +++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp @@ -2548,11 +2548,11 @@ void RequestMessage::processRecycleTapeFile_Restore(cta::xrd::Response& response void RequestMessage::processChangeStorageClass(cta::xrd::Response& response) { using namespace cta::admin; - std::string storageClassName = getRequired(OptionString::STORAGE_CLASS_NAME); + std::string newStorageClassName = getRequired(OptionString::STORAGE_CLASS_NAME); auto archiveFileIds = getRequired(OptionStrList::FILE_ID); XrdCtaChangeStorageClass xrdCtaChangeStorageClass(m_catalogue, m_lc); - xrdCtaChangeStorageClass.updateCatalogue(storageClassName, archiveFileIds); + xrdCtaChangeStorageClass.updateCatalogue(archiveFileIds, newStorageClassName); response.set_type(cta::xrd::Response::RSP_SUCCESS); } -- GitLab From 7aeba0f3c1a7c3bedc50e721e97c5c473ed603f4 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 13 Sep 2022 11:37:45 +0200 Subject: [PATCH 032/126] 78 - modify query and add query error handeling --- catalogue/RdbmsCatalogue.cpp | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index d8ca036dde..23f125b945 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -1420,10 +1420,10 @@ void RdbmsCatalogue::modifyStorageClassName(const common::dataStructures::Securi void RdbmsCatalogue::modifyArchiveFileStorageClassId(const uint64_t& archiveFileId, const std::string& newStorageClassName) const { try { const char *const sql = - "UPDATE ARCHIVE_FILE " - "SET ARCHIVE_FILE.STORAGE_CLASS_ID = STORAGE_CLASS.STORAGE_CLASS_ID " - "JOIN STORAGE_CLASS ON" - "STORAGE_CLASS.STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME " + "UPDATE ARCHIVE_FILE " + "SET ARCHIVE_FILE.STORAGE_CLASS_ID = (" + "SELECT STORAGE_CLASS_ID FROM STORAGE_CLASS WHERE STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME " + ") " "WHERE " "ARCHIVE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID"; @@ -1432,6 +1432,16 @@ void RdbmsCatalogue::modifyArchiveFileStorageClassId(const uint64_t& archiveFile stmt.bindString(":STORAGE_CLASS_NAME", newStorageClassName); stmt.bindUint64(":ARCHIVE_FILE_ID", archiveFileId); auto rset = stmt.executeQuery(); + + if(!storageClassExists(conn, newStorageClassName)) { + exception::UserError ue; + ue.getMessage() << "Cannot modify archive file " << ": " << archiveFileId << " because storage class " + << ":" << newStorageClassName << " does not exist"; + throw ue; + } + if(0 == stmt.getNbAffectedRows()) { + throw exception::UserError(std::string("Cannot archive file ") + archiveFileId + " because it does not exist"); + } } catch(exception::UserError &) { throw; } catch(exception::Exception &ex) { @@ -2176,7 +2186,6 @@ bool RdbmsCatalogue::tapePoolExists(rdbms::Conn &conn, const std::string &tapePo } } - //------------------------------------------------------------------------------ // tapePoolUsedInAnArchiveRoute //------------------------------------------------------------------------------ -- GitLab From cb4160daee9f413e4b7fef3c0eddacb6ab762ab3 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 13 Sep 2022 13:37:36 +0200 Subject: [PATCH 033/126] 78 - add error query error handeling --- catalogue/RdbmsCatalogue.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index 23f125b945..54a8d469c9 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -1440,7 +1440,9 @@ void RdbmsCatalogue::modifyArchiveFileStorageClassId(const uint64_t& archiveFile throw ue; } if(0 == stmt.getNbAffectedRows()) { - throw exception::UserError(std::string("Cannot archive file ") + archiveFileId + " because it does not exist"); + exception::UserError ue; + ue.getMessage() << "Cannot archive file " << archiveFileId << " because it does not exist"; + throw ue; } } catch(exception::UserError &) { throw; -- GitLab From 535a71de5b75f47da347584f31568f3d6bba3237 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Mon, 3 Oct 2022 16:03:57 +0200 Subject: [PATCH 034/126] Merge branch 'main' into 78-create-a-tool-to-change-the-storage-class-of-data-which-has-already-been-written-to-tape --- .gitlab/ci/analysis.gitlab-ci.yml | 2 +- .gitlab/ci/kube-tests.gitlab-ci.yml | 2 +- ReleaseNotes.md | 36 +- catalogue/12.0/oracle_catalogue_schema.sql | 530 ++++++ catalogue/12.0/postgres_catalogue_schema.sql | 515 ++++++ catalogue/12.0/sqlite_catalogue_schema.sql | 481 ++++++ catalogue/AllCatalogueSchema.hpp | 1532 +++++++++++++++++ catalogue/Catalogue.hpp | 47 +- catalogue/DummyCatalogue.hpp | 11 +- catalogue/RdbmsCatalogue.cpp | 85 +- catalogue/RdbmsCatalogue.hpp | 3 + catalogue/TapeDrivesCatalogueState.cpp | 12 - catalogue/TapeDrivesCatalogueState.hpp | 1 - catalogue/common_catalogue_schema.sql | 33 +- .../liquibase/oracle/11.0to12.0.sql | 84 + .../liquibase/postgres/10.0to11.0.sql | 42 + .../liquibase/postgres/11.0to12.0.sql | 84 + cmake/CTAVersions.cmake | 2 +- cmdline/CtaAdminCmd.cpp | 2 - cmdline/CtaAdminCmdParse.hpp | 4 - cmdline/CtaAdminTextFormatter.cpp | 45 - cmdline/CtaAdminTextFormatter.hpp | 2 - cmdline/cta-admin.1cta | 3 - common/Configuration.hpp | 4 +- common/dataStructures/Tape.cpp | 2 +- common/dataStructures/TapeDrive.cpp | 2 +- .../Dockerfile | 2 +- .../cc7/etc/yum/pluginconf.d/versionlock.list | 34 +- .../orchestration/tests/update_db_test.sh | 2 + cta.spec.in | 7 + frontend-grpc/FrontendGRpcSvc.cpp | 2 +- rdbms/Login.cpp | 2 +- scheduler/CMakeLists.txt | 1 - scheduler/GenericSchedulerTest.cpp | 249 --- scheduler/OStoreDB/OStoreDB.cpp | 2 +- scheduler/Scheduler.cpp | 57 +- scheduler/Scheduler.hpp | 8 - scheduler/SchedulerTest.cpp | 249 --- scheduler/SchedulingInfos.cpp | 49 - scheduler/SchedulingInfos.hpp | 73 - xroot_plugins/XrdSsiCtaRequestMessage.cpp | 13 - xroot_plugins/XrdSsiCtaRequestMessage.hpp | 1 - 42 files changed, 3463 insertions(+), 854 deletions(-) create mode 100644 catalogue/12.0/oracle_catalogue_schema.sql create mode 100644 catalogue/12.0/postgres_catalogue_schema.sql create mode 100644 catalogue/12.0/sqlite_catalogue_schema.sql create mode 100644 catalogue/migrations/liquibase/oracle/11.0to12.0.sql create mode 100644 catalogue/migrations/liquibase/postgres/10.0to11.0.sql create mode 100644 catalogue/migrations/liquibase/postgres/11.0to12.0.sql delete mode 100644 scheduler/SchedulingInfos.cpp delete mode 100644 scheduler/SchedulingInfos.hpp diff --git a/.gitlab/ci/analysis.gitlab-ci.yml b/.gitlab/ci/analysis.gitlab-ci.yml index aceb88d9c4..626ee47599 100644 --- a/.gitlab/ci/analysis.gitlab-ci.yml +++ b/.gitlab/ci/analysis.gitlab-ci.yml @@ -23,7 +23,7 @@ cppcheck_report: - yum -y install python3-pip - python3 -m pip install -U cppcheck_codequality - cppcheck-codequality --input-file=cppcheck_out.xml --output-file=cppcheck.json - - if cat cppcheck_out.xml | grep -q "\"error\""; then echo "ERRORS have been detected"; exit 1; fi; + - if cat cppcheck_out.xml | grep -q "<error\s"; then echo "ERRORS have been detected"; exit 1; fi; artifacts: reports: codequality: cppcheck.json diff --git a/.gitlab/ci/kube-tests.gitlab-ci.yml b/.gitlab/ci/kube-tests.gitlab-ci.yml index 27d17cf018..cd4220c281 100644 --- a/.gitlab/ci/kube-tests.gitlab-ci.yml +++ b/.gitlab/ci/kube-tests.gitlab-ci.yml @@ -78,7 +78,7 @@ dbunittests-oracle: - tags variables: TEST_SCRIPT: "/usr/bin/true" - EXTENDED_OPTIONS: "-O -D -t 600 -C" + EXTENDED_OPTIONS: "-O -D -U -t 600 -C" liquibase-update: stage: test diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 7dfea26b67..a128f03747 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -1,17 +1,46 @@ # v4.NEXT +## Summary +### Bug fixes +- cta/CTA#160 - Improve DB access to get all the Tape Drive States +- cta/CTA#164 - Cppcheck job is not failing when there are errors +- cta/CTA#165 - Fix oracle dbunittests + +# v4.7.11-1 + ## Summary ### Features - cta/CTA#89 - Create stubs for Enstore tape label format -### Bug fixes -- cta/CTA#127 - DataTransferSession keeps busy while waiting for a mount to be required +- cta/CTA#126 - Remove cta-admin schedulinginfo subcommand ### Continuous Integration - cta/CTA#7 - Use same versionlock.list file for xrootd4 and 5 - cta/CTA#18 - CI - Testing of DB schema upgrade script - cta/CTA#49 - Clean up orchestration test scripts -- cta/CTA#101 - CI should cancel running jobs for a branch after a force push +- cta/CTA#101 - CI should cancel running jobs for a branch after a force pushi +- cta/CTA#130 - Do not fail liquibase-update test when CTA is updated without oracle support +- cta/CTA#112 - Add postgres 10.0 to 11.0 liquibase Schema migration script +- cta/CTA#132 - Support postgres migrations ### Building and Packaging - cta/CTA#92 - Refactor CTA code so that it can be build without Oracle dependencies +- cta/CTA#63 - CTA Frontend protobuf changes to support dCache +### Bug fixes +- cta/CTA#30 - Check size of comments before commit them to the Catalogue +- cta/CTA#142 - Save logs in liquibase-update test +- cta/CTA#127 - DataTransferSession keeps busy while waiting for a mount to be required +- cta/CTA#130 - Liquibase-update test fails when CTA is update without oracle support +- cta/CTA#127 - DataTransferSession keeps busy while waiting for a mount to be required +- cta/CTA#125 - Problem with reading OSM tapes using position by block id + +# v4.7.10-1 + +## Summary +- This is a schema upgrade release for catalogue version 12.0. +- For details on how to do the upgrade check: https://eoscta.docs.cern.ch/catalogue/upgrading_the_schema/ +### Catalogue Schema +- cta/CTA#135 - Added EXPORTED and EXPORTED_PENDING states to catalogue +- cta/CTA#131 - Fix indexes on columns in CTA Catalogue which use LOWER() +- cta/CTA#140 - Added REPACKING_DISABLED state to catalogue +- cta/CTA#137 - Added a new column ENCRYPTION_KEY_NAME in the table TAPE_POOL # v4.7.9-2 @@ -24,7 +53,6 @@ See all details and fix in cta/CTA#119 ### Bug fixes - cta/CTA#119 [new repo] - Packaging issue with CTA 4.7.8-1/4.7.9-1 -- #125 - Problem with reading OSM tapes using position by block id # v4.7.9-1 diff --git a/catalogue/12.0/oracle_catalogue_schema.sql b/catalogue/12.0/oracle_catalogue_schema.sql new file mode 100644 index 0000000000..e835a40c13 --- /dev/null +++ b/catalogue/12.0/oracle_catalogue_schema.sql @@ -0,0 +1,530 @@ +CREATE SEQUENCE ARCHIVE_FILE_ID_SEQ + INCREMENT BY 1 + START WITH 4294967296 + NOMAXVALUE + MINVALUE 1 + NOCYCLE + CACHE 20 + NOORDER; +CREATE SEQUENCE LOGICAL_LIBRARY_ID_SEQ + INCREMENT BY 1 + START WITH 1 + NOMAXVALUE + MINVALUE 1 + NOCYCLE + CACHE 20 + NOORDER; +CREATE SEQUENCE MEDIA_TYPE_ID_SEQ + INCREMENT BY 1 + START WITH 1 + NOMAXVALUE + MINVALUE 1 + NOCYCLE + CACHE 20 + NOORDER; +CREATE SEQUENCE STORAGE_CLASS_ID_SEQ + INCREMENT BY 1 + START WITH 10000 + NOMAXVALUE + MINVALUE 1 + NOCYCLE + CACHE 20 + NOORDER; +CREATE SEQUENCE TAPE_POOL_ID_SEQ + INCREMENT BY 1 + START WITH 1 + NOMAXVALUE + MINVALUE 1 + NOCYCLE + CACHE 20 + NOORDER; +CREATE SEQUENCE VIRTUAL_ORGANIZATION_ID_SEQ + INCREMENT BY 1 + START WITH 1 + NOMAXVALUE + MINVALUE 1 + NOCYCLE + CACHE 20 + NOORDER; +CREATE SEQUENCE FILE_RECYCLE_LOG_ID_SEQ + INCREMENT BY 1 + START WITH 1 + NOMAXVALUE + MINVALUE 1 + NOCYCLE + CACHE 20 + NOORDER; +CREATE GLOBAL TEMPORARY TABLE TEMP_TAPE_FILE_INSERTION_BATCH( + VID VARCHAR2(100), + FSEQ NUMERIC(20, 0) , + BLOCK_ID NUMERIC(20, 0) , + LOGICAL_SIZE_IN_BYTES NUMERIC(20, 0) , + COPY_NB NUMERIC(3, 0) , + CREATION_TIME NUMERIC(20, 0) , + ARCHIVE_FILE_ID NUMERIC(20, 0) +) +ON COMMIT DELETE ROWS; +CREATE INDEX TEMP_T_F_I_B_AFI_IDX ON TEMP_TAPE_FILE_INSERTION_BATCH(ARCHIVE_FILE_ID); +CREATE TABLE CTA_CATALOGUE( + SCHEMA_VERSION_MAJOR NUMERIC(20, 0) CONSTRAINT CTA_CATALOGUE_SVM1_NN NOT NULL, + SCHEMA_VERSION_MINOR NUMERIC(20, 0) CONSTRAINT CTA_CATALOGUE_SVM2_NN NOT NULL, + NEXT_SCHEMA_VERSION_MAJOR NUMERIC(20, 0), + NEXT_SCHEMA_VERSION_MINOR NUMERIC(20, 0), + STATUS VARCHAR2(100), + IS_PRODUCTION CHAR(1) DEFAULT '0' CONSTRAINT CTA_CATALOGUE_IP_NN NOT NULL, + CONSTRAINT CTA_CATALOGUE_IP_BOOL_CK CHECK(IS_PRODUCTION IN ('0','1')) +); +CREATE TABLE ADMIN_USER( + ADMIN_USER_NAME VARCHAR2(100) CONSTRAINT ADMIN_USER_AUN_NN NOT NULL, + USER_COMMENT VARCHAR2(1000) CONSTRAINT ADMIN_USER_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT ADMIN_USER_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT ADMIN_USER_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT ADMIN_USER_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT ADMIN_USER_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT ADMIN_USER_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT ADMIN_USER_LUT_NN NOT NULL, + CONSTRAINT ADMIN_USER_PK PRIMARY KEY(ADMIN_USER_NAME) +); +CREATE UNIQUE INDEX ADMIN_USER_AUN_CI_UN_IDX ON ADMIN_USER(LOWER(ADMIN_USER_NAME)); +CREATE TABLE DISK_INSTANCE( + DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_DINM_NN NOT NULL, + USER_COMMENT VARCHAR2(1000) CONSTRAINT DISK_INSTANCE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_LUT_NN NOT NULL, + CONSTRAINT DISK_INSTANCE_PK PRIMARY KEY(DISK_INSTANCE_NAME) +); +CREATE UNIQUE INDEX DISK_INSTANCE_DIN_CI_UN_IDX ON DISK_INSTANCE(LOWER(DISK_INSTANCE_NAME)); +CREATE TABLE DISK_INSTANCE_SPACE( + DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_SPACE_DINM_NN NOT NULL, + DISK_INSTANCE_SPACE_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_SPACE_DISNM_NN NOT NULL, + FREE_SPACE_QUERY_URL VARCHAR2(1000) CONSTRAINT DISK_INSTANCE_SPACE_FSQU_NN NOT NULL, + REFRESH_INTERVAL NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_RI_NN NOT NULL, + LAST_REFRESH_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_LRT_NN NOT NULL, + FREE_SPACE NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_TFS_NN NOT NULL, + USER_COMMENT VARCHAR2(1000) CONSTRAINT DISK_INSTANCE_SPACE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_SPACE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_SPACE_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_SPACE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_SPACE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_LUT_NN NOT NULL, + CONSTRAINT DISK_INSTANCE_SPACE_PK PRIMARY KEY(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME), + CONSTRAINT DISK_INSTANCE_SPACE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME) +); +CREATE UNIQUE INDEX DISK_INSTANCE_SPACE_DISN_CI_UN_IDX ON DISK_INSTANCE_SPACE(LOWER(DISK_INSTANCE_SPACE_NAME)); +CREATE UNIQUE INDEX DISK_INSTANCE_SPACE_DISN_UN_IDX ON DISK_INSTANCE_SPACE(DISK_INSTANCE_SPACE_NAME); + +CREATE TABLE DISK_SYSTEM( + DISK_SYSTEM_NAME VARCHAR2(100) CONSTRAINT DISK_SYSTEM_DSNM_NN NOT NULL, + DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT DISK_SYSTEM_DIN_NN NOT NULL, + DISK_INSTANCE_SPACE_NAME VARCHAR2(100) CONSTRAINT DISK_SYSTEM_DISN_NN NOT NULL, + FILE_REGEXP VARCHAR2(100) CONSTRAINT DISK_SYSTEM_FR_NN NOT NULL, + TARGETED_FREE_SPACE NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_TFS_NN NOT NULL, + SLEEP_TIME NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_ST_NN NOT NULL, + USER_COMMENT VARCHAR2(1000) CONSTRAINT DISK_SYSTEM_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT DISK_SYSTEM_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT DISK_SYSTEM_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT DISK_SYSTEM_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT DISK_SYSTEM_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_LUT_NN NOT NULL, + CONSTRAINT NAME_PK PRIMARY KEY(DISK_SYSTEM_NAME), + CONSTRAINT DISK_SYSTEM_DIN_DISN_FK FOREIGN KEY(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME) REFERENCES DISK_INSTANCE_SPACE(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME) +); +CREATE UNIQUE INDEX DISK_SYSTEM_DSN_CI_UN_IDX ON DISK_SYSTEM(LOWER(DISK_SYSTEM_NAME)); +CREATE INDEX DISK_SYSTEM_DIN_DISN_IDX ON DISK_SYSTEM(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME); +CREATE TABLE VIRTUAL_ORGANIZATION( + VIRTUAL_ORGANIZATION_ID NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_VOI_NN NOT NULL, + VIRTUAL_ORGANIZATION_NAME VARCHAR2(100) CONSTRAINT VIRTUAL_ORGANIZATION_VON_NN NOT NULL, + READ_MAX_DRIVES NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_RMD_NN NOT NULL, + WRITE_MAX_DRIVES NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_WMD_NN NOT NULL, + MAX_FILE_SIZE NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_MFS_NN NOT NULL, + USER_COMMENT VARCHAR2(1000) CONSTRAINT VIRTUAL_ORGANIZATION_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT VIRTUAL_ORGANIZATION_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT VIRTUAL_ORGANIZATION_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT VIRTUAL_ORGANIZATION_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT VIRTUAL_ORGANIZATION_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_LUT_NN NOT NULL, + DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT VIRTUAL_ORGANIZATION_DIN_NN NOT NULL, + CONSTRAINT VIRTUAL_ORGANIZATION_PK PRIMARY KEY(VIRTUAL_ORGANIZATION_ID), + CONSTRAINT VIRTUAL_ORGANIZATION_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME) +); +CREATE UNIQUE INDEX VIRTUAL_ORG_VON_CI_UN_IDX ON VIRTUAL_ORGANIZATION(LOWER(VIRTUAL_ORGANIZATION_NAME)); +CREATE UNIQUE INDEX VIRTUAL_ORG_VON_UN_IDX ON VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_NAME); +CREATE INDEX VIRTUAL_ORG_DIN_IDX ON VIRTUAL_ORGANIZATION(DISK_INSTANCE_NAME); + +CREATE TABLE STORAGE_CLASS( + STORAGE_CLASS_ID NUMERIC(20, 0) CONSTRAINT STORAGE_CLASS_SCI_NN NOT NULL, + STORAGE_CLASS_NAME VARCHAR2(100) CONSTRAINT STORAGE_CLASS_SCN_NN NOT NULL, + NB_COPIES NUMERIC(3, 0) CONSTRAINT STORAGE_CLASS_NC_NN NOT NULL, + VIRTUAL_ORGANIZATION_ID NUMERIC(20, 0) CONSTRAINT STORAGE_CLASS_VOI_NN NOT NULL, + USER_COMMENT VARCHAR2(1000) CONSTRAINT STORAGE_CLASS_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT STORAGE_CLASS_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT STORAGE_CLASS_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT STORAGE_CLASS_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT STORAGE_CLASS_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT STORAGE_CLASS_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT STORAGE_CLASS_LUT_NN NOT NULL, + CONSTRAINT STORAGE_CLASS_PK PRIMARY KEY(STORAGE_CLASS_ID), + CONSTRAINT STORAGE_CLASS_VOI_FK FOREIGN KEY(VIRTUAL_ORGANIZATION_ID) REFERENCES VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_ID) +); +CREATE UNIQUE INDEX STORAGE_CLASS_SCN_CI_UN_IDX ON STORAGE_CLASS(LOWER(STORAGE_CLASS_NAME)); +CREATE UNIQUE INDEX STORAGE_CLASS_SCN_UN_IDX ON STORAGE_CLASS(STORAGE_CLASS_NAME); +CREATE INDEX STORAGE_CLASS_VOI_IDX ON STORAGE_CLASS(VIRTUAL_ORGANIZATION_ID); +CREATE TABLE TAPE_POOL( + TAPE_POOL_ID NUMERIC(20, 0) CONSTRAINT TAPE_POOL_TPI_NN NOT NULL, + TAPE_POOL_NAME VARCHAR2(100) CONSTRAINT TAPE_POOL_TPN_NN NOT NULL, + VIRTUAL_ORGANIZATION_ID NUMERIC(20, 0) CONSTRAINT TAPE_POOL_VOI_NN NOT NULL, + NB_PARTIAL_TAPES NUMERIC(20, 0) CONSTRAINT TAPE_POOL_NPT_NN NOT NULL, + IS_ENCRYPTED CHAR(1) CONSTRAINT TAPE_POOL_IE_NN NOT NULL, + ENCRYPTION_KEY_NAME VARCHAR2(100), + SUPPLY VARCHAR2(100), + USER_COMMENT VARCHAR2(1000) CONSTRAINT TAPE_POOL_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT TAPE_POOL_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT TAPE_POOL_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT TAPE_POOL_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT TAPE_POOL_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT TAPE_POOL_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT TAPE_POOL_LUT_NN NOT NULL, + CONSTRAINT TAPE_POOL_PK PRIMARY KEY(TAPE_POOL_ID), + CONSTRAINT TAPE_POOL_IS_ENCRYPTED_BOOL_CK CHECK(IS_ENCRYPTED IN ('0', '1')), + CONSTRAINT TAPE_POOL_VO_FK FOREIGN KEY(VIRTUAL_ORGANIZATION_ID) REFERENCES VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_ID) +); +CREATE UNIQUE INDEX TAPE_POOL_TPN_CI_UN_IDX ON TAPE_POOL(LOWER(TAPE_POOL_NAME)); +CREATE UNIQUE INDEX TAPE_POOL_TPN_UN_IDX ON TAPE_POOL(TAPE_POOL_NAME); +CREATE INDEX TAPE_POOL_VOI_IDX ON TAPE_POOL(VIRTUAL_ORGANIZATION_ID); +CREATE TABLE ARCHIVE_ROUTE( + STORAGE_CLASS_ID NUMERIC(20, 0) CONSTRAINT ARCHIVE_ROUTE_SCI_NN NOT NULL, + COPY_NB NUMERIC(3, 0) CONSTRAINT ARCHIVE_ROUTE_CN_NN NOT NULL, + TAPE_POOL_ID NUMERIC(20, 0) CONSTRAINT ARCHIVE_ROUTE_TPI_NN NOT NULL, + USER_COMMENT VARCHAR2(1000) CONSTRAINT ARCHIVE_ROUTE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT ARCHIVE_ROUTE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT ARCHIVE_ROUTE_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT ARCHIVE_ROUTE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT ARCHIVE_ROUTE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT ARCHIVE_ROUTE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT ARCHIVE_ROUTE_LUT_NN NOT NULL, + CONSTRAINT ARCHIVE_ROUTE_PK PRIMARY KEY(STORAGE_CLASS_ID, COPY_NB), + CONSTRAINT ARCHIVE_ROUTE_STORAGE_CLASS_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID), + CONSTRAINT ARCHIVE_ROUTE_TAPE_POOL_FK FOREIGN KEY(TAPE_POOL_ID) REFERENCES TAPE_POOL(TAPE_POOL_ID), + CONSTRAINT ARCHIVE_ROUTE_COPY_NB_GT_0_CK CHECK(COPY_NB > 0), + CONSTRAINT ARCHIVE_ROUTE_SCI_TPI_UN UNIQUE(STORAGE_CLASS_ID, TAPE_POOL_ID) +); +CREATE TABLE MEDIA_TYPE( + MEDIA_TYPE_ID NUMERIC(20, 0) CONSTRAINT MEDIA_TYPE_MTI_NN NOT NULL, + MEDIA_TYPE_NAME VARCHAR2(100) CONSTRAINT MEDIA_TYPE_MTN_NN NOT NULL, + CARTRIDGE VARCHAR2(100) CONSTRAINT MEDIA_TYPE_C_NN NOT NULL, + CAPACITY_IN_BYTES NUMERIC(20, 0) CONSTRAINT MEDIA_TYPE_CIB_NN NOT NULL, + PRIMARY_DENSITY_CODE NUMERIC(3, 0), + SECONDARY_DENSITY_CODE NUMERIC(3, 0), + NB_WRAPS NUMERIC(10, 0), + MIN_LPOS NUMERIC(20, 0), + MAX_LPOS NUMERIC(20, 0), + USER_COMMENT VARCHAR2(1000) CONSTRAINT MEDIA_TYPE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT MEDIA_TYPE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT MEDIA_TYPE_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT MEDIA_TYPE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT MEDIA_TYPE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT MEDIA_TYPE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT MEDIA_TYPE_LUT_NN NOT NULL, + CONSTRAINT MEDIA_TYPE_PK PRIMARY KEY(MEDIA_TYPE_ID) +); +CREATE UNIQUE INDEX MEDIA_TYPE_MTN_CI_UN_IDX ON MEDIA_TYPE(LOWER(MEDIA_TYPE_NAME)); +CREATE UNIQUE INDEX MEDIA_TYPE_MTN_UN_IDX ON MEDIA_TYPE(MEDIA_TYPE_NAME); +CREATE TABLE LOGICAL_LIBRARY( + LOGICAL_LIBRARY_ID NUMERIC(20, 0) CONSTRAINT LOGICAL_LIBRARY_LLI_NN NOT NULL, + LOGICAL_LIBRARY_NAME VARCHAR2(100) CONSTRAINT LOGICAL_LIBRARY_LLN_NN NOT NULL, + IS_DISABLED CHAR(1) DEFAULT '0' CONSTRAINT LOGICAL_LIBRARY_ID_NN NOT NULL, + DISABLED_REASON VARCHAR2(1000) , + USER_COMMENT VARCHAR2(1000) CONSTRAINT LOGICAL_LIBRARY_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT LOGICAL_LIBRARY_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT LOGICAL_LIBRARY_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT LOGICAL_LIBRARY_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT LOGICAL_LIBRARY_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT LOGICAL_LIBRARY_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT LOGICAL_LIBRARY_LUT_NN NOT NULL, + CONSTRAINT LOGICAL_LIBRARY_PK PRIMARY KEY(LOGICAL_LIBRARY_ID), + CONSTRAINT LOGICAL_LIBRARY_ID_BOOL_CK CHECK(IS_DISABLED IN ('0', '1')) +); +CREATE UNIQUE INDEX LOGICAL_LIBRARY_LLN_CI_UN_IDX ON LOGICAL_LIBRARY(LOWER(LOGICAL_LIBRARY_NAME)); +CREATE UNIQUE INDEX LOGICAL_LIBRARY_LLN_UN_IDX ON LOGICAL_LIBRARY(LOGICAL_LIBRARY_NAME); +CREATE TABLE TAPE( + VID VARCHAR2(100) CONSTRAINT TAPE_V_NN NOT NULL, + MEDIA_TYPE_ID NUMERIC(20, 0) CONSTRAINT TAPE_MTID_NN NOT NULL, + VENDOR VARCHAR2(100) CONSTRAINT TAPE_V2_NN NOT NULL, + LOGICAL_LIBRARY_ID NUMERIC(20, 0) CONSTRAINT TAPE_LLI_NN NOT NULL, + TAPE_POOL_ID NUMERIC(20, 0) CONSTRAINT TAPE_TPI_NN NOT NULL, + ENCRYPTION_KEY_NAME VARCHAR2(100), + DATA_IN_BYTES NUMERIC(20, 0) CONSTRAINT TAPE_DIB_NN NOT NULL, + LAST_FSEQ NUMERIC(20, 0) CONSTRAINT TAPE_LF_NN NOT NULL, + NB_MASTER_FILES NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_NB_MASTER_FILES_NN NOT NULL, + MASTER_DATA_IN_BYTES NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_MASTER_DATA_IN_BYTES_NN NOT NULL, + IS_FULL CHAR(1) CONSTRAINT TAPE_IF_NN NOT NULL, + IS_FROM_CASTOR CHAR(1) CONSTRAINT TAPE_IFC_NN NOT NULL, + DIRTY CHAR(1) DEFAULT '1' CONSTRAINT TAPE_DIRTY_NN NOT NULL, + NB_COPY_NB_1 NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_NB_COPY_NB_1_NN NOT NULL, + COPY_NB_1_IN_BYTES NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_COPY_NB_1_IN_BYTES_NN NOT NULL, + NB_COPY_NB_GT_1 NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_NB_COPY_NB_GT_1_NN NOT NULL, + COPY_NB_GT_1_IN_BYTES NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_COPY_NB_GT_1_IN_BYTES_NN NOT NULL, + LABEL_FORMAT CHAR(1), + LABEL_DRIVE VARCHAR2(100), + LABEL_TIME NUMERIC(20, 0), + LAST_READ_DRIVE VARCHAR2(100), + LAST_READ_TIME NUMERIC(20, 0), + LAST_WRITE_DRIVE VARCHAR2(100), + LAST_WRITE_TIME NUMERIC(20, 0), + READ_MOUNT_COUNT NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_RMC_NN NOT NULL, + WRITE_MOUNT_COUNT NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_WMC_NN NOT NULL, + USER_COMMENT VARCHAR2(1000), + TAPE_STATE VARCHAR2(100) CONSTRAINT TAPE_TS_NN NOT NULL, + STATE_REASON VARCHAR2(1000), + STATE_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT TAPE_SUT_NN NOT NULL, + STATE_MODIFIED_BY VARCHAR2(100) CONSTRAINT TAPE_SMB_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT TAPE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT TAPE_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT TAPE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT TAPE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT TAPE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT TAPE_LUT_NN NOT NULL, + VERIFICATION_STATUS VARCHAR2(1000), + CONSTRAINT TAPE_PK PRIMARY KEY(VID), + CONSTRAINT TAPE_LOGICAL_LIBRARY_FK FOREIGN KEY(LOGICAL_LIBRARY_ID) REFERENCES LOGICAL_LIBRARY(LOGICAL_LIBRARY_ID), + CONSTRAINT TAPE_TAPE_POOL_FK FOREIGN KEY(TAPE_POOL_ID) REFERENCES TAPE_POOL(TAPE_POOL_ID), + CONSTRAINT TAPE_IS_FULL_BOOL_CK CHECK(IS_FULL IN ('0', '1')), + CONSTRAINT TAPE_IS_FROM_CASTOR_BOOL_CK CHECK(IS_FROM_CASTOR IN ('0', '1')), + CONSTRAINT TAPE_DIRTY_BOOL_CK CHECK(DIRTY IN ('0','1')), + CONSTRAINT TAPE_STATE_CK CHECK(TAPE_STATE IN ('ACTIVE', 'REPACKING_PENDING', 'REPACKING', 'REPACKING_DISABLED', 'DISABLED', 'BROKEN_PENDING', 'BROKEN', 'EXPORTED_PENDING', 'EXPORTED')), + CONSTRAINT TAPE_MEDIA_TYPE_FK FOREIGN KEY(MEDIA_TYPE_ID) REFERENCES MEDIA_TYPE(MEDIA_TYPE_ID) +); +CREATE UNIQUE INDEX TAPE_VID_CI_UN_IDX ON TAPE(LOWER(VID)); +CREATE INDEX TAPE_TAPE_POOL_ID_IDX ON TAPE(TAPE_POOL_ID); +CREATE INDEX TAPE_STATE_IDX ON TAPE(TAPE_STATE); +CREATE INDEX TAPE_LLI_IDX ON TAPE(LOGICAL_LIBRARY_ID); +CREATE INDEX TAPE_MTI_IDX ON TAPE(MEDIA_TYPE_ID); +CREATE TABLE MOUNT_POLICY( + MOUNT_POLICY_NAME VARCHAR2(100) CONSTRAINT MOUNT_POLICY_MPN_NN NOT NULL, + ARCHIVE_PRIORITY NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_AP_NN NOT NULL, + ARCHIVE_MIN_REQUEST_AGE NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_AMRA_NN NOT NULL, + RETRIEVE_PRIORITY NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_RP_NN NOT NULL, + RETRIEVE_MIN_REQUEST_AGE NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_RMRA_NN NOT NULL, + USER_COMMENT VARCHAR2(1000) CONSTRAINT MOUNT_POLICY_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT MOUNT_POLICY_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT MOUNT_POLICY_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT MOUNT_POLICY_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT MOUNT_POLICY_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_LUT_NN NOT NULL, + CONSTRAINT MOUNT_POLICY_PK PRIMARY KEY(MOUNT_POLICY_NAME) +); +CREATE UNIQUE INDEX MOUNT_POLICY_MPN_CI_UN_IDX ON MOUNT_POLICY(LOWER(MOUNT_POLICY_NAME)); +CREATE TABLE REQUESTER_ACTIVITY_MOUNT_RULE( + DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT RQSTER_ACT_RULE_DIN_NN NOT NULL, + REQUESTER_NAME VARCHAR2(100) CONSTRAINT RQSTER_ACT_RULE_RN_NN NOT NULL, + ACTIVITY_REGEX VARCHAR2(100) CONSTRAINT RQSTER_ACT_RULE_AR_NN NOT NULL, + MOUNT_POLICY_NAME VARCHAR2(100) CONSTRAINT RQSTER_ACT_RULE_MPN_NN NOT NULL, + USER_COMMENT VARCHAR2(1000) CONSTRAINT RQSTER_ACT_RULE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT RQSTER_ACT_RULE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT RQSTER_ACT_RULE_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_ACT_RULE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT RQSTER_ACT_RULE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT RQSTER_ACT_RULE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_ACT_RULE_LUT_NN NOT NULL, + CONSTRAINT RQSTER_ACT_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_NAME, ACTIVITY_REGEX), + CONSTRAINT RQSTER_ACT_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME), + CONSTRAINT RQSTER_ACT_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME) +); +CREATE INDEX REQ_ACT_MNT_RULE_MPN_IDX ON REQUESTER_ACTIVITY_MOUNT_RULE(MOUNT_POLICY_NAME); +CREATE INDEX REQ_ACT_MNT_RULE_DIN_IDX ON REQUESTER_ACTIVITY_MOUNT_RULE(DISK_INSTANCE_NAME); + +CREATE TABLE REQUESTER_MOUNT_RULE( + DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT RQSTER_RULE_DIN_NN NOT NULL, + REQUESTER_NAME VARCHAR2(100) CONSTRAINT RQSTER_RULE_RN_NN NOT NULL, + MOUNT_POLICY_NAME VARCHAR2(100) CONSTRAINT RQSTER_RULE_MPN_NN NOT NULL, + USER_COMMENT VARCHAR2(1000) CONSTRAINT RQSTER_RULE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT RQSTER_RULE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT RQSTER_RULE_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_RULE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT RQSTER_RULE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT RQSTER_RULE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_RULE_LUT_NN NOT NULL, + CONSTRAINT RQSTER_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_NAME), + CONSTRAINT RQSTER_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME), + CONSTRAINT RQSTER_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME) +); +CREATE INDEX REQ_MNT_RULE_MPN_IDX ON REQUESTER_MOUNT_RULE(MOUNT_POLICY_NAME); +CREATE INDEX REQ_MNT_RULE_DIN_IDX ON REQUESTER_MOUNT_RULE(DISK_INSTANCE_NAME); + +CREATE TABLE REQUESTER_GROUP_MOUNT_RULE( + DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT RQSTER_GRP_RULE_DIN_NN NOT NULL, + REQUESTER_GROUP_NAME VARCHAR2(100) CONSTRAINT RQSTER_GRP_RULE_RGN_NN NOT NULL, + MOUNT_POLICY_NAME VARCHAR2(100) CONSTRAINT RQSTER_GRP_RULE_MPN_NN NOT NULL, + USER_COMMENT VARCHAR2(1000) CONSTRAINT RQSTER_GRP_RULE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT RQSTER_GRP_RULE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT RQSTER_GRP_RULE_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_GRP_RULE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT RQSTER_GRP_RULE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT RQSTER_GRP_RULE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_GRP_RULE_LUT_NN NOT NULL, + CONSTRAINT RQSTER_GRP_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_GROUP_NAME), + CONSTRAINT RQSTER_GRP_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME), + CONSTRAINT RQSTER_GRP_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME) + +); +CREATE INDEX REQ_GRP_MNT_RULE_MPN_IDX ON REQUESTER_GROUP_MOUNT_RULE(MOUNT_POLICY_NAME); +CREATE INDEX REQ_GRP_MNT_RULE_DIN_IDX ON REQUESTER_GROUP_MOUNT_RULE(DISK_INSTANCE_NAME); + +CREATE TABLE ARCHIVE_FILE( + ARCHIVE_FILE_ID NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_AFI_NN NOT NULL, + DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT ARCHIVE_FILE_DIN_NN NOT NULL, + DISK_FILE_ID VARCHAR2(100) CONSTRAINT ARCHIVE_FILE_DFI_NN NOT NULL, + DISK_FILE_UID NUMERIC(10, 0) CONSTRAINT ARCHIVE_FILE_DFUID_NN NOT NULL, + DISK_FILE_GID NUMERIC(10, 0) CONSTRAINT ARCHIVE_FILE_DFGID_NN NOT NULL, + SIZE_IN_BYTES NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_SIB_NN NOT NULL, + CHECKSUM_BLOB RAW(200), + CHECKSUM_ADLER32 NUMERIC(10, 0) CONSTRAINT ARCHIVE_FILE_CB2_NN NOT NULL, + STORAGE_CLASS_ID NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_SCI_NN NOT NULL, + CREATION_TIME NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_CT2_NN NOT NULL, + RECONCILIATION_TIME NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_RT_NN NOT NULL, + IS_DELETED CHAR(1) DEFAULT '0' CONSTRAINT ARCHIVE_FILE_ID_NN NOT NULL, + COLLOCATION_HINT VARCHAR2(100), + CONSTRAINT ARCHIVE_FILE_PK PRIMARY KEY(ARCHIVE_FILE_ID), + CONSTRAINT ARCHIVE_FILE_STORAGE_CLASS_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID), + CONSTRAINT ARCHIVE_FILE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME), + CONSTRAINT ARCHIVE_FILE_DIN_DFI_UN UNIQUE(DISK_INSTANCE_NAME, DISK_FILE_ID), + CONSTRAINT ARCHIVE_FILE_ID_BOOL_CK CHECK(IS_DELETED IN ('0', '1')) +); +CREATE INDEX ARCHIVE_FILE_DIN_IDX ON ARCHIVE_FILE(DISK_INSTANCE_NAME); +CREATE INDEX ARCHIVE_FILE_DFI_IDX ON ARCHIVE_FILE(DISK_FILE_ID); +CREATE INDEX ARCHIVE_FILE_SCI_IDX ON ARCHIVE_FILE(STORAGE_CLASS_ID); +CREATE TABLE TAPE_FILE( + VID VARCHAR2(100) CONSTRAINT TAPE_FILE_V_NN NOT NULL, + FSEQ NUMERIC(20, 0) CONSTRAINT TAPE_FILE_F_NN NOT NULL, + BLOCK_ID NUMERIC(20, 0) CONSTRAINT TAPE_FILE_BI_NN NOT NULL, + LOGICAL_SIZE_IN_BYTES NUMERIC(20, 0) CONSTRAINT TAPE_FILE_CSIB_NN NOT NULL, + COPY_NB NUMERIC(3, 0) CONSTRAINT TAPE_FILE_CN_NN NOT NULL, + CREATION_TIME NUMERIC(20, 0) CONSTRAINT TAPE_FILE_CT_NN NOT NULL, + ARCHIVE_FILE_ID NUMERIC(20, 0) CONSTRAINT TAPE_FILE_AFI_NN NOT NULL, + CONSTRAINT TAPE_FILE_PK PRIMARY KEY(VID, FSEQ), + CONSTRAINT TAPE_FILE_TAPE_FK FOREIGN KEY(VID) + REFERENCES TAPE(VID), + CONSTRAINT TAPE_FILE_ARCHIVE_FILE_FK FOREIGN KEY(ARCHIVE_FILE_ID) + REFERENCES ARCHIVE_FILE(ARCHIVE_FILE_ID), + CONSTRAINT TAPE_FILE_VID_BLOCK_ID_UN UNIQUE(VID, BLOCK_ID), + CONSTRAINT TAPE_FILE_COPY_NB_GT_0_CK CHECK(COPY_NB > 0) +); +CREATE INDEX TAPE_FILE_VID_IDX ON TAPE_FILE(VID); +CREATE INDEX TAPE_FILE_ARCHIVE_FILE_ID_IDX ON TAPE_FILE(ARCHIVE_FILE_ID); +CREATE TABLE FILE_RECYCLE_LOG( + FILE_RECYCLE_LOG_ID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_ID_NN NOT NULL, + VID VARCHAR2(100) CONSTRAINT FILE_RECYCLE_LOG_VID_NN NOT NULL, + FSEQ NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_FSEQ_NN NOT NULL, + BLOCK_ID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_BID_NN NOT NULL, + COPY_NB NUMERIC(3, 0) CONSTRAINT FILE_RECYCLE_LOG_COPY_NB_NN NOT NULL, + TAPE_FILE_CREATION_TIME NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_TFCT_NN NOT NULL, + ARCHIVE_FILE_ID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_AFI_NN NOT NULL, + DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT FILE_RECYCLE_LOG_DIN_NN NOT NULL, + DISK_FILE_ID VARCHAR2(100) CONSTRAINT FILE_RECYCLE_LOG_DFI_NN NOT NULL, + DISK_FILE_ID_WHEN_DELETED VARCHAR2(100) CONSTRAINT FILE_RECYCLE_LOG_DFIWD_NN NOT NULL, + DISK_FILE_UID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_DFU_NN NOT NULL, + DISK_FILE_GID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_DFG_NN NOT NULL, + SIZE_IN_BYTES NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_SIB_NN NOT NULL, + CHECKSUM_BLOB RAW(200), + CHECKSUM_ADLER32 NUMERIC(10, 0) CONSTRAINT FILE_RECYCLE_LOG_CA_NN NOT NULL, + STORAGE_CLASS_ID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_SCI_NN NOT NULL, + ARCHIVE_FILE_CREATION_TIME NUMERIC(20, 0) CONSTRAINT FILE_RECYLE_LOG_CT_NN NOT NULL, + RECONCILIATION_TIME NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_RT_NN NOT NULL, + COLLOCATION_HINT VARCHAR2(100), + DISK_FILE_PATH VARCHAR2(2000), + REASON_LOG VARCHAR2(1000) CONSTRAINT FILE_RECYCLE_LOG_RL_NN NOT NULL, + RECYCLE_LOG_TIME NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_RLT_NN NOT NULL, + CONSTRAINT FILE_RECYCLE_LOG_PK PRIMARY KEY(FILE_RECYCLE_LOG_ID), + CONSTRAINT FILE_RECYCLE_LOG_VID_FK FOREIGN KEY(VID) REFERENCES TAPE(VID), + CONSTRAINT FILE_RECYCLE_LOG_SC_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID) +); +CREATE INDEX FILE_RECYCLE_LOG_DFI_IDX ON FILE_RECYCLE_LOG(DISK_FILE_ID); +CREATE INDEX FILE_RECYCLE_LOG_SCD_IDX ON FILE_RECYCLE_LOG(STORAGE_CLASS_ID); +CREATE INDEX FILE_RECYCLE_LOG_VID_IDX ON FILE_RECYCLE_LOG(VID); +CREATE TABLE DRIVE_CONFIG ( + DRIVE_NAME VARCHAR2(100) CONSTRAINT DRIVE_CONFIG_DN_NN NOT NULL, + CATEGORY VARCHAR2(100) CONSTRAINT DRIVE_CONFIG_C_NN NOT NULL, + KEY_NAME VARCHAR2(100) CONSTRAINT DRIVE_CONFIG_KN_NN NOT NULL, + VALUE VARCHAR2(1000) CONSTRAINT DRIVE_CONFIG_V_NN NOT NULL, + SOURCE VARCHAR2(100) CONSTRAINT DRIVE_CONFIG_S_NN NOT NULL, + CONSTRAINT DRIVE_CONFIG_DN_PK PRIMARY KEY(KEY_NAME, DRIVE_NAME) +); +CREATE TABLE DRIVE_STATE ( + DRIVE_NAME VARCHAR2(100) CONSTRAINT DRIVE_DN_NN NOT NULL, + HOST VARCHAR2(100) CONSTRAINT DRIVE_H_NN NOT NULL, + LOGICAL_LIBRARY VARCHAR2(100) CONSTRAINT DRIVE_LL_NN NOT NULL, + SESSION_ID NUMERIC(20, 0), + BYTES_TRANSFERED_IN_SESSION NUMERIC(20, 0), + FILES_TRANSFERED_IN_SESSION NUMERIC(20, 0), + SESSION_START_TIME NUMERIC(20, 0), + SESSION_ELAPSED_TIME NUMERIC(20, 0), + MOUNT_START_TIME NUMERIC(20, 0), + TRANSFER_START_TIME NUMERIC(20, 0), + UNLOAD_START_TIME NUMERIC(20, 0), + UNMOUNT_START_TIME NUMERIC(20, 0), + DRAINING_START_TIME NUMERIC(20, 0), + DOWN_OR_UP_START_TIME NUMERIC(20, 0), + PROBE_START_TIME NUMERIC(20, 0), + CLEANUP_START_TIME NUMERIC(20, 0), + START_START_TIME NUMERIC(20, 0), + SHUTDOWN_TIME NUMERIC(20, 0), + MOUNT_TYPE VARCHAR2(100) DEFAULT 'NO_MOUNT' CONSTRAINT DRIVE_MT_NN NOT NULL, + DRIVE_STATUS VARCHAR2(100) DEFAULT 'UNKNOWN' CONSTRAINT DRIVE_DS_NN NOT NULL, + DESIRED_UP CHAR(1) DEFAULT '0' CONSTRAINT DRIVE_DU_NN NOT NULL, + DESIRED_FORCE_DOWN CHAR(1) DEFAULT '0' CONSTRAINT DRIVE_DFD_NN NOT NULL, + REASON_UP_DOWN VARCHAR2(1000), + CURRENT_VID VARCHAR2(100), + CTA_VERSION VARCHAR2(100), + CURRENT_PRIORITY NUMERIC(20, 0), + CURRENT_ACTIVITY VARCHAR2(100), + CURRENT_TAPE_POOL VARCHAR2(100), + NEXT_MOUNT_TYPE VARCHAR2(100) DEFAULT 'NO_MOUNT' CONSTRAINT DRIVE_NMT_NN NOT NULL, + NEXT_VID VARCHAR2(100), + NEXT_PRIORITY NUMERIC(20, 0), + NEXT_ACTIVITY VARCHAR2(100), + NEXT_TAPE_POOL VARCHAR2(100), + DEV_FILE_NAME VARCHAR2(100), + RAW_LIBRARY_SLOT VARCHAR2(100), + CURRENT_VO VARCHAR2(100), + NEXT_VO VARCHAR2(100), + USER_COMMENT VARCHAR2(1000), + CREATION_LOG_USER_NAME VARCHAR2(100), + CREATION_LOG_HOST_NAME VARCHAR2(100), + CREATION_LOG_TIME NUMERIC(20, 0), + LAST_UPDATE_USER_NAME VARCHAR2(100), + LAST_UPDATE_HOST_NAME VARCHAR2(100), + LAST_UPDATE_TIME NUMERIC(20, 0), + DISK_SYSTEM_NAME VARCHAR2(100), + RESERVED_BYTES NUMERIC(20, 0), + RESERVATION_SESSION_ID NUMERIC(20, 0), + CONSTRAINT DRIVE_DN_PK PRIMARY KEY(DRIVE_NAME), + CONSTRAINT DRIVE_DU_BOOL_CK CHECK(DESIRED_UP IN ('0', '1')), + CONSTRAINT DRIVE_DFD_BOOL_CK CHECK(DESIRED_FORCE_DOWN IN ('0', '1')), + CONSTRAINT DRIVE_DS_STRING_CK CHECK(DRIVE_STATUS IN ('DOWN', 'UP', 'PROBING', 'STARTING', + 'MOUNTING', 'TRANSFERING', 'UNLOADING', 'UNMOUNTING', 'DRAININGTODISK', 'CLEANINGUP', 'SHUTDOWN', + 'UNKNOWN')), + CONSTRAINT DRIVE_MT_STRING_CK CHECK(MOUNT_TYPE IN ('NO_MOUNT', 'ARCHIVE_FOR_USER', + 'ARCHIVE_FOR_REPACK', 'RETRIEVE', 'LABEL', 'ARCHIVE_ALL_TYPES')), + CONSTRAINT DRIVE_NMT_STRING_CK CHECK(NEXT_MOUNT_TYPE IN ('NO_MOUNT', 'ARCHIVE_FOR_USER', + 'ARCHIVE_FOR_REPACK', 'RETRIEVE', 'LABEL', 'ARCHIVE_ALL_TYPES')) +); +CREATE UNIQUE INDEX DRIVE_STATE_DN_CI_UN_IDX ON DRIVE_STATE(LOWER(DRIVE_NAME)); +INSERT INTO CTA_CATALOGUE( + SCHEMA_VERSION_MAJOR, + SCHEMA_VERSION_MINOR, + STATUS) +VALUES( + 12, + 0, + 'PRODUCTION'); +ALTER TABLE CTA_CATALOGUE ADD CONSTRAINT CATALOGUE_STATUS_CONTENT_CK CHECK((NEXT_SCHEMA_VERSION_MAJOR IS NULL AND NEXT_SCHEMA_VERSION_MINOR IS NULL AND STATUS='PRODUCTION') OR (NEXT_SCHEMA_VERSION_MAJOR IS NOT NULL AND NEXT_SCHEMA_VERSION_MINOR IS NOT NULL AND STATUS='UPGRADING')) INITIALLY DEFERRED; + +COMMIT; diff --git a/catalogue/12.0/postgres_catalogue_schema.sql b/catalogue/12.0/postgres_catalogue_schema.sql new file mode 100644 index 0000000000..6b150277c8 --- /dev/null +++ b/catalogue/12.0/postgres_catalogue_schema.sql @@ -0,0 +1,515 @@ +CREATE SEQUENCE ARCHIVE_FILE_ID_SEQ + INCREMENT BY 1 + START WITH 1 + NO MAXVALUE + MINVALUE 1 + NO CYCLE + CACHE 20; +CREATE SEQUENCE LOGICAL_LIBRARY_ID_SEQ + INCREMENT BY 1 + START WITH 1 + NO MAXVALUE + MINVALUE 1 + NO CYCLE + CACHE 20; +CREATE SEQUENCE MEDIA_TYPE_ID_SEQ + INCREMENT BY 1 + START WITH 1 + NO MAXVALUE + MINVALUE 1 + NO CYCLE + CACHE 20; +CREATE SEQUENCE STORAGE_CLASS_ID_SEQ + INCREMENT BY 1 + START WITH 1 + NO MAXVALUE + MINVALUE 1 + NO CYCLE + CACHE 20; +CREATE SEQUENCE TAPE_POOL_ID_SEQ + INCREMENT BY 1 + START WITH 1 + NO MAXVALUE + MINVALUE 1 + NO CYCLE + CACHE 20; +CREATE SEQUENCE VIRTUAL_ORGANIZATION_ID_SEQ + INCREMENT BY 1 + START WITH 1 + NO MAXVALUE + MINVALUE 1 + NO CYCLE + CACHE 20; +CREATE SEQUENCE FILE_RECYCLE_LOG_ID_SEQ + INCREMENT BY 1 + START WITH 1 + NO MAXVALUE + MINVALUE 1 + NO CYCLE + CACHE 20; +CREATE TABLE CTA_CATALOGUE( + SCHEMA_VERSION_MAJOR NUMERIC(20, 0) CONSTRAINT CTA_CATALOGUE_SVM1_NN NOT NULL, + SCHEMA_VERSION_MINOR NUMERIC(20, 0) CONSTRAINT CTA_CATALOGUE_SVM2_NN NOT NULL, + NEXT_SCHEMA_VERSION_MAJOR NUMERIC(20, 0), + NEXT_SCHEMA_VERSION_MINOR NUMERIC(20, 0), + STATUS VARCHAR(100), + IS_PRODUCTION CHAR(1) DEFAULT '0' CONSTRAINT CTA_CATALOGUE_IP_NN NOT NULL, + CONSTRAINT CTA_CATALOGUE_IP_BOOL_CK CHECK(IS_PRODUCTION IN ('0','1')) +); +CREATE TABLE ADMIN_USER( + ADMIN_USER_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_AUN_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT ADMIN_USER_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT ADMIN_USER_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT ADMIN_USER_LUT_NN NOT NULL, + CONSTRAINT ADMIN_USER_PK PRIMARY KEY(ADMIN_USER_NAME) +); +CREATE UNIQUE INDEX ADMIN_USER_AUN_CI_UN_IDX ON ADMIN_USER(LOWER(ADMIN_USER_NAME)); +CREATE TABLE DISK_INSTANCE( + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_DINM_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT DISK_INSTANCE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_LUT_NN NOT NULL, + CONSTRAINT DISK_INSTANCE_PK PRIMARY KEY(DISK_INSTANCE_NAME) +); +CREATE UNIQUE INDEX DISK_INSTANCE_DIN_CI_UN_IDX ON DISK_INSTANCE(LOWER(DISK_INSTANCE_NAME)); +CREATE TABLE DISK_INSTANCE_SPACE( + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_DINM_NN NOT NULL, + DISK_INSTANCE_SPACE_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_DISNM_NN NOT NULL, + FREE_SPACE_QUERY_URL VARCHAR(1000) CONSTRAINT DISK_INSTANCE_SPACE_FSQU_NN NOT NULL, + REFRESH_INTERVAL NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_RI_NN NOT NULL, + LAST_REFRESH_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_LRT_NN NOT NULL, + FREE_SPACE NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_TFS_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT DISK_INSTANCE_SPACE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_LUT_NN NOT NULL, + CONSTRAINT DISK_INSTANCE_SPACE_PK PRIMARY KEY(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME), + CONSTRAINT DISK_INSTANCE_SPACE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME) +); +CREATE UNIQUE INDEX DISK_INSTANCE_SPACE_DISN_CI_UN_IDX ON DISK_INSTANCE_SPACE(LOWER(DISK_INSTANCE_SPACE_NAME)); +CREATE UNIQUE INDEX DISK_INSTANCE_SPACE_DISN_UN_IDX ON DISK_INSTANCE_SPACE(DISK_INSTANCE_SPACE_NAME); + +CREATE TABLE DISK_SYSTEM( + DISK_SYSTEM_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_DSNM_NN NOT NULL, + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_DIN_NN NOT NULL, + DISK_INSTANCE_SPACE_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_DISN_NN NOT NULL, + FILE_REGEXP VARCHAR(100) CONSTRAINT DISK_SYSTEM_FR_NN NOT NULL, + TARGETED_FREE_SPACE NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_TFS_NN NOT NULL, + SLEEP_TIME NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_ST_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT DISK_SYSTEM_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_LUT_NN NOT NULL, + CONSTRAINT NAME_PK PRIMARY KEY(DISK_SYSTEM_NAME), + CONSTRAINT DISK_SYSTEM_DIN_DISN_FK FOREIGN KEY(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME) REFERENCES DISK_INSTANCE_SPACE(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME) +); +CREATE UNIQUE INDEX DISK_SYSTEM_DSN_CI_UN_IDX ON DISK_SYSTEM(LOWER(DISK_SYSTEM_NAME)); +CREATE INDEX DISK_SYSTEM_DIN_DISN_IDX ON DISK_SYSTEM(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME); +CREATE TABLE VIRTUAL_ORGANIZATION( + VIRTUAL_ORGANIZATION_ID NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_VOI_NN NOT NULL, + VIRTUAL_ORGANIZATION_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_VON_NN NOT NULL, + READ_MAX_DRIVES NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_RMD_NN NOT NULL, + WRITE_MAX_DRIVES NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_WMD_NN NOT NULL, + MAX_FILE_SIZE NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_MFS_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT VIRTUAL_ORGANIZATION_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_LUT_NN NOT NULL, + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_DIN_NN NOT NULL, + CONSTRAINT VIRTUAL_ORGANIZATION_PK PRIMARY KEY(VIRTUAL_ORGANIZATION_ID), + CONSTRAINT VIRTUAL_ORGANIZATION_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME) +); +CREATE UNIQUE INDEX VIRTUAL_ORG_VON_CI_UN_IDX ON VIRTUAL_ORGANIZATION(LOWER(VIRTUAL_ORGANIZATION_NAME)); +CREATE UNIQUE INDEX VIRTUAL_ORG_VON_UN_IDX ON VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_NAME); +CREATE INDEX VIRTUAL_ORG_DIN_IDX ON VIRTUAL_ORGANIZATION(DISK_INSTANCE_NAME); + +CREATE TABLE STORAGE_CLASS( + STORAGE_CLASS_ID NUMERIC(20, 0) CONSTRAINT STORAGE_CLASS_SCI_NN NOT NULL, + STORAGE_CLASS_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_SCN_NN NOT NULL, + NB_COPIES NUMERIC(3, 0) CONSTRAINT STORAGE_CLASS_NC_NN NOT NULL, + VIRTUAL_ORGANIZATION_ID NUMERIC(20, 0) CONSTRAINT STORAGE_CLASS_VOI_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT STORAGE_CLASS_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT STORAGE_CLASS_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT STORAGE_CLASS_LUT_NN NOT NULL, + CONSTRAINT STORAGE_CLASS_PK PRIMARY KEY(STORAGE_CLASS_ID), + CONSTRAINT STORAGE_CLASS_VOI_FK FOREIGN KEY(VIRTUAL_ORGANIZATION_ID) REFERENCES VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_ID) +); +CREATE UNIQUE INDEX STORAGE_CLASS_SCN_CI_UN_IDX ON STORAGE_CLASS(LOWER(STORAGE_CLASS_NAME)); +CREATE UNIQUE INDEX STORAGE_CLASS_SCN_UN_IDX ON STORAGE_CLASS(STORAGE_CLASS_NAME); +CREATE INDEX STORAGE_CLASS_VOI_IDX ON STORAGE_CLASS(VIRTUAL_ORGANIZATION_ID); +CREATE TABLE TAPE_POOL( + TAPE_POOL_ID NUMERIC(20, 0) CONSTRAINT TAPE_POOL_TPI_NN NOT NULL, + TAPE_POOL_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_TPN_NN NOT NULL, + VIRTUAL_ORGANIZATION_ID NUMERIC(20, 0) CONSTRAINT TAPE_POOL_VOI_NN NOT NULL, + NB_PARTIAL_TAPES NUMERIC(20, 0) CONSTRAINT TAPE_POOL_NPT_NN NOT NULL, + IS_ENCRYPTED CHAR(1) CONSTRAINT TAPE_POOL_IE_NN NOT NULL, + ENCRYPTION_KEY_NAME VARCHAR(100), + SUPPLY VARCHAR(100), + USER_COMMENT VARCHAR(1000) CONSTRAINT TAPE_POOL_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT TAPE_POOL_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT TAPE_POOL_LUT_NN NOT NULL, + CONSTRAINT TAPE_POOL_PK PRIMARY KEY(TAPE_POOL_ID), + CONSTRAINT TAPE_POOL_IS_ENCRYPTED_BOOL_CK CHECK(IS_ENCRYPTED IN ('0', '1')), + CONSTRAINT TAPE_POOL_VO_FK FOREIGN KEY(VIRTUAL_ORGANIZATION_ID) REFERENCES VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_ID) +); +CREATE UNIQUE INDEX TAPE_POOL_TPN_CI_UN_IDX ON TAPE_POOL(LOWER(TAPE_POOL_NAME)); +CREATE UNIQUE INDEX TAPE_POOL_TPN_UN_IDX ON TAPE_POOL(TAPE_POOL_NAME); +CREATE INDEX TAPE_POOL_VOI_IDX ON TAPE_POOL(VIRTUAL_ORGANIZATION_ID); +CREATE TABLE ARCHIVE_ROUTE( + STORAGE_CLASS_ID NUMERIC(20, 0) CONSTRAINT ARCHIVE_ROUTE_SCI_NN NOT NULL, + COPY_NB NUMERIC(3, 0) CONSTRAINT ARCHIVE_ROUTE_CN_NN NOT NULL, + TAPE_POOL_ID NUMERIC(20, 0) CONSTRAINT ARCHIVE_ROUTE_TPI_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT ARCHIVE_ROUTE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT ARCHIVE_ROUTE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT ARCHIVE_ROUTE_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT ARCHIVE_ROUTE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT ARCHIVE_ROUTE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT ARCHIVE_ROUTE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT ARCHIVE_ROUTE_LUT_NN NOT NULL, + CONSTRAINT ARCHIVE_ROUTE_PK PRIMARY KEY(STORAGE_CLASS_ID, COPY_NB), + CONSTRAINT ARCHIVE_ROUTE_STORAGE_CLASS_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID), + CONSTRAINT ARCHIVE_ROUTE_TAPE_POOL_FK FOREIGN KEY(TAPE_POOL_ID) REFERENCES TAPE_POOL(TAPE_POOL_ID), + CONSTRAINT ARCHIVE_ROUTE_COPY_NB_GT_0_CK CHECK(COPY_NB > 0), + CONSTRAINT ARCHIVE_ROUTE_SCI_TPI_UN UNIQUE(STORAGE_CLASS_ID, TAPE_POOL_ID) +); +CREATE TABLE MEDIA_TYPE( + MEDIA_TYPE_ID NUMERIC(20, 0) CONSTRAINT MEDIA_TYPE_MTI_NN NOT NULL, + MEDIA_TYPE_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_MTN_NN NOT NULL, + CARTRIDGE VARCHAR(100) CONSTRAINT MEDIA_TYPE_C_NN NOT NULL, + CAPACITY_IN_BYTES NUMERIC(20, 0) CONSTRAINT MEDIA_TYPE_CIB_NN NOT NULL, + PRIMARY_DENSITY_CODE NUMERIC(3, 0), + SECONDARY_DENSITY_CODE NUMERIC(3, 0), + NB_WRAPS NUMERIC(10, 0), + MIN_LPOS NUMERIC(20, 0), + MAX_LPOS NUMERIC(20, 0), + USER_COMMENT VARCHAR(1000) CONSTRAINT MEDIA_TYPE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT MEDIA_TYPE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT MEDIA_TYPE_LUT_NN NOT NULL, + CONSTRAINT MEDIA_TYPE_PK PRIMARY KEY(MEDIA_TYPE_ID) +); +CREATE UNIQUE INDEX MEDIA_TYPE_MTN_CI_UN_IDX ON MEDIA_TYPE(LOWER(MEDIA_TYPE_NAME)); +CREATE UNIQUE INDEX MEDIA_TYPE_MTN_UN_IDX ON MEDIA_TYPE(MEDIA_TYPE_NAME); +CREATE TABLE LOGICAL_LIBRARY( + LOGICAL_LIBRARY_ID NUMERIC(20, 0) CONSTRAINT LOGICAL_LIBRARY_LLI_NN NOT NULL, + LOGICAL_LIBRARY_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_LLN_NN NOT NULL, + IS_DISABLED CHAR(1) DEFAULT '0' CONSTRAINT LOGICAL_LIBRARY_ID_NN NOT NULL, + DISABLED_REASON VARCHAR(1000) , + USER_COMMENT VARCHAR(1000) CONSTRAINT LOGICAL_LIBRARY_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT LOGICAL_LIBRARY_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT LOGICAL_LIBRARY_LUT_NN NOT NULL, + CONSTRAINT LOGICAL_LIBRARY_PK PRIMARY KEY(LOGICAL_LIBRARY_ID), + CONSTRAINT LOGICAL_LIBRARY_ID_BOOL_CK CHECK(IS_DISABLED IN ('0', '1')) +); +CREATE UNIQUE INDEX LOGICAL_LIBRARY_LLN_CI_UN_IDX ON LOGICAL_LIBRARY(LOWER(LOGICAL_LIBRARY_NAME)); +CREATE UNIQUE INDEX LOGICAL_LIBRARY_LLN_UN_IDX ON LOGICAL_LIBRARY(LOGICAL_LIBRARY_NAME); +CREATE TABLE TAPE( + VID VARCHAR(100) CONSTRAINT TAPE_V_NN NOT NULL, + MEDIA_TYPE_ID NUMERIC(20, 0) CONSTRAINT TAPE_MTID_NN NOT NULL, + VENDOR VARCHAR(100) CONSTRAINT TAPE_V2_NN NOT NULL, + LOGICAL_LIBRARY_ID NUMERIC(20, 0) CONSTRAINT TAPE_LLI_NN NOT NULL, + TAPE_POOL_ID NUMERIC(20, 0) CONSTRAINT TAPE_TPI_NN NOT NULL, + ENCRYPTION_KEY_NAME VARCHAR(100), + DATA_IN_BYTES NUMERIC(20, 0) CONSTRAINT TAPE_DIB_NN NOT NULL, + LAST_FSEQ NUMERIC(20, 0) CONSTRAINT TAPE_LF_NN NOT NULL, + NB_MASTER_FILES NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_NB_MASTER_FILES_NN NOT NULL, + MASTER_DATA_IN_BYTES NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_MASTER_DATA_IN_BYTES_NN NOT NULL, + IS_FULL CHAR(1) CONSTRAINT TAPE_IF_NN NOT NULL, + IS_FROM_CASTOR CHAR(1) CONSTRAINT TAPE_IFC_NN NOT NULL, + DIRTY CHAR(1) DEFAULT '1' CONSTRAINT TAPE_DIRTY_NN NOT NULL, + NB_COPY_NB_1 NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_NB_COPY_NB_1_NN NOT NULL, + COPY_NB_1_IN_BYTES NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_COPY_NB_1_IN_BYTES_NN NOT NULL, + NB_COPY_NB_GT_1 NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_NB_COPY_NB_GT_1_NN NOT NULL, + COPY_NB_GT_1_IN_BYTES NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_COPY_NB_GT_1_IN_BYTES_NN NOT NULL, + LABEL_FORMAT CHAR(1), + LABEL_DRIVE VARCHAR(100), + LABEL_TIME NUMERIC(20, 0), + LAST_READ_DRIVE VARCHAR(100), + LAST_READ_TIME NUMERIC(20, 0), + LAST_WRITE_DRIVE VARCHAR(100), + LAST_WRITE_TIME NUMERIC(20, 0), + READ_MOUNT_COUNT NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_RMC_NN NOT NULL, + WRITE_MOUNT_COUNT NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_WMC_NN NOT NULL, + USER_COMMENT VARCHAR(1000), + TAPE_STATE VARCHAR(100) CONSTRAINT TAPE_TS_NN NOT NULL, + STATE_REASON VARCHAR(1000), + STATE_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT TAPE_SUT_NN NOT NULL, + STATE_MODIFIED_BY VARCHAR(100) CONSTRAINT TAPE_SMB_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT TAPE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT TAPE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT TAPE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT TAPE_LUT_NN NOT NULL, + VERIFICATION_STATUS VARCHAR(1000), + CONSTRAINT TAPE_PK PRIMARY KEY(VID), + CONSTRAINT TAPE_LOGICAL_LIBRARY_FK FOREIGN KEY(LOGICAL_LIBRARY_ID) REFERENCES LOGICAL_LIBRARY(LOGICAL_LIBRARY_ID), + CONSTRAINT TAPE_TAPE_POOL_FK FOREIGN KEY(TAPE_POOL_ID) REFERENCES TAPE_POOL(TAPE_POOL_ID), + CONSTRAINT TAPE_IS_FULL_BOOL_CK CHECK(IS_FULL IN ('0', '1')), + CONSTRAINT TAPE_IS_FROM_CASTOR_BOOL_CK CHECK(IS_FROM_CASTOR IN ('0', '1')), + CONSTRAINT TAPE_DIRTY_BOOL_CK CHECK(DIRTY IN ('0','1')), + CONSTRAINT TAPE_STATE_CK CHECK(TAPE_STATE IN ('ACTIVE', 'REPACKING_PENDING', 'REPACKING', 'REPACKING_DISABLED', 'DISABLED', 'BROKEN_PENDING', 'BROKEN', 'EXPORTED_PENDING', 'EXPORTED')), + CONSTRAINT TAPE_MEDIA_TYPE_FK FOREIGN KEY(MEDIA_TYPE_ID) REFERENCES MEDIA_TYPE(MEDIA_TYPE_ID) +); +CREATE UNIQUE INDEX TAPE_VID_CI_UN_IDX ON TAPE(LOWER(VID)); +CREATE INDEX TAPE_TAPE_POOL_ID_IDX ON TAPE(TAPE_POOL_ID); +CREATE INDEX TAPE_STATE_IDX ON TAPE(TAPE_STATE); +CREATE INDEX TAPE_LLI_IDX ON TAPE(LOGICAL_LIBRARY_ID); +CREATE INDEX TAPE_MTI_IDX ON TAPE(MEDIA_TYPE_ID); +CREATE TABLE MOUNT_POLICY( + MOUNT_POLICY_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_MPN_NN NOT NULL, + ARCHIVE_PRIORITY NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_AP_NN NOT NULL, + ARCHIVE_MIN_REQUEST_AGE NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_AMRA_NN NOT NULL, + RETRIEVE_PRIORITY NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_RP_NN NOT NULL, + RETRIEVE_MIN_REQUEST_AGE NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_RMRA_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT MOUNT_POLICY_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_LUT_NN NOT NULL, + CONSTRAINT MOUNT_POLICY_PK PRIMARY KEY(MOUNT_POLICY_NAME) +); +CREATE UNIQUE INDEX MOUNT_POLICY_MPN_CI_UN_IDX ON MOUNT_POLICY(LOWER(MOUNT_POLICY_NAME)); +CREATE TABLE REQUESTER_ACTIVITY_MOUNT_RULE( + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_DIN_NN NOT NULL, + REQUESTER_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_RN_NN NOT NULL, + ACTIVITY_REGEX VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_AR_NN NOT NULL, + MOUNT_POLICY_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_MPN_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT RQSTER_ACT_RULE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_ACT_RULE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_ACT_RULE_LUT_NN NOT NULL, + CONSTRAINT RQSTER_ACT_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_NAME, ACTIVITY_REGEX), + CONSTRAINT RQSTER_ACT_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME), + CONSTRAINT RQSTER_ACT_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME) +); +CREATE INDEX REQ_ACT_MNT_RULE_MPN_IDX ON REQUESTER_ACTIVITY_MOUNT_RULE(MOUNT_POLICY_NAME); +CREATE INDEX REQ_ACT_MNT_RULE_DIN_IDX ON REQUESTER_ACTIVITY_MOUNT_RULE(DISK_INSTANCE_NAME); + +CREATE TABLE REQUESTER_MOUNT_RULE( + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_DIN_NN NOT NULL, + REQUESTER_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_RN_NN NOT NULL, + MOUNT_POLICY_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_MPN_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT RQSTER_RULE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_RULE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_RULE_LUT_NN NOT NULL, + CONSTRAINT RQSTER_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_NAME), + CONSTRAINT RQSTER_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME), + CONSTRAINT RQSTER_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME) +); +CREATE INDEX REQ_MNT_RULE_MPN_IDX ON REQUESTER_MOUNT_RULE(MOUNT_POLICY_NAME); +CREATE INDEX REQ_MNT_RULE_DIN_IDX ON REQUESTER_MOUNT_RULE(DISK_INSTANCE_NAME); + +CREATE TABLE REQUESTER_GROUP_MOUNT_RULE( + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_DIN_NN NOT NULL, + REQUESTER_GROUP_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_RGN_NN NOT NULL, + MOUNT_POLICY_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_MPN_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT RQSTER_GRP_RULE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_CLHN_NN NOT NULL, + CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_GRP_RULE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_GRP_RULE_LUT_NN NOT NULL, + CONSTRAINT RQSTER_GRP_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_GROUP_NAME), + CONSTRAINT RQSTER_GRP_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME), + CONSTRAINT RQSTER_GRP_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME) + +); +CREATE INDEX REQ_GRP_MNT_RULE_MPN_IDX ON REQUESTER_GROUP_MOUNT_RULE(MOUNT_POLICY_NAME); +CREATE INDEX REQ_GRP_MNT_RULE_DIN_IDX ON REQUESTER_GROUP_MOUNT_RULE(DISK_INSTANCE_NAME); + +CREATE TABLE ARCHIVE_FILE( + ARCHIVE_FILE_ID NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_AFI_NN NOT NULL, + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT ARCHIVE_FILE_DIN_NN NOT NULL, + DISK_FILE_ID VARCHAR(100) CONSTRAINT ARCHIVE_FILE_DFI_NN NOT NULL, + DISK_FILE_UID NUMERIC(10, 0) CONSTRAINT ARCHIVE_FILE_DFUID_NN NOT NULL, + DISK_FILE_GID NUMERIC(10, 0) CONSTRAINT ARCHIVE_FILE_DFGID_NN NOT NULL, + SIZE_IN_BYTES NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_SIB_NN NOT NULL, + CHECKSUM_BLOB BYTEA, + CHECKSUM_ADLER32 NUMERIC(10, 0) CONSTRAINT ARCHIVE_FILE_CB2_NN NOT NULL, + STORAGE_CLASS_ID NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_SCI_NN NOT NULL, + CREATION_TIME NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_CT2_NN NOT NULL, + RECONCILIATION_TIME NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_RT_NN NOT NULL, + IS_DELETED CHAR(1) DEFAULT '0' CONSTRAINT ARCHIVE_FILE_ID_NN NOT NULL, + COLLOCATION_HINT VARCHAR(100), + CONSTRAINT ARCHIVE_FILE_PK PRIMARY KEY(ARCHIVE_FILE_ID), + CONSTRAINT ARCHIVE_FILE_STORAGE_CLASS_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID), + CONSTRAINT ARCHIVE_FILE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME), + CONSTRAINT ARCHIVE_FILE_DIN_DFI_UN UNIQUE(DISK_INSTANCE_NAME, DISK_FILE_ID), + CONSTRAINT ARCHIVE_FILE_ID_BOOL_CK CHECK(IS_DELETED IN ('0', '1')) +); +CREATE INDEX ARCHIVE_FILE_DIN_IDX ON ARCHIVE_FILE(DISK_INSTANCE_NAME); +CREATE INDEX ARCHIVE_FILE_DFI_IDX ON ARCHIVE_FILE(DISK_FILE_ID); +CREATE INDEX ARCHIVE_FILE_SCI_IDX ON ARCHIVE_FILE(STORAGE_CLASS_ID); +CREATE TABLE TAPE_FILE( + VID VARCHAR(100) CONSTRAINT TAPE_FILE_V_NN NOT NULL, + FSEQ NUMERIC(20, 0) CONSTRAINT TAPE_FILE_F_NN NOT NULL, + BLOCK_ID NUMERIC(20, 0) CONSTRAINT TAPE_FILE_BI_NN NOT NULL, + LOGICAL_SIZE_IN_BYTES NUMERIC(20, 0) CONSTRAINT TAPE_FILE_CSIB_NN NOT NULL, + COPY_NB NUMERIC(3, 0) CONSTRAINT TAPE_FILE_CN_NN NOT NULL, + CREATION_TIME NUMERIC(20, 0) CONSTRAINT TAPE_FILE_CT_NN NOT NULL, + ARCHIVE_FILE_ID NUMERIC(20, 0) CONSTRAINT TAPE_FILE_AFI_NN NOT NULL, + CONSTRAINT TAPE_FILE_PK PRIMARY KEY(VID, FSEQ), + CONSTRAINT TAPE_FILE_TAPE_FK FOREIGN KEY(VID) + REFERENCES TAPE(VID), + CONSTRAINT TAPE_FILE_ARCHIVE_FILE_FK FOREIGN KEY(ARCHIVE_FILE_ID) + REFERENCES ARCHIVE_FILE(ARCHIVE_FILE_ID), + CONSTRAINT TAPE_FILE_VID_BLOCK_ID_UN UNIQUE(VID, BLOCK_ID), + CONSTRAINT TAPE_FILE_COPY_NB_GT_0_CK CHECK(COPY_NB > 0) +); +CREATE INDEX TAPE_FILE_VID_IDX ON TAPE_FILE(VID); +CREATE INDEX TAPE_FILE_ARCHIVE_FILE_ID_IDX ON TAPE_FILE(ARCHIVE_FILE_ID); +CREATE TABLE FILE_RECYCLE_LOG( + FILE_RECYCLE_LOG_ID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_ID_NN NOT NULL, + VID VARCHAR(100) CONSTRAINT FILE_RECYCLE_LOG_VID_NN NOT NULL, + FSEQ NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_FSEQ_NN NOT NULL, + BLOCK_ID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_BID_NN NOT NULL, + COPY_NB NUMERIC(3, 0) CONSTRAINT FILE_RECYCLE_LOG_COPY_NB_NN NOT NULL, + TAPE_FILE_CREATION_TIME NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_TFCT_NN NOT NULL, + ARCHIVE_FILE_ID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_AFI_NN NOT NULL, + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT FILE_RECYCLE_LOG_DIN_NN NOT NULL, + DISK_FILE_ID VARCHAR(100) CONSTRAINT FILE_RECYCLE_LOG_DFI_NN NOT NULL, + DISK_FILE_ID_WHEN_DELETED VARCHAR(100) CONSTRAINT FILE_RECYCLE_LOG_DFIWD_NN NOT NULL, + DISK_FILE_UID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_DFU_NN NOT NULL, + DISK_FILE_GID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_DFG_NN NOT NULL, + SIZE_IN_BYTES NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_SIB_NN NOT NULL, + CHECKSUM_BLOB BYTEA, + CHECKSUM_ADLER32 NUMERIC(10, 0) CONSTRAINT FILE_RECYCLE_LOG_CA_NN NOT NULL, + STORAGE_CLASS_ID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_SCI_NN NOT NULL, + ARCHIVE_FILE_CREATION_TIME NUMERIC(20, 0) CONSTRAINT FILE_RECYLE_LOG_CT_NN NOT NULL, + RECONCILIATION_TIME NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_RT_NN NOT NULL, + COLLOCATION_HINT VARCHAR(100), + DISK_FILE_PATH VARCHAR(2000), + REASON_LOG VARCHAR(1000) CONSTRAINT FILE_RECYCLE_LOG_RL_NN NOT NULL, + RECYCLE_LOG_TIME NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_RLT_NN NOT NULL, + CONSTRAINT FILE_RECYCLE_LOG_PK PRIMARY KEY(FILE_RECYCLE_LOG_ID), + CONSTRAINT FILE_RECYCLE_LOG_VID_FK FOREIGN KEY(VID) REFERENCES TAPE(VID), + CONSTRAINT FILE_RECYCLE_LOG_SC_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID) +); +CREATE INDEX FILE_RECYCLE_LOG_DFI_IDX ON FILE_RECYCLE_LOG(DISK_FILE_ID); +CREATE INDEX FILE_RECYCLE_LOG_SCD_IDX ON FILE_RECYCLE_LOG(STORAGE_CLASS_ID); +CREATE INDEX FILE_RECYCLE_LOG_VID_IDX ON FILE_RECYCLE_LOG(VID); +CREATE TABLE DRIVE_CONFIG ( + DRIVE_NAME VARCHAR(100) CONSTRAINT DRIVE_CONFIG_DN_NN NOT NULL, + CATEGORY VARCHAR(100) CONSTRAINT DRIVE_CONFIG_C_NN NOT NULL, + KEY_NAME VARCHAR(100) CONSTRAINT DRIVE_CONFIG_KN_NN NOT NULL, + VALUE VARCHAR(1000) CONSTRAINT DRIVE_CONFIG_V_NN NOT NULL, + SOURCE VARCHAR(100) CONSTRAINT DRIVE_CONFIG_S_NN NOT NULL, + CONSTRAINT DRIVE_CONFIG_DN_PK PRIMARY KEY(KEY_NAME, DRIVE_NAME) +); +CREATE TABLE DRIVE_STATE ( + DRIVE_NAME VARCHAR(100) CONSTRAINT DRIVE_DN_NN NOT NULL, + HOST VARCHAR(100) CONSTRAINT DRIVE_H_NN NOT NULL, + LOGICAL_LIBRARY VARCHAR(100) CONSTRAINT DRIVE_LL_NN NOT NULL, + SESSION_ID NUMERIC(20, 0), + BYTES_TRANSFERED_IN_SESSION NUMERIC(20, 0), + FILES_TRANSFERED_IN_SESSION NUMERIC(20, 0), + SESSION_START_TIME NUMERIC(20, 0), + SESSION_ELAPSED_TIME NUMERIC(20, 0), + MOUNT_START_TIME NUMERIC(20, 0), + TRANSFER_START_TIME NUMERIC(20, 0), + UNLOAD_START_TIME NUMERIC(20, 0), + UNMOUNT_START_TIME NUMERIC(20, 0), + DRAINING_START_TIME NUMERIC(20, 0), + DOWN_OR_UP_START_TIME NUMERIC(20, 0), + PROBE_START_TIME NUMERIC(20, 0), + CLEANUP_START_TIME NUMERIC(20, 0), + START_START_TIME NUMERIC(20, 0), + SHUTDOWN_TIME NUMERIC(20, 0), + MOUNT_TYPE VARCHAR(100) DEFAULT 'NO_MOUNT' CONSTRAINT DRIVE_MT_NN NOT NULL, + DRIVE_STATUS VARCHAR(100) DEFAULT 'UNKNOWN' CONSTRAINT DRIVE_DS_NN NOT NULL, + DESIRED_UP CHAR(1) DEFAULT '0' CONSTRAINT DRIVE_DU_NN NOT NULL, + DESIRED_FORCE_DOWN CHAR(1) DEFAULT '0' CONSTRAINT DRIVE_DFD_NN NOT NULL, + REASON_UP_DOWN VARCHAR(1000), + CURRENT_VID VARCHAR(100), + CTA_VERSION VARCHAR(100), + CURRENT_PRIORITY NUMERIC(20, 0), + CURRENT_ACTIVITY VARCHAR(100), + CURRENT_TAPE_POOL VARCHAR(100), + NEXT_MOUNT_TYPE VARCHAR(100) DEFAULT 'NO_MOUNT' CONSTRAINT DRIVE_NMT_NN NOT NULL, + NEXT_VID VARCHAR(100), + NEXT_PRIORITY NUMERIC(20, 0), + NEXT_ACTIVITY VARCHAR(100), + NEXT_TAPE_POOL VARCHAR(100), + DEV_FILE_NAME VARCHAR(100), + RAW_LIBRARY_SLOT VARCHAR(100), + CURRENT_VO VARCHAR(100), + NEXT_VO VARCHAR(100), + USER_COMMENT VARCHAR(1000), + CREATION_LOG_USER_NAME VARCHAR(100), + CREATION_LOG_HOST_NAME VARCHAR(100), + CREATION_LOG_TIME NUMERIC(20, 0), + LAST_UPDATE_USER_NAME VARCHAR(100), + LAST_UPDATE_HOST_NAME VARCHAR(100), + LAST_UPDATE_TIME NUMERIC(20, 0), + DISK_SYSTEM_NAME VARCHAR(100), + RESERVED_BYTES NUMERIC(20, 0), + RESERVATION_SESSION_ID NUMERIC(20, 0), + CONSTRAINT DRIVE_DN_PK PRIMARY KEY(DRIVE_NAME), + CONSTRAINT DRIVE_DU_BOOL_CK CHECK(DESIRED_UP IN ('0', '1')), + CONSTRAINT DRIVE_DFD_BOOL_CK CHECK(DESIRED_FORCE_DOWN IN ('0', '1')), + CONSTRAINT DRIVE_DS_STRING_CK CHECK(DRIVE_STATUS IN ('DOWN', 'UP', 'PROBING', 'STARTING', + 'MOUNTING', 'TRANSFERING', 'UNLOADING', 'UNMOUNTING', 'DRAININGTODISK', 'CLEANINGUP', 'SHUTDOWN', + 'UNKNOWN')), + CONSTRAINT DRIVE_MT_STRING_CK CHECK(MOUNT_TYPE IN ('NO_MOUNT', 'ARCHIVE_FOR_USER', + 'ARCHIVE_FOR_REPACK', 'RETRIEVE', 'LABEL', 'ARCHIVE_ALL_TYPES')), + CONSTRAINT DRIVE_NMT_STRING_CK CHECK(NEXT_MOUNT_TYPE IN ('NO_MOUNT', 'ARCHIVE_FOR_USER', + 'ARCHIVE_FOR_REPACK', 'RETRIEVE', 'LABEL', 'ARCHIVE_ALL_TYPES')) +); +CREATE UNIQUE INDEX DRIVE_STATE_DN_CI_UN_IDX ON DRIVE_STATE(LOWER(DRIVE_NAME)); +INSERT INTO CTA_CATALOGUE( + SCHEMA_VERSION_MAJOR, + SCHEMA_VERSION_MINOR, + STATUS) +VALUES( + 12, + 0, + 'PRODUCTION'); +ALTER TABLE ARCHIVE_FILE DROP CONSTRAINT + ARCHIVE_FILE_DIN_DFI_UN; +ALTER TABLE ARCHIVE_FILE ADD CONSTRAINT + ARCHIVE_FILE_DIN_DFI_UN UNIQUE(DISK_INSTANCE_NAME, DISK_FILE_ID) DEFERRABLE INITIALLY IMMEDIATE; +ALTER TABLE CTA_CATALOGUE ADD CONSTRAINT + CATALOGUE_STATUS_CONTENT_CK CHECK((NEXT_SCHEMA_VERSION_MAJOR IS NULL AND NEXT_SCHEMA_VERSION_MINOR IS NULL AND STATUS='PRODUCTION') OR (STATUS='UPGRADING')); diff --git a/catalogue/12.0/sqlite_catalogue_schema.sql b/catalogue/12.0/sqlite_catalogue_schema.sql new file mode 100644 index 0000000000..c0afd68d4e --- /dev/null +++ b/catalogue/12.0/sqlite_catalogue_schema.sql @@ -0,0 +1,481 @@ +CREATE TABLE ARCHIVE_FILE_ID( + ID INTEGER PRIMARY KEY AUTOINCREMENT +); +CREATE TABLE LOGICAL_LIBRARY_ID( + ID INTEGER PRIMARY KEY AUTOINCREMENT +); +CREATE TABLE MEDIA_TYPE_ID( + ID INTEGER PRIMARY KEY AUTOINCREMENT +); +CREATE TABLE STORAGE_CLASS_ID( + ID INTEGER PRIMARY KEY AUTOINCREMENT +); +CREATE TABLE TAPE_POOL_ID( + ID INTEGER PRIMARY KEY AUTOINCREMENT +); +CREATE TABLE VIRTUAL_ORGANIZATION_ID( + ID INTEGER PRIMARY KEY AUTOINCREMENT +); +CREATE TABLE FILE_RECYCLE_LOG_ID( + ID INTEGER PRIMARY KEY AUTOINCREMENT +); +CREATE TABLE CTA_CATALOGUE( + SCHEMA_VERSION_MAJOR INTEGER CONSTRAINT CTA_CATALOGUE_SVM1_NN NOT NULL, + SCHEMA_VERSION_MINOR INTEGER CONSTRAINT CTA_CATALOGUE_SVM2_NN NOT NULL, + NEXT_SCHEMA_VERSION_MAJOR INTEGER, + NEXT_SCHEMA_VERSION_MINOR INTEGER, + STATUS VARCHAR(100), + IS_PRODUCTION CHAR(1) DEFAULT '0' CONSTRAINT CTA_CATALOGUE_IP_NN NOT NULL, + CONSTRAINT CTA_CATALOGUE_IP_BOOL_CK CHECK(IS_PRODUCTION IN ('0','1')) +); +CREATE TABLE ADMIN_USER( + ADMIN_USER_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_AUN_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT ADMIN_USER_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_CLHN_NN NOT NULL, + CREATION_LOG_TIME INTEGER CONSTRAINT ADMIN_USER_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_LUHN_NN NOT NULL, + LAST_UPDATE_TIME INTEGER CONSTRAINT ADMIN_USER_LUT_NN NOT NULL, + CONSTRAINT ADMIN_USER_PK PRIMARY KEY(ADMIN_USER_NAME) +); +CREATE UNIQUE INDEX ADMIN_USER_AUN_CI_UN_IDX ON ADMIN_USER(ADMIN_USER_NAME); +CREATE TABLE DISK_INSTANCE( + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_DINM_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT DISK_INSTANCE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_CLHN_NN NOT NULL, + CREATION_LOG_TIME INTEGER CONSTRAINT DISK_INSTANCE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME INTEGER CONSTRAINT DISK_INSTANCE_LUT_NN NOT NULL, + CONSTRAINT DISK_INSTANCE_PK PRIMARY KEY(DISK_INSTANCE_NAME) +); +CREATE UNIQUE INDEX DISK_INSTANCE_DIN_CI_UN_IDX ON DISK_INSTANCE(DISK_INSTANCE_NAME); +CREATE TABLE DISK_INSTANCE_SPACE( + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_DINM_NN NOT NULL, + DISK_INSTANCE_SPACE_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_DISNM_NN NOT NULL, + FREE_SPACE_QUERY_URL VARCHAR(1000) CONSTRAINT DISK_INSTANCE_SPACE_FSQU_NN NOT NULL, + REFRESH_INTERVAL INTEGER CONSTRAINT DISK_INSTANCE_SPACE_RI_NN NOT NULL, + LAST_REFRESH_TIME INTEGER CONSTRAINT DISK_INSTANCE_SPACE_LRT_NN NOT NULL, + FREE_SPACE INTEGER CONSTRAINT DISK_INSTANCE_SPACE_TFS_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT DISK_INSTANCE_SPACE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_CLHN_NN NOT NULL, + CREATION_LOG_TIME INTEGER CONSTRAINT DISK_INSTANCE_SPACE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME INTEGER CONSTRAINT DISK_INSTANCE_SPACE_LUT_NN NOT NULL, + CONSTRAINT DISK_INSTANCE_SPACE_PK PRIMARY KEY(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME), + CONSTRAINT DISK_INSTANCE_SPACE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME) +); +CREATE UNIQUE INDEX DISK_INSTANCE_SPACE_DISN_CI_UN_IDX ON DISK_INSTANCE_SPACE(DISK_INSTANCE_SPACE_NAME); +CREATE UNIQUE INDEX DISK_INSTANCE_SPACE_DISN_UN_IDX ON DISK_INSTANCE_SPACE(DISK_INSTANCE_SPACE_NAME); + +CREATE TABLE DISK_SYSTEM( + DISK_SYSTEM_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_DSNM_NN NOT NULL, + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_DIN_NN NOT NULL, + DISK_INSTANCE_SPACE_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_DISN_NN NOT NULL, + FILE_REGEXP VARCHAR(100) CONSTRAINT DISK_SYSTEM_FR_NN NOT NULL, + TARGETED_FREE_SPACE INTEGER CONSTRAINT DISK_SYSTEM_TFS_NN NOT NULL, + SLEEP_TIME INTEGER CONSTRAINT DISK_SYSTEM_ST_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT DISK_SYSTEM_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_CLHN_NN NOT NULL, + CREATION_LOG_TIME INTEGER CONSTRAINT DISK_SYSTEM_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_LUHN_NN NOT NULL, + LAST_UPDATE_TIME INTEGER CONSTRAINT DISK_SYSTEM_LUT_NN NOT NULL, + CONSTRAINT NAME_PK PRIMARY KEY(DISK_SYSTEM_NAME), + CONSTRAINT DISK_SYSTEM_DIN_DISN_FK FOREIGN KEY(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME) REFERENCES DISK_INSTANCE_SPACE(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME) +); +CREATE UNIQUE INDEX DISK_SYSTEM_DSN_CI_UN_IDX ON DISK_SYSTEM(DISK_SYSTEM_NAME); +CREATE INDEX DISK_SYSTEM_DIN_DISN_IDX ON DISK_SYSTEM(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME); +CREATE TABLE VIRTUAL_ORGANIZATION( + VIRTUAL_ORGANIZATION_ID INTEGER CONSTRAINT VIRTUAL_ORGANIZATION_VOI_NN NOT NULL, + VIRTUAL_ORGANIZATION_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_VON_NN NOT NULL, + READ_MAX_DRIVES INTEGER CONSTRAINT VIRTUAL_ORGANIZATION_RMD_NN NOT NULL, + WRITE_MAX_DRIVES INTEGER CONSTRAINT VIRTUAL_ORGANIZATION_WMD_NN NOT NULL, + MAX_FILE_SIZE INTEGER CONSTRAINT VIRTUAL_ORGANIZATION_MFS_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT VIRTUAL_ORGANIZATION_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_CLHN_NN NOT NULL, + CREATION_LOG_TIME INTEGER CONSTRAINT VIRTUAL_ORGANIZATION_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_LUHN_NN NOT NULL, + LAST_UPDATE_TIME INTEGER CONSTRAINT VIRTUAL_ORGANIZATION_LUT_NN NOT NULL, + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_DIN_NN NOT NULL, + CONSTRAINT VIRTUAL_ORGANIZATION_PK PRIMARY KEY(VIRTUAL_ORGANIZATION_ID), + CONSTRAINT VIRTUAL_ORGANIZATION_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME) +); +CREATE UNIQUE INDEX VIRTUAL_ORG_VON_CI_UN_IDX ON VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_NAME); +CREATE UNIQUE INDEX VIRTUAL_ORG_VON_UN_IDX ON VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_NAME); +CREATE INDEX VIRTUAL_ORG_DIN_IDX ON VIRTUAL_ORGANIZATION(DISK_INSTANCE_NAME); + +CREATE TABLE STORAGE_CLASS( + STORAGE_CLASS_ID INTEGER CONSTRAINT STORAGE_CLASS_SCI_NN NOT NULL, + STORAGE_CLASS_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_SCN_NN NOT NULL, + NB_COPIES INTEGER CONSTRAINT STORAGE_CLASS_NC_NN NOT NULL, + VIRTUAL_ORGANIZATION_ID INTEGER CONSTRAINT STORAGE_CLASS_VOI_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT STORAGE_CLASS_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_CLHN_NN NOT NULL, + CREATION_LOG_TIME INTEGER CONSTRAINT STORAGE_CLASS_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_LUHN_NN NOT NULL, + LAST_UPDATE_TIME INTEGER CONSTRAINT STORAGE_CLASS_LUT_NN NOT NULL, + CONSTRAINT STORAGE_CLASS_PK PRIMARY KEY(STORAGE_CLASS_ID), + CONSTRAINT STORAGE_CLASS_VOI_FK FOREIGN KEY(VIRTUAL_ORGANIZATION_ID) REFERENCES VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_ID) +); +CREATE UNIQUE INDEX STORAGE_CLASS_SCN_CI_UN_IDX ON STORAGE_CLASS(STORAGE_CLASS_NAME); +CREATE UNIQUE INDEX STORAGE_CLASS_SCN_UN_IDX ON STORAGE_CLASS(STORAGE_CLASS_NAME); +CREATE INDEX STORAGE_CLASS_VOI_IDX ON STORAGE_CLASS(VIRTUAL_ORGANIZATION_ID); +CREATE TABLE TAPE_POOL( + TAPE_POOL_ID INTEGER CONSTRAINT TAPE_POOL_TPI_NN NOT NULL, + TAPE_POOL_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_TPN_NN NOT NULL, + VIRTUAL_ORGANIZATION_ID INTEGER CONSTRAINT TAPE_POOL_VOI_NN NOT NULL, + NB_PARTIAL_TAPES INTEGER CONSTRAINT TAPE_POOL_NPT_NN NOT NULL, + IS_ENCRYPTED CHAR(1) CONSTRAINT TAPE_POOL_IE_NN NOT NULL, + ENCRYPTION_KEY_NAME VARCHAR(100), + SUPPLY VARCHAR(100), + USER_COMMENT VARCHAR(1000) CONSTRAINT TAPE_POOL_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_CLHN_NN NOT NULL, + CREATION_LOG_TIME INTEGER CONSTRAINT TAPE_POOL_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_LUHN_NN NOT NULL, + LAST_UPDATE_TIME INTEGER CONSTRAINT TAPE_POOL_LUT_NN NOT NULL, + CONSTRAINT TAPE_POOL_PK PRIMARY KEY(TAPE_POOL_ID), + CONSTRAINT TAPE_POOL_IS_ENCRYPTED_BOOL_CK CHECK(IS_ENCRYPTED IN ('0', '1')), + CONSTRAINT TAPE_POOL_VO_FK FOREIGN KEY(VIRTUAL_ORGANIZATION_ID) REFERENCES VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_ID) +); +CREATE UNIQUE INDEX TAPE_POOL_TPN_CI_UN_IDX ON TAPE_POOL(TAPE_POOL_NAME); +CREATE UNIQUE INDEX TAPE_POOL_TPN_UN_IDX ON TAPE_POOL(TAPE_POOL_NAME); +CREATE INDEX TAPE_POOL_VOI_IDX ON TAPE_POOL(VIRTUAL_ORGANIZATION_ID); +CREATE TABLE ARCHIVE_ROUTE( + STORAGE_CLASS_ID INTEGER CONSTRAINT ARCHIVE_ROUTE_SCI_NN NOT NULL, + COPY_NB INTEGER CONSTRAINT ARCHIVE_ROUTE_CN_NN NOT NULL, + TAPE_POOL_ID INTEGER CONSTRAINT ARCHIVE_ROUTE_TPI_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT ARCHIVE_ROUTE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT ARCHIVE_ROUTE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT ARCHIVE_ROUTE_CLHN_NN NOT NULL, + CREATION_LOG_TIME INTEGER CONSTRAINT ARCHIVE_ROUTE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT ARCHIVE_ROUTE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT ARCHIVE_ROUTE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME INTEGER CONSTRAINT ARCHIVE_ROUTE_LUT_NN NOT NULL, + CONSTRAINT ARCHIVE_ROUTE_PK PRIMARY KEY(STORAGE_CLASS_ID, COPY_NB), + CONSTRAINT ARCHIVE_ROUTE_STORAGE_CLASS_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID), + CONSTRAINT ARCHIVE_ROUTE_TAPE_POOL_FK FOREIGN KEY(TAPE_POOL_ID) REFERENCES TAPE_POOL(TAPE_POOL_ID), + CONSTRAINT ARCHIVE_ROUTE_COPY_NB_GT_0_CK CHECK(COPY_NB > 0), + CONSTRAINT ARCHIVE_ROUTE_SCI_TPI_UN UNIQUE(STORAGE_CLASS_ID, TAPE_POOL_ID) +); +CREATE TABLE MEDIA_TYPE( + MEDIA_TYPE_ID INTEGER CONSTRAINT MEDIA_TYPE_MTI_NN NOT NULL, + MEDIA_TYPE_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_MTN_NN NOT NULL, + CARTRIDGE VARCHAR(100) CONSTRAINT MEDIA_TYPE_C_NN NOT NULL, + CAPACITY_IN_BYTES INTEGER CONSTRAINT MEDIA_TYPE_CIB_NN NOT NULL, + PRIMARY_DENSITY_CODE INTEGER, + SECONDARY_DENSITY_CODE INTEGER, + NB_WRAPS INTEGER, + MIN_LPOS INTEGER, + MAX_LPOS INTEGER, + USER_COMMENT VARCHAR(1000) CONSTRAINT MEDIA_TYPE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_CLHN_NN NOT NULL, + CREATION_LOG_TIME INTEGER CONSTRAINT MEDIA_TYPE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME INTEGER CONSTRAINT MEDIA_TYPE_LUT_NN NOT NULL, + CONSTRAINT MEDIA_TYPE_PK PRIMARY KEY(MEDIA_TYPE_ID) +); +CREATE UNIQUE INDEX MEDIA_TYPE_MTN_CI_UN_IDX ON MEDIA_TYPE(MEDIA_TYPE_NAME); +CREATE UNIQUE INDEX MEDIA_TYPE_MTN_UN_IDX ON MEDIA_TYPE(MEDIA_TYPE_NAME); +CREATE TABLE LOGICAL_LIBRARY( + LOGICAL_LIBRARY_ID INTEGER CONSTRAINT LOGICAL_LIBRARY_LLI_NN NOT NULL, + LOGICAL_LIBRARY_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_LLN_NN NOT NULL, + IS_DISABLED CHAR(1) DEFAULT '0' CONSTRAINT LOGICAL_LIBRARY_ID_NN NOT NULL, + DISABLED_REASON VARCHAR(1000) , + USER_COMMENT VARCHAR(1000) CONSTRAINT LOGICAL_LIBRARY_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_CLHN_NN NOT NULL, + CREATION_LOG_TIME INTEGER CONSTRAINT LOGICAL_LIBRARY_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_LUHN_NN NOT NULL, + LAST_UPDATE_TIME INTEGER CONSTRAINT LOGICAL_LIBRARY_LUT_NN NOT NULL, + CONSTRAINT LOGICAL_LIBRARY_PK PRIMARY KEY(LOGICAL_LIBRARY_ID), + CONSTRAINT LOGICAL_LIBRARY_ID_BOOL_CK CHECK(IS_DISABLED IN ('0', '1')) +); +CREATE UNIQUE INDEX LOGICAL_LIBRARY_LLN_CI_UN_IDX ON LOGICAL_LIBRARY(LOGICAL_LIBRARY_NAME); +CREATE UNIQUE INDEX LOGICAL_LIBRARY_LLN_UN_IDX ON LOGICAL_LIBRARY(LOGICAL_LIBRARY_NAME); +CREATE TABLE TAPE( + VID VARCHAR(100) CONSTRAINT TAPE_V_NN NOT NULL, + MEDIA_TYPE_ID INTEGER CONSTRAINT TAPE_MTID_NN NOT NULL, + VENDOR VARCHAR(100) CONSTRAINT TAPE_V2_NN NOT NULL, + LOGICAL_LIBRARY_ID INTEGER CONSTRAINT TAPE_LLI_NN NOT NULL, + TAPE_POOL_ID INTEGER CONSTRAINT TAPE_TPI_NN NOT NULL, + ENCRYPTION_KEY_NAME VARCHAR(100), + DATA_IN_BYTES INTEGER CONSTRAINT TAPE_DIB_NN NOT NULL, + LAST_FSEQ INTEGER CONSTRAINT TAPE_LF_NN NOT NULL, + NB_MASTER_FILES INTEGER DEFAULT 0 CONSTRAINT TAPE_NB_MASTER_FILES_NN NOT NULL, + MASTER_DATA_IN_BYTES INTEGER DEFAULT 0 CONSTRAINT TAPE_MASTER_DATA_IN_BYTES_NN NOT NULL, + IS_FULL CHAR(1) CONSTRAINT TAPE_IF_NN NOT NULL, + IS_FROM_CASTOR CHAR(1) CONSTRAINT TAPE_IFC_NN NOT NULL, + DIRTY CHAR(1) DEFAULT '1' CONSTRAINT TAPE_DIRTY_NN NOT NULL, + NB_COPY_NB_1 INTEGER DEFAULT 0 CONSTRAINT TAPE_NB_COPY_NB_1_NN NOT NULL, + COPY_NB_1_IN_BYTES INTEGER DEFAULT 0 CONSTRAINT TAPE_COPY_NB_1_IN_BYTES_NN NOT NULL, + NB_COPY_NB_GT_1 INTEGER DEFAULT 0 CONSTRAINT TAPE_NB_COPY_NB_GT_1_NN NOT NULL, + COPY_NB_GT_1_IN_BYTES INTEGER DEFAULT 0 CONSTRAINT TAPE_COPY_NB_GT_1_IN_BYTES_NN NOT NULL, + LABEL_FORMAT CHAR(1), + LABEL_DRIVE VARCHAR(100), + LABEL_TIME INTEGER, + LAST_READ_DRIVE VARCHAR(100), + LAST_READ_TIME INTEGER, + LAST_WRITE_DRIVE VARCHAR(100), + LAST_WRITE_TIME INTEGER, + READ_MOUNT_COUNT INTEGER DEFAULT 0 CONSTRAINT TAPE_RMC_NN NOT NULL, + WRITE_MOUNT_COUNT INTEGER DEFAULT 0 CONSTRAINT TAPE_WMC_NN NOT NULL, + USER_COMMENT VARCHAR(1000), + TAPE_STATE VARCHAR(100) CONSTRAINT TAPE_TS_NN NOT NULL, + STATE_REASON VARCHAR(1000), + STATE_UPDATE_TIME INTEGER CONSTRAINT TAPE_SUT_NN NOT NULL, + STATE_MODIFIED_BY VARCHAR(100) CONSTRAINT TAPE_SMB_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT TAPE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_CLHN_NN NOT NULL, + CREATION_LOG_TIME INTEGER CONSTRAINT TAPE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT TAPE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME INTEGER CONSTRAINT TAPE_LUT_NN NOT NULL, + VERIFICATION_STATUS VARCHAR(1000), + CONSTRAINT TAPE_PK PRIMARY KEY(VID), + CONSTRAINT TAPE_LOGICAL_LIBRARY_FK FOREIGN KEY(LOGICAL_LIBRARY_ID) REFERENCES LOGICAL_LIBRARY(LOGICAL_LIBRARY_ID), + CONSTRAINT TAPE_TAPE_POOL_FK FOREIGN KEY(TAPE_POOL_ID) REFERENCES TAPE_POOL(TAPE_POOL_ID), + CONSTRAINT TAPE_IS_FULL_BOOL_CK CHECK(IS_FULL IN ('0', '1')), + CONSTRAINT TAPE_IS_FROM_CASTOR_BOOL_CK CHECK(IS_FROM_CASTOR IN ('0', '1')), + CONSTRAINT TAPE_DIRTY_BOOL_CK CHECK(DIRTY IN ('0','1')), + CONSTRAINT TAPE_STATE_CK CHECK(TAPE_STATE IN ('ACTIVE', 'REPACKING_PENDING', 'REPACKING', 'REPACKING_DISABLED', 'DISABLED', 'BROKEN_PENDING', 'BROKEN', 'EXPORTED_PENDING', 'EXPORTED')), + CONSTRAINT TAPE_MEDIA_TYPE_FK FOREIGN KEY(MEDIA_TYPE_ID) REFERENCES MEDIA_TYPE(MEDIA_TYPE_ID) +); +CREATE UNIQUE INDEX TAPE_VID_CI_UN_IDX ON TAPE(VID); +CREATE INDEX TAPE_TAPE_POOL_ID_IDX ON TAPE(TAPE_POOL_ID); +CREATE INDEX TAPE_STATE_IDX ON TAPE(TAPE_STATE); +CREATE INDEX TAPE_LLI_IDX ON TAPE(LOGICAL_LIBRARY_ID); +CREATE INDEX TAPE_MTI_IDX ON TAPE(MEDIA_TYPE_ID); +CREATE TABLE MOUNT_POLICY( + MOUNT_POLICY_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_MPN_NN NOT NULL, + ARCHIVE_PRIORITY INTEGER CONSTRAINT MOUNT_POLICY_AP_NN NOT NULL, + ARCHIVE_MIN_REQUEST_AGE INTEGER CONSTRAINT MOUNT_POLICY_AMRA_NN NOT NULL, + RETRIEVE_PRIORITY INTEGER CONSTRAINT MOUNT_POLICY_RP_NN NOT NULL, + RETRIEVE_MIN_REQUEST_AGE INTEGER CONSTRAINT MOUNT_POLICY_RMRA_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT MOUNT_POLICY_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_CLHN_NN NOT NULL, + CREATION_LOG_TIME INTEGER CONSTRAINT MOUNT_POLICY_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_LUHN_NN NOT NULL, + LAST_UPDATE_TIME INTEGER CONSTRAINT MOUNT_POLICY_LUT_NN NOT NULL, + CONSTRAINT MOUNT_POLICY_PK PRIMARY KEY(MOUNT_POLICY_NAME) +); +CREATE UNIQUE INDEX MOUNT_POLICY_MPN_CI_UN_IDX ON MOUNT_POLICY(MOUNT_POLICY_NAME); +CREATE TABLE REQUESTER_ACTIVITY_MOUNT_RULE( + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_DIN_NN NOT NULL, + REQUESTER_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_RN_NN NOT NULL, + ACTIVITY_REGEX VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_AR_NN NOT NULL, + MOUNT_POLICY_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_MPN_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT RQSTER_ACT_RULE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_CLHN_NN NOT NULL, + CREATION_LOG_TIME INTEGER CONSTRAINT RQSTER_ACT_RULE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME INTEGER CONSTRAINT RQSTER_ACT_RULE_LUT_NN NOT NULL, + CONSTRAINT RQSTER_ACT_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_NAME, ACTIVITY_REGEX), + CONSTRAINT RQSTER_ACT_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME), + CONSTRAINT RQSTER_ACT_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME) +); +CREATE INDEX REQ_ACT_MNT_RULE_MPN_IDX ON REQUESTER_ACTIVITY_MOUNT_RULE(MOUNT_POLICY_NAME); +CREATE INDEX REQ_ACT_MNT_RULE_DIN_IDX ON REQUESTER_ACTIVITY_MOUNT_RULE(DISK_INSTANCE_NAME); + +CREATE TABLE REQUESTER_MOUNT_RULE( + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_DIN_NN NOT NULL, + REQUESTER_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_RN_NN NOT NULL, + MOUNT_POLICY_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_MPN_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT RQSTER_RULE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_CLHN_NN NOT NULL, + CREATION_LOG_TIME INTEGER CONSTRAINT RQSTER_RULE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME INTEGER CONSTRAINT RQSTER_RULE_LUT_NN NOT NULL, + CONSTRAINT RQSTER_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_NAME), + CONSTRAINT RQSTER_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME), + CONSTRAINT RQSTER_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME) +); +CREATE INDEX REQ_MNT_RULE_MPN_IDX ON REQUESTER_MOUNT_RULE(MOUNT_POLICY_NAME); +CREATE INDEX REQ_MNT_RULE_DIN_IDX ON REQUESTER_MOUNT_RULE(DISK_INSTANCE_NAME); + +CREATE TABLE REQUESTER_GROUP_MOUNT_RULE( + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_DIN_NN NOT NULL, + REQUESTER_GROUP_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_RGN_NN NOT NULL, + MOUNT_POLICY_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_MPN_NN NOT NULL, + USER_COMMENT VARCHAR(1000) CONSTRAINT RQSTER_GRP_RULE_UC_NN NOT NULL, + CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_CLUN_NN NOT NULL, + CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_CLHN_NN NOT NULL, + CREATION_LOG_TIME INTEGER CONSTRAINT RQSTER_GRP_RULE_CLT_NN NOT NULL, + LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_LUUN_NN NOT NULL, + LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_LUHN_NN NOT NULL, + LAST_UPDATE_TIME INTEGER CONSTRAINT RQSTER_GRP_RULE_LUT_NN NOT NULL, + CONSTRAINT RQSTER_GRP_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_GROUP_NAME), + CONSTRAINT RQSTER_GRP_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME), + CONSTRAINT RQSTER_GRP_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME) + +); +CREATE INDEX REQ_GRP_MNT_RULE_MPN_IDX ON REQUESTER_GROUP_MOUNT_RULE(MOUNT_POLICY_NAME); +CREATE INDEX REQ_GRP_MNT_RULE_DIN_IDX ON REQUESTER_GROUP_MOUNT_RULE(DISK_INSTANCE_NAME); + +CREATE TABLE ARCHIVE_FILE( + ARCHIVE_FILE_ID INTEGER CONSTRAINT ARCHIVE_FILE_AFI_NN NOT NULL, + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT ARCHIVE_FILE_DIN_NN NOT NULL, + DISK_FILE_ID VARCHAR(100) CONSTRAINT ARCHIVE_FILE_DFI_NN NOT NULL, + DISK_FILE_UID INTEGER CONSTRAINT ARCHIVE_FILE_DFUID_NN NOT NULL, + DISK_FILE_GID INTEGER CONSTRAINT ARCHIVE_FILE_DFGID_NN NOT NULL, + SIZE_IN_BYTES INTEGER CONSTRAINT ARCHIVE_FILE_SIB_NN NOT NULL, + CHECKSUM_BLOB BLOB(200), + CHECKSUM_ADLER32 INTEGER CONSTRAINT ARCHIVE_FILE_CB2_NN NOT NULL, + STORAGE_CLASS_ID INTEGER CONSTRAINT ARCHIVE_FILE_SCI_NN NOT NULL, + CREATION_TIME INTEGER CONSTRAINT ARCHIVE_FILE_CT2_NN NOT NULL, + RECONCILIATION_TIME INTEGER CONSTRAINT ARCHIVE_FILE_RT_NN NOT NULL, + IS_DELETED CHAR(1) DEFAULT '0' CONSTRAINT ARCHIVE_FILE_ID_NN NOT NULL, + COLLOCATION_HINT VARCHAR(100), + CONSTRAINT ARCHIVE_FILE_PK PRIMARY KEY(ARCHIVE_FILE_ID), + CONSTRAINT ARCHIVE_FILE_STORAGE_CLASS_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID), + CONSTRAINT ARCHIVE_FILE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME), + CONSTRAINT ARCHIVE_FILE_DIN_DFI_UN UNIQUE(DISK_INSTANCE_NAME, DISK_FILE_ID), + CONSTRAINT ARCHIVE_FILE_ID_BOOL_CK CHECK(IS_DELETED IN ('0', '1')) +); +CREATE INDEX ARCHIVE_FILE_DIN_IDX ON ARCHIVE_FILE(DISK_INSTANCE_NAME); +CREATE INDEX ARCHIVE_FILE_DFI_IDX ON ARCHIVE_FILE(DISK_FILE_ID); +CREATE INDEX ARCHIVE_FILE_SCI_IDX ON ARCHIVE_FILE(STORAGE_CLASS_ID); +CREATE TABLE TAPE_FILE( + VID VARCHAR(100) CONSTRAINT TAPE_FILE_V_NN NOT NULL, + FSEQ INTEGER CONSTRAINT TAPE_FILE_F_NN NOT NULL, + BLOCK_ID INTEGER CONSTRAINT TAPE_FILE_BI_NN NOT NULL, + LOGICAL_SIZE_IN_BYTES INTEGER CONSTRAINT TAPE_FILE_CSIB_NN NOT NULL, + COPY_NB INTEGER CONSTRAINT TAPE_FILE_CN_NN NOT NULL, + CREATION_TIME INTEGER CONSTRAINT TAPE_FILE_CT_NN NOT NULL, + ARCHIVE_FILE_ID INTEGER CONSTRAINT TAPE_FILE_AFI_NN NOT NULL, + CONSTRAINT TAPE_FILE_PK PRIMARY KEY(VID, FSEQ), + CONSTRAINT TAPE_FILE_TAPE_FK FOREIGN KEY(VID) + REFERENCES TAPE(VID), + CONSTRAINT TAPE_FILE_ARCHIVE_FILE_FK FOREIGN KEY(ARCHIVE_FILE_ID) + REFERENCES ARCHIVE_FILE(ARCHIVE_FILE_ID), + CONSTRAINT TAPE_FILE_VID_BLOCK_ID_UN UNIQUE(VID, BLOCK_ID), + CONSTRAINT TAPE_FILE_COPY_NB_GT_0_CK CHECK(COPY_NB > 0) +); +CREATE INDEX TAPE_FILE_VID_IDX ON TAPE_FILE(VID); +CREATE INDEX TAPE_FILE_ARCHIVE_FILE_ID_IDX ON TAPE_FILE(ARCHIVE_FILE_ID); +CREATE TABLE FILE_RECYCLE_LOG( + FILE_RECYCLE_LOG_ID INTEGER CONSTRAINT FILE_RECYCLE_LOG_ID_NN NOT NULL, + VID VARCHAR(100) CONSTRAINT FILE_RECYCLE_LOG_VID_NN NOT NULL, + FSEQ INTEGER CONSTRAINT FILE_RECYCLE_LOG_FSEQ_NN NOT NULL, + BLOCK_ID INTEGER CONSTRAINT FILE_RECYCLE_LOG_BID_NN NOT NULL, + COPY_NB INTEGER CONSTRAINT FILE_RECYCLE_LOG_COPY_NB_NN NOT NULL, + TAPE_FILE_CREATION_TIME INTEGER CONSTRAINT FILE_RECYCLE_LOG_TFCT_NN NOT NULL, + ARCHIVE_FILE_ID INTEGER CONSTRAINT FILE_RECYCLE_LOG_AFI_NN NOT NULL, + DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT FILE_RECYCLE_LOG_DIN_NN NOT NULL, + DISK_FILE_ID VARCHAR(100) CONSTRAINT FILE_RECYCLE_LOG_DFI_NN NOT NULL, + DISK_FILE_ID_WHEN_DELETED VARCHAR(100) CONSTRAINT FILE_RECYCLE_LOG_DFIWD_NN NOT NULL, + DISK_FILE_UID INTEGER CONSTRAINT FILE_RECYCLE_LOG_DFU_NN NOT NULL, + DISK_FILE_GID INTEGER CONSTRAINT FILE_RECYCLE_LOG_DFG_NN NOT NULL, + SIZE_IN_BYTES INTEGER CONSTRAINT FILE_RECYCLE_LOG_SIB_NN NOT NULL, + CHECKSUM_BLOB BLOB(200), + CHECKSUM_ADLER32 INTEGER CONSTRAINT FILE_RECYCLE_LOG_CA_NN NOT NULL, + STORAGE_CLASS_ID INTEGER CONSTRAINT FILE_RECYCLE_LOG_SCI_NN NOT NULL, + ARCHIVE_FILE_CREATION_TIME INTEGER CONSTRAINT FILE_RECYLE_LOG_CT_NN NOT NULL, + RECONCILIATION_TIME INTEGER CONSTRAINT FILE_RECYCLE_LOG_RT_NN NOT NULL, + COLLOCATION_HINT VARCHAR(100), + DISK_FILE_PATH VARCHAR(2000), + REASON_LOG VARCHAR(1000) CONSTRAINT FILE_RECYCLE_LOG_RL_NN NOT NULL, + RECYCLE_LOG_TIME INTEGER CONSTRAINT FILE_RECYCLE_LOG_RLT_NN NOT NULL, + CONSTRAINT FILE_RECYCLE_LOG_PK PRIMARY KEY(FILE_RECYCLE_LOG_ID), + CONSTRAINT FILE_RECYCLE_LOG_VID_FK FOREIGN KEY(VID) REFERENCES TAPE(VID), + CONSTRAINT FILE_RECYCLE_LOG_SC_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID) +); +CREATE INDEX FILE_RECYCLE_LOG_DFI_IDX ON FILE_RECYCLE_LOG(DISK_FILE_ID); +CREATE INDEX FILE_RECYCLE_LOG_SCD_IDX ON FILE_RECYCLE_LOG(STORAGE_CLASS_ID); +CREATE INDEX FILE_RECYCLE_LOG_VID_IDX ON FILE_RECYCLE_LOG(VID); +CREATE TABLE DRIVE_CONFIG ( + DRIVE_NAME VARCHAR(100) CONSTRAINT DRIVE_CONFIG_DN_NN NOT NULL, + CATEGORY VARCHAR(100) CONSTRAINT DRIVE_CONFIG_C_NN NOT NULL, + KEY_NAME VARCHAR(100) CONSTRAINT DRIVE_CONFIG_KN_NN NOT NULL, + VALUE VARCHAR(1000) CONSTRAINT DRIVE_CONFIG_V_NN NOT NULL, + SOURCE VARCHAR(100) CONSTRAINT DRIVE_CONFIG_S_NN NOT NULL, + CONSTRAINT DRIVE_CONFIG_DN_PK PRIMARY KEY(KEY_NAME, DRIVE_NAME) +); +CREATE TABLE DRIVE_STATE ( + DRIVE_NAME VARCHAR(100) CONSTRAINT DRIVE_DN_NN NOT NULL, + HOST VARCHAR(100) CONSTRAINT DRIVE_H_NN NOT NULL, + LOGICAL_LIBRARY VARCHAR(100) CONSTRAINT DRIVE_LL_NN NOT NULL, + SESSION_ID INTEGER, + BYTES_TRANSFERED_IN_SESSION INTEGER, + FILES_TRANSFERED_IN_SESSION INTEGER, + SESSION_START_TIME INTEGER, + SESSION_ELAPSED_TIME INTEGER, + MOUNT_START_TIME INTEGER, + TRANSFER_START_TIME INTEGER, + UNLOAD_START_TIME INTEGER, + UNMOUNT_START_TIME INTEGER, + DRAINING_START_TIME INTEGER, + DOWN_OR_UP_START_TIME INTEGER, + PROBE_START_TIME INTEGER, + CLEANUP_START_TIME INTEGER, + START_START_TIME INTEGER, + SHUTDOWN_TIME INTEGER, + MOUNT_TYPE VARCHAR(100) DEFAULT 'NO_MOUNT' CONSTRAINT DRIVE_MT_NN NOT NULL, + DRIVE_STATUS VARCHAR(100) DEFAULT 'UNKNOWN' CONSTRAINT DRIVE_DS_NN NOT NULL, + DESIRED_UP CHAR(1) DEFAULT '0' CONSTRAINT DRIVE_DU_NN NOT NULL, + DESIRED_FORCE_DOWN CHAR(1) DEFAULT '0' CONSTRAINT DRIVE_DFD_NN NOT NULL, + REASON_UP_DOWN VARCHAR(1000), + CURRENT_VID VARCHAR(100), + CTA_VERSION VARCHAR(100), + CURRENT_PRIORITY INTEGER, + CURRENT_ACTIVITY VARCHAR(100), + CURRENT_TAPE_POOL VARCHAR(100), + NEXT_MOUNT_TYPE VARCHAR(100) DEFAULT 'NO_MOUNT' CONSTRAINT DRIVE_NMT_NN NOT NULL, + NEXT_VID VARCHAR(100), + NEXT_PRIORITY INTEGER, + NEXT_ACTIVITY VARCHAR(100), + NEXT_TAPE_POOL VARCHAR(100), + DEV_FILE_NAME VARCHAR(100), + RAW_LIBRARY_SLOT VARCHAR(100), + CURRENT_VO VARCHAR(100), + NEXT_VO VARCHAR(100), + USER_COMMENT VARCHAR(1000), + CREATION_LOG_USER_NAME VARCHAR(100), + CREATION_LOG_HOST_NAME VARCHAR(100), + CREATION_LOG_TIME INTEGER, + LAST_UPDATE_USER_NAME VARCHAR(100), + LAST_UPDATE_HOST_NAME VARCHAR(100), + LAST_UPDATE_TIME INTEGER, + DISK_SYSTEM_NAME VARCHAR(100), + RESERVED_BYTES INTEGER, + RESERVATION_SESSION_ID INTEGER, + CONSTRAINT DRIVE_DN_PK PRIMARY KEY(DRIVE_NAME), + CONSTRAINT DRIVE_DU_BOOL_CK CHECK(DESIRED_UP IN ('0', '1')), + CONSTRAINT DRIVE_DFD_BOOL_CK CHECK(DESIRED_FORCE_DOWN IN ('0', '1')), + CONSTRAINT DRIVE_DS_STRING_CK CHECK(DRIVE_STATUS IN ('DOWN', 'UP', 'PROBING', 'STARTING', + 'MOUNTING', 'TRANSFERING', 'UNLOADING', 'UNMOUNTING', 'DRAININGTODISK', 'CLEANINGUP', 'SHUTDOWN', + 'UNKNOWN')), + CONSTRAINT DRIVE_MT_STRING_CK CHECK(MOUNT_TYPE IN ('NO_MOUNT', 'ARCHIVE_FOR_USER', + 'ARCHIVE_FOR_REPACK', 'RETRIEVE', 'LABEL', 'ARCHIVE_ALL_TYPES')), + CONSTRAINT DRIVE_NMT_STRING_CK CHECK(NEXT_MOUNT_TYPE IN ('NO_MOUNT', 'ARCHIVE_FOR_USER', + 'ARCHIVE_FOR_REPACK', 'RETRIEVE', 'LABEL', 'ARCHIVE_ALL_TYPES')) +); +CREATE UNIQUE INDEX DRIVE_STATE_DN_CI_UN_IDX ON DRIVE_STATE(DRIVE_NAME); +INSERT INTO CTA_CATALOGUE( + SCHEMA_VERSION_MAJOR, + SCHEMA_VERSION_MINOR, + STATUS) +VALUES( + 12, + 0, + 'PRODUCTION'); diff --git a/catalogue/AllCatalogueSchema.hpp b/catalogue/AllCatalogueSchema.hpp index 157ed215ab..f0d9a1458a 100644 --- a/catalogue/AllCatalogueSchema.hpp +++ b/catalogue/AllCatalogueSchema.hpp @@ -4893,6 +4893,1538 @@ namespace catalogue{ "ALTER TABLE ARCHIVE_FILE ADD CONSTRAINT" " ARCHIVE_FILE_DIN_DFI_UN UNIQUE(DISK_INSTANCE_NAME, DISK_FILE_ID) DEFERRABLE INITIALLY IMMEDIATE;" "ALTER TABLE CTA_CATALOGUE ADD CONSTRAINT" + " CATALOGUE_STATUS_CONTENT_CK CHECK((NEXT_SCHEMA_VERSION_MAJOR IS NULL AND NEXT_SCHEMA_VERSION_MINOR IS NULL AND STATUS='PRODUCTION') OR (STATUS='UPGRADING'));" + }, + } }, + {"12.0", + { + {"oracle", "CREATE SEQUENCE ARCHIVE_FILE_ID_SEQ" + " INCREMENT BY 1" + " START WITH 4294967296" + " NOMAXVALUE" + " MINVALUE 1" + " NOCYCLE" + " CACHE 20" + " NOORDER;" + "CREATE SEQUENCE LOGICAL_LIBRARY_ID_SEQ" + " INCREMENT BY 1" + " START WITH 1" + " NOMAXVALUE" + " MINVALUE 1" + " NOCYCLE" + " CACHE 20" + " NOORDER;" + "CREATE SEQUENCE MEDIA_TYPE_ID_SEQ" + " INCREMENT BY 1" + " START WITH 1" + " NOMAXVALUE" + " MINVALUE 1" + " NOCYCLE" + " CACHE 20" + " NOORDER;" + "CREATE SEQUENCE STORAGE_CLASS_ID_SEQ" + " INCREMENT BY 1" + " START WITH 10000" + " NOMAXVALUE" + " MINVALUE 1" + " NOCYCLE" + " CACHE 20" + " NOORDER;" + "CREATE SEQUENCE TAPE_POOL_ID_SEQ" + " INCREMENT BY 1" + " START WITH 1" + " NOMAXVALUE" + " MINVALUE 1" + " NOCYCLE" + " CACHE 20" + " NOORDER;" + "CREATE SEQUENCE VIRTUAL_ORGANIZATION_ID_SEQ" + " INCREMENT BY 1" + " START WITH 1" + " NOMAXVALUE" + " MINVALUE 1" + " NOCYCLE" + " CACHE 20" + " NOORDER;" + "CREATE SEQUENCE FILE_RECYCLE_LOG_ID_SEQ" + " INCREMENT BY 1" + " START WITH 1" + " NOMAXVALUE" + " MINVALUE 1" + " NOCYCLE" + " CACHE 20" + " NOORDER;" + "CREATE GLOBAL TEMPORARY TABLE TEMP_TAPE_FILE_INSERTION_BATCH(" + " VID VARCHAR2(100)," + " FSEQ NUMERIC(20, 0) ," + " BLOCK_ID NUMERIC(20, 0) ," + " LOGICAL_SIZE_IN_BYTES NUMERIC(20, 0) ," + " COPY_NB NUMERIC(3, 0) ," + " CREATION_TIME NUMERIC(20, 0) ," + " ARCHIVE_FILE_ID NUMERIC(20, 0)" + ")" + "ON COMMIT DELETE ROWS;" + "CREATE INDEX TEMP_T_F_I_B_AFI_IDX ON TEMP_TAPE_FILE_INSERTION_BATCH(ARCHIVE_FILE_ID);" + "CREATE TABLE CTA_CATALOGUE(" + " SCHEMA_VERSION_MAJOR NUMERIC(20, 0) CONSTRAINT CTA_CATALOGUE_SVM1_NN NOT NULL," + " SCHEMA_VERSION_MINOR NUMERIC(20, 0) CONSTRAINT CTA_CATALOGUE_SVM2_NN NOT NULL," + " NEXT_SCHEMA_VERSION_MAJOR NUMERIC(20, 0)," + " NEXT_SCHEMA_VERSION_MINOR NUMERIC(20, 0)," + " STATUS VARCHAR2(100)," + " IS_PRODUCTION CHAR(1) DEFAULT '0' CONSTRAINT CTA_CATALOGUE_IP_NN NOT NULL," + " CONSTRAINT CTA_CATALOGUE_IP_BOOL_CK CHECK(IS_PRODUCTION IN ('0','1'))" + ");" + "CREATE TABLE ADMIN_USER(" + " ADMIN_USER_NAME VARCHAR2(100) CONSTRAINT ADMIN_USER_AUN_NN NOT NULL," + " USER_COMMENT VARCHAR2(1000) CONSTRAINT ADMIN_USER_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT ADMIN_USER_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT ADMIN_USER_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT ADMIN_USER_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT ADMIN_USER_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT ADMIN_USER_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT ADMIN_USER_LUT_NN NOT NULL," + " CONSTRAINT ADMIN_USER_PK PRIMARY KEY(ADMIN_USER_NAME)" + ");" + "CREATE UNIQUE INDEX ADMIN_USER_AUN_CI_UN_IDX ON ADMIN_USER(LOWER(ADMIN_USER_NAME));" + "CREATE TABLE DISK_INSTANCE(" + " DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_DINM_NN NOT NULL," + " USER_COMMENT VARCHAR2(1000) CONSTRAINT DISK_INSTANCE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_LUT_NN NOT NULL," + " CONSTRAINT DISK_INSTANCE_PK PRIMARY KEY(DISK_INSTANCE_NAME)" + ");" + "CREATE UNIQUE INDEX DISK_INSTANCE_DIN_CI_UN_IDX ON DISK_INSTANCE(LOWER(DISK_INSTANCE_NAME));" + "CREATE TABLE DISK_INSTANCE_SPACE(" + " DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_SPACE_DINM_NN NOT NULL," + " DISK_INSTANCE_SPACE_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_SPACE_DISNM_NN NOT NULL," + " FREE_SPACE_QUERY_URL VARCHAR2(1000) CONSTRAINT DISK_INSTANCE_SPACE_FSQU_NN NOT NULL," + " REFRESH_INTERVAL NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_RI_NN NOT NULL," + " LAST_REFRESH_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_LRT_NN NOT NULL," + " FREE_SPACE NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_TFS_NN NOT NULL," + " USER_COMMENT VARCHAR2(1000) CONSTRAINT DISK_INSTANCE_SPACE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_SPACE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_SPACE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_SPACE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT DISK_INSTANCE_SPACE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_LUT_NN NOT NULL," + " CONSTRAINT DISK_INSTANCE_SPACE_PK PRIMARY KEY(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME)," + " CONSTRAINT DISK_INSTANCE_SPACE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)" + ");" + "CREATE UNIQUE INDEX DISK_INSTANCE_SPACE_DISN_CI_UN_IDX ON DISK_INSTANCE_SPACE(LOWER(DISK_INSTANCE_SPACE_NAME));" + "CREATE UNIQUE INDEX DISK_INSTANCE_SPACE_DISN_UN_IDX ON DISK_INSTANCE_SPACE(DISK_INSTANCE_SPACE_NAME);" + "" + "CREATE TABLE DISK_SYSTEM(" + " DISK_SYSTEM_NAME VARCHAR2(100) CONSTRAINT DISK_SYSTEM_DSNM_NN NOT NULL," + " DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT DISK_SYSTEM_DIN_NN NOT NULL, " + " DISK_INSTANCE_SPACE_NAME VARCHAR2(100) CONSTRAINT DISK_SYSTEM_DISN_NN NOT NULL, " + " FILE_REGEXP VARCHAR2(100) CONSTRAINT DISK_SYSTEM_FR_NN NOT NULL," + " TARGETED_FREE_SPACE NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_TFS_NN NOT NULL," + " SLEEP_TIME NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_ST_NN NOT NULL," + " USER_COMMENT VARCHAR2(1000) CONSTRAINT DISK_SYSTEM_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT DISK_SYSTEM_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT DISK_SYSTEM_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT DISK_SYSTEM_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT DISK_SYSTEM_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_LUT_NN NOT NULL," + " CONSTRAINT NAME_PK PRIMARY KEY(DISK_SYSTEM_NAME)," + " CONSTRAINT DISK_SYSTEM_DIN_DISN_FK FOREIGN KEY(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME) REFERENCES DISK_INSTANCE_SPACE(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME)" + ");" + "CREATE UNIQUE INDEX DISK_SYSTEM_DSN_CI_UN_IDX ON DISK_SYSTEM(LOWER(DISK_SYSTEM_NAME));" + "CREATE INDEX DISK_SYSTEM_DIN_DISN_IDX ON DISK_SYSTEM(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME);" + "CREATE TABLE VIRTUAL_ORGANIZATION(" + " VIRTUAL_ORGANIZATION_ID NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_VOI_NN NOT NULL," + " VIRTUAL_ORGANIZATION_NAME VARCHAR2(100) CONSTRAINT VIRTUAL_ORGANIZATION_VON_NN NOT NULL," + " READ_MAX_DRIVES NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_RMD_NN NOT NULL," + " WRITE_MAX_DRIVES NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_WMD_NN NOT NULL," + " MAX_FILE_SIZE NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_MFS_NN NOT NULL," + " USER_COMMENT VARCHAR2(1000) CONSTRAINT VIRTUAL_ORGANIZATION_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT VIRTUAL_ORGANIZATION_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT VIRTUAL_ORGANIZATION_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT VIRTUAL_ORGANIZATION_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT VIRTUAL_ORGANIZATION_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_LUT_NN NOT NULL," + " DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT VIRTUAL_ORGANIZATION_DIN_NN NOT NULL," + " CONSTRAINT VIRTUAL_ORGANIZATION_PK PRIMARY KEY(VIRTUAL_ORGANIZATION_ID)," + " CONSTRAINT VIRTUAL_ORGANIZATION_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)" + ");" + "CREATE UNIQUE INDEX VIRTUAL_ORG_VON_CI_UN_IDX ON VIRTUAL_ORGANIZATION(LOWER(VIRTUAL_ORGANIZATION_NAME));" + "CREATE UNIQUE INDEX VIRTUAL_ORG_VON_UN_IDX ON VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_NAME);" + "CREATE INDEX VIRTUAL_ORG_DIN_IDX ON VIRTUAL_ORGANIZATION(DISK_INSTANCE_NAME);" + "" + "CREATE TABLE STORAGE_CLASS(" + " STORAGE_CLASS_ID NUMERIC(20, 0) CONSTRAINT STORAGE_CLASS_SCI_NN NOT NULL," + " STORAGE_CLASS_NAME VARCHAR2(100) CONSTRAINT STORAGE_CLASS_SCN_NN NOT NULL," + " NB_COPIES NUMERIC(3, 0) CONSTRAINT STORAGE_CLASS_NC_NN NOT NULL," + " VIRTUAL_ORGANIZATION_ID NUMERIC(20, 0) CONSTRAINT STORAGE_CLASS_VOI_NN NOT NULL," + " USER_COMMENT VARCHAR2(1000) CONSTRAINT STORAGE_CLASS_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT STORAGE_CLASS_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT STORAGE_CLASS_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT STORAGE_CLASS_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT STORAGE_CLASS_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT STORAGE_CLASS_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT STORAGE_CLASS_LUT_NN NOT NULL," + " CONSTRAINT STORAGE_CLASS_PK PRIMARY KEY(STORAGE_CLASS_ID)," + " CONSTRAINT STORAGE_CLASS_VOI_FK FOREIGN KEY(VIRTUAL_ORGANIZATION_ID) REFERENCES VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_ID)" + ");" + "CREATE UNIQUE INDEX STORAGE_CLASS_SCN_CI_UN_IDX ON STORAGE_CLASS(LOWER(STORAGE_CLASS_NAME));" + "CREATE UNIQUE INDEX STORAGE_CLASS_SCN_UN_IDX ON STORAGE_CLASS(STORAGE_CLASS_NAME);" + "CREATE INDEX STORAGE_CLASS_VOI_IDX ON STORAGE_CLASS(VIRTUAL_ORGANIZATION_ID);" + "CREATE TABLE TAPE_POOL(" + " TAPE_POOL_ID NUMERIC(20, 0) CONSTRAINT TAPE_POOL_TPI_NN NOT NULL," + " TAPE_POOL_NAME VARCHAR2(100) CONSTRAINT TAPE_POOL_TPN_NN NOT NULL," + " VIRTUAL_ORGANIZATION_ID NUMERIC(20, 0) CONSTRAINT TAPE_POOL_VOI_NN NOT NULL," + " NB_PARTIAL_TAPES NUMERIC(20, 0) CONSTRAINT TAPE_POOL_NPT_NN NOT NULL," + " IS_ENCRYPTED CHAR(1) CONSTRAINT TAPE_POOL_IE_NN NOT NULL," + " ENCRYPTION_KEY_NAME VARCHAR2(100)," + " SUPPLY VARCHAR2(100)," + " USER_COMMENT VARCHAR2(1000) CONSTRAINT TAPE_POOL_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT TAPE_POOL_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT TAPE_POOL_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT TAPE_POOL_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT TAPE_POOL_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT TAPE_POOL_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT TAPE_POOL_LUT_NN NOT NULL," + " CONSTRAINT TAPE_POOL_PK PRIMARY KEY(TAPE_POOL_ID)," + " CONSTRAINT TAPE_POOL_IS_ENCRYPTED_BOOL_CK CHECK(IS_ENCRYPTED IN ('0', '1'))," + " CONSTRAINT TAPE_POOL_VO_FK FOREIGN KEY(VIRTUAL_ORGANIZATION_ID) REFERENCES VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_ID)" + ");" + "CREATE UNIQUE INDEX TAPE_POOL_TPN_CI_UN_IDX ON TAPE_POOL(LOWER(TAPE_POOL_NAME));" + "CREATE UNIQUE INDEX TAPE_POOL_TPN_UN_IDX ON TAPE_POOL(TAPE_POOL_NAME);" + "CREATE INDEX TAPE_POOL_VOI_IDX ON TAPE_POOL(VIRTUAL_ORGANIZATION_ID);" + "CREATE TABLE ARCHIVE_ROUTE(" + " STORAGE_CLASS_ID NUMERIC(20, 0) CONSTRAINT ARCHIVE_ROUTE_SCI_NN NOT NULL," + " COPY_NB NUMERIC(3, 0) CONSTRAINT ARCHIVE_ROUTE_CN_NN NOT NULL," + " TAPE_POOL_ID NUMERIC(20, 0) CONSTRAINT ARCHIVE_ROUTE_TPI_NN NOT NULL," + " USER_COMMENT VARCHAR2(1000) CONSTRAINT ARCHIVE_ROUTE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT ARCHIVE_ROUTE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT ARCHIVE_ROUTE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT ARCHIVE_ROUTE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT ARCHIVE_ROUTE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT ARCHIVE_ROUTE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT ARCHIVE_ROUTE_LUT_NN NOT NULL," + " CONSTRAINT ARCHIVE_ROUTE_PK PRIMARY KEY(STORAGE_CLASS_ID, COPY_NB)," + " CONSTRAINT ARCHIVE_ROUTE_STORAGE_CLASS_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID)," + " CONSTRAINT ARCHIVE_ROUTE_TAPE_POOL_FK FOREIGN KEY(TAPE_POOL_ID) REFERENCES TAPE_POOL(TAPE_POOL_ID)," + " CONSTRAINT ARCHIVE_ROUTE_COPY_NB_GT_0_CK CHECK(COPY_NB > 0)," + " CONSTRAINT ARCHIVE_ROUTE_SCI_TPI_UN UNIQUE(STORAGE_CLASS_ID, TAPE_POOL_ID)" + ");" + "CREATE TABLE MEDIA_TYPE(" + " MEDIA_TYPE_ID NUMERIC(20, 0) CONSTRAINT MEDIA_TYPE_MTI_NN NOT NULL," + " MEDIA_TYPE_NAME VARCHAR2(100) CONSTRAINT MEDIA_TYPE_MTN_NN NOT NULL," + " CARTRIDGE VARCHAR2(100) CONSTRAINT MEDIA_TYPE_C_NN NOT NULL," + " CAPACITY_IN_BYTES NUMERIC(20, 0) CONSTRAINT MEDIA_TYPE_CIB_NN NOT NULL," + " PRIMARY_DENSITY_CODE NUMERIC(3, 0)," + " SECONDARY_DENSITY_CODE NUMERIC(3, 0)," + " NB_WRAPS NUMERIC(10, 0)," + " MIN_LPOS NUMERIC(20, 0)," + " MAX_LPOS NUMERIC(20, 0)," + " USER_COMMENT VARCHAR2(1000) CONSTRAINT MEDIA_TYPE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT MEDIA_TYPE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT MEDIA_TYPE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT MEDIA_TYPE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT MEDIA_TYPE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT MEDIA_TYPE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT MEDIA_TYPE_LUT_NN NOT NULL," + " CONSTRAINT MEDIA_TYPE_PK PRIMARY KEY(MEDIA_TYPE_ID)" + ");" + "CREATE UNIQUE INDEX MEDIA_TYPE_MTN_CI_UN_IDX ON MEDIA_TYPE(LOWER(MEDIA_TYPE_NAME));" + "CREATE UNIQUE INDEX MEDIA_TYPE_MTN_UN_IDX ON MEDIA_TYPE(MEDIA_TYPE_NAME);" + "CREATE TABLE LOGICAL_LIBRARY(" + " LOGICAL_LIBRARY_ID NUMERIC(20, 0) CONSTRAINT LOGICAL_LIBRARY_LLI_NN NOT NULL," + " LOGICAL_LIBRARY_NAME VARCHAR2(100) CONSTRAINT LOGICAL_LIBRARY_LLN_NN NOT NULL," + " IS_DISABLED CHAR(1) DEFAULT '0' CONSTRAINT LOGICAL_LIBRARY_ID_NN NOT NULL," + " DISABLED_REASON VARCHAR2(1000) ," + " USER_COMMENT VARCHAR2(1000) CONSTRAINT LOGICAL_LIBRARY_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT LOGICAL_LIBRARY_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT LOGICAL_LIBRARY_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT LOGICAL_LIBRARY_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT LOGICAL_LIBRARY_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT LOGICAL_LIBRARY_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT LOGICAL_LIBRARY_LUT_NN NOT NULL," + " CONSTRAINT LOGICAL_LIBRARY_PK PRIMARY KEY(LOGICAL_LIBRARY_ID)," + " CONSTRAINT LOGICAL_LIBRARY_ID_BOOL_CK CHECK(IS_DISABLED IN ('0', '1'))" + ");" + "CREATE UNIQUE INDEX LOGICAL_LIBRARY_LLN_CI_UN_IDX ON LOGICAL_LIBRARY(LOWER(LOGICAL_LIBRARY_NAME));" + "CREATE UNIQUE INDEX LOGICAL_LIBRARY_LLN_UN_IDX ON LOGICAL_LIBRARY(LOGICAL_LIBRARY_NAME);" + "CREATE TABLE TAPE(" + " VID VARCHAR2(100) CONSTRAINT TAPE_V_NN NOT NULL," + " MEDIA_TYPE_ID NUMERIC(20, 0) CONSTRAINT TAPE_MTID_NN NOT NULL," + " VENDOR VARCHAR2(100) CONSTRAINT TAPE_V2_NN NOT NULL," + " LOGICAL_LIBRARY_ID NUMERIC(20, 0) CONSTRAINT TAPE_LLI_NN NOT NULL," + " TAPE_POOL_ID NUMERIC(20, 0) CONSTRAINT TAPE_TPI_NN NOT NULL," + " ENCRYPTION_KEY_NAME VARCHAR2(100)," + " DATA_IN_BYTES NUMERIC(20, 0) CONSTRAINT TAPE_DIB_NN NOT NULL," + " LAST_FSEQ NUMERIC(20, 0) CONSTRAINT TAPE_LF_NN NOT NULL," + " NB_MASTER_FILES NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_NB_MASTER_FILES_NN NOT NULL," + " MASTER_DATA_IN_BYTES NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_MASTER_DATA_IN_BYTES_NN NOT NULL," + " IS_FULL CHAR(1) CONSTRAINT TAPE_IF_NN NOT NULL," + " IS_FROM_CASTOR CHAR(1) CONSTRAINT TAPE_IFC_NN NOT NULL," + " DIRTY CHAR(1) DEFAULT '1' CONSTRAINT TAPE_DIRTY_NN NOT NULL," + " NB_COPY_NB_1 NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_NB_COPY_NB_1_NN NOT NULL," + " COPY_NB_1_IN_BYTES NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_COPY_NB_1_IN_BYTES_NN NOT NULL," + " NB_COPY_NB_GT_1 NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_NB_COPY_NB_GT_1_NN NOT NULL," + " COPY_NB_GT_1_IN_BYTES NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_COPY_NB_GT_1_IN_BYTES_NN NOT NULL," + " LABEL_FORMAT CHAR(1)," + " LABEL_DRIVE VARCHAR2(100)," + " LABEL_TIME NUMERIC(20, 0)," + " LAST_READ_DRIVE VARCHAR2(100)," + " LAST_READ_TIME NUMERIC(20, 0)," + " LAST_WRITE_DRIVE VARCHAR2(100)," + " LAST_WRITE_TIME NUMERIC(20, 0)," + " READ_MOUNT_COUNT NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_RMC_NN NOT NULL," + " WRITE_MOUNT_COUNT NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_WMC_NN NOT NULL," + " USER_COMMENT VARCHAR2(1000)," + " TAPE_STATE VARCHAR2(100) CONSTRAINT TAPE_TS_NN NOT NULL," + " STATE_REASON VARCHAR2(1000)," + " STATE_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT TAPE_SUT_NN NOT NULL," + " STATE_MODIFIED_BY VARCHAR2(100) CONSTRAINT TAPE_SMB_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT TAPE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT TAPE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT TAPE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT TAPE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT TAPE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT TAPE_LUT_NN NOT NULL," + " VERIFICATION_STATUS VARCHAR2(1000)," + " CONSTRAINT TAPE_PK PRIMARY KEY(VID)," + " CONSTRAINT TAPE_LOGICAL_LIBRARY_FK FOREIGN KEY(LOGICAL_LIBRARY_ID) REFERENCES LOGICAL_LIBRARY(LOGICAL_LIBRARY_ID)," + " CONSTRAINT TAPE_TAPE_POOL_FK FOREIGN KEY(TAPE_POOL_ID) REFERENCES TAPE_POOL(TAPE_POOL_ID)," + " CONSTRAINT TAPE_IS_FULL_BOOL_CK CHECK(IS_FULL IN ('0', '1'))," + " CONSTRAINT TAPE_IS_FROM_CASTOR_BOOL_CK CHECK(IS_FROM_CASTOR IN ('0', '1'))," + " CONSTRAINT TAPE_DIRTY_BOOL_CK CHECK(DIRTY IN ('0','1'))," + " CONSTRAINT TAPE_STATE_CK CHECK(TAPE_STATE IN ('ACTIVE', 'REPACKING_PENDING', 'REPACKING', 'REPACKING_DISABLED', 'DISABLED', 'BROKEN_PENDING', 'BROKEN', 'EXPORTED_PENDING', 'EXPORTED'))," + " CONSTRAINT TAPE_MEDIA_TYPE_FK FOREIGN KEY(MEDIA_TYPE_ID) REFERENCES MEDIA_TYPE(MEDIA_TYPE_ID)" + ");" + "CREATE UNIQUE INDEX TAPE_VID_CI_UN_IDX ON TAPE(LOWER(VID));" + "CREATE INDEX TAPE_TAPE_POOL_ID_IDX ON TAPE(TAPE_POOL_ID);" + "CREATE INDEX TAPE_STATE_IDX ON TAPE(TAPE_STATE);" + "CREATE INDEX TAPE_LLI_IDX ON TAPE(LOGICAL_LIBRARY_ID);" + "CREATE INDEX TAPE_MTI_IDX ON TAPE(MEDIA_TYPE_ID);" + "CREATE TABLE MOUNT_POLICY(" + " MOUNT_POLICY_NAME VARCHAR2(100) CONSTRAINT MOUNT_POLICY_MPN_NN NOT NULL," + " ARCHIVE_PRIORITY NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_AP_NN NOT NULL," + " ARCHIVE_MIN_REQUEST_AGE NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_AMRA_NN NOT NULL," + " RETRIEVE_PRIORITY NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_RP_NN NOT NULL," + " RETRIEVE_MIN_REQUEST_AGE NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_RMRA_NN NOT NULL," + " USER_COMMENT VARCHAR2(1000) CONSTRAINT MOUNT_POLICY_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT MOUNT_POLICY_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT MOUNT_POLICY_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT MOUNT_POLICY_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT MOUNT_POLICY_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_LUT_NN NOT NULL," + " CONSTRAINT MOUNT_POLICY_PK PRIMARY KEY(MOUNT_POLICY_NAME)" + ");" + "CREATE UNIQUE INDEX MOUNT_POLICY_MPN_CI_UN_IDX ON MOUNT_POLICY(LOWER(MOUNT_POLICY_NAME));" + "CREATE TABLE REQUESTER_ACTIVITY_MOUNT_RULE(" + " DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT RQSTER_ACT_RULE_DIN_NN NOT NULL," + " REQUESTER_NAME VARCHAR2(100) CONSTRAINT RQSTER_ACT_RULE_RN_NN NOT NULL," + " ACTIVITY_REGEX VARCHAR2(100) CONSTRAINT RQSTER_ACT_RULE_AR_NN NOT NULL," + " MOUNT_POLICY_NAME VARCHAR2(100) CONSTRAINT RQSTER_ACT_RULE_MPN_NN NOT NULL," + " USER_COMMENT VARCHAR2(1000) CONSTRAINT RQSTER_ACT_RULE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT RQSTER_ACT_RULE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT RQSTER_ACT_RULE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_ACT_RULE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT RQSTER_ACT_RULE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT RQSTER_ACT_RULE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_ACT_RULE_LUT_NN NOT NULL," + " CONSTRAINT RQSTER_ACT_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_NAME, ACTIVITY_REGEX)," + " CONSTRAINT RQSTER_ACT_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME)," + " CONSTRAINT RQSTER_ACT_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)" + ");" + "CREATE INDEX REQ_ACT_MNT_RULE_MPN_IDX ON REQUESTER_ACTIVITY_MOUNT_RULE(MOUNT_POLICY_NAME);" + "CREATE INDEX REQ_ACT_MNT_RULE_DIN_IDX ON REQUESTER_ACTIVITY_MOUNT_RULE(DISK_INSTANCE_NAME);" + "" + "CREATE TABLE REQUESTER_MOUNT_RULE(" + " DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT RQSTER_RULE_DIN_NN NOT NULL," + " REQUESTER_NAME VARCHAR2(100) CONSTRAINT RQSTER_RULE_RN_NN NOT NULL," + " MOUNT_POLICY_NAME VARCHAR2(100) CONSTRAINT RQSTER_RULE_MPN_NN NOT NULL," + " USER_COMMENT VARCHAR2(1000) CONSTRAINT RQSTER_RULE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT RQSTER_RULE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT RQSTER_RULE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_RULE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT RQSTER_RULE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT RQSTER_RULE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_RULE_LUT_NN NOT NULL," + " CONSTRAINT RQSTER_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_NAME)," + " CONSTRAINT RQSTER_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME)," + " CONSTRAINT RQSTER_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)" + ");" + "CREATE INDEX REQ_MNT_RULE_MPN_IDX ON REQUESTER_MOUNT_RULE(MOUNT_POLICY_NAME);" + "CREATE INDEX REQ_MNT_RULE_DIN_IDX ON REQUESTER_MOUNT_RULE(DISK_INSTANCE_NAME);" + "" + "CREATE TABLE REQUESTER_GROUP_MOUNT_RULE(" + " DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT RQSTER_GRP_RULE_DIN_NN NOT NULL," + " REQUESTER_GROUP_NAME VARCHAR2(100) CONSTRAINT RQSTER_GRP_RULE_RGN_NN NOT NULL," + " MOUNT_POLICY_NAME VARCHAR2(100) CONSTRAINT RQSTER_GRP_RULE_MPN_NN NOT NULL," + " USER_COMMENT VARCHAR2(1000) CONSTRAINT RQSTER_GRP_RULE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR2(100) CONSTRAINT RQSTER_GRP_RULE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR2(100) CONSTRAINT RQSTER_GRP_RULE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_GRP_RULE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR2(100) CONSTRAINT RQSTER_GRP_RULE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR2(100) CONSTRAINT RQSTER_GRP_RULE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_GRP_RULE_LUT_NN NOT NULL," + " CONSTRAINT RQSTER_GRP_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_GROUP_NAME)," + " CONSTRAINT RQSTER_GRP_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME)," + " CONSTRAINT RQSTER_GRP_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)" + "" + ");" + "CREATE INDEX REQ_GRP_MNT_RULE_MPN_IDX ON REQUESTER_GROUP_MOUNT_RULE(MOUNT_POLICY_NAME);" + "CREATE INDEX REQ_GRP_MNT_RULE_DIN_IDX ON REQUESTER_GROUP_MOUNT_RULE(DISK_INSTANCE_NAME);" + "" + "CREATE TABLE ARCHIVE_FILE(" + " ARCHIVE_FILE_ID NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_AFI_NN NOT NULL," + " DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT ARCHIVE_FILE_DIN_NN NOT NULL," + " DISK_FILE_ID VARCHAR2(100) CONSTRAINT ARCHIVE_FILE_DFI_NN NOT NULL," + " DISK_FILE_UID NUMERIC(10, 0) CONSTRAINT ARCHIVE_FILE_DFUID_NN NOT NULL," + " DISK_FILE_GID NUMERIC(10, 0) CONSTRAINT ARCHIVE_FILE_DFGID_NN NOT NULL," + " SIZE_IN_BYTES NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_SIB_NN NOT NULL," + " CHECKSUM_BLOB RAW(200)," + " CHECKSUM_ADLER32 NUMERIC(10, 0) CONSTRAINT ARCHIVE_FILE_CB2_NN NOT NULL," + " STORAGE_CLASS_ID NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_SCI_NN NOT NULL," + " CREATION_TIME NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_CT2_NN NOT NULL," + " RECONCILIATION_TIME NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_RT_NN NOT NULL," + " IS_DELETED CHAR(1) DEFAULT '0' CONSTRAINT ARCHIVE_FILE_ID_NN NOT NULL," + " COLLOCATION_HINT VARCHAR2(100)," + " CONSTRAINT ARCHIVE_FILE_PK PRIMARY KEY(ARCHIVE_FILE_ID)," + " CONSTRAINT ARCHIVE_FILE_STORAGE_CLASS_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID)," + " CONSTRAINT ARCHIVE_FILE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)," + " CONSTRAINT ARCHIVE_FILE_DIN_DFI_UN UNIQUE(DISK_INSTANCE_NAME, DISK_FILE_ID)," + " CONSTRAINT ARCHIVE_FILE_ID_BOOL_CK CHECK(IS_DELETED IN ('0', '1'))" + ");" + "CREATE INDEX ARCHIVE_FILE_DIN_IDX ON ARCHIVE_FILE(DISK_INSTANCE_NAME);" + "CREATE INDEX ARCHIVE_FILE_DFI_IDX ON ARCHIVE_FILE(DISK_FILE_ID);" + "CREATE INDEX ARCHIVE_FILE_SCI_IDX ON ARCHIVE_FILE(STORAGE_CLASS_ID);" + "CREATE TABLE TAPE_FILE(" + " VID VARCHAR2(100) CONSTRAINT TAPE_FILE_V_NN NOT NULL," + " FSEQ NUMERIC(20, 0) CONSTRAINT TAPE_FILE_F_NN NOT NULL," + " BLOCK_ID NUMERIC(20, 0) CONSTRAINT TAPE_FILE_BI_NN NOT NULL," + " LOGICAL_SIZE_IN_BYTES NUMERIC(20, 0) CONSTRAINT TAPE_FILE_CSIB_NN NOT NULL," + " COPY_NB NUMERIC(3, 0) CONSTRAINT TAPE_FILE_CN_NN NOT NULL," + " CREATION_TIME NUMERIC(20, 0) CONSTRAINT TAPE_FILE_CT_NN NOT NULL," + " ARCHIVE_FILE_ID NUMERIC(20, 0) CONSTRAINT TAPE_FILE_AFI_NN NOT NULL," + " CONSTRAINT TAPE_FILE_PK PRIMARY KEY(VID, FSEQ)," + " CONSTRAINT TAPE_FILE_TAPE_FK FOREIGN KEY(VID)" + " REFERENCES TAPE(VID)," + " CONSTRAINT TAPE_FILE_ARCHIVE_FILE_FK FOREIGN KEY(ARCHIVE_FILE_ID)" + " REFERENCES ARCHIVE_FILE(ARCHIVE_FILE_ID)," + " CONSTRAINT TAPE_FILE_VID_BLOCK_ID_UN UNIQUE(VID, BLOCK_ID)," + " CONSTRAINT TAPE_FILE_COPY_NB_GT_0_CK CHECK(COPY_NB > 0)" + ");" + "CREATE INDEX TAPE_FILE_VID_IDX ON TAPE_FILE(VID);" + "CREATE INDEX TAPE_FILE_ARCHIVE_FILE_ID_IDX ON TAPE_FILE(ARCHIVE_FILE_ID);" + "CREATE TABLE FILE_RECYCLE_LOG(" + " FILE_RECYCLE_LOG_ID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_ID_NN NOT NULL," + " VID VARCHAR2(100) CONSTRAINT FILE_RECYCLE_LOG_VID_NN NOT NULL," + " FSEQ NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_FSEQ_NN NOT NULL," + " BLOCK_ID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_BID_NN NOT NULL," + " COPY_NB NUMERIC(3, 0) CONSTRAINT FILE_RECYCLE_LOG_COPY_NB_NN NOT NULL," + " TAPE_FILE_CREATION_TIME NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_TFCT_NN NOT NULL," + " ARCHIVE_FILE_ID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_AFI_NN NOT NULL," + " DISK_INSTANCE_NAME VARCHAR2(100) CONSTRAINT FILE_RECYCLE_LOG_DIN_NN NOT NULL," + " DISK_FILE_ID VARCHAR2(100) CONSTRAINT FILE_RECYCLE_LOG_DFI_NN NOT NULL," + " DISK_FILE_ID_WHEN_DELETED VARCHAR2(100) CONSTRAINT FILE_RECYCLE_LOG_DFIWD_NN NOT NULL," + " DISK_FILE_UID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_DFU_NN NOT NULL," + " DISK_FILE_GID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_DFG_NN NOT NULL," + " SIZE_IN_BYTES NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_SIB_NN NOT NULL," + " CHECKSUM_BLOB RAW(200)," + " CHECKSUM_ADLER32 NUMERIC(10, 0) CONSTRAINT FILE_RECYCLE_LOG_CA_NN NOT NULL," + " STORAGE_CLASS_ID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_SCI_NN NOT NULL," + " ARCHIVE_FILE_CREATION_TIME NUMERIC(20, 0) CONSTRAINT FILE_RECYLE_LOG_CT_NN NOT NULL," + " RECONCILIATION_TIME NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_RT_NN NOT NULL," + " COLLOCATION_HINT VARCHAR2(100)," + " DISK_FILE_PATH VARCHAR2(2000)," + " REASON_LOG VARCHAR2(1000) CONSTRAINT FILE_RECYCLE_LOG_RL_NN NOT NULL," + " RECYCLE_LOG_TIME NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_RLT_NN NOT NULL," + " CONSTRAINT FILE_RECYCLE_LOG_PK PRIMARY KEY(FILE_RECYCLE_LOG_ID)," + " CONSTRAINT FILE_RECYCLE_LOG_VID_FK FOREIGN KEY(VID) REFERENCES TAPE(VID)," + " CONSTRAINT FILE_RECYCLE_LOG_SC_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID)" + ");" + "CREATE INDEX FILE_RECYCLE_LOG_DFI_IDX ON FILE_RECYCLE_LOG(DISK_FILE_ID);" + "CREATE INDEX FILE_RECYCLE_LOG_SCD_IDX ON FILE_RECYCLE_LOG(STORAGE_CLASS_ID);" + "CREATE INDEX FILE_RECYCLE_LOG_VID_IDX ON FILE_RECYCLE_LOG(VID);" + "CREATE TABLE DRIVE_CONFIG (" + " DRIVE_NAME VARCHAR2(100) CONSTRAINT DRIVE_CONFIG_DN_NN NOT NULL," + " CATEGORY VARCHAR2(100) CONSTRAINT DRIVE_CONFIG_C_NN NOT NULL," + " KEY_NAME VARCHAR2(100) CONSTRAINT DRIVE_CONFIG_KN_NN NOT NULL," + " VALUE VARCHAR2(1000) CONSTRAINT DRIVE_CONFIG_V_NN NOT NULL," + " SOURCE VARCHAR2(100) CONSTRAINT DRIVE_CONFIG_S_NN NOT NULL," + " CONSTRAINT DRIVE_CONFIG_DN_PK PRIMARY KEY(KEY_NAME, DRIVE_NAME)" + ");" + "CREATE TABLE DRIVE_STATE (" + " DRIVE_NAME VARCHAR2(100) CONSTRAINT DRIVE_DN_NN NOT NULL," + " HOST VARCHAR2(100) CONSTRAINT DRIVE_H_NN NOT NULL," + " LOGICAL_LIBRARY VARCHAR2(100) CONSTRAINT DRIVE_LL_NN NOT NULL," + " SESSION_ID NUMERIC(20, 0)," + " BYTES_TRANSFERED_IN_SESSION NUMERIC(20, 0)," + " FILES_TRANSFERED_IN_SESSION NUMERIC(20, 0)," + " SESSION_START_TIME NUMERIC(20, 0)," + " SESSION_ELAPSED_TIME NUMERIC(20, 0)," + " MOUNT_START_TIME NUMERIC(20, 0)," + " TRANSFER_START_TIME NUMERIC(20, 0)," + " UNLOAD_START_TIME NUMERIC(20, 0)," + " UNMOUNT_START_TIME NUMERIC(20, 0)," + " DRAINING_START_TIME NUMERIC(20, 0)," + " DOWN_OR_UP_START_TIME NUMERIC(20, 0)," + " PROBE_START_TIME NUMERIC(20, 0)," + " CLEANUP_START_TIME NUMERIC(20, 0)," + " START_START_TIME NUMERIC(20, 0)," + " SHUTDOWN_TIME NUMERIC(20, 0)," + " MOUNT_TYPE VARCHAR2(100) DEFAULT 'NO_MOUNT' CONSTRAINT DRIVE_MT_NN NOT NULL," + " DRIVE_STATUS VARCHAR2(100) DEFAULT 'UNKNOWN' CONSTRAINT DRIVE_DS_NN NOT NULL," + " DESIRED_UP CHAR(1) DEFAULT '0' CONSTRAINT DRIVE_DU_NN NOT NULL," + " DESIRED_FORCE_DOWN CHAR(1) DEFAULT '0' CONSTRAINT DRIVE_DFD_NN NOT NULL," + " REASON_UP_DOWN VARCHAR2(1000)," + " CURRENT_VID VARCHAR2(100)," + " CTA_VERSION VARCHAR2(100)," + " CURRENT_PRIORITY NUMERIC(20, 0)," + " CURRENT_ACTIVITY VARCHAR2(100)," + " CURRENT_TAPE_POOL VARCHAR2(100)," + " NEXT_MOUNT_TYPE VARCHAR2(100) DEFAULT 'NO_MOUNT' CONSTRAINT DRIVE_NMT_NN NOT NULL," + " NEXT_VID VARCHAR2(100)," + " NEXT_PRIORITY NUMERIC(20, 0)," + " NEXT_ACTIVITY VARCHAR2(100)," + " NEXT_TAPE_POOL VARCHAR2(100)," + " DEV_FILE_NAME VARCHAR2(100)," + " RAW_LIBRARY_SLOT VARCHAR2(100)," + " CURRENT_VO VARCHAR2(100)," + " NEXT_VO VARCHAR2(100)," + " USER_COMMENT VARCHAR2(1000)," + " CREATION_LOG_USER_NAME VARCHAR2(100)," + " CREATION_LOG_HOST_NAME VARCHAR2(100)," + " CREATION_LOG_TIME NUMERIC(20, 0)," + " LAST_UPDATE_USER_NAME VARCHAR2(100)," + " LAST_UPDATE_HOST_NAME VARCHAR2(100)," + " LAST_UPDATE_TIME NUMERIC(20, 0)," + " DISK_SYSTEM_NAME VARCHAR2(100)," + " RESERVED_BYTES NUMERIC(20, 0)," + " RESERVATION_SESSION_ID NUMERIC(20, 0)," + " CONSTRAINT DRIVE_DN_PK PRIMARY KEY(DRIVE_NAME)," + " CONSTRAINT DRIVE_DU_BOOL_CK CHECK(DESIRED_UP IN ('0', '1'))," + " CONSTRAINT DRIVE_DFD_BOOL_CK CHECK(DESIRED_FORCE_DOWN IN ('0', '1'))," + " CONSTRAINT DRIVE_DS_STRING_CK CHECK(DRIVE_STATUS IN ('DOWN', 'UP', 'PROBING', 'STARTING'," + " 'MOUNTING', 'TRANSFERING', 'UNLOADING', 'UNMOUNTING', 'DRAININGTODISK', 'CLEANINGUP', 'SHUTDOWN'," + " 'UNKNOWN'))," + " CONSTRAINT DRIVE_MT_STRING_CK CHECK(MOUNT_TYPE IN ('NO_MOUNT', 'ARCHIVE_FOR_USER'," + " 'ARCHIVE_FOR_REPACK', 'RETRIEVE', 'LABEL', 'ARCHIVE_ALL_TYPES'))," + " CONSTRAINT DRIVE_NMT_STRING_CK CHECK(NEXT_MOUNT_TYPE IN ('NO_MOUNT', 'ARCHIVE_FOR_USER'," + " 'ARCHIVE_FOR_REPACK', 'RETRIEVE', 'LABEL', 'ARCHIVE_ALL_TYPES'))" + ");" + "CREATE UNIQUE INDEX DRIVE_STATE_DN_CI_UN_IDX ON DRIVE_STATE(LOWER(DRIVE_NAME));" + "INSERT INTO CTA_CATALOGUE(" + " SCHEMA_VERSION_MAJOR," + " SCHEMA_VERSION_MINOR," + " STATUS)" + "VALUES(" + " 12," + " 0," + " 'PRODUCTION');" + "ALTER TABLE CTA_CATALOGUE ADD CONSTRAINT CATALOGUE_STATUS_CONTENT_CK CHECK((NEXT_SCHEMA_VERSION_MAJOR IS NULL AND NEXT_SCHEMA_VERSION_MINOR IS NULL AND STATUS='PRODUCTION') OR (NEXT_SCHEMA_VERSION_MAJOR IS NOT NULL AND NEXT_SCHEMA_VERSION_MINOR IS NOT NULL AND STATUS='UPGRADING')) INITIALLY DEFERRED;" + "" + "COMMIT;" + }, + {"sqlite", "CREATE TABLE ARCHIVE_FILE_ID(" + " ID INTEGER PRIMARY KEY AUTOINCREMENT" + ");" + "CREATE TABLE LOGICAL_LIBRARY_ID(" + " ID INTEGER PRIMARY KEY AUTOINCREMENT" + ");" + "CREATE TABLE MEDIA_TYPE_ID(" + " ID INTEGER PRIMARY KEY AUTOINCREMENT" + ");" + "CREATE TABLE STORAGE_CLASS_ID(" + " ID INTEGER PRIMARY KEY AUTOINCREMENT" + ");" + "CREATE TABLE TAPE_POOL_ID(" + " ID INTEGER PRIMARY KEY AUTOINCREMENT" + ");" + "CREATE TABLE VIRTUAL_ORGANIZATION_ID(" + " ID INTEGER PRIMARY KEY AUTOINCREMENT" + ");" + "CREATE TABLE FILE_RECYCLE_LOG_ID(" + " ID INTEGER PRIMARY KEY AUTOINCREMENT" + ");" + "CREATE TABLE CTA_CATALOGUE(" + " SCHEMA_VERSION_MAJOR INTEGER CONSTRAINT CTA_CATALOGUE_SVM1_NN NOT NULL," + " SCHEMA_VERSION_MINOR INTEGER CONSTRAINT CTA_CATALOGUE_SVM2_NN NOT NULL," + " NEXT_SCHEMA_VERSION_MAJOR INTEGER," + " NEXT_SCHEMA_VERSION_MINOR INTEGER," + " STATUS VARCHAR(100)," + " IS_PRODUCTION CHAR(1) DEFAULT '0' CONSTRAINT CTA_CATALOGUE_IP_NN NOT NULL," + " CONSTRAINT CTA_CATALOGUE_IP_BOOL_CK CHECK(IS_PRODUCTION IN ('0','1'))" + ");" + "CREATE TABLE ADMIN_USER(" + " ADMIN_USER_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_AUN_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT ADMIN_USER_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_CLHN_NN NOT NULL," + " CREATION_LOG_TIME INTEGER CONSTRAINT ADMIN_USER_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME INTEGER CONSTRAINT ADMIN_USER_LUT_NN NOT NULL," + " CONSTRAINT ADMIN_USER_PK PRIMARY KEY(ADMIN_USER_NAME)" + ");" + "CREATE UNIQUE INDEX ADMIN_USER_AUN_CI_UN_IDX ON ADMIN_USER(ADMIN_USER_NAME);" + "CREATE TABLE DISK_INSTANCE(" + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_DINM_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT DISK_INSTANCE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME INTEGER CONSTRAINT DISK_INSTANCE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME INTEGER CONSTRAINT DISK_INSTANCE_LUT_NN NOT NULL," + " CONSTRAINT DISK_INSTANCE_PK PRIMARY KEY(DISK_INSTANCE_NAME)" + ");" + "CREATE UNIQUE INDEX DISK_INSTANCE_DIN_CI_UN_IDX ON DISK_INSTANCE(DISK_INSTANCE_NAME);" + "CREATE TABLE DISK_INSTANCE_SPACE(" + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_DINM_NN NOT NULL," + " DISK_INSTANCE_SPACE_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_DISNM_NN NOT NULL," + " FREE_SPACE_QUERY_URL VARCHAR(1000) CONSTRAINT DISK_INSTANCE_SPACE_FSQU_NN NOT NULL," + " REFRESH_INTERVAL INTEGER CONSTRAINT DISK_INSTANCE_SPACE_RI_NN NOT NULL," + " LAST_REFRESH_TIME INTEGER CONSTRAINT DISK_INSTANCE_SPACE_LRT_NN NOT NULL," + " FREE_SPACE INTEGER CONSTRAINT DISK_INSTANCE_SPACE_TFS_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT DISK_INSTANCE_SPACE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME INTEGER CONSTRAINT DISK_INSTANCE_SPACE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME INTEGER CONSTRAINT DISK_INSTANCE_SPACE_LUT_NN NOT NULL," + " CONSTRAINT DISK_INSTANCE_SPACE_PK PRIMARY KEY(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME)," + " CONSTRAINT DISK_INSTANCE_SPACE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)" + ");" + "CREATE UNIQUE INDEX DISK_INSTANCE_SPACE_DISN_CI_UN_IDX ON DISK_INSTANCE_SPACE(DISK_INSTANCE_SPACE_NAME);" + "CREATE UNIQUE INDEX DISK_INSTANCE_SPACE_DISN_UN_IDX ON DISK_INSTANCE_SPACE(DISK_INSTANCE_SPACE_NAME);" + "" + "CREATE TABLE DISK_SYSTEM(" + " DISK_SYSTEM_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_DSNM_NN NOT NULL," + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_DIN_NN NOT NULL, " + " DISK_INSTANCE_SPACE_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_DISN_NN NOT NULL, " + " FILE_REGEXP VARCHAR(100) CONSTRAINT DISK_SYSTEM_FR_NN NOT NULL," + " TARGETED_FREE_SPACE INTEGER CONSTRAINT DISK_SYSTEM_TFS_NN NOT NULL," + " SLEEP_TIME INTEGER CONSTRAINT DISK_SYSTEM_ST_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT DISK_SYSTEM_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_CLHN_NN NOT NULL," + " CREATION_LOG_TIME INTEGER CONSTRAINT DISK_SYSTEM_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME INTEGER CONSTRAINT DISK_SYSTEM_LUT_NN NOT NULL," + " CONSTRAINT NAME_PK PRIMARY KEY(DISK_SYSTEM_NAME)," + " CONSTRAINT DISK_SYSTEM_DIN_DISN_FK FOREIGN KEY(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME) REFERENCES DISK_INSTANCE_SPACE(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME)" + ");" + "CREATE UNIQUE INDEX DISK_SYSTEM_DSN_CI_UN_IDX ON DISK_SYSTEM(DISK_SYSTEM_NAME);" + "CREATE INDEX DISK_SYSTEM_DIN_DISN_IDX ON DISK_SYSTEM(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME);" + "CREATE TABLE VIRTUAL_ORGANIZATION(" + " VIRTUAL_ORGANIZATION_ID INTEGER CONSTRAINT VIRTUAL_ORGANIZATION_VOI_NN NOT NULL," + " VIRTUAL_ORGANIZATION_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_VON_NN NOT NULL," + " READ_MAX_DRIVES INTEGER CONSTRAINT VIRTUAL_ORGANIZATION_RMD_NN NOT NULL," + " WRITE_MAX_DRIVES INTEGER CONSTRAINT VIRTUAL_ORGANIZATION_WMD_NN NOT NULL," + " MAX_FILE_SIZE INTEGER CONSTRAINT VIRTUAL_ORGANIZATION_MFS_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT VIRTUAL_ORGANIZATION_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_CLHN_NN NOT NULL," + " CREATION_LOG_TIME INTEGER CONSTRAINT VIRTUAL_ORGANIZATION_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME INTEGER CONSTRAINT VIRTUAL_ORGANIZATION_LUT_NN NOT NULL," + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_DIN_NN NOT NULL," + " CONSTRAINT VIRTUAL_ORGANIZATION_PK PRIMARY KEY(VIRTUAL_ORGANIZATION_ID)," + " CONSTRAINT VIRTUAL_ORGANIZATION_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)" + ");" + "CREATE UNIQUE INDEX VIRTUAL_ORG_VON_CI_UN_IDX ON VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_NAME);" + "CREATE UNIQUE INDEX VIRTUAL_ORG_VON_UN_IDX ON VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_NAME);" + "CREATE INDEX VIRTUAL_ORG_DIN_IDX ON VIRTUAL_ORGANIZATION(DISK_INSTANCE_NAME);" + "" + "CREATE TABLE STORAGE_CLASS(" + " STORAGE_CLASS_ID INTEGER CONSTRAINT STORAGE_CLASS_SCI_NN NOT NULL," + " STORAGE_CLASS_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_SCN_NN NOT NULL," + " NB_COPIES INTEGER CONSTRAINT STORAGE_CLASS_NC_NN NOT NULL," + " VIRTUAL_ORGANIZATION_ID INTEGER CONSTRAINT STORAGE_CLASS_VOI_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT STORAGE_CLASS_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_CLHN_NN NOT NULL," + " CREATION_LOG_TIME INTEGER CONSTRAINT STORAGE_CLASS_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME INTEGER CONSTRAINT STORAGE_CLASS_LUT_NN NOT NULL," + " CONSTRAINT STORAGE_CLASS_PK PRIMARY KEY(STORAGE_CLASS_ID)," + " CONSTRAINT STORAGE_CLASS_VOI_FK FOREIGN KEY(VIRTUAL_ORGANIZATION_ID) REFERENCES VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_ID)" + ");" + "CREATE UNIQUE INDEX STORAGE_CLASS_SCN_CI_UN_IDX ON STORAGE_CLASS(STORAGE_CLASS_NAME);" + "CREATE UNIQUE INDEX STORAGE_CLASS_SCN_UN_IDX ON STORAGE_CLASS(STORAGE_CLASS_NAME);" + "CREATE INDEX STORAGE_CLASS_VOI_IDX ON STORAGE_CLASS(VIRTUAL_ORGANIZATION_ID);" + "CREATE TABLE TAPE_POOL(" + " TAPE_POOL_ID INTEGER CONSTRAINT TAPE_POOL_TPI_NN NOT NULL," + " TAPE_POOL_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_TPN_NN NOT NULL," + " VIRTUAL_ORGANIZATION_ID INTEGER CONSTRAINT TAPE_POOL_VOI_NN NOT NULL," + " NB_PARTIAL_TAPES INTEGER CONSTRAINT TAPE_POOL_NPT_NN NOT NULL," + " IS_ENCRYPTED CHAR(1) CONSTRAINT TAPE_POOL_IE_NN NOT NULL," + " ENCRYPTION_KEY_NAME VARCHAR(100)," + " SUPPLY VARCHAR(100)," + " USER_COMMENT VARCHAR(1000) CONSTRAINT TAPE_POOL_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_CLHN_NN NOT NULL," + " CREATION_LOG_TIME INTEGER CONSTRAINT TAPE_POOL_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME INTEGER CONSTRAINT TAPE_POOL_LUT_NN NOT NULL," + " CONSTRAINT TAPE_POOL_PK PRIMARY KEY(TAPE_POOL_ID)," + " CONSTRAINT TAPE_POOL_IS_ENCRYPTED_BOOL_CK CHECK(IS_ENCRYPTED IN ('0', '1'))," + " CONSTRAINT TAPE_POOL_VO_FK FOREIGN KEY(VIRTUAL_ORGANIZATION_ID) REFERENCES VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_ID)" + ");" + "CREATE UNIQUE INDEX TAPE_POOL_TPN_CI_UN_IDX ON TAPE_POOL(TAPE_POOL_NAME);" + "CREATE UNIQUE INDEX TAPE_POOL_TPN_UN_IDX ON TAPE_POOL(TAPE_POOL_NAME);" + "CREATE INDEX TAPE_POOL_VOI_IDX ON TAPE_POOL(VIRTUAL_ORGANIZATION_ID);" + "CREATE TABLE ARCHIVE_ROUTE(" + " STORAGE_CLASS_ID INTEGER CONSTRAINT ARCHIVE_ROUTE_SCI_NN NOT NULL," + " COPY_NB INTEGER CONSTRAINT ARCHIVE_ROUTE_CN_NN NOT NULL," + " TAPE_POOL_ID INTEGER CONSTRAINT ARCHIVE_ROUTE_TPI_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT ARCHIVE_ROUTE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT ARCHIVE_ROUTE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT ARCHIVE_ROUTE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME INTEGER CONSTRAINT ARCHIVE_ROUTE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT ARCHIVE_ROUTE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT ARCHIVE_ROUTE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME INTEGER CONSTRAINT ARCHIVE_ROUTE_LUT_NN NOT NULL," + " CONSTRAINT ARCHIVE_ROUTE_PK PRIMARY KEY(STORAGE_CLASS_ID, COPY_NB)," + " CONSTRAINT ARCHIVE_ROUTE_STORAGE_CLASS_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID)," + " CONSTRAINT ARCHIVE_ROUTE_TAPE_POOL_FK FOREIGN KEY(TAPE_POOL_ID) REFERENCES TAPE_POOL(TAPE_POOL_ID)," + " CONSTRAINT ARCHIVE_ROUTE_COPY_NB_GT_0_CK CHECK(COPY_NB > 0)," + " CONSTRAINT ARCHIVE_ROUTE_SCI_TPI_UN UNIQUE(STORAGE_CLASS_ID, TAPE_POOL_ID)" + ");" + "CREATE TABLE MEDIA_TYPE(" + " MEDIA_TYPE_ID INTEGER CONSTRAINT MEDIA_TYPE_MTI_NN NOT NULL," + " MEDIA_TYPE_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_MTN_NN NOT NULL," + " CARTRIDGE VARCHAR(100) CONSTRAINT MEDIA_TYPE_C_NN NOT NULL," + " CAPACITY_IN_BYTES INTEGER CONSTRAINT MEDIA_TYPE_CIB_NN NOT NULL," + " PRIMARY_DENSITY_CODE INTEGER," + " SECONDARY_DENSITY_CODE INTEGER," + " NB_WRAPS INTEGER," + " MIN_LPOS INTEGER," + " MAX_LPOS INTEGER," + " USER_COMMENT VARCHAR(1000) CONSTRAINT MEDIA_TYPE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME INTEGER CONSTRAINT MEDIA_TYPE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME INTEGER CONSTRAINT MEDIA_TYPE_LUT_NN NOT NULL," + " CONSTRAINT MEDIA_TYPE_PK PRIMARY KEY(MEDIA_TYPE_ID)" + ");" + "CREATE UNIQUE INDEX MEDIA_TYPE_MTN_CI_UN_IDX ON MEDIA_TYPE(MEDIA_TYPE_NAME);" + "CREATE UNIQUE INDEX MEDIA_TYPE_MTN_UN_IDX ON MEDIA_TYPE(MEDIA_TYPE_NAME);" + "CREATE TABLE LOGICAL_LIBRARY(" + " LOGICAL_LIBRARY_ID INTEGER CONSTRAINT LOGICAL_LIBRARY_LLI_NN NOT NULL," + " LOGICAL_LIBRARY_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_LLN_NN NOT NULL," + " IS_DISABLED CHAR(1) DEFAULT '0' CONSTRAINT LOGICAL_LIBRARY_ID_NN NOT NULL," + " DISABLED_REASON VARCHAR(1000) ," + " USER_COMMENT VARCHAR(1000) CONSTRAINT LOGICAL_LIBRARY_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_CLHN_NN NOT NULL," + " CREATION_LOG_TIME INTEGER CONSTRAINT LOGICAL_LIBRARY_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME INTEGER CONSTRAINT LOGICAL_LIBRARY_LUT_NN NOT NULL," + " CONSTRAINT LOGICAL_LIBRARY_PK PRIMARY KEY(LOGICAL_LIBRARY_ID)," + " CONSTRAINT LOGICAL_LIBRARY_ID_BOOL_CK CHECK(IS_DISABLED IN ('0', '1'))" + ");" + "CREATE UNIQUE INDEX LOGICAL_LIBRARY_LLN_CI_UN_IDX ON LOGICAL_LIBRARY(LOGICAL_LIBRARY_NAME);" + "CREATE UNIQUE INDEX LOGICAL_LIBRARY_LLN_UN_IDX ON LOGICAL_LIBRARY(LOGICAL_LIBRARY_NAME);" + "CREATE TABLE TAPE(" + " VID VARCHAR(100) CONSTRAINT TAPE_V_NN NOT NULL," + " MEDIA_TYPE_ID INTEGER CONSTRAINT TAPE_MTID_NN NOT NULL," + " VENDOR VARCHAR(100) CONSTRAINT TAPE_V2_NN NOT NULL," + " LOGICAL_LIBRARY_ID INTEGER CONSTRAINT TAPE_LLI_NN NOT NULL," + " TAPE_POOL_ID INTEGER CONSTRAINT TAPE_TPI_NN NOT NULL," + " ENCRYPTION_KEY_NAME VARCHAR(100)," + " DATA_IN_BYTES INTEGER CONSTRAINT TAPE_DIB_NN NOT NULL," + " LAST_FSEQ INTEGER CONSTRAINT TAPE_LF_NN NOT NULL," + " NB_MASTER_FILES INTEGER DEFAULT 0 CONSTRAINT TAPE_NB_MASTER_FILES_NN NOT NULL," + " MASTER_DATA_IN_BYTES INTEGER DEFAULT 0 CONSTRAINT TAPE_MASTER_DATA_IN_BYTES_NN NOT NULL," + " IS_FULL CHAR(1) CONSTRAINT TAPE_IF_NN NOT NULL," + " IS_FROM_CASTOR CHAR(1) CONSTRAINT TAPE_IFC_NN NOT NULL," + " DIRTY CHAR(1) DEFAULT '1' CONSTRAINT TAPE_DIRTY_NN NOT NULL," + " NB_COPY_NB_1 INTEGER DEFAULT 0 CONSTRAINT TAPE_NB_COPY_NB_1_NN NOT NULL," + " COPY_NB_1_IN_BYTES INTEGER DEFAULT 0 CONSTRAINT TAPE_COPY_NB_1_IN_BYTES_NN NOT NULL," + " NB_COPY_NB_GT_1 INTEGER DEFAULT 0 CONSTRAINT TAPE_NB_COPY_NB_GT_1_NN NOT NULL," + " COPY_NB_GT_1_IN_BYTES INTEGER DEFAULT 0 CONSTRAINT TAPE_COPY_NB_GT_1_IN_BYTES_NN NOT NULL," + " LABEL_FORMAT CHAR(1)," + " LABEL_DRIVE VARCHAR(100)," + " LABEL_TIME INTEGER," + " LAST_READ_DRIVE VARCHAR(100)," + " LAST_READ_TIME INTEGER," + " LAST_WRITE_DRIVE VARCHAR(100)," + " LAST_WRITE_TIME INTEGER," + " READ_MOUNT_COUNT INTEGER DEFAULT 0 CONSTRAINT TAPE_RMC_NN NOT NULL," + " WRITE_MOUNT_COUNT INTEGER DEFAULT 0 CONSTRAINT TAPE_WMC_NN NOT NULL," + " USER_COMMENT VARCHAR(1000)," + " TAPE_STATE VARCHAR(100) CONSTRAINT TAPE_TS_NN NOT NULL," + " STATE_REASON VARCHAR(1000)," + " STATE_UPDATE_TIME INTEGER CONSTRAINT TAPE_SUT_NN NOT NULL," + " STATE_MODIFIED_BY VARCHAR(100) CONSTRAINT TAPE_SMB_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT TAPE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME INTEGER CONSTRAINT TAPE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT TAPE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME INTEGER CONSTRAINT TAPE_LUT_NN NOT NULL," + " VERIFICATION_STATUS VARCHAR(1000)," + " CONSTRAINT TAPE_PK PRIMARY KEY(VID)," + " CONSTRAINT TAPE_LOGICAL_LIBRARY_FK FOREIGN KEY(LOGICAL_LIBRARY_ID) REFERENCES LOGICAL_LIBRARY(LOGICAL_LIBRARY_ID)," + " CONSTRAINT TAPE_TAPE_POOL_FK FOREIGN KEY(TAPE_POOL_ID) REFERENCES TAPE_POOL(TAPE_POOL_ID)," + " CONSTRAINT TAPE_IS_FULL_BOOL_CK CHECK(IS_FULL IN ('0', '1'))," + " CONSTRAINT TAPE_IS_FROM_CASTOR_BOOL_CK CHECK(IS_FROM_CASTOR IN ('0', '1'))," + " CONSTRAINT TAPE_DIRTY_BOOL_CK CHECK(DIRTY IN ('0','1'))," + " CONSTRAINT TAPE_STATE_CK CHECK(TAPE_STATE IN ('ACTIVE', 'REPACKING_PENDING', 'REPACKING', 'REPACKING_DISABLED', 'DISABLED', 'BROKEN_PENDING', 'BROKEN', 'EXPORTED_PENDING', 'EXPORTED'))," + " CONSTRAINT TAPE_MEDIA_TYPE_FK FOREIGN KEY(MEDIA_TYPE_ID) REFERENCES MEDIA_TYPE(MEDIA_TYPE_ID)" + ");" + "CREATE UNIQUE INDEX TAPE_VID_CI_UN_IDX ON TAPE(VID);" + "CREATE INDEX TAPE_TAPE_POOL_ID_IDX ON TAPE(TAPE_POOL_ID);" + "CREATE INDEX TAPE_STATE_IDX ON TAPE(TAPE_STATE);" + "CREATE INDEX TAPE_LLI_IDX ON TAPE(LOGICAL_LIBRARY_ID);" + "CREATE INDEX TAPE_MTI_IDX ON TAPE(MEDIA_TYPE_ID);" + "CREATE TABLE MOUNT_POLICY(" + " MOUNT_POLICY_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_MPN_NN NOT NULL," + " ARCHIVE_PRIORITY INTEGER CONSTRAINT MOUNT_POLICY_AP_NN NOT NULL," + " ARCHIVE_MIN_REQUEST_AGE INTEGER CONSTRAINT MOUNT_POLICY_AMRA_NN NOT NULL," + " RETRIEVE_PRIORITY INTEGER CONSTRAINT MOUNT_POLICY_RP_NN NOT NULL," + " RETRIEVE_MIN_REQUEST_AGE INTEGER CONSTRAINT MOUNT_POLICY_RMRA_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT MOUNT_POLICY_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_CLHN_NN NOT NULL," + " CREATION_LOG_TIME INTEGER CONSTRAINT MOUNT_POLICY_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME INTEGER CONSTRAINT MOUNT_POLICY_LUT_NN NOT NULL," + " CONSTRAINT MOUNT_POLICY_PK PRIMARY KEY(MOUNT_POLICY_NAME)" + ");" + "CREATE UNIQUE INDEX MOUNT_POLICY_MPN_CI_UN_IDX ON MOUNT_POLICY(MOUNT_POLICY_NAME);" + "CREATE TABLE REQUESTER_ACTIVITY_MOUNT_RULE(" + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_DIN_NN NOT NULL," + " REQUESTER_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_RN_NN NOT NULL," + " ACTIVITY_REGEX VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_AR_NN NOT NULL," + " MOUNT_POLICY_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_MPN_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT RQSTER_ACT_RULE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME INTEGER CONSTRAINT RQSTER_ACT_RULE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME INTEGER CONSTRAINT RQSTER_ACT_RULE_LUT_NN NOT NULL," + " CONSTRAINT RQSTER_ACT_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_NAME, ACTIVITY_REGEX)," + " CONSTRAINT RQSTER_ACT_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME)," + " CONSTRAINT RQSTER_ACT_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)" + ");" + "CREATE INDEX REQ_ACT_MNT_RULE_MPN_IDX ON REQUESTER_ACTIVITY_MOUNT_RULE(MOUNT_POLICY_NAME);" + "CREATE INDEX REQ_ACT_MNT_RULE_DIN_IDX ON REQUESTER_ACTIVITY_MOUNT_RULE(DISK_INSTANCE_NAME);" + "" + "CREATE TABLE REQUESTER_MOUNT_RULE(" + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_DIN_NN NOT NULL," + " REQUESTER_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_RN_NN NOT NULL," + " MOUNT_POLICY_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_MPN_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT RQSTER_RULE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME INTEGER CONSTRAINT RQSTER_RULE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME INTEGER CONSTRAINT RQSTER_RULE_LUT_NN NOT NULL," + " CONSTRAINT RQSTER_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_NAME)," + " CONSTRAINT RQSTER_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME)," + " CONSTRAINT RQSTER_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)" + ");" + "CREATE INDEX REQ_MNT_RULE_MPN_IDX ON REQUESTER_MOUNT_RULE(MOUNT_POLICY_NAME);" + "CREATE INDEX REQ_MNT_RULE_DIN_IDX ON REQUESTER_MOUNT_RULE(DISK_INSTANCE_NAME);" + "" + "CREATE TABLE REQUESTER_GROUP_MOUNT_RULE(" + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_DIN_NN NOT NULL," + " REQUESTER_GROUP_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_RGN_NN NOT NULL," + " MOUNT_POLICY_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_MPN_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT RQSTER_GRP_RULE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME INTEGER CONSTRAINT RQSTER_GRP_RULE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME INTEGER CONSTRAINT RQSTER_GRP_RULE_LUT_NN NOT NULL," + " CONSTRAINT RQSTER_GRP_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_GROUP_NAME)," + " CONSTRAINT RQSTER_GRP_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME)," + " CONSTRAINT RQSTER_GRP_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)" + "" + ");" + "CREATE INDEX REQ_GRP_MNT_RULE_MPN_IDX ON REQUESTER_GROUP_MOUNT_RULE(MOUNT_POLICY_NAME);" + "CREATE INDEX REQ_GRP_MNT_RULE_DIN_IDX ON REQUESTER_GROUP_MOUNT_RULE(DISK_INSTANCE_NAME);" + "" + "CREATE TABLE ARCHIVE_FILE(" + " ARCHIVE_FILE_ID INTEGER CONSTRAINT ARCHIVE_FILE_AFI_NN NOT NULL," + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT ARCHIVE_FILE_DIN_NN NOT NULL," + " DISK_FILE_ID VARCHAR(100) CONSTRAINT ARCHIVE_FILE_DFI_NN NOT NULL," + " DISK_FILE_UID INTEGER CONSTRAINT ARCHIVE_FILE_DFUID_NN NOT NULL," + " DISK_FILE_GID INTEGER CONSTRAINT ARCHIVE_FILE_DFGID_NN NOT NULL," + " SIZE_IN_BYTES INTEGER CONSTRAINT ARCHIVE_FILE_SIB_NN NOT NULL," + " CHECKSUM_BLOB BLOB(200)," + " CHECKSUM_ADLER32 INTEGER CONSTRAINT ARCHIVE_FILE_CB2_NN NOT NULL," + " STORAGE_CLASS_ID INTEGER CONSTRAINT ARCHIVE_FILE_SCI_NN NOT NULL," + " CREATION_TIME INTEGER CONSTRAINT ARCHIVE_FILE_CT2_NN NOT NULL," + " RECONCILIATION_TIME INTEGER CONSTRAINT ARCHIVE_FILE_RT_NN NOT NULL," + " IS_DELETED CHAR(1) DEFAULT '0' CONSTRAINT ARCHIVE_FILE_ID_NN NOT NULL," + " COLLOCATION_HINT VARCHAR(100)," + " CONSTRAINT ARCHIVE_FILE_PK PRIMARY KEY(ARCHIVE_FILE_ID)," + " CONSTRAINT ARCHIVE_FILE_STORAGE_CLASS_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID)," + " CONSTRAINT ARCHIVE_FILE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)," + " CONSTRAINT ARCHIVE_FILE_DIN_DFI_UN UNIQUE(DISK_INSTANCE_NAME, DISK_FILE_ID)," + " CONSTRAINT ARCHIVE_FILE_ID_BOOL_CK CHECK(IS_DELETED IN ('0', '1'))" + ");" + "CREATE INDEX ARCHIVE_FILE_DIN_IDX ON ARCHIVE_FILE(DISK_INSTANCE_NAME);" + "CREATE INDEX ARCHIVE_FILE_DFI_IDX ON ARCHIVE_FILE(DISK_FILE_ID);" + "CREATE INDEX ARCHIVE_FILE_SCI_IDX ON ARCHIVE_FILE(STORAGE_CLASS_ID);" + "CREATE TABLE TAPE_FILE(" + " VID VARCHAR(100) CONSTRAINT TAPE_FILE_V_NN NOT NULL," + " FSEQ INTEGER CONSTRAINT TAPE_FILE_F_NN NOT NULL," + " BLOCK_ID INTEGER CONSTRAINT TAPE_FILE_BI_NN NOT NULL," + " LOGICAL_SIZE_IN_BYTES INTEGER CONSTRAINT TAPE_FILE_CSIB_NN NOT NULL," + " COPY_NB INTEGER CONSTRAINT TAPE_FILE_CN_NN NOT NULL," + " CREATION_TIME INTEGER CONSTRAINT TAPE_FILE_CT_NN NOT NULL," + " ARCHIVE_FILE_ID INTEGER CONSTRAINT TAPE_FILE_AFI_NN NOT NULL," + " CONSTRAINT TAPE_FILE_PK PRIMARY KEY(VID, FSEQ)," + " CONSTRAINT TAPE_FILE_TAPE_FK FOREIGN KEY(VID)" + " REFERENCES TAPE(VID)," + " CONSTRAINT TAPE_FILE_ARCHIVE_FILE_FK FOREIGN KEY(ARCHIVE_FILE_ID)" + " REFERENCES ARCHIVE_FILE(ARCHIVE_FILE_ID)," + " CONSTRAINT TAPE_FILE_VID_BLOCK_ID_UN UNIQUE(VID, BLOCK_ID)," + " CONSTRAINT TAPE_FILE_COPY_NB_GT_0_CK CHECK(COPY_NB > 0)" + ");" + "CREATE INDEX TAPE_FILE_VID_IDX ON TAPE_FILE(VID);" + "CREATE INDEX TAPE_FILE_ARCHIVE_FILE_ID_IDX ON TAPE_FILE(ARCHIVE_FILE_ID);" + "CREATE TABLE FILE_RECYCLE_LOG(" + " FILE_RECYCLE_LOG_ID INTEGER CONSTRAINT FILE_RECYCLE_LOG_ID_NN NOT NULL," + " VID VARCHAR(100) CONSTRAINT FILE_RECYCLE_LOG_VID_NN NOT NULL," + " FSEQ INTEGER CONSTRAINT FILE_RECYCLE_LOG_FSEQ_NN NOT NULL," + " BLOCK_ID INTEGER CONSTRAINT FILE_RECYCLE_LOG_BID_NN NOT NULL," + " COPY_NB INTEGER CONSTRAINT FILE_RECYCLE_LOG_COPY_NB_NN NOT NULL," + " TAPE_FILE_CREATION_TIME INTEGER CONSTRAINT FILE_RECYCLE_LOG_TFCT_NN NOT NULL," + " ARCHIVE_FILE_ID INTEGER CONSTRAINT FILE_RECYCLE_LOG_AFI_NN NOT NULL," + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT FILE_RECYCLE_LOG_DIN_NN NOT NULL," + " DISK_FILE_ID VARCHAR(100) CONSTRAINT FILE_RECYCLE_LOG_DFI_NN NOT NULL," + " DISK_FILE_ID_WHEN_DELETED VARCHAR(100) CONSTRAINT FILE_RECYCLE_LOG_DFIWD_NN NOT NULL," + " DISK_FILE_UID INTEGER CONSTRAINT FILE_RECYCLE_LOG_DFU_NN NOT NULL," + " DISK_FILE_GID INTEGER CONSTRAINT FILE_RECYCLE_LOG_DFG_NN NOT NULL," + " SIZE_IN_BYTES INTEGER CONSTRAINT FILE_RECYCLE_LOG_SIB_NN NOT NULL," + " CHECKSUM_BLOB BLOB(200)," + " CHECKSUM_ADLER32 INTEGER CONSTRAINT FILE_RECYCLE_LOG_CA_NN NOT NULL," + " STORAGE_CLASS_ID INTEGER CONSTRAINT FILE_RECYCLE_LOG_SCI_NN NOT NULL," + " ARCHIVE_FILE_CREATION_TIME INTEGER CONSTRAINT FILE_RECYLE_LOG_CT_NN NOT NULL," + " RECONCILIATION_TIME INTEGER CONSTRAINT FILE_RECYCLE_LOG_RT_NN NOT NULL," + " COLLOCATION_HINT VARCHAR(100)," + " DISK_FILE_PATH VARCHAR(2000)," + " REASON_LOG VARCHAR(1000) CONSTRAINT FILE_RECYCLE_LOG_RL_NN NOT NULL," + " RECYCLE_LOG_TIME INTEGER CONSTRAINT FILE_RECYCLE_LOG_RLT_NN NOT NULL," + " CONSTRAINT FILE_RECYCLE_LOG_PK PRIMARY KEY(FILE_RECYCLE_LOG_ID)," + " CONSTRAINT FILE_RECYCLE_LOG_VID_FK FOREIGN KEY(VID) REFERENCES TAPE(VID)," + " CONSTRAINT FILE_RECYCLE_LOG_SC_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID)" + ");" + "CREATE INDEX FILE_RECYCLE_LOG_DFI_IDX ON FILE_RECYCLE_LOG(DISK_FILE_ID);" + "CREATE INDEX FILE_RECYCLE_LOG_SCD_IDX ON FILE_RECYCLE_LOG(STORAGE_CLASS_ID);" + "CREATE INDEX FILE_RECYCLE_LOG_VID_IDX ON FILE_RECYCLE_LOG(VID);" + "CREATE TABLE DRIVE_CONFIG (" + " DRIVE_NAME VARCHAR(100) CONSTRAINT DRIVE_CONFIG_DN_NN NOT NULL," + " CATEGORY VARCHAR(100) CONSTRAINT DRIVE_CONFIG_C_NN NOT NULL," + " KEY_NAME VARCHAR(100) CONSTRAINT DRIVE_CONFIG_KN_NN NOT NULL," + " VALUE VARCHAR(1000) CONSTRAINT DRIVE_CONFIG_V_NN NOT NULL," + " SOURCE VARCHAR(100) CONSTRAINT DRIVE_CONFIG_S_NN NOT NULL," + " CONSTRAINT DRIVE_CONFIG_DN_PK PRIMARY KEY(KEY_NAME, DRIVE_NAME)" + ");" + "CREATE TABLE DRIVE_STATE (" + " DRIVE_NAME VARCHAR(100) CONSTRAINT DRIVE_DN_NN NOT NULL," + " HOST VARCHAR(100) CONSTRAINT DRIVE_H_NN NOT NULL," + " LOGICAL_LIBRARY VARCHAR(100) CONSTRAINT DRIVE_LL_NN NOT NULL," + " SESSION_ID INTEGER," + " BYTES_TRANSFERED_IN_SESSION INTEGER," + " FILES_TRANSFERED_IN_SESSION INTEGER," + " SESSION_START_TIME INTEGER," + " SESSION_ELAPSED_TIME INTEGER," + " MOUNT_START_TIME INTEGER," + " TRANSFER_START_TIME INTEGER," + " UNLOAD_START_TIME INTEGER," + " UNMOUNT_START_TIME INTEGER," + " DRAINING_START_TIME INTEGER," + " DOWN_OR_UP_START_TIME INTEGER," + " PROBE_START_TIME INTEGER," + " CLEANUP_START_TIME INTEGER," + " START_START_TIME INTEGER," + " SHUTDOWN_TIME INTEGER," + " MOUNT_TYPE VARCHAR(100) DEFAULT 'NO_MOUNT' CONSTRAINT DRIVE_MT_NN NOT NULL," + " DRIVE_STATUS VARCHAR(100) DEFAULT 'UNKNOWN' CONSTRAINT DRIVE_DS_NN NOT NULL," + " DESIRED_UP CHAR(1) DEFAULT '0' CONSTRAINT DRIVE_DU_NN NOT NULL," + " DESIRED_FORCE_DOWN CHAR(1) DEFAULT '0' CONSTRAINT DRIVE_DFD_NN NOT NULL," + " REASON_UP_DOWN VARCHAR(1000)," + " CURRENT_VID VARCHAR(100)," + " CTA_VERSION VARCHAR(100)," + " CURRENT_PRIORITY INTEGER," + " CURRENT_ACTIVITY VARCHAR(100)," + " CURRENT_TAPE_POOL VARCHAR(100)," + " NEXT_MOUNT_TYPE VARCHAR(100) DEFAULT 'NO_MOUNT' CONSTRAINT DRIVE_NMT_NN NOT NULL," + " NEXT_VID VARCHAR(100)," + " NEXT_PRIORITY INTEGER," + " NEXT_ACTIVITY VARCHAR(100)," + " NEXT_TAPE_POOL VARCHAR(100)," + " DEV_FILE_NAME VARCHAR(100)," + " RAW_LIBRARY_SLOT VARCHAR(100)," + " CURRENT_VO VARCHAR(100)," + " NEXT_VO VARCHAR(100)," + " USER_COMMENT VARCHAR(1000)," + " CREATION_LOG_USER_NAME VARCHAR(100)," + " CREATION_LOG_HOST_NAME VARCHAR(100)," + " CREATION_LOG_TIME INTEGER," + " LAST_UPDATE_USER_NAME VARCHAR(100)," + " LAST_UPDATE_HOST_NAME VARCHAR(100)," + " LAST_UPDATE_TIME INTEGER," + " DISK_SYSTEM_NAME VARCHAR(100)," + " RESERVED_BYTES INTEGER," + " RESERVATION_SESSION_ID INTEGER," + " CONSTRAINT DRIVE_DN_PK PRIMARY KEY(DRIVE_NAME)," + " CONSTRAINT DRIVE_DU_BOOL_CK CHECK(DESIRED_UP IN ('0', '1'))," + " CONSTRAINT DRIVE_DFD_BOOL_CK CHECK(DESIRED_FORCE_DOWN IN ('0', '1'))," + " CONSTRAINT DRIVE_DS_STRING_CK CHECK(DRIVE_STATUS IN ('DOWN', 'UP', 'PROBING', 'STARTING'," + " 'MOUNTING', 'TRANSFERING', 'UNLOADING', 'UNMOUNTING', 'DRAININGTODISK', 'CLEANINGUP', 'SHUTDOWN'," + " 'UNKNOWN'))," + " CONSTRAINT DRIVE_MT_STRING_CK CHECK(MOUNT_TYPE IN ('NO_MOUNT', 'ARCHIVE_FOR_USER'," + " 'ARCHIVE_FOR_REPACK', 'RETRIEVE', 'LABEL', 'ARCHIVE_ALL_TYPES'))," + " CONSTRAINT DRIVE_NMT_STRING_CK CHECK(NEXT_MOUNT_TYPE IN ('NO_MOUNT', 'ARCHIVE_FOR_USER'," + " 'ARCHIVE_FOR_REPACK', 'RETRIEVE', 'LABEL', 'ARCHIVE_ALL_TYPES'))" + ");" + "CREATE UNIQUE INDEX DRIVE_STATE_DN_CI_UN_IDX ON DRIVE_STATE(DRIVE_NAME);" + "INSERT INTO CTA_CATALOGUE(" + " SCHEMA_VERSION_MAJOR," + " SCHEMA_VERSION_MINOR," + " STATUS)" + "VALUES(" + " 12," + " 0," + " 'PRODUCTION');" + }, + {"postgres", "CREATE SEQUENCE ARCHIVE_FILE_ID_SEQ" + " INCREMENT BY 1" + " START WITH 1" + " NO MAXVALUE" + " MINVALUE 1" + " NO CYCLE" + " CACHE 20;" + "CREATE SEQUENCE LOGICAL_LIBRARY_ID_SEQ" + " INCREMENT BY 1" + " START WITH 1" + " NO MAXVALUE" + " MINVALUE 1" + " NO CYCLE" + " CACHE 20;" + "CREATE SEQUENCE MEDIA_TYPE_ID_SEQ" + " INCREMENT BY 1" + " START WITH 1" + " NO MAXVALUE" + " MINVALUE 1" + " NO CYCLE" + " CACHE 20;" + "CREATE SEQUENCE STORAGE_CLASS_ID_SEQ" + " INCREMENT BY 1" + " START WITH 1" + " NO MAXVALUE" + " MINVALUE 1" + " NO CYCLE" + " CACHE 20;" + "CREATE SEQUENCE TAPE_POOL_ID_SEQ" + " INCREMENT BY 1" + " START WITH 1" + " NO MAXVALUE" + " MINVALUE 1" + " NO CYCLE" + " CACHE 20;" + "CREATE SEQUENCE VIRTUAL_ORGANIZATION_ID_SEQ" + " INCREMENT BY 1" + " START WITH 1" + " NO MAXVALUE" + " MINVALUE 1" + " NO CYCLE" + " CACHE 20;" + "CREATE SEQUENCE FILE_RECYCLE_LOG_ID_SEQ" + " INCREMENT BY 1" + " START WITH 1" + " NO MAXVALUE" + " MINVALUE 1" + " NO CYCLE" + " CACHE 20;" + "CREATE TABLE CTA_CATALOGUE(" + " SCHEMA_VERSION_MAJOR NUMERIC(20, 0) CONSTRAINT CTA_CATALOGUE_SVM1_NN NOT NULL," + " SCHEMA_VERSION_MINOR NUMERIC(20, 0) CONSTRAINT CTA_CATALOGUE_SVM2_NN NOT NULL," + " NEXT_SCHEMA_VERSION_MAJOR NUMERIC(20, 0)," + " NEXT_SCHEMA_VERSION_MINOR NUMERIC(20, 0)," + " STATUS VARCHAR(100)," + " IS_PRODUCTION CHAR(1) DEFAULT '0' CONSTRAINT CTA_CATALOGUE_IP_NN NOT NULL," + " CONSTRAINT CTA_CATALOGUE_IP_BOOL_CK CHECK(IS_PRODUCTION IN ('0','1'))" + ");" + "CREATE TABLE ADMIN_USER(" + " ADMIN_USER_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_AUN_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT ADMIN_USER_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT ADMIN_USER_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT ADMIN_USER_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT ADMIN_USER_LUT_NN NOT NULL," + " CONSTRAINT ADMIN_USER_PK PRIMARY KEY(ADMIN_USER_NAME)" + ");" + "CREATE UNIQUE INDEX ADMIN_USER_AUN_CI_UN_IDX ON ADMIN_USER(LOWER(ADMIN_USER_NAME));" + "CREATE TABLE DISK_INSTANCE(" + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_DINM_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT DISK_INSTANCE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_LUT_NN NOT NULL," + " CONSTRAINT DISK_INSTANCE_PK PRIMARY KEY(DISK_INSTANCE_NAME)" + ");" + "CREATE UNIQUE INDEX DISK_INSTANCE_DIN_CI_UN_IDX ON DISK_INSTANCE(LOWER(DISK_INSTANCE_NAME));" + "CREATE TABLE DISK_INSTANCE_SPACE(" + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_DINM_NN NOT NULL," + " DISK_INSTANCE_SPACE_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_DISNM_NN NOT NULL," + " FREE_SPACE_QUERY_URL VARCHAR(1000) CONSTRAINT DISK_INSTANCE_SPACE_FSQU_NN NOT NULL," + " REFRESH_INTERVAL NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_RI_NN NOT NULL," + " LAST_REFRESH_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_LRT_NN NOT NULL," + " FREE_SPACE NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_TFS_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT DISK_INSTANCE_SPACE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT DISK_INSTANCE_SPACE_LUT_NN NOT NULL," + " CONSTRAINT DISK_INSTANCE_SPACE_PK PRIMARY KEY(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME)," + " CONSTRAINT DISK_INSTANCE_SPACE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)" + ");" + "CREATE UNIQUE INDEX DISK_INSTANCE_SPACE_DISN_CI_UN_IDX ON DISK_INSTANCE_SPACE(LOWER(DISK_INSTANCE_SPACE_NAME));" + "CREATE UNIQUE INDEX DISK_INSTANCE_SPACE_DISN_UN_IDX ON DISK_INSTANCE_SPACE(DISK_INSTANCE_SPACE_NAME);" + "" + "CREATE TABLE DISK_SYSTEM(" + " DISK_SYSTEM_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_DSNM_NN NOT NULL," + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_DIN_NN NOT NULL, " + " DISK_INSTANCE_SPACE_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_DISN_NN NOT NULL, " + " FILE_REGEXP VARCHAR(100) CONSTRAINT DISK_SYSTEM_FR_NN NOT NULL," + " TARGETED_FREE_SPACE NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_TFS_NN NOT NULL," + " SLEEP_TIME NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_ST_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT DISK_SYSTEM_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT DISK_SYSTEM_LUT_NN NOT NULL," + " CONSTRAINT NAME_PK PRIMARY KEY(DISK_SYSTEM_NAME)," + " CONSTRAINT DISK_SYSTEM_DIN_DISN_FK FOREIGN KEY(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME) REFERENCES DISK_INSTANCE_SPACE(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME)" + ");" + "CREATE UNIQUE INDEX DISK_SYSTEM_DSN_CI_UN_IDX ON DISK_SYSTEM(LOWER(DISK_SYSTEM_NAME));" + "CREATE INDEX DISK_SYSTEM_DIN_DISN_IDX ON DISK_SYSTEM(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME);" + "CREATE TABLE VIRTUAL_ORGANIZATION(" + " VIRTUAL_ORGANIZATION_ID NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_VOI_NN NOT NULL," + " VIRTUAL_ORGANIZATION_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_VON_NN NOT NULL," + " READ_MAX_DRIVES NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_RMD_NN NOT NULL," + " WRITE_MAX_DRIVES NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_WMD_NN NOT NULL," + " MAX_FILE_SIZE NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_MFS_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT VIRTUAL_ORGANIZATION_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT VIRTUAL_ORGANIZATION_LUT_NN NOT NULL," + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT VIRTUAL_ORGANIZATION_DIN_NN NOT NULL," + " CONSTRAINT VIRTUAL_ORGANIZATION_PK PRIMARY KEY(VIRTUAL_ORGANIZATION_ID)," + " CONSTRAINT VIRTUAL_ORGANIZATION_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)" + ");" + "CREATE UNIQUE INDEX VIRTUAL_ORG_VON_CI_UN_IDX ON VIRTUAL_ORGANIZATION(LOWER(VIRTUAL_ORGANIZATION_NAME));" + "CREATE UNIQUE INDEX VIRTUAL_ORG_VON_UN_IDX ON VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_NAME);" + "CREATE INDEX VIRTUAL_ORG_DIN_IDX ON VIRTUAL_ORGANIZATION(DISK_INSTANCE_NAME);" + "" + "CREATE TABLE STORAGE_CLASS(" + " STORAGE_CLASS_ID NUMERIC(20, 0) CONSTRAINT STORAGE_CLASS_SCI_NN NOT NULL," + " STORAGE_CLASS_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_SCN_NN NOT NULL," + " NB_COPIES NUMERIC(3, 0) CONSTRAINT STORAGE_CLASS_NC_NN NOT NULL," + " VIRTUAL_ORGANIZATION_ID NUMERIC(20, 0) CONSTRAINT STORAGE_CLASS_VOI_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT STORAGE_CLASS_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT STORAGE_CLASS_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT STORAGE_CLASS_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT STORAGE_CLASS_LUT_NN NOT NULL," + " CONSTRAINT STORAGE_CLASS_PK PRIMARY KEY(STORAGE_CLASS_ID)," + " CONSTRAINT STORAGE_CLASS_VOI_FK FOREIGN KEY(VIRTUAL_ORGANIZATION_ID) REFERENCES VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_ID)" + ");" + "CREATE UNIQUE INDEX STORAGE_CLASS_SCN_CI_UN_IDX ON STORAGE_CLASS(LOWER(STORAGE_CLASS_NAME));" + "CREATE UNIQUE INDEX STORAGE_CLASS_SCN_UN_IDX ON STORAGE_CLASS(STORAGE_CLASS_NAME);" + "CREATE INDEX STORAGE_CLASS_VOI_IDX ON STORAGE_CLASS(VIRTUAL_ORGANIZATION_ID);" + "CREATE TABLE TAPE_POOL(" + " TAPE_POOL_ID NUMERIC(20, 0) CONSTRAINT TAPE_POOL_TPI_NN NOT NULL," + " TAPE_POOL_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_TPN_NN NOT NULL," + " VIRTUAL_ORGANIZATION_ID NUMERIC(20, 0) CONSTRAINT TAPE_POOL_VOI_NN NOT NULL," + " NB_PARTIAL_TAPES NUMERIC(20, 0) CONSTRAINT TAPE_POOL_NPT_NN NOT NULL," + " IS_ENCRYPTED CHAR(1) CONSTRAINT TAPE_POOL_IE_NN NOT NULL," + " ENCRYPTION_KEY_NAME VARCHAR(100)," + " SUPPLY VARCHAR(100)," + " USER_COMMENT VARCHAR(1000) CONSTRAINT TAPE_POOL_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT TAPE_POOL_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT TAPE_POOL_LUT_NN NOT NULL," + " CONSTRAINT TAPE_POOL_PK PRIMARY KEY(TAPE_POOL_ID)," + " CONSTRAINT TAPE_POOL_IS_ENCRYPTED_BOOL_CK CHECK(IS_ENCRYPTED IN ('0', '1'))," + " CONSTRAINT TAPE_POOL_VO_FK FOREIGN KEY(VIRTUAL_ORGANIZATION_ID) REFERENCES VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_ID)" + ");" + "CREATE UNIQUE INDEX TAPE_POOL_TPN_CI_UN_IDX ON TAPE_POOL(LOWER(TAPE_POOL_NAME));" + "CREATE UNIQUE INDEX TAPE_POOL_TPN_UN_IDX ON TAPE_POOL(TAPE_POOL_NAME);" + "CREATE INDEX TAPE_POOL_VOI_IDX ON TAPE_POOL(VIRTUAL_ORGANIZATION_ID);" + "CREATE TABLE ARCHIVE_ROUTE(" + " STORAGE_CLASS_ID NUMERIC(20, 0) CONSTRAINT ARCHIVE_ROUTE_SCI_NN NOT NULL," + " COPY_NB NUMERIC(3, 0) CONSTRAINT ARCHIVE_ROUTE_CN_NN NOT NULL," + " TAPE_POOL_ID NUMERIC(20, 0) CONSTRAINT ARCHIVE_ROUTE_TPI_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT ARCHIVE_ROUTE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT ARCHIVE_ROUTE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT ARCHIVE_ROUTE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT ARCHIVE_ROUTE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT ARCHIVE_ROUTE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT ARCHIVE_ROUTE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT ARCHIVE_ROUTE_LUT_NN NOT NULL," + " CONSTRAINT ARCHIVE_ROUTE_PK PRIMARY KEY(STORAGE_CLASS_ID, COPY_NB)," + " CONSTRAINT ARCHIVE_ROUTE_STORAGE_CLASS_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID)," + " CONSTRAINT ARCHIVE_ROUTE_TAPE_POOL_FK FOREIGN KEY(TAPE_POOL_ID) REFERENCES TAPE_POOL(TAPE_POOL_ID)," + " CONSTRAINT ARCHIVE_ROUTE_COPY_NB_GT_0_CK CHECK(COPY_NB > 0)," + " CONSTRAINT ARCHIVE_ROUTE_SCI_TPI_UN UNIQUE(STORAGE_CLASS_ID, TAPE_POOL_ID)" + ");" + "CREATE TABLE MEDIA_TYPE(" + " MEDIA_TYPE_ID NUMERIC(20, 0) CONSTRAINT MEDIA_TYPE_MTI_NN NOT NULL," + " MEDIA_TYPE_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_MTN_NN NOT NULL," + " CARTRIDGE VARCHAR(100) CONSTRAINT MEDIA_TYPE_C_NN NOT NULL," + " CAPACITY_IN_BYTES NUMERIC(20, 0) CONSTRAINT MEDIA_TYPE_CIB_NN NOT NULL," + " PRIMARY_DENSITY_CODE NUMERIC(3, 0)," + " SECONDARY_DENSITY_CODE NUMERIC(3, 0)," + " NB_WRAPS NUMERIC(10, 0)," + " MIN_LPOS NUMERIC(20, 0)," + " MAX_LPOS NUMERIC(20, 0)," + " USER_COMMENT VARCHAR(1000) CONSTRAINT MEDIA_TYPE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT MEDIA_TYPE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT MEDIA_TYPE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT MEDIA_TYPE_LUT_NN NOT NULL," + " CONSTRAINT MEDIA_TYPE_PK PRIMARY KEY(MEDIA_TYPE_ID)" + ");" + "CREATE UNIQUE INDEX MEDIA_TYPE_MTN_CI_UN_IDX ON MEDIA_TYPE(LOWER(MEDIA_TYPE_NAME));" + "CREATE UNIQUE INDEX MEDIA_TYPE_MTN_UN_IDX ON MEDIA_TYPE(MEDIA_TYPE_NAME);" + "CREATE TABLE LOGICAL_LIBRARY(" + " LOGICAL_LIBRARY_ID NUMERIC(20, 0) CONSTRAINT LOGICAL_LIBRARY_LLI_NN NOT NULL," + " LOGICAL_LIBRARY_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_LLN_NN NOT NULL," + " IS_DISABLED CHAR(1) DEFAULT '0' CONSTRAINT LOGICAL_LIBRARY_ID_NN NOT NULL," + " DISABLED_REASON VARCHAR(1000) ," + " USER_COMMENT VARCHAR(1000) CONSTRAINT LOGICAL_LIBRARY_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT LOGICAL_LIBRARY_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT LOGICAL_LIBRARY_LUT_NN NOT NULL," + " CONSTRAINT LOGICAL_LIBRARY_PK PRIMARY KEY(LOGICAL_LIBRARY_ID)," + " CONSTRAINT LOGICAL_LIBRARY_ID_BOOL_CK CHECK(IS_DISABLED IN ('0', '1'))" + ");" + "CREATE UNIQUE INDEX LOGICAL_LIBRARY_LLN_CI_UN_IDX ON LOGICAL_LIBRARY(LOWER(LOGICAL_LIBRARY_NAME));" + "CREATE UNIQUE INDEX LOGICAL_LIBRARY_LLN_UN_IDX ON LOGICAL_LIBRARY(LOGICAL_LIBRARY_NAME);" + "CREATE TABLE TAPE(" + " VID VARCHAR(100) CONSTRAINT TAPE_V_NN NOT NULL," + " MEDIA_TYPE_ID NUMERIC(20, 0) CONSTRAINT TAPE_MTID_NN NOT NULL," + " VENDOR VARCHAR(100) CONSTRAINT TAPE_V2_NN NOT NULL," + " LOGICAL_LIBRARY_ID NUMERIC(20, 0) CONSTRAINT TAPE_LLI_NN NOT NULL," + " TAPE_POOL_ID NUMERIC(20, 0) CONSTRAINT TAPE_TPI_NN NOT NULL," + " ENCRYPTION_KEY_NAME VARCHAR(100)," + " DATA_IN_BYTES NUMERIC(20, 0) CONSTRAINT TAPE_DIB_NN NOT NULL," + " LAST_FSEQ NUMERIC(20, 0) CONSTRAINT TAPE_LF_NN NOT NULL," + " NB_MASTER_FILES NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_NB_MASTER_FILES_NN NOT NULL," + " MASTER_DATA_IN_BYTES NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_MASTER_DATA_IN_BYTES_NN NOT NULL," + " IS_FULL CHAR(1) CONSTRAINT TAPE_IF_NN NOT NULL," + " IS_FROM_CASTOR CHAR(1) CONSTRAINT TAPE_IFC_NN NOT NULL," + " DIRTY CHAR(1) DEFAULT '1' CONSTRAINT TAPE_DIRTY_NN NOT NULL," + " NB_COPY_NB_1 NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_NB_COPY_NB_1_NN NOT NULL," + " COPY_NB_1_IN_BYTES NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_COPY_NB_1_IN_BYTES_NN NOT NULL," + " NB_COPY_NB_GT_1 NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_NB_COPY_NB_GT_1_NN NOT NULL," + " COPY_NB_GT_1_IN_BYTES NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_COPY_NB_GT_1_IN_BYTES_NN NOT NULL," + " LABEL_FORMAT CHAR(1)," + " LABEL_DRIVE VARCHAR(100)," + " LABEL_TIME NUMERIC(20, 0)," + " LAST_READ_DRIVE VARCHAR(100)," + " LAST_READ_TIME NUMERIC(20, 0)," + " LAST_WRITE_DRIVE VARCHAR(100)," + " LAST_WRITE_TIME NUMERIC(20, 0)," + " READ_MOUNT_COUNT NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_RMC_NN NOT NULL," + " WRITE_MOUNT_COUNT NUMERIC(20, 0) DEFAULT 0 CONSTRAINT TAPE_WMC_NN NOT NULL," + " USER_COMMENT VARCHAR(1000)," + " TAPE_STATE VARCHAR(100) CONSTRAINT TAPE_TS_NN NOT NULL," + " STATE_REASON VARCHAR(1000)," + " STATE_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT TAPE_SUT_NN NOT NULL," + " STATE_MODIFIED_BY VARCHAR(100) CONSTRAINT TAPE_SMB_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT TAPE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT TAPE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT TAPE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT TAPE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT TAPE_LUT_NN NOT NULL," + " VERIFICATION_STATUS VARCHAR(1000)," + " CONSTRAINT TAPE_PK PRIMARY KEY(VID)," + " CONSTRAINT TAPE_LOGICAL_LIBRARY_FK FOREIGN KEY(LOGICAL_LIBRARY_ID) REFERENCES LOGICAL_LIBRARY(LOGICAL_LIBRARY_ID)," + " CONSTRAINT TAPE_TAPE_POOL_FK FOREIGN KEY(TAPE_POOL_ID) REFERENCES TAPE_POOL(TAPE_POOL_ID)," + " CONSTRAINT TAPE_IS_FULL_BOOL_CK CHECK(IS_FULL IN ('0', '1'))," + " CONSTRAINT TAPE_IS_FROM_CASTOR_BOOL_CK CHECK(IS_FROM_CASTOR IN ('0', '1'))," + " CONSTRAINT TAPE_DIRTY_BOOL_CK CHECK(DIRTY IN ('0','1'))," + " CONSTRAINT TAPE_STATE_CK CHECK(TAPE_STATE IN ('ACTIVE', 'REPACKING_PENDING', 'REPACKING', 'REPACKING_DISABLED', 'DISABLED', 'BROKEN_PENDING', 'BROKEN', 'EXPORTED_PENDING', 'EXPORTED'))," + " CONSTRAINT TAPE_MEDIA_TYPE_FK FOREIGN KEY(MEDIA_TYPE_ID) REFERENCES MEDIA_TYPE(MEDIA_TYPE_ID)" + ");" + "CREATE UNIQUE INDEX TAPE_VID_CI_UN_IDX ON TAPE(LOWER(VID));" + "CREATE INDEX TAPE_TAPE_POOL_ID_IDX ON TAPE(TAPE_POOL_ID);" + "CREATE INDEX TAPE_STATE_IDX ON TAPE(TAPE_STATE);" + "CREATE INDEX TAPE_LLI_IDX ON TAPE(LOGICAL_LIBRARY_ID);" + "CREATE INDEX TAPE_MTI_IDX ON TAPE(MEDIA_TYPE_ID);" + "CREATE TABLE MOUNT_POLICY(" + " MOUNT_POLICY_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_MPN_NN NOT NULL," + " ARCHIVE_PRIORITY NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_AP_NN NOT NULL," + " ARCHIVE_MIN_REQUEST_AGE NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_AMRA_NN NOT NULL," + " RETRIEVE_PRIORITY NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_RP_NN NOT NULL," + " RETRIEVE_MIN_REQUEST_AGE NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_RMRA_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT MOUNT_POLICY_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT MOUNT_POLICY_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT MOUNT_POLICY_LUT_NN NOT NULL," + " CONSTRAINT MOUNT_POLICY_PK PRIMARY KEY(MOUNT_POLICY_NAME)" + ");" + "CREATE UNIQUE INDEX MOUNT_POLICY_MPN_CI_UN_IDX ON MOUNT_POLICY(LOWER(MOUNT_POLICY_NAME));" + "CREATE TABLE REQUESTER_ACTIVITY_MOUNT_RULE(" + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_DIN_NN NOT NULL," + " REQUESTER_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_RN_NN NOT NULL," + " ACTIVITY_REGEX VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_AR_NN NOT NULL," + " MOUNT_POLICY_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_MPN_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT RQSTER_ACT_RULE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_ACT_RULE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_ACT_RULE_LUT_NN NOT NULL," + " CONSTRAINT RQSTER_ACT_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_NAME, ACTIVITY_REGEX)," + " CONSTRAINT RQSTER_ACT_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME)," + " CONSTRAINT RQSTER_ACT_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)" + ");" + "CREATE INDEX REQ_ACT_MNT_RULE_MPN_IDX ON REQUESTER_ACTIVITY_MOUNT_RULE(MOUNT_POLICY_NAME);" + "CREATE INDEX REQ_ACT_MNT_RULE_DIN_IDX ON REQUESTER_ACTIVITY_MOUNT_RULE(DISK_INSTANCE_NAME);" + "" + "CREATE TABLE REQUESTER_MOUNT_RULE(" + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_DIN_NN NOT NULL," + " REQUESTER_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_RN_NN NOT NULL," + " MOUNT_POLICY_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_MPN_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT RQSTER_RULE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_RULE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_RULE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_RULE_LUT_NN NOT NULL," + " CONSTRAINT RQSTER_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_NAME)," + " CONSTRAINT RQSTER_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME)," + " CONSTRAINT RQSTER_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)" + ");" + "CREATE INDEX REQ_MNT_RULE_MPN_IDX ON REQUESTER_MOUNT_RULE(MOUNT_POLICY_NAME);" + "CREATE INDEX REQ_MNT_RULE_DIN_IDX ON REQUESTER_MOUNT_RULE(DISK_INSTANCE_NAME);" + "" + "CREATE TABLE REQUESTER_GROUP_MOUNT_RULE(" + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_DIN_NN NOT NULL," + " REQUESTER_GROUP_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_RGN_NN NOT NULL," + " MOUNT_POLICY_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_MPN_NN NOT NULL," + " USER_COMMENT VARCHAR(1000) CONSTRAINT RQSTER_GRP_RULE_UC_NN NOT NULL," + " CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_CLUN_NN NOT NULL," + " CREATION_LOG_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_CLHN_NN NOT NULL," + " CREATION_LOG_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_GRP_RULE_CLT_NN NOT NULL," + " LAST_UPDATE_USER_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_LUUN_NN NOT NULL," + " LAST_UPDATE_HOST_NAME VARCHAR(100) CONSTRAINT RQSTER_GRP_RULE_LUHN_NN NOT NULL," + " LAST_UPDATE_TIME NUMERIC(20, 0) CONSTRAINT RQSTER_GRP_RULE_LUT_NN NOT NULL," + " CONSTRAINT RQSTER_GRP_RULE_PK PRIMARY KEY(DISK_INSTANCE_NAME, REQUESTER_GROUP_NAME)," + " CONSTRAINT RQSTER_GRP_RULE_MNT_PLC_FK FOREIGN KEY(MOUNT_POLICY_NAME) REFERENCES MOUNT_POLICY(MOUNT_POLICY_NAME)," + " CONSTRAINT RQSTER_GRP_RULE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)" + "" + ");" + "CREATE INDEX REQ_GRP_MNT_RULE_MPN_IDX ON REQUESTER_GROUP_MOUNT_RULE(MOUNT_POLICY_NAME);" + "CREATE INDEX REQ_GRP_MNT_RULE_DIN_IDX ON REQUESTER_GROUP_MOUNT_RULE(DISK_INSTANCE_NAME);" + "" + "CREATE TABLE ARCHIVE_FILE(" + " ARCHIVE_FILE_ID NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_AFI_NN NOT NULL," + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT ARCHIVE_FILE_DIN_NN NOT NULL," + " DISK_FILE_ID VARCHAR(100) CONSTRAINT ARCHIVE_FILE_DFI_NN NOT NULL," + " DISK_FILE_UID NUMERIC(10, 0) CONSTRAINT ARCHIVE_FILE_DFUID_NN NOT NULL," + " DISK_FILE_GID NUMERIC(10, 0) CONSTRAINT ARCHIVE_FILE_DFGID_NN NOT NULL," + " SIZE_IN_BYTES NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_SIB_NN NOT NULL," + " CHECKSUM_BLOB BYTEA," + " CHECKSUM_ADLER32 NUMERIC(10, 0) CONSTRAINT ARCHIVE_FILE_CB2_NN NOT NULL," + " STORAGE_CLASS_ID NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_SCI_NN NOT NULL," + " CREATION_TIME NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_CT2_NN NOT NULL," + " RECONCILIATION_TIME NUMERIC(20, 0) CONSTRAINT ARCHIVE_FILE_RT_NN NOT NULL," + " IS_DELETED CHAR(1) DEFAULT '0' CONSTRAINT ARCHIVE_FILE_ID_NN NOT NULL," + " COLLOCATION_HINT VARCHAR(100)," + " CONSTRAINT ARCHIVE_FILE_PK PRIMARY KEY(ARCHIVE_FILE_ID)," + " CONSTRAINT ARCHIVE_FILE_STORAGE_CLASS_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID)," + " CONSTRAINT ARCHIVE_FILE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME)," + " CONSTRAINT ARCHIVE_FILE_DIN_DFI_UN UNIQUE(DISK_INSTANCE_NAME, DISK_FILE_ID)," + " CONSTRAINT ARCHIVE_FILE_ID_BOOL_CK CHECK(IS_DELETED IN ('0', '1'))" + ");" + "CREATE INDEX ARCHIVE_FILE_DIN_IDX ON ARCHIVE_FILE(DISK_INSTANCE_NAME);" + "CREATE INDEX ARCHIVE_FILE_DFI_IDX ON ARCHIVE_FILE(DISK_FILE_ID);" + "CREATE INDEX ARCHIVE_FILE_SCI_IDX ON ARCHIVE_FILE(STORAGE_CLASS_ID);" + "CREATE TABLE TAPE_FILE(" + " VID VARCHAR(100) CONSTRAINT TAPE_FILE_V_NN NOT NULL," + " FSEQ NUMERIC(20, 0) CONSTRAINT TAPE_FILE_F_NN NOT NULL," + " BLOCK_ID NUMERIC(20, 0) CONSTRAINT TAPE_FILE_BI_NN NOT NULL," + " LOGICAL_SIZE_IN_BYTES NUMERIC(20, 0) CONSTRAINT TAPE_FILE_CSIB_NN NOT NULL," + " COPY_NB NUMERIC(3, 0) CONSTRAINT TAPE_FILE_CN_NN NOT NULL," + " CREATION_TIME NUMERIC(20, 0) CONSTRAINT TAPE_FILE_CT_NN NOT NULL," + " ARCHIVE_FILE_ID NUMERIC(20, 0) CONSTRAINT TAPE_FILE_AFI_NN NOT NULL," + " CONSTRAINT TAPE_FILE_PK PRIMARY KEY(VID, FSEQ)," + " CONSTRAINT TAPE_FILE_TAPE_FK FOREIGN KEY(VID)" + " REFERENCES TAPE(VID)," + " CONSTRAINT TAPE_FILE_ARCHIVE_FILE_FK FOREIGN KEY(ARCHIVE_FILE_ID)" + " REFERENCES ARCHIVE_FILE(ARCHIVE_FILE_ID)," + " CONSTRAINT TAPE_FILE_VID_BLOCK_ID_UN UNIQUE(VID, BLOCK_ID)," + " CONSTRAINT TAPE_FILE_COPY_NB_GT_0_CK CHECK(COPY_NB > 0)" + ");" + "CREATE INDEX TAPE_FILE_VID_IDX ON TAPE_FILE(VID);" + "CREATE INDEX TAPE_FILE_ARCHIVE_FILE_ID_IDX ON TAPE_FILE(ARCHIVE_FILE_ID);" + "CREATE TABLE FILE_RECYCLE_LOG(" + " FILE_RECYCLE_LOG_ID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_ID_NN NOT NULL," + " VID VARCHAR(100) CONSTRAINT FILE_RECYCLE_LOG_VID_NN NOT NULL," + " FSEQ NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_FSEQ_NN NOT NULL," + " BLOCK_ID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_BID_NN NOT NULL," + " COPY_NB NUMERIC(3, 0) CONSTRAINT FILE_RECYCLE_LOG_COPY_NB_NN NOT NULL," + " TAPE_FILE_CREATION_TIME NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_TFCT_NN NOT NULL," + " ARCHIVE_FILE_ID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_AFI_NN NOT NULL," + " DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT FILE_RECYCLE_LOG_DIN_NN NOT NULL," + " DISK_FILE_ID VARCHAR(100) CONSTRAINT FILE_RECYCLE_LOG_DFI_NN NOT NULL," + " DISK_FILE_ID_WHEN_DELETED VARCHAR(100) CONSTRAINT FILE_RECYCLE_LOG_DFIWD_NN NOT NULL," + " DISK_FILE_UID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_DFU_NN NOT NULL," + " DISK_FILE_GID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_DFG_NN NOT NULL," + " SIZE_IN_BYTES NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_SIB_NN NOT NULL," + " CHECKSUM_BLOB BYTEA," + " CHECKSUM_ADLER32 NUMERIC(10, 0) CONSTRAINT FILE_RECYCLE_LOG_CA_NN NOT NULL," + " STORAGE_CLASS_ID NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_SCI_NN NOT NULL," + " ARCHIVE_FILE_CREATION_TIME NUMERIC(20, 0) CONSTRAINT FILE_RECYLE_LOG_CT_NN NOT NULL," + " RECONCILIATION_TIME NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_RT_NN NOT NULL," + " COLLOCATION_HINT VARCHAR(100)," + " DISK_FILE_PATH VARCHAR(2000)," + " REASON_LOG VARCHAR(1000) CONSTRAINT FILE_RECYCLE_LOG_RL_NN NOT NULL," + " RECYCLE_LOG_TIME NUMERIC(20, 0) CONSTRAINT FILE_RECYCLE_LOG_RLT_NN NOT NULL," + " CONSTRAINT FILE_RECYCLE_LOG_PK PRIMARY KEY(FILE_RECYCLE_LOG_ID)," + " CONSTRAINT FILE_RECYCLE_LOG_VID_FK FOREIGN KEY(VID) REFERENCES TAPE(VID)," + " CONSTRAINT FILE_RECYCLE_LOG_SC_FK FOREIGN KEY(STORAGE_CLASS_ID) REFERENCES STORAGE_CLASS(STORAGE_CLASS_ID)" + ");" + "CREATE INDEX FILE_RECYCLE_LOG_DFI_IDX ON FILE_RECYCLE_LOG(DISK_FILE_ID);" + "CREATE INDEX FILE_RECYCLE_LOG_SCD_IDX ON FILE_RECYCLE_LOG(STORAGE_CLASS_ID);" + "CREATE INDEX FILE_RECYCLE_LOG_VID_IDX ON FILE_RECYCLE_LOG(VID);" + "CREATE TABLE DRIVE_CONFIG (" + " DRIVE_NAME VARCHAR(100) CONSTRAINT DRIVE_CONFIG_DN_NN NOT NULL," + " CATEGORY VARCHAR(100) CONSTRAINT DRIVE_CONFIG_C_NN NOT NULL," + " KEY_NAME VARCHAR(100) CONSTRAINT DRIVE_CONFIG_KN_NN NOT NULL," + " VALUE VARCHAR(1000) CONSTRAINT DRIVE_CONFIG_V_NN NOT NULL," + " SOURCE VARCHAR(100) CONSTRAINT DRIVE_CONFIG_S_NN NOT NULL," + " CONSTRAINT DRIVE_CONFIG_DN_PK PRIMARY KEY(KEY_NAME, DRIVE_NAME)" + ");" + "CREATE TABLE DRIVE_STATE (" + " DRIVE_NAME VARCHAR(100) CONSTRAINT DRIVE_DN_NN NOT NULL," + " HOST VARCHAR(100) CONSTRAINT DRIVE_H_NN NOT NULL," + " LOGICAL_LIBRARY VARCHAR(100) CONSTRAINT DRIVE_LL_NN NOT NULL," + " SESSION_ID NUMERIC(20, 0)," + " BYTES_TRANSFERED_IN_SESSION NUMERIC(20, 0)," + " FILES_TRANSFERED_IN_SESSION NUMERIC(20, 0)," + " SESSION_START_TIME NUMERIC(20, 0)," + " SESSION_ELAPSED_TIME NUMERIC(20, 0)," + " MOUNT_START_TIME NUMERIC(20, 0)," + " TRANSFER_START_TIME NUMERIC(20, 0)," + " UNLOAD_START_TIME NUMERIC(20, 0)," + " UNMOUNT_START_TIME NUMERIC(20, 0)," + " DRAINING_START_TIME NUMERIC(20, 0)," + " DOWN_OR_UP_START_TIME NUMERIC(20, 0)," + " PROBE_START_TIME NUMERIC(20, 0)," + " CLEANUP_START_TIME NUMERIC(20, 0)," + " START_START_TIME NUMERIC(20, 0)," + " SHUTDOWN_TIME NUMERIC(20, 0)," + " MOUNT_TYPE VARCHAR(100) DEFAULT 'NO_MOUNT' CONSTRAINT DRIVE_MT_NN NOT NULL," + " DRIVE_STATUS VARCHAR(100) DEFAULT 'UNKNOWN' CONSTRAINT DRIVE_DS_NN NOT NULL," + " DESIRED_UP CHAR(1) DEFAULT '0' CONSTRAINT DRIVE_DU_NN NOT NULL," + " DESIRED_FORCE_DOWN CHAR(1) DEFAULT '0' CONSTRAINT DRIVE_DFD_NN NOT NULL," + " REASON_UP_DOWN VARCHAR(1000)," + " CURRENT_VID VARCHAR(100)," + " CTA_VERSION VARCHAR(100)," + " CURRENT_PRIORITY NUMERIC(20, 0)," + " CURRENT_ACTIVITY VARCHAR(100)," + " CURRENT_TAPE_POOL VARCHAR(100)," + " NEXT_MOUNT_TYPE VARCHAR(100) DEFAULT 'NO_MOUNT' CONSTRAINT DRIVE_NMT_NN NOT NULL," + " NEXT_VID VARCHAR(100)," + " NEXT_PRIORITY NUMERIC(20, 0)," + " NEXT_ACTIVITY VARCHAR(100)," + " NEXT_TAPE_POOL VARCHAR(100)," + " DEV_FILE_NAME VARCHAR(100)," + " RAW_LIBRARY_SLOT VARCHAR(100)," + " CURRENT_VO VARCHAR(100)," + " NEXT_VO VARCHAR(100)," + " USER_COMMENT VARCHAR(1000)," + " CREATION_LOG_USER_NAME VARCHAR(100)," + " CREATION_LOG_HOST_NAME VARCHAR(100)," + " CREATION_LOG_TIME NUMERIC(20, 0)," + " LAST_UPDATE_USER_NAME VARCHAR(100)," + " LAST_UPDATE_HOST_NAME VARCHAR(100)," + " LAST_UPDATE_TIME NUMERIC(20, 0)," + " DISK_SYSTEM_NAME VARCHAR(100)," + " RESERVED_BYTES NUMERIC(20, 0)," + " RESERVATION_SESSION_ID NUMERIC(20, 0)," + " CONSTRAINT DRIVE_DN_PK PRIMARY KEY(DRIVE_NAME)," + " CONSTRAINT DRIVE_DU_BOOL_CK CHECK(DESIRED_UP IN ('0', '1'))," + " CONSTRAINT DRIVE_DFD_BOOL_CK CHECK(DESIRED_FORCE_DOWN IN ('0', '1'))," + " CONSTRAINT DRIVE_DS_STRING_CK CHECK(DRIVE_STATUS IN ('DOWN', 'UP', 'PROBING', 'STARTING'," + " 'MOUNTING', 'TRANSFERING', 'UNLOADING', 'UNMOUNTING', 'DRAININGTODISK', 'CLEANINGUP', 'SHUTDOWN'," + " 'UNKNOWN'))," + " CONSTRAINT DRIVE_MT_STRING_CK CHECK(MOUNT_TYPE IN ('NO_MOUNT', 'ARCHIVE_FOR_USER'," + " 'ARCHIVE_FOR_REPACK', 'RETRIEVE', 'LABEL', 'ARCHIVE_ALL_TYPES'))," + " CONSTRAINT DRIVE_NMT_STRING_CK CHECK(NEXT_MOUNT_TYPE IN ('NO_MOUNT', 'ARCHIVE_FOR_USER'," + " 'ARCHIVE_FOR_REPACK', 'RETRIEVE', 'LABEL', 'ARCHIVE_ALL_TYPES'))" + ");" + "CREATE UNIQUE INDEX DRIVE_STATE_DN_CI_UN_IDX ON DRIVE_STATE(LOWER(DRIVE_NAME));" + "INSERT INTO CTA_CATALOGUE(" + " SCHEMA_VERSION_MAJOR," + " SCHEMA_VERSION_MINOR," + " STATUS)" + "VALUES(" + " 12," + " 0," + " 'PRODUCTION');" + "ALTER TABLE ARCHIVE_FILE DROP CONSTRAINT" + " ARCHIVE_FILE_DIN_DFI_UN;" + "ALTER TABLE ARCHIVE_FILE ADD CONSTRAINT" + " ARCHIVE_FILE_DIN_DFI_UN UNIQUE(DISK_INSTANCE_NAME, DISK_FILE_ID) DEFERRABLE INITIALLY IMMEDIATE;" + "ALTER TABLE CTA_CATALOGUE ADD CONSTRAINT" " CATALOGUE_STATUS_CONTENT_CK CHECK((NEXT_SCHEMA_VERSION_MAJOR IS NULL AND NEXT_SCHEMA_VERSION_MINOR IS NULL AND STATUS='PRODUCTION') OR (STATUS='UPGRADING'));" }, } }, diff --git a/catalogue/Catalogue.hpp b/catalogue/Catalogue.hpp index 3751ed4332..8b9c4ff27e 100644 --- a/catalogue/Catalogue.hpp +++ b/catalogue/Catalogue.hpp @@ -101,35 +101,21 @@ namespace cta { namespace catalogue { -CTA_GENERATE_EXCEPTION_CLASS(WrongSchemaVersionException); +CTA_GENERATE_EXCEPTION_CLASS(CommentOrReasonWithMoreSizeThanMaximunAllowed); CTA_GENERATE_EXCEPTION_CLASS(NegativeDiskSpaceReservationReached); +CTA_GENERATE_EXCEPTION_CLASS(WrongSchemaVersionException); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentDiskSystem); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonEmptyDiskSystemAfterDelete); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentDiskInstance); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonEmptyDiskInstanceAfterDelete); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentDiskInstanceSpace); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonEmptyDiskInstanceSpaceAfterDelete); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonEmptyLogicalLibrary); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonEmptyTape); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentArchiveRoute); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentLogicalLibrary); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentTape); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentTapePool); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentVirtualOrganization); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringCartridge); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringComment); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringDiskInstanceName); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringDiskInstanceSpaceName); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringDiskSystemName); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringFileRegexp); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringFreeSpaceQueryURL); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAZeroRefreshInterval); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAZeroSleepTime); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAZeroTargetedFreeSpace); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringCartridge); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringDiskInstanceName); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringDiskInstanceSpaceName); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringLogicalLibraryName); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringMediaType); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringMediaTypeName); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringReasonWhenTapeStateNotActive); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringStorageClassName); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringTapePoolName); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringUsername); @@ -137,16 +123,31 @@ CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringVendor); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringVid); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringVo); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyTapePool); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonEmptyDiskInstanceAfterDelete); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonEmptyDiskInstanceSpaceAfterDelete); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonEmptyDiskSystemAfterDelete); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonEmptyLogicalLibrary); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonEmptyTape); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentArchiveRoute); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentDiskInstance); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentDiskInstanceSpace); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentDiskSystem); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentLogicalLibrary); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentTape); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentTapePool); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentTapeState); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentVirtualOrganization); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAZeroCapacity); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAZeroCopyNb); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAZeroRefreshInterval); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAZeroSleepTime); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAZeroTargetedFreeSpace); +CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedExistingDeletedFileCopy); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedMediaTypeUsedByTapes); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedStorageClassUsedByArchiveFiles); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedStorageClassUsedByArchiveRoutes); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedStorageClassUsedByFileRecycleLogs); CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedTapePoolUsedInAnArchiveRoute); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedExistingDeletedFileCopy); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedANonExistentTapeState); -CTA_GENERATE_USER_EXCEPTION_CLASS(UserSpecifiedAnEmptyStringReasonWhenTapeStateNotActive); /** * Abstract class defining the interface to the CTA catalogue responsible for diff --git a/catalogue/DummyCatalogue.hpp b/catalogue/DummyCatalogue.hpp index dc611d1734..64bae3555b 100644 --- a/catalogue/DummyCatalogue.hpp +++ b/catalogue/DummyCatalogue.hpp @@ -18,6 +18,7 @@ #pragma once #include <iostream> +#include <list> #include "Catalogue.hpp" @@ -93,7 +94,6 @@ public: void deleteFileFromRecycleBin(const uint64_t archiveFileId, log::LogContext &lc) {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");} void deleteFilesFromRecycleLog(const std::string & vid, log::LogContext & lc) {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");} void createTapeDrive(const common::dataStructures::TapeDrive &tapeDrive) {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");} - std::list<common::dataStructures::TapeDrive> getTapeDrives() const {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");} void deleteTapeDrive(const std::string &tapeDriveName) {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");} void createTapeDriveConfig(const std::string &tapeDriveName, const std::string &category, const std::string &keyName, const std::string &value, const std::string &source) {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");} std::list<cta::catalogue::Catalogue::DriveConfig> getTapeDriveConfigs() const {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");} @@ -267,7 +267,7 @@ public: return {m_tapeDriveStatus.driveName}; } - std::optional<common::dataStructures::TapeDrive> getTapeDrive(const std::string &tapeDriveName) const { + std::optional<common::dataStructures::TapeDrive> getTapeDrive(const std::string &tapeDriveName) const override { if (m_tapeDriveStatus.driveName != "") return m_tapeDriveStatus; common::dataStructures::TapeDrive tapeDriveStatus; const time_t reportTime = time(nullptr); @@ -291,6 +291,13 @@ public: return tapeDriveStatus; } + std::list<common::dataStructures::TapeDrive> getTapeDrives() const override { + std::list<common::dataStructures::TapeDrive> tapeDrives; + const auto tapeDrive = getTapeDrive(m_tapeDriveStatus.driveName); + if (tapeDrive.has_value()) tapeDrives.push_back(tapeDrive.value()); + return tapeDrives; + } + void setDesiredTapeDriveState(const std::string&, const common::dataStructures::DesiredDriveState &desiredState) override { m_tapeDriveStatus.desiredUp = desiredState.up; diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index 54a8d469c9..64fea8e4df 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -84,13 +84,14 @@ void RdbmsCatalogue::createAdminUser( const std::string &username, const std::string &comment) { try { - if(username.empty()) { + if (username.empty()) { throw UserSpecifiedAnEmptyStringUsername("Cannot create admin user because the username is an empty string"); } - if(comment.empty()) { + if (comment.empty()) { throw UserSpecifiedAnEmptyStringComment("Cannot create admin user because the comment is an empty string"); } + checkCommentOrReasonMaxLength(comment); auto conn = m_connPool.getConn(); if (adminUserExists(conn, username)) { @@ -272,13 +273,14 @@ std::list<common::dataStructures::AdminUser> RdbmsCatalogue::getAdminUsers() con void RdbmsCatalogue::modifyAdminUserComment(const common::dataStructures::SecurityIdentity &admin, const std::string &username, const std::string &comment) { try { - if(username.empty()) { + if (username.empty()) { throw UserSpecifiedAnEmptyStringUsername("Cannot modify admin user because the username is an empty string"); } - if(comment.empty()) { + if (comment.empty()) { throw UserSpecifiedAnEmptyStringComment("Cannot modify admin user because the comment is an empty string"); } + checkCommentOrReasonMaxLength(comment); const time_t now = time(nullptr); const char *const sql = @@ -313,19 +315,20 @@ void RdbmsCatalogue::modifyAdminUserComment(const common::dataStructures::Securi // createVirtualOrganization //------------------------------------------------------------------------------ void RdbmsCatalogue::createVirtualOrganization(const common::dataStructures::SecurityIdentity &admin, const common::dataStructures::VirtualOrganization &vo){ - try{ - if(vo.name.empty()){ + try { + if (vo.name.empty()){ throw UserSpecifiedAnEmptyStringVo("Cannot create virtual organization because the name is an empty string"); } - if(vo.comment.empty()) { + if (vo.comment.empty()) { throw UserSpecifiedAnEmptyStringComment("Cannot create virtual organization because the comment is an empty string"); } - if(vo.diskInstanceName.empty()) { + checkCommentOrReasonMaxLength(vo.comment); + if (vo.diskInstanceName.empty()) { throw UserSpecifiedAnEmptyStringDiskInstanceName("Cannot create virtual organization because the disk instance is an empty string"); } auto conn = m_connPool.getConn(); - if(virtualOrganizationExists(conn, vo.name)) { + if (virtualOrganizationExists(conn, vo.name)) { throw exception::UserError(std::string("Cannot create vo : ") + vo.name + " because it already exists"); } @@ -751,6 +754,7 @@ void RdbmsCatalogue::modifyVirtualOrganizationMaxFileSize(const common::dataStru void RdbmsCatalogue::modifyVirtualOrganizationComment(const common::dataStructures::SecurityIdentity& admin, const std::string& voName, const std::string& comment) { try { + checkCommentOrReasonMaxLength(comment); const time_t now = time(nullptr); const char *const sql = "UPDATE VIRTUAL_ORGANIZATION SET " @@ -832,10 +836,10 @@ void RdbmsCatalogue::createStorageClass( " an empty string"); } - if(storageClass.comment.empty()) { + if (storageClass.comment.empty()) { throw UserSpecifiedAnEmptyStringComment("Cannot create storage class because the comment is an empty string"); } - + checkCommentOrReasonMaxLength(storageClass.comment); std::string vo = storageClass.vo.name; if(vo.empty()) { @@ -1304,6 +1308,8 @@ void RdbmsCatalogue::modifyStorageClassNbCopies(const common::dataStructures::Se void RdbmsCatalogue::modifyStorageClassComment(const common::dataStructures::SecurityIdentity &admin, const std::string &name, const std::string &comment) { try { + checkCommentOrReasonMaxLength(comment); + const time_t now = time(nullptr); const char *const sql = "UPDATE STORAGE_CLASS SET " @@ -1458,25 +1464,26 @@ void RdbmsCatalogue::createMediaType( const common::dataStructures::SecurityIdentity &admin, const MediaType &mediaType) { try { - if(mediaType.name.empty()) { + if (mediaType.name.empty()) { throw UserSpecifiedAnEmptyStringMediaTypeName("Cannot create media type because the media type name is an" " empty string"); } - if(mediaType.cartridge.empty()) { + if (mediaType.cartridge.empty()) { throw UserSpecifiedAnEmptyStringCartridge(std::string("Cannot create media type ") + mediaType.name + " because the cartridge is an empty string"); } - if(mediaType.comment.empty()) { + if (mediaType.comment.empty()) { throw UserSpecifiedAnEmptyStringComment(std::string("Cannot create media type ") + mediaType.name + " because the comment is an empty string"); } - if(mediaType.capacityInBytes == 0){ + checkCommentOrReasonMaxLength(mediaType.comment); + if (mediaType.capacityInBytes == 0){ throw UserSpecifiedAZeroCapacity(std::string("Cannot create media type ") + mediaType.name + " because the capacity is zero"); } auto conn = m_connPool.getConn(); - if(mediaTypeExists(conn, mediaType.name)) { + if (mediaTypeExists(conn, mediaType.name)) { throw exception::UserError(std::string("Cannot create media type ") + mediaType.name + " because it already exists"); } @@ -2017,6 +2024,7 @@ void RdbmsCatalogue::modifyMediaTypeMaxLPos(const common::dataStructures::Securi void RdbmsCatalogue::modifyMediaTypeComment(const common::dataStructures::SecurityIdentity &admin, const std::string &name, const std::string &comment) { try { + checkCommentOrReasonMaxLength(comment); const time_t now = time(nullptr); const char *const sql = "UPDATE MEDIA_TYPE SET " @@ -2066,9 +2074,10 @@ void RdbmsCatalogue::createTapePool( throw UserSpecifiedAnEmptyStringVo("Cannot create tape pool because the VO is an empty string"); } - if(comment.empty()) { + if (comment.empty()) { throw UserSpecifiedAnEmptyStringComment("Cannot create tape pool because the comment is an empty string"); } + checkCommentOrReasonMaxLength(comment); auto conn = m_connPool.getConn(); @@ -2757,6 +2766,7 @@ void RdbmsCatalogue::modifyTapePoolComment(const common::dataStructures::Securit if(comment.empty()) { throw UserSpecifiedAnEmptyStringComment("Cannot modify tape pool because the new comment is an empty string"); } + checkCommentOrReasonMaxLength(comment); const time_t now = time(nullptr); const char *const sql = @@ -2938,6 +2948,7 @@ void RdbmsCatalogue::createArchiveRoute( if(comment.empty()) { throw UserSpecifiedAnEmptyStringComment("Cannot create archive route because comment is an empty string"); } + checkCommentOrReasonMaxLength(comment); const time_t now = time(nullptr); auto conn = m_connPool.getConn(); @@ -3268,6 +3279,7 @@ void RdbmsCatalogue::modifyArchiveRouteComment(const common::dataStructures::Sec const std::string &storageClassName, const uint32_t copyNb, const std::string &comment) { try { + checkCommentOrReasonMaxLength(comment); const time_t now = time(nullptr); const char *const sql = "UPDATE ARCHIVE_ROUTE SET " @@ -3317,6 +3329,7 @@ void RdbmsCatalogue::createLogicalLibrary( const bool isDisabled, const std::string &comment) { try { + checkCommentOrReasonMaxLength(comment); auto conn = m_connPool.getConn(); if(logicalLibraryExists(conn, name)) { throw exception::UserError(std::string("Cannot create logical library ") + name + @@ -3547,6 +3560,7 @@ void RdbmsCatalogue::modifyLogicalLibraryName(const common::dataStructures::Secu void RdbmsCatalogue::modifyLogicalLibraryComment(const common::dataStructures::SecurityIdentity &admin, const std::string &name, const std::string &comment) { try { + checkCommentOrReasonMaxLength(comment); const time_t now = time(nullptr); const char *const sql = "UPDATE LOGICAL_LIBRARY SET " @@ -3582,6 +3596,7 @@ void RdbmsCatalogue::modifyLogicalLibraryComment(const common::dataStructures::S void RdbmsCatalogue::modifyLogicalLibraryDisabledReason(const common::dataStructures::SecurityIdentity &admin, const std::string &name, const std::string &disabledReason) { try { + checkCommentOrReasonMaxLength(disabledReason); const time_t now = time(nullptr); const char *const sql = "UPDATE LOGICAL_LIBRARY SET " @@ -3663,8 +3678,9 @@ void RdbmsCatalogue::createTape( bool full = tape.full; // Translate an empty comment string to a NULL database value const std::optional<std::string> tapeComment = tape.comment && tape.comment->empty() ? std::nullopt : tape.comment; + checkCommentOrReasonMaxLength(tapeComment); const std::optional<std::string> stateReason = tape.stateReason && cta::utils::trimString(tape.stateReason.value()).empty() ? std::nullopt : tape.stateReason; - + checkCommentOrReasonMaxLength(stateReason); if(vid.empty()) { throw UserSpecifiedAnEmptyStringVid("Cannot create tape because the VID is an empty string"); } @@ -5167,6 +5183,7 @@ void RdbmsCatalogue::modifyTapeState(const common::dataStructures::SecurityIdent const time_t now = time(nullptr); const std::optional<std::string> stateReasonCopy = stateReason && cta::utils::trimString(stateReason.value()).empty() ? std::nullopt : stateReason; + checkCommentOrReasonMaxLength(stateReasonCopy); std::string stateStr; try { @@ -5482,6 +5499,7 @@ void RdbmsCatalogue::setTapeDirty(const std::string& vid) { void RdbmsCatalogue::modifyTapeComment(const common::dataStructures::SecurityIdentity &admin, const std::string &vid, const std::optional<std::string> &comment) { try { + checkCommentOrReasonMaxLength(comment); const time_t now = time(nullptr); const char *const sql = "UPDATE TAPE SET " @@ -5567,6 +5585,7 @@ void RdbmsCatalogue::modifyRequesterActivityMountRulePolicy(const common::dataSt void RdbmsCatalogue::modifyRequesterActivityMountRuleComment(const common::dataStructures::SecurityIdentity &admin, const std::string &instanceName, const std::string &requesterName, const std::string &activityRegex, const std::string &comment) { try { + checkCommentOrReasonMaxLength(comment); const time_t now = time(nullptr); const char *const sql = "UPDATE REQUESTER_ACTIVITY_MOUNT_RULE SET " @@ -5645,6 +5664,7 @@ void RdbmsCatalogue::modifyRequesterMountRulePolicy(const common::dataStructures void RdbmsCatalogue::modifyRequesteMountRuleComment(const common::dataStructures::SecurityIdentity &admin, const std::string &instanceName, const std::string &requesterName, const std::string &comment) { try { + checkCommentOrReasonMaxLength(comment); const time_t now = time(nullptr); const char *const sql = "UPDATE REQUESTER_MOUNT_RULE SET " @@ -5721,6 +5741,7 @@ void RdbmsCatalogue::modifyRequesterGroupMountRulePolicy(const common::dataStruc void RdbmsCatalogue::modifyRequesterGroupMountRuleComment(const common::dataStructures::SecurityIdentity &admin, const std::string &instanceName, const std::string &requesterGroupName, const std::string &comment) { try { + checkCommentOrReasonMaxLength(comment); const time_t now = time(nullptr); const char *const sql = "UPDATE REQUESTER_GROUP_MOUNT_RULE SET " @@ -5759,8 +5780,9 @@ void RdbmsCatalogue::modifyRequesterGroupMountRuleComment(const common::dataStru void RdbmsCatalogue::createMountPolicy(const common::dataStructures::SecurityIdentity &admin, const CreateMountPolicyAttributes & mountPolicy) { std::string name = mountPolicy.name; try { + checkCommentOrReasonMaxLength(mountPolicy.comment); auto conn = m_connPool.getConn(); - if(mountPolicyExists(conn, name)) { + if (mountPolicyExists(conn, name)) { throw exception::UserError(std::string("Cannot create mount policy ") + name + " because a mount policy with the same name already exists"); } @@ -5846,6 +5868,7 @@ void RdbmsCatalogue::createRequesterActivityMountRule( const std::string &activityRegex, const std::string &comment) { try { + checkCommentOrReasonMaxLength(comment); auto conn = m_connPool.getConn(); if(requesterActivityMountRuleExists(conn, diskInstanceName, requesterName, activityRegex)) { throw exception::UserError(std::string("Cannot create rule to assign mount-policy ") + mountPolicyName + @@ -6022,6 +6045,7 @@ void RdbmsCatalogue::createRequesterMountRule( const std::string &requesterName, const std::string &comment) { try { + checkCommentOrReasonMaxLength(comment); const auto user = User(diskInstanceName, requesterName); auto conn = m_connPool.getConn(); const auto mountPolicy = getRequesterMountPolicy(conn, user); @@ -6193,6 +6217,7 @@ void RdbmsCatalogue::createRequesterGroupMountRule( const std::string &requesterGroupName, const std::string &comment) { try { + checkCommentOrReasonMaxLength(comment); auto conn = m_connPool.getConn(); { const auto group = Group(diskInstanceName, requesterGroupName); @@ -7008,6 +7033,7 @@ void RdbmsCatalogue::modifyMountPolicyRetrieveMinRequestAge(const common::dataSt void RdbmsCatalogue::modifyMountPolicyComment(const common::dataStructures::SecurityIdentity &admin, const std::string &name, const std::string &comment) { try { + checkCommentOrReasonMaxLength(comment); const time_t now = time(nullptr); const char *const sql = "UPDATE MOUNT_POLICY SET " @@ -7076,6 +7102,7 @@ void RdbmsCatalogue::createDiskSystem( if(comment.empty()) { throw UserSpecifiedAnEmptyStringComment("Cannot create disk system because the comment is an empty string"); } + checkCommentOrReasonMaxLength(comment); auto conn = m_connPool.getConn(); if(diskSystemExists(conn, name)) { @@ -7344,6 +7371,7 @@ void RdbmsCatalogue::modifyDiskSystemComment(const common::dataStructures::Secur throw UserSpecifiedAnEmptyStringComment("Cannot modify disk system " "because the new comment is an empty string"); } + checkCommentOrReasonMaxLength(comment); const time_t now = time(nullptr); const char *const sql = @@ -7516,6 +7544,7 @@ void RdbmsCatalogue::createDiskInstance( if(comment.empty()) { throw UserSpecifiedAnEmptyStringComment("Cannot create disk system because the comment is an empty string"); } + checkCommentOrReasonMaxLength(comment); auto conn = m_connPool.getConn(); if(diskInstanceExists(conn, name)) { @@ -7666,6 +7695,7 @@ void RdbmsCatalogue::modifyDiskInstanceComment(const common::dataStructures::Sec throw UserSpecifiedAnEmptyStringComment("Cannot modify disk instance " "because the new comment is an empty string"); } + checkCommentOrReasonMaxLength(comment); const time_t now = time(nullptr); const char *const sql = @@ -7715,6 +7745,7 @@ void RdbmsCatalogue::modifyDiskInstanceComment(const common::dataStructures::Sec if(comment.empty()) { throw UserSpecifiedAnEmptyStringComment("Cannot create disk instance space because the comment is an empty string"); } + checkCommentOrReasonMaxLength(comment); auto conn = m_connPool.getConn(); if(!diskInstanceExists(conn, diskInstance)) { @@ -7939,6 +7970,7 @@ void RdbmsCatalogue::modifyDiskInstanceSpaceComment(const common::dataStructures throw UserSpecifiedAnEmptyStringComment("Cannot modify disk instance space " "because the new comment is an empty string"); } + checkCommentOrReasonMaxLength(comment); const time_t now = time(nullptr); const char *const sql = "UPDATE DISK_INSTANCE_SPACE SET " @@ -10368,6 +10400,7 @@ void RdbmsCatalogue::copyTapeFilesToFileRecycleLog(rdbms::Conn & conn, const com //------------------------------------------------------------------------------ void RdbmsCatalogue::insertFileInFileRecycleLog(rdbms::Conn& conn, const InsertFileRecycleLog& fileRecycleLog){ try{ + checkCommentOrReasonMaxLength(fileRecycleLog.reasonLog); uint64_t fileRecycleLogId = getNextFileRecyleLogId(conn); const char *const sql = "INSERT INTO FILE_RECYCLE_LOG(" @@ -11290,6 +11323,7 @@ std::optional<common::dataStructures::TapeDrive> RdbmsCatalogue::getTapeDrive(co void RdbmsCatalogue::setDesiredTapeDriveState(const std::string& tapeDriveName, const common::dataStructures::DesiredDriveState &desiredState) { try { + checkCommentOrReasonMaxLength(desiredState.reason); std::string sql = "UPDATE DRIVE_STATE SET " "DESIRED_UP = :DESIRED_UP," @@ -11935,5 +11969,18 @@ void RdbmsCatalogue::releaseDiskSpace(const std::string& driveName, const uint64 } } +void RdbmsCatalogue::checkCommentOrReasonMaxLength(const std::optional<std::string>& str) const { + const size_t MAX_CHAR_COMMENT = 1000; + if (!str.has_value()) return; + if (str.value().length() > MAX_CHAR_COMMENT) { + log::LogContext lc(m_log); + log::ScopedParamContainer spc(lc); + spc.add("Large_Message: ", str.value()); + lc.log(log::ERR, "The reason or comment has more characters than the maximun allowed."); + throw CommentOrReasonWithMoreSizeThanMaximunAllowed( + "The comment or reason string value has more than 1000 characters"); + } +} + } // namespace catalogue } // namespace cta diff --git a/catalogue/RdbmsCatalogue.hpp b/catalogue/RdbmsCatalogue.hpp index 8458b0e92c..c54fabea66 100644 --- a/catalogue/RdbmsCatalogue.hpp +++ b/catalogue/RdbmsCatalogue.hpp @@ -26,6 +26,7 @@ #include "InsertFileRecycleLog.hpp" #include <memory> +#include <string> namespace cta { namespace common { @@ -2457,6 +2458,8 @@ private: void settingSqlTapeDriveValues(cta::rdbms::Stmt *stmt, const common::dataStructures::TapeDrive &tapeDrive) const; common::dataStructures::TapeDrive gettingSqlTapeDriveValues(cta::rdbms::Rset* rset) const; + + void checkCommentOrReasonMaxLength(const std::optional<std::string>& comment) const; }; // class RdbmsCatalogue } // namespace catalogue diff --git a/catalogue/TapeDrivesCatalogueState.cpp b/catalogue/TapeDrivesCatalogueState.cpp index e91aafce44..972ac56ffa 100644 --- a/catalogue/TapeDrivesCatalogueState.cpp +++ b/catalogue/TapeDrivesCatalogueState.cpp @@ -66,18 +66,6 @@ void TapeDrivesCatalogueState::checkDriveCanBeCreated(const cta::common::dataStr } } -std::list<cta::common::dataStructures::TapeDrive> TapeDrivesCatalogueState::getDriveStates( - log::LogContext & lc) const { - std::list<cta::common::dataStructures::TapeDrive> tapeDrivesList; - const auto driveNames = m_catalogue.getTapeDriveNames(); - for (const auto& driveName : driveNames) { - const auto tapeDrive = m_catalogue.getTapeDrive(driveName); - if (!tapeDrive) continue; - tapeDrivesList.push_back(tapeDrive.value()); - } - return tapeDrivesList; -} - void TapeDrivesCatalogueState::removeDrive(const std::string& drive, log::LogContext &lc) { try { m_catalogue.deleteTapeDrive(drive); diff --git a/catalogue/TapeDrivesCatalogueState.hpp b/catalogue/TapeDrivesCatalogueState.hpp index 4d53954e2f..e935d28d36 100644 --- a/catalogue/TapeDrivesCatalogueState.hpp +++ b/catalogue/TapeDrivesCatalogueState.hpp @@ -71,7 +71,6 @@ public: const common::dataStructures::SecurityIdentity& identity, log::LogContext & lc); CTA_GENERATE_EXCEPTION_CLASS(DriveAlreadyExistsException); void checkDriveCanBeCreated(const cta::common::dataStructures::DriveInfo & driveInfo); - std::list<cta::common::dataStructures::TapeDrive> getDriveStates(log::LogContext & lc) const; void removeDrive(const std::string& drive, log::LogContext &lc); void setDesiredDriveState(const std::string& drive, const common::dataStructures::DesiredDriveState & desiredState, log::LogContext &lc); diff --git a/catalogue/common_catalogue_schema.sql b/catalogue/common_catalogue_schema.sql index 786780e8c8..f110330744 100644 --- a/catalogue/common_catalogue_schema.sql +++ b/catalogue/common_catalogue_schema.sql @@ -18,7 +18,7 @@ CREATE TABLE ADMIN_USER( LAST_UPDATE_TIME UINT64TYPE CONSTRAINT ADMIN_USER_LUT_NN NOT NULL, CONSTRAINT ADMIN_USER_PK PRIMARY KEY(ADMIN_USER_NAME) ); -CREATE UNIQUE INDEX ADMIN_USER_AUN_UN_IDX ON ADMIN_USER(LOWER(ADMIN_USER_NAME)); +CREATE UNIQUE INDEX ADMIN_USER_AUN_CI_UN_IDX ON ADMIN_USER(LOWER(ADMIN_USER_NAME)); CREATE TABLE DISK_INSTANCE( DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_DINM_NN NOT NULL, USER_COMMENT VARCHAR(1000) CONSTRAINT DISK_INSTANCE_UC_NN NOT NULL, @@ -30,7 +30,7 @@ CREATE TABLE DISK_INSTANCE( LAST_UPDATE_TIME UINT64TYPE CONSTRAINT DISK_INSTANCE_LUT_NN NOT NULL, CONSTRAINT DISK_INSTANCE_PK PRIMARY KEY(DISK_INSTANCE_NAME) ); -CREATE UNIQUE INDEX DISK_INSTANCE_DIN_UN_IDX ON DISK_INSTANCE(LOWER(DISK_INSTANCE_NAME)); +CREATE UNIQUE INDEX DISK_INSTANCE_DIN_CI_UN_IDX ON DISK_INSTANCE(LOWER(DISK_INSTANCE_NAME)); CREATE TABLE DISK_INSTANCE_SPACE( DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_DINM_NN NOT NULL, DISK_INSTANCE_SPACE_NAME VARCHAR(100) CONSTRAINT DISK_INSTANCE_SPACE_DISNM_NN NOT NULL, @@ -48,7 +48,8 @@ CREATE TABLE DISK_INSTANCE_SPACE( CONSTRAINT DISK_INSTANCE_SPACE_PK PRIMARY KEY(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME), CONSTRAINT DISK_INSTANCE_SPACE_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME) ); -CREATE UNIQUE INDEX DISK_INSTNCE_SPCE_DISN_UN_IDX ON DISK_INSTANCE_SPACE(LOWER(DISK_INSTANCE_SPACE_NAME)); +CREATE UNIQUE INDEX DISK_INSTANCE_SPACE_DISN_CI_UN_IDX ON DISK_INSTANCE_SPACE(LOWER(DISK_INSTANCE_SPACE_NAME)); +CREATE UNIQUE INDEX DISK_INSTANCE_SPACE_DISN_UN_IDX ON DISK_INSTANCE_SPACE(DISK_INSTANCE_SPACE_NAME); CREATE TABLE DISK_SYSTEM( DISK_SYSTEM_NAME VARCHAR(100) CONSTRAINT DISK_SYSTEM_DSNM_NN NOT NULL, @@ -67,7 +68,7 @@ CREATE TABLE DISK_SYSTEM( CONSTRAINT NAME_PK PRIMARY KEY(DISK_SYSTEM_NAME), CONSTRAINT DISK_SYSTEM_DIN_DISN_FK FOREIGN KEY(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME) REFERENCES DISK_INSTANCE_SPACE(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME) ); -CREATE UNIQUE INDEX DISK_SYSTEM_DSN_UN_IDX ON DISK_SYSTEM(LOWER(DISK_SYSTEM_NAME)); +CREATE UNIQUE INDEX DISK_SYSTEM_DSN_CI_UN_IDX ON DISK_SYSTEM(LOWER(DISK_SYSTEM_NAME)); CREATE INDEX DISK_SYSTEM_DIN_DISN_IDX ON DISK_SYSTEM(DISK_INSTANCE_NAME, DISK_INSTANCE_SPACE_NAME); CREATE TABLE VIRTUAL_ORGANIZATION( VIRTUAL_ORGANIZATION_ID UINT64TYPE CONSTRAINT VIRTUAL_ORGANIZATION_VOI_NN NOT NULL, @@ -86,7 +87,8 @@ CREATE TABLE VIRTUAL_ORGANIZATION( CONSTRAINT VIRTUAL_ORGANIZATION_PK PRIMARY KEY(VIRTUAL_ORGANIZATION_ID), CONSTRAINT VIRTUAL_ORGANIZATION_DIN_FK FOREIGN KEY(DISK_INSTANCE_NAME) REFERENCES DISK_INSTANCE(DISK_INSTANCE_NAME) ); -CREATE UNIQUE INDEX VIRTUAL_ORG_VON_UN_IDX ON VIRTUAL_ORGANIZATION(LOWER(VIRTUAL_ORGANIZATION_NAME)); +CREATE UNIQUE INDEX VIRTUAL_ORG_VON_CI_UN_IDX ON VIRTUAL_ORGANIZATION(LOWER(VIRTUAL_ORGANIZATION_NAME)); +CREATE UNIQUE INDEX VIRTUAL_ORG_VON_UN_IDX ON VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_NAME); CREATE INDEX VIRTUAL_ORG_DIN_IDX ON VIRTUAL_ORGANIZATION(DISK_INSTANCE_NAME); CREATE TABLE STORAGE_CLASS( @@ -104,7 +106,8 @@ CREATE TABLE STORAGE_CLASS( CONSTRAINT STORAGE_CLASS_PK PRIMARY KEY(STORAGE_CLASS_ID), CONSTRAINT STORAGE_CLASS_VOI_FK FOREIGN KEY(VIRTUAL_ORGANIZATION_ID) REFERENCES VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_ID) ); -CREATE UNIQUE INDEX STORAGE_CLASS_SCN_UN_IDX ON STORAGE_CLASS(LOWER(STORAGE_CLASS_NAME)); +CREATE UNIQUE INDEX STORAGE_CLASS_SCN_CI_UN_IDX ON STORAGE_CLASS(LOWER(STORAGE_CLASS_NAME)); +CREATE UNIQUE INDEX STORAGE_CLASS_SCN_UN_IDX ON STORAGE_CLASS(STORAGE_CLASS_NAME); CREATE INDEX STORAGE_CLASS_VOI_IDX ON STORAGE_CLASS(VIRTUAL_ORGANIZATION_ID); CREATE TABLE TAPE_POOL( TAPE_POOL_ID UINT64TYPE CONSTRAINT TAPE_POOL_TPI_NN NOT NULL, @@ -112,6 +115,7 @@ CREATE TABLE TAPE_POOL( VIRTUAL_ORGANIZATION_ID UINT64TYPE CONSTRAINT TAPE_POOL_VOI_NN NOT NULL, NB_PARTIAL_TAPES UINT64TYPE CONSTRAINT TAPE_POOL_NPT_NN NOT NULL, IS_ENCRYPTED CHAR(1) CONSTRAINT TAPE_POOL_IE_NN NOT NULL, + ENCRYPTION_KEY_NAME VARCHAR(100), SUPPLY VARCHAR(100), USER_COMMENT VARCHAR(1000) CONSTRAINT TAPE_POOL_UC_NN NOT NULL, CREATION_LOG_USER_NAME VARCHAR(100) CONSTRAINT TAPE_POOL_CLUN_NN NOT NULL, @@ -124,7 +128,8 @@ CREATE TABLE TAPE_POOL( CONSTRAINT TAPE_POOL_IS_ENCRYPTED_BOOL_CK CHECK(IS_ENCRYPTED IN ('0', '1')), CONSTRAINT TAPE_POOL_VO_FK FOREIGN KEY(VIRTUAL_ORGANIZATION_ID) REFERENCES VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_ID) ); -CREATE UNIQUE INDEX TAPE_POOL_TPN_UN_IDX ON TAPE_POOL(LOWER(TAPE_POOL_NAME)); +CREATE UNIQUE INDEX TAPE_POOL_TPN_CI_UN_IDX ON TAPE_POOL(LOWER(TAPE_POOL_NAME)); +CREATE UNIQUE INDEX TAPE_POOL_TPN_UN_IDX ON TAPE_POOL(TAPE_POOL_NAME); CREATE INDEX TAPE_POOL_VOI_IDX ON TAPE_POOL(VIRTUAL_ORGANIZATION_ID); CREATE TABLE ARCHIVE_ROUTE( STORAGE_CLASS_ID UINT64TYPE CONSTRAINT ARCHIVE_ROUTE_SCI_NN NOT NULL, @@ -162,7 +167,8 @@ CREATE TABLE MEDIA_TYPE( LAST_UPDATE_TIME UINT64TYPE CONSTRAINT MEDIA_TYPE_LUT_NN NOT NULL, CONSTRAINT MEDIA_TYPE_PK PRIMARY KEY(MEDIA_TYPE_ID) ); -CREATE UNIQUE INDEX MEDIA_TYPE_MTN_UN_IDX ON MEDIA_TYPE(LOWER(MEDIA_TYPE_NAME)); +CREATE UNIQUE INDEX MEDIA_TYPE_MTN_CI_UN_IDX ON MEDIA_TYPE(LOWER(MEDIA_TYPE_NAME)); +CREATE UNIQUE INDEX MEDIA_TYPE_MTN_UN_IDX ON MEDIA_TYPE(MEDIA_TYPE_NAME); CREATE TABLE LOGICAL_LIBRARY( LOGICAL_LIBRARY_ID UINT64TYPE CONSTRAINT LOGICAL_LIBRARY_LLI_NN NOT NULL, LOGICAL_LIBRARY_NAME VARCHAR(100) CONSTRAINT LOGICAL_LIBRARY_LLN_NN NOT NULL, @@ -178,7 +184,8 @@ CREATE TABLE LOGICAL_LIBRARY( CONSTRAINT LOGICAL_LIBRARY_PK PRIMARY KEY(LOGICAL_LIBRARY_ID), CONSTRAINT LOGICAL_LIBRARY_ID_BOOL_CK CHECK(IS_DISABLED IN ('0', '1')) ); -CREATE UNIQUE INDEX LOGICAL_LIBRARY_LLN_UN_IDX ON LOGICAL_LIBRARY(LOWER(LOGICAL_LIBRARY_NAME)); +CREATE UNIQUE INDEX LOGICAL_LIBRARY_LLN_CI_UN_IDX ON LOGICAL_LIBRARY(LOWER(LOGICAL_LIBRARY_NAME)); +CREATE UNIQUE INDEX LOGICAL_LIBRARY_LLN_UN_IDX ON LOGICAL_LIBRARY(LOGICAL_LIBRARY_NAME); CREATE TABLE TAPE( VID VARCHAR(100) CONSTRAINT TAPE_V_NN NOT NULL, MEDIA_TYPE_ID UINT64TYPE CONSTRAINT TAPE_MTID_NN NOT NULL, @@ -224,10 +231,10 @@ CREATE TABLE TAPE( CONSTRAINT TAPE_IS_FULL_BOOL_CK CHECK(IS_FULL IN ('0', '1')), CONSTRAINT TAPE_IS_FROM_CASTOR_BOOL_CK CHECK(IS_FROM_CASTOR IN ('0', '1')), CONSTRAINT TAPE_DIRTY_BOOL_CK CHECK(DIRTY IN ('0','1')), - CONSTRAINT TAPE_STATE_CK CHECK(TAPE_STATE IN ('ACTIVE', 'REPACKING_PENDING', 'REPACKING', 'DISABLED', 'BROKEN_PENDING', 'BROKEN')), + CONSTRAINT TAPE_STATE_CK CHECK(TAPE_STATE IN ('ACTIVE', 'REPACKING_PENDING', 'REPACKING', 'REPACKING_DISABLED', 'DISABLED', 'BROKEN_PENDING', 'BROKEN', 'EXPORTED_PENDING', 'EXPORTED')), CONSTRAINT TAPE_MEDIA_TYPE_FK FOREIGN KEY(MEDIA_TYPE_ID) REFERENCES MEDIA_TYPE(MEDIA_TYPE_ID) ); -CREATE UNIQUE INDEX TAPE_VID_UN_IDX ON TAPE(LOWER(VID)); +CREATE UNIQUE INDEX TAPE_VID_CI_UN_IDX ON TAPE(LOWER(VID)); CREATE INDEX TAPE_TAPE_POOL_ID_IDX ON TAPE(TAPE_POOL_ID); CREATE INDEX TAPE_STATE_IDX ON TAPE(TAPE_STATE); CREATE INDEX TAPE_LLI_IDX ON TAPE(LOGICAL_LIBRARY_ID); @@ -247,7 +254,7 @@ CREATE TABLE MOUNT_POLICY( LAST_UPDATE_TIME UINT64TYPE CONSTRAINT MOUNT_POLICY_LUT_NN NOT NULL, CONSTRAINT MOUNT_POLICY_PK PRIMARY KEY(MOUNT_POLICY_NAME) ); -CREATE UNIQUE INDEX MOUNT_POLICY_MPN_UN_IDX ON MOUNT_POLICY(LOWER(MOUNT_POLICY_NAME)); +CREATE UNIQUE INDEX MOUNT_POLICY_MPN_CI_UN_IDX ON MOUNT_POLICY(LOWER(MOUNT_POLICY_NAME)); CREATE TABLE REQUESTER_ACTIVITY_MOUNT_RULE( DISK_INSTANCE_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_DIN_NN NOT NULL, REQUESTER_NAME VARCHAR(100) CONSTRAINT RQSTER_ACT_RULE_RN_NN NOT NULL, @@ -442,4 +449,4 @@ CREATE TABLE DRIVE_STATE ( CONSTRAINT DRIVE_NMT_STRING_CK CHECK(NEXT_MOUNT_TYPE IN ('NO_MOUNT', 'ARCHIVE_FOR_USER', 'ARCHIVE_FOR_REPACK', 'RETRIEVE', 'LABEL', 'ARCHIVE_ALL_TYPES')) ); -CREATE UNIQUE INDEX DRIVE_STATE_DN_UN_IDX ON DRIVE_STATE(LOWER(DRIVE_NAME)); +CREATE UNIQUE INDEX DRIVE_STATE_DN_CI_UN_IDX ON DRIVE_STATE(LOWER(DRIVE_NAME)); diff --git a/catalogue/migrations/liquibase/oracle/11.0to12.0.sql b/catalogue/migrations/liquibase/oracle/11.0to12.0.sql new file mode 100644 index 0000000000..d3a17dad68 --- /dev/null +++ b/catalogue/migrations/liquibase/oracle/11.0to12.0.sql @@ -0,0 +1,84 @@ +--liquibase formatted sql + +--changeset afonso:1 failOnError:true dbms:oracle +--preconditions onFail:HALT onError:HALT +--precondition-sql-check expectedResult:"11.0" SELECT CONCAT(CONCAT(CAST(SCHEMA_VERSION_MAJOR as VARCHAR(10)),'.'), CAST(SCHEMA_VERSION_MINOR AS VARCHAR(10))) AS CATALOGUE_VERSION FROM CTA_CATALOGUE; +--!!!THIS FALSE PRECONDITION IS HERE TO BLOCK AN UPGRADE WHILE THE DEVELOPMENT OF THE NEW CATALOGUE VERSION IS BEING DEVELOPED!!! +UPDATE CTA_CATALOGUE SET STATUS='UPGRADING'; +UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MAJOR=12; +UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MINOR=0; +--rollback UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MAJOR=NULL; +--rollback UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MINOR=NULL; +--rollback UPDATE CTA_CATALOGUE SET STATUS='PRODUCTION'; + +--changeset afonso:2 failOnError:true dbms:oracle +--preconditions onFail:HALT onError:HALT +--precondition-sql-check expectedResult:"11.0" SELECT CONCAT(CONCAT(CAST(SCHEMA_VERSION_MAJOR as VARCHAR(10)),'.'), CAST(SCHEMA_VERSION_MINOR AS VARCHAR(10))) AS CATALOGUE_VERSION FROM CTA_CATALOGUE; +ALTER TABLE TAPE DROP CONSTRAINT TAPE_STATE_CK; +ALTER TABLE TAPE ADD CONSTRAINT TAPE_STATE_CK CHECK(TAPE_STATE IN ('ACTIVE', 'REPACKING_PENDING', 'REPACKING', 'REPACKING_DISABLED', 'DISABLED', 'BROKEN_PENDING', 'BROKEN', 'EXPORTED_PENDING', 'EXPORTED')); +--rollback ALTER TABLE TAPE DROP CONSTRAINT TAPE_STATE_CK; +--rollback ALTER TABLE TAPE ADD CONSTRAINT TAPE_STATE_CK CHECK(TAPE_STATE IN ('ACTIVE', 'REPACKING_PENDING', 'REPACKING', 'DISABLED', 'BROKEN_PENDING', 'BROKEN')); + +--changeset afonso:3 failOnError:true dbms:oracle +--preconditions onFail:HALT onError:HALT +--precondition-sql-check expectedResult:"11.0" SELECT CONCAT(CONCAT(CAST(SCHEMA_VERSION_MAJOR as VARCHAR(10)),'.'), CAST(SCHEMA_VERSION_MINOR AS VARCHAR(10))) AS CATALOGUE_VERSION FROM CTA_CATALOGUE; +ALTER INDEX ADMIN_USER_AUN_UN_IDX RENAME TO ADMIN_USER_AUN_CI_UN_IDX; +ALTER INDEX DISK_INSTANCE_DIN_UN_IDX RENAME TO DISK_INSTANCE_DIN_CI_UN_IDX; +ALTER INDEX DISK_INSTNCE_SPCE_DISN_UN_IDX RENAME TO DISK_INSTANCE_SPACE_DISN_CI_UN_IDX; +ALTER INDEX DISK_SYSTEM_DSN_UN_IDX RENAME TO DISK_SYSTEM_DSN_CI_UN_IDX; +ALTER INDEX VIRTUAL_ORG_VON_UN_IDX RENAME TO VIRTUAL_ORG_VON_CI_UN_IDX; +ALTER INDEX STORAGE_CLASS_SCN_UN_IDX RENAME TO STORAGE_CLASS_SCN_CI_UN_IDX; +ALTER INDEX TAPE_POOL_TPN_UN_IDX RENAME TO TAPE_POOL_TPN_CI_UN_IDX; +ALTER INDEX MEDIA_TYPE_MTN_UN_IDX RENAME TO MEDIA_TYPE_MTN_CI_UN_IDX; +ALTER INDEX LOGICAL_LIBRARY_LLN_UN_IDX RENAME TO LOGICAL_LIBRARY_LLN_CI_UN_IDX; +ALTER INDEX TAPE_VID_UN_IDX RENAME TO TAPE_VID_CI_UN_IDX; +ALTER INDEX MOUNT_POLICY_MPN_UN_IDX RENAME TO MOUNT_POLICY_MPN_CI_UN_IDX; +ALTER INDEX DRIVE_STATE_DN_UN_IDX RENAME TO DRIVE_STATE_DN_CI_UN_IDX; +--rollback ALTER INDEX ADMIN_USER_AUN_CI_UN_IDX RENAME TO ADMIN_USER_AUN_UN_IDX; +--rollback ALTER INDEX DISK_INSTANCE_DIN_CI_UN_IDX RENAME TO DISK_INSTANCE_DIN_UN_IDX; +--rollback ALTER INDEX DISK_INSTANCE_SPACE_DISN_CI_UN_IDX RENAME TO DISK_INSTNCE_SPCE_DISN_UN_IDX; +--rollback ALTER INDEX DISK_SYSTEM_DSN_CI_UN_IDX RENAME TO DISK_SYSTEM_DSN_UN_IDX; +--rollback ALTER INDEX VIRTUAL_ORG_VON_CI_UN_IDX RENAME TO VIRTUAL_ORG_VON_UN_IDX; +--rollback ALTER INDEX STORAGE_CLASS_SCN_CI_UN_IDX RENAME TO STORAGE_CLASS_SCN_UN_IDX; +--rollback ALTER INDEX TAPE_POOL_TPN_CI_UN_IDX RENAME TO TAPE_POOL_TPN_UN_IDX; +--rollback ALTER INDEX MEDIA_TYPE_MTN_CI_UN_IDX RENAME TO MEDIA_TYPE_MTN_UN_IDX; +--rollback ALTER INDEX LOGICAL_LIBRARY_LLN_CI_UN_IDX RENAME TO LOGICAL_LIBRARY_LLN_UN_IDX; +--rollback ALTER INDEX TAPE_VID_CI_UN_IDX RENAME TO TAPE_VID_UN_IDX; +--rollback ALTER INDEX MOUNT_POLICY_MPN_CI_UN_IDX RENAME TO MOUNT_POLICY_MPN_UN_IDX; +--rollback ALTER INDEX DRIVE_STATE_DN_CI_UN_IDX RENAME TO DRIVE_STATE_DN_UN_IDX; + +--changeset afonso:4 failOnError:true dbms:oracle +--preconditions onFail:HALT onError:HALT +--precondition-sql-check expectedResult:"11.0" SELECT CONCAT(CONCAT(CAST(SCHEMA_VERSION_MAJOR as VARCHAR(10)),'.'), CAST(SCHEMA_VERSION_MINOR AS VARCHAR(10))) AS CATALOGUE_VERSION FROM CTA_CATALOGUE; +CREATE UNIQUE INDEX DISK_INSTANCE_SPACE_DISN_UN_IDX ON DISK_INSTANCE_SPACE(DISK_INSTANCE_SPACE_NAME); +CREATE UNIQUE INDEX VIRTUAL_ORG_VON_UN_IDX ON VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_NAME); +CREATE UNIQUE INDEX STORAGE_CLASS_SCN_UN_IDX ON STORAGE_CLASS(STORAGE_CLASS_NAME); +CREATE UNIQUE INDEX TAPE_POOL_TPN_UN_IDX ON TAPE_POOL(TAPE_POOL_NAME); +CREATE UNIQUE INDEX MEDIA_TYPE_MTN_UN_IDX ON MEDIA_TYPE(MEDIA_TYPE_NAME); +CREATE UNIQUE INDEX LOGICAL_LIBRARY_LLN_UN_IDX ON LOGICAL_LIBRARY(LOGICAL_LIBRARY_NAME); +--rollback DROP INDEX DISK_INSTANCE_SPACE_DISN_UN_IDX; +--rollback DROP INDEX VIRTUAL_ORG_VON_UN_IDX; +--rollback DROP INDEX STORAGE_CLASS_SCN_UN_IDX; +--rollback DROP INDEX TAPE_POOL_TPN_UN_IDX; +--rollback DROP INDEX MEDIA_TYPE_MTN_UN_IDX; +--rollback DROP INDEX LOGICAL_LIBRARY_LLN_UN_IDX; + +--changeset afonso:5 failOnError:true dbms:oracle +--preconditions onFail:HALT onError:HALT +--precondition-sql-check expectedResult:"11.0" SELECT CONCAT(CONCAT(CAST(SCHEMA_VERSION_MAJOR as VARCHAR(10)),'.'), CAST(SCHEMA_VERSION_MINOR AS VARCHAR(10))) AS CATALOGUE_VERSION FROM CTA_CATALOGUE; +ALTER TABLE TAPE_POOL ADD ENCRYPTION_KEY_NAME VARCHAR2(100); +--rollback ALTER TABLE TAPE_POOL DROP COLUMN ENCRYPTION_KEY_NAME; + +--changeset afonso:6 failOnError:true dbms:oracle +--preconditions onFail:HALT onError:HALT +--precondition-sql-check expectedResult:"11.0" SELECT CONCAT(CONCAT(CAST(SCHEMA_VERSION_MAJOR as VARCHAR(10)),'.'), CAST(SCHEMA_VERSION_MINOR AS VARCHAR(10))) AS CATALOGUE_VERSION FROM CTA_CATALOGUE; +UPDATE CTA_CATALOGUE SET STATUS='PRODUCTION'; +UPDATE CTA_CATALOGUE SET SCHEMA_VERSION_MAJOR=12; +UPDATE CTA_CATALOGUE SET SCHEMA_VERSION_MINOR=0; +UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MAJOR=NULL; +UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MINOR=NULL; +--rollback UPDATE CTA_CATALOGUE SET STATUS='UPGRADING'; +--rollback UPDATE CTA_CATALOGUE SET SCHEMA_VERSION_MAJOR=11; +--rollback UPDATE CTA_CATALOGUE SET SCHEMA_VERSION_MINOR=0; +--rollback UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MAJOR=12; +--rollback UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MINOR=0; diff --git a/catalogue/migrations/liquibase/postgres/10.0to11.0.sql b/catalogue/migrations/liquibase/postgres/10.0to11.0.sql new file mode 100644 index 0000000000..977b49c76e --- /dev/null +++ b/catalogue/migrations/liquibase/postgres/10.0to11.0.sql @@ -0,0 +1,42 @@ +--liquibase formatted sql + +--changeset afonso:1 failOnError:true dbms:postgresql +--preconditions onFail:HALT onError:HALT +--precondition-sql-check expectedResult:"10.0" SELECT CONCAT(CONCAT(CAST(SCHEMA_VERSION_MAJOR as VARCHAR(10)),'.'), CAST(SCHEMA_VERSION_MINOR AS VARCHAR(10))) AS CATALOGUE_VERSION FROM CTA_CATALOGUE; +--!!!THIS FALSE PRECONDITION IS HERE TO BLOCK AN UPGRADE WHILE THE DEVELOPMENT OF THE NEW CATALOGUE VERSION IS BEING DEVELOPED!!! +UPDATE CTA_CATALOGUE SET STATUS='UPGRADING'; +UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MAJOR=11; +UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MINOR=0; +--rollback UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MAJOR=NULL; +--rollback UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MINOR=NULL; +--rollback UPDATE CTA_CATALOGUE SET STATUS='PRODUCTION'; + +--changeset afonso:2 failOnError:true dbms:postgresql +--preconditions onFail:HALT onError:HALT +--precondition-sql-check expectedResult:"10.0" SELECT CONCAT(CONCAT(CAST(SCHEMA_VERSION_MAJOR as VARCHAR(10)),'.'), CAST(SCHEMA_VERSION_MINOR AS VARCHAR(10))) AS CATALOGUE_VERSION FROM CTA_CATALOGUE; +ALTER TABLE TAPE DROP CONSTRAINT TAPE_STATE_CK; +ALTER TABLE TAPE ADD CONSTRAINT TAPE_STATE_CK CHECK(TAPE_STATE IN ('ACTIVE', 'REPACKING_PENDING', 'REPACKING', 'DISABLED', 'BROKEN_PENDING', 'BROKEN')); +--rollback ALTER TABLE TAPE DROP CONSTRAINT TAPE_STATE_CK; +--rollback ALTER TABLE TAPE ADD CONSTRAINT TAPE_STATE_CK CHECK(TAPE_STATE IN ('ACTIVE', 'REPACKING', 'DISABLED', 'BROKEN')); + +--changeset afonso:3 failOnError:true dbms:postgresql +--preconditions onFail:HALT onError:HALT +--precondition-sql-check expectedResult:"10.0" SELECT CONCAT(CONCAT(CAST(SCHEMA_VERSION_MAJOR as VARCHAR(10)),'.'), CAST(SCHEMA_VERSION_MINOR AS VARCHAR(10))) AS CATALOGUE_VERSION FROM CTA_CATALOGUE; +ALTER TABLE DISK_SYSTEM DROP COLUMN FREE_SPACE_QUERY_URL; +ALTER TABLE DISK_SYSTEM DROP COLUMN REFRESH_INTERVAL; +--rollback ALTER TABLE DISK_SYSTEM ADD COLUMN FREE_SPACE_QUERY_URL VARCHAR(1000); +--rollback ALTER TABLE DISK_SYSTEM ADD COLUMN REFRESH_INTERVAL UINT64TYPE; + +--changeset afonso:4 failOnError:true dbms:postgresql +--preconditions onFail:HALT onError:HALT +--precondition-sql-check expectedResult:"10.0" SELECT CONCAT(CONCAT(CAST(SCHEMA_VERSION_MAJOR as VARCHAR(10)),'.'), CAST(SCHEMA_VERSION_MINOR AS VARCHAR(10))) AS CATALOGUE_VERSION FROM CTA_CATALOGUE; +UPDATE CTA_CATALOGUE SET SCHEMA_VERSION_MAJOR=11; +UPDATE CTA_CATALOGUE SET SCHEMA_VERSION_MINOR=0; +UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MAJOR=NULL; +UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MINOR=NULL; +UPDATE CTA_CATALOGUE SET STATUS='PRODUCTION'; +--rollback UPDATE CTA_CATALOGUE SET STATUS='UPGRADING'; +--rollback UPDATE CTA_CATALOGUE SET SCHEMA_VERSION_MAJOR=10; +--rollback UPDATE CTA_CATALOGUE SET SCHEMA_VERSION_MINOR=0; +--rollback UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MAJOR=11; +--rollback UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MINOR=0; diff --git a/catalogue/migrations/liquibase/postgres/11.0to12.0.sql b/catalogue/migrations/liquibase/postgres/11.0to12.0.sql new file mode 100644 index 0000000000..bf8ac30871 --- /dev/null +++ b/catalogue/migrations/liquibase/postgres/11.0to12.0.sql @@ -0,0 +1,84 @@ +--liquibase formatted sql + +--changeset afonso:1 failOnError:true dbms:postgresql +--preconditions onFail:HALT onError:HALT +--precondition-sql-check expectedResult:"11.0" SELECT CONCAT(CONCAT(CAST(SCHEMA_VERSION_MAJOR as VARCHAR(10)),'.'), CAST(SCHEMA_VERSION_MINOR AS VARCHAR(10))) AS CATALOGUE_VERSION FROM CTA_CATALOGUE; +--!!!THIS FALSE PRECONDITION IS HERE TO BLOCK AN UPGRADE WHILE THE DEVELOPMENT OF THE NEW CATALOGUE VERSION IS BEING DEVELOPED!!! +UPDATE CTA_CATALOGUE SET STATUS='UPGRADING'; +UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MAJOR=12; +UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MINOR=0; +--rollback UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MAJOR=NULL; +--rollback UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MINOR=NULL; +--rollback UPDATE CTA_CATALOGUE SET STATUS='PRODUCTION'; + +--changeset afonso:2 failOnError:true dbms:postgresql +--preconditions onFail:HALT onError:HALT +--precondition-sql-check expectedResult:"11.0" SELECT CONCAT(CONCAT(CAST(SCHEMA_VERSION_MAJOR as VARCHAR(10)),'.'), CAST(SCHEMA_VERSION_MINOR AS VARCHAR(10))) AS CATALOGUE_VERSION FROM CTA_CATALOGUE; +ALTER TABLE TAPE DROP CONSTRAINT TAPE_STATE_CK; +ALTER TABLE TAPE ADD CONSTRAINT TAPE_STATE_CK CHECK(TAPE_STATE IN ('ACTIVE', 'REPACKING_PENDING', 'REPACKING', 'REPACKING_DISABLED', 'DISABLED', 'BROKEN_PENDING', 'BROKEN', 'EXPORTED_PENDING', 'EXPORTED')); +--rollback ALTER TABLE TAPE DROP CONSTRAINT TAPE_STATE_CK; +--rollback ALTER TABLE TAPE ADD CONSTRAINT TAPE_STATE_CK CHECK(TAPE_STATE IN ('ACTIVE', 'REPACKING_PENDING', 'REPACKING', 'DISABLED', 'BROKEN_PENDING', 'BROKEN')); + +--changeset afonso:3 failOnError:true dbms:postgresql +--preconditions onFail:HALT onError:HALT +--precondition-sql-check expectedResult:"11.0" SELECT CONCAT(CONCAT(CAST(SCHEMA_VERSION_MAJOR as VARCHAR(10)),'.'), CAST(SCHEMA_VERSION_MINOR AS VARCHAR(10))) AS CATALOGUE_VERSION FROM CTA_CATALOGUE; +ALTER INDEX ADMIN_USER_AUN_UN_IDX RENAME TO ADMIN_USER_AUN_CI_UN_IDX; +ALTER INDEX DISK_INSTANCE_DIN_UN_IDX RENAME TO DISK_INSTANCE_DIN_CI_UN_IDX; +ALTER INDEX DISK_INSTNCE_SPCE_DISN_UN_IDX RENAME TO DISK_INSTANCE_SPACE_DISN_CI_UN_IDX; +ALTER INDEX DISK_SYSTEM_DSN_UN_IDX RENAME TO DISK_SYSTEM_DSN_CI_UN_IDX; +ALTER INDEX VIRTUAL_ORG_VON_UN_IDX RENAME TO VIRTUAL_ORG_VON_CI_UN_IDX; +ALTER INDEX STORAGE_CLASS_SCN_UN_IDX RENAME TO STORAGE_CLASS_SCN_CI_UN_IDX; +ALTER INDEX TAPE_POOL_TPN_UN_IDX RENAME TO TAPE_POOL_TPN_CI_UN_IDX; +ALTER INDEX MEDIA_TYPE_MTN_UN_IDX RENAME TO MEDIA_TYPE_MTN_CI_UN_IDX; +ALTER INDEX LOGICAL_LIBRARY_LLN_UN_IDX RENAME TO LOGICAL_LIBRARY_LLN_CI_UN_IDX; +ALTER INDEX TAPE_VID_UN_IDX RENAME TO TAPE_VID_CI_UN_IDX; +ALTER INDEX MOUNT_POLICY_MPN_UN_IDX RENAME TO MOUNT_POLICY_MPN_CI_UN_IDX; +ALTER INDEX DRIVE_STATE_DN_UN_IDX RENAME TO DRIVE_STATE_DN_CI_UN_IDX; +--rollback ALTER INDEX ADMIN_USER_AUN_CI_UN_IDX RENAME TO ADMIN_USER_AUN_UN_IDX; +--rollback ALTER INDEX DISK_INSTANCE_DIN_CI_UN_IDX RENAME TO DISK_INSTANCE_DIN_UN_IDX; +--rollback ALTER INDEX DISK_INSTANCE_SPACE_DISN_CI_UN_IDX RENAME TO DISK_INSTNCE_SPCE_DISN_UN_IDX; +--rollback ALTER INDEX DISK_SYSTEM_DSN_CI_UN_IDX RENAME TO DISK_SYSTEM_DSN_UN_IDX; +--rollback ALTER INDEX VIRTUAL_ORG_VON_CI_UN_IDX RENAME TO VIRTUAL_ORG_VON_UN_IDX; +--rollback ALTER INDEX STORAGE_CLASS_SCN_CI_UN_IDX RENAME TO STORAGE_CLASS_SCN_UN_IDX; +--rollback ALTER INDEX TAPE_POOL_TPN_CI_UN_IDX RENAME TO TAPE_POOL_TPN_UN_IDX; +--rollback ALTER INDEX MEDIA_TYPE_MTN_CI_UN_IDX RENAME TO MEDIA_TYPE_MTN_UN_IDX; +--rollback ALTER INDEX LOGICAL_LIBRARY_LLN_CI_UN_IDX RENAME TO LOGICAL_LIBRARY_LLN_UN_IDX; +--rollback ALTER INDEX TAPE_VID_CI_UN_IDX RENAME TO TAPE_VID_UN_IDX; +--rollback ALTER INDEX MOUNT_POLICY_MPN_CI_UN_IDX RENAME TO MOUNT_POLICY_MPN_UN_IDX; +--rollback ALTER INDEX DRIVE_STATE_DN_CI_UN_IDX RENAME TO DRIVE_STATE_DN_UN_IDX; + +--changeset afonso:4 failOnError:true dbms:postgresql +--preconditions onFail:HALT onError:HALT +--precondition-sql-check expectedResult:"11.0" SELECT CONCAT(CONCAT(CAST(SCHEMA_VERSION_MAJOR as VARCHAR(10)),'.'), CAST(SCHEMA_VERSION_MINOR AS VARCHAR(10))) AS CATALOGUE_VERSION FROM CTA_CATALOGUE; +CREATE UNIQUE INDEX DISK_INSTANCE_SPACE_DISN_UN_IDX ON DISK_INSTANCE_SPACE(DISK_INSTANCE_SPACE_NAME); +CREATE UNIQUE INDEX VIRTUAL_ORG_VON_UN_IDX ON VIRTUAL_ORGANIZATION(VIRTUAL_ORGANIZATION_NAME); +CREATE UNIQUE INDEX STORAGE_CLASS_SCN_UN_IDX ON STORAGE_CLASS(STORAGE_CLASS_NAME); +CREATE UNIQUE INDEX TAPE_POOL_TPN_UN_IDX ON TAPE_POOL(TAPE_POOL_NAME); +CREATE UNIQUE INDEX MEDIA_TYPE_MTN_UN_IDX ON MEDIA_TYPE(MEDIA_TYPE_NAME); +CREATE UNIQUE INDEX LOGICAL_LIBRARY_LLN_UN_IDX ON LOGICAL_LIBRARY(LOGICAL_LIBRARY_NAME); +--rollback DROP INDEX DISK_INSTANCE_SPACE_DISN_UN_IDX; +--rollback DROP INDEX VIRTUAL_ORG_VON_UN_IDX; +--rollback DROP INDEX STORAGE_CLASS_SCN_UN_IDX; +--rollback DROP INDEX TAPE_POOL_TPN_UN_IDX; +--rollback DROP INDEX MEDIA_TYPE_MTN_UN_IDX; +--rollback DROP INDEX LOGICAL_LIBRARY_LLN_UN_IDX; + +--changeset afonso:5 failOnError:true dbms:postgresql +--preconditions onFail:HALT onError:HALT +--precondition-sql-check expectedResult:"11.0" SELECT CONCAT(CONCAT(CAST(SCHEMA_VERSION_MAJOR as VARCHAR(10)),'.'), CAST(SCHEMA_VERSION_MINOR AS VARCHAR(10))) AS CATALOGUE_VERSION FROM CTA_CATALOGUE; +ALTER TABLE TAPE_POOL ADD ENCRYPTION_KEY_NAME VARCHAR(100); +--rollback ALTER TABLE TAPE_POOL DROP COLUMN ENCRYPTION_KEY_NAME; + +--changeset afonso:6 failOnError:true dbms:postgresql +--preconditions onFail:HALT onError:HALT +--precondition-sql-check expectedResult:"11.0" SELECT CONCAT(CONCAT(CAST(SCHEMA_VERSION_MAJOR as VARCHAR(10)),'.'), CAST(SCHEMA_VERSION_MINOR AS VARCHAR(10))) AS CATALOGUE_VERSION FROM CTA_CATALOGUE; +UPDATE CTA_CATALOGUE SET SCHEMA_VERSION_MAJOR=12; +UPDATE CTA_CATALOGUE SET SCHEMA_VERSION_MINOR=0; +UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MAJOR=NULL; +UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MINOR=NULL; +UPDATE CTA_CATALOGUE SET STATUS='PRODUCTION'; +--rollback UPDATE CTA_CATALOGUE SET STATUS='UPGRADING'; +--rollback UPDATE CTA_CATALOGUE SET SCHEMA_VERSION_MAJOR=11; +--rollback UPDATE CTA_CATALOGUE SET SCHEMA_VERSION_MINOR=0; +--rollback UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MAJOR=12; +--rollback UPDATE CTA_CATALOGUE SET NEXT_SCHEMA_VERSION_MINOR=0; diff --git a/cmake/CTAVersions.cmake b/cmake/CTAVersions.cmake index 89f7c93ec5..0fba36ac5f 100644 --- a/cmake/CTAVersions.cmake +++ b/cmake/CTAVersions.cmake @@ -18,7 +18,7 @@ set(CTA_RELEASE 1) set(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION "v0.0") # Catalogue Schema Version -set(CTA_CATALOGUE_SCHEMA_VERSION_MAJOR 11) +set(CTA_CATALOGUE_SCHEMA_VERSION_MAJOR 12) set(CTA_CATALOGUE_SCHEMA_VERSION_MINOR 0) # Shared object internal version (used in SONAME) diff --git a/cmdline/CtaAdminCmd.cpp b/cmdline/CtaAdminCmd.cpp index 3825d20e1c..fca4cdf396 100644 --- a/cmdline/CtaAdminCmd.cpp +++ b/cmdline/CtaAdminCmd.cpp @@ -144,7 +144,6 @@ void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const case Data::kVersionItem: formattedText.print(record.version_item()); break; case Data::kMtlsItem: formattedText.print(record.mtls_item()); break; case Data::kRtflsItem: formattedText.print(record.rtfls_item()); break; - case Data::kSilsItem: formattedText.print(record.sils_item()); break; default: throw std::runtime_error("Received invalid stream data from CTA Frontend."); } @@ -307,7 +306,6 @@ void CtaAdminCmd::send() const case HeaderType::VIRTUALORGANIZATION_LS: formattedText.printVirtualOrganizationLsHeader(); break; case HeaderType::VERSION_CMD: formattedText.printVersionHeader(); break; case HeaderType::MEDIATYPE_LS: formattedText.printMediaTypeLsHeader(); break; - case HeaderType::SCHEDULINGINFOS_LS: formattedText.printSchedulingInfoLsHeader(); break; case HeaderType::RECYLETAPEFILE_LS: formattedText.printRecycleTapeFileLsHeader(); break; case HeaderType::NONE: default: break; diff --git a/cmdline/CtaAdminCmdParse.hpp b/cmdline/CtaAdminCmdParse.hpp index bbb10c43ca..7abc1abe35 100644 --- a/cmdline/CtaAdminCmdParse.hpp +++ b/cmdline/CtaAdminCmdParse.hpp @@ -228,8 +228,6 @@ const cmdLookup_t cmdLookup = { { "vo", AdminCmd::CMD_VIRTUALORGANIZATION }, { "version", AdminCmd::CMD_VERSION}, { "v", AdminCmd::CMD_VERSION}, - { "schedulinginfo", AdminCmd::CMD_SCHEDULINGINFOS}, - { "si", AdminCmd::CMD_SCHEDULINGINFOS}, { "recycletf", AdminCmd::CMD_RECYCLETAPEFILE}, { "rtf", AdminCmd::CMD_RECYCLETAPEFILE}, { "activitymountrule", AdminCmd::CMD_ACTIVITYMOUNTRULE }, @@ -468,7 +466,6 @@ const std::map<AdminCmd::Cmd, CmdHelp> cmdHelp = { " * Specify the maximum file size (--maxfilesize) for this virtual organization (optional, 0 means no limit)\n\n" }}, { AdminCmd::CMD_VERSION, { "version", "v", { } }}, - { AdminCmd::CMD_SCHEDULINGINFOS, { "schedulinginfo", "si", { "ls" } }}, { AdminCmd::CMD_RECYCLETAPEFILE, { "recycletf", "rtf", { "ls" }, "\n Tape files in the recycle log can be listed by VID, EOS disk file ID, EOS disk instance,\n" " ArchiveFileId or copy number. Disk file IDs should be provided in hexadecimal format (fxid).\n\n" @@ -700,7 +697,6 @@ const std::map<cmd_key_t, cmd_val_t> cmdOptions = { {{ AdminCmd::CMD_VIRTUALORGANIZATION, AdminCmd::SUBCMD_LS }, { }}, {{ AdminCmd::CMD_VERSION, AdminCmd::SUBCMD_NONE }, { }}, - {{ AdminCmd::CMD_SCHEDULINGINFOS, AdminCmd::SUBCMD_LS }, { }}, {{ AdminCmd::CMD_RECYCLETAPEFILE, AdminCmd::SUBCMD_LS }, { opt_vid.optional(), opt_fid.optional(), opt_fidfile.optional(), opt_copynb.optional(), opt_archivefileid.optional(), opt_instance.optional() }}, {{ AdminCmd::CMD_RECYCLETAPEFILE, AdminCmd::SUBCMD_RESTORE }, diff --git a/cmdline/CtaAdminTextFormatter.cpp b/cmdline/CtaAdminTextFormatter.cpp index 41f1aec175..e30aa67ab5 100644 --- a/cmdline/CtaAdminTextFormatter.cpp +++ b/cmdline/CtaAdminTextFormatter.cpp @@ -1157,51 +1157,6 @@ void TextFormatter::print(const VersionItem & version_item){ ); } -void TextFormatter::printSchedulingInfoLsHeader(){ - push_back("HEADER"); - push_back( - "logical library", - "vid", - "tapepool", - "vo", - "media type", - "vendor", - "mount type", - "tape capacity", - "priority", - "min request age", - "files queued", - "bytes queued", - "oldest job start time", - "sleeping mount", - "sleep time", - "disk system slept for", - "mount count" - ); -} - -void TextFormatter::print(const SchedulingInfoLsItem& sils_item) { - push_back( - sils_item.logical_library(), - sils_item.vid(), - sils_item.tapepool(), - sils_item.vo(), - sils_item.media_type(), - sils_item.vendor(), - sils_item.mount_type(), - sils_item.tape_capacity_in_bytes(), - sils_item.mount_policy_priority(), - sils_item.mount_policy_min_request_age(), - sils_item.files_queued(), - sils_item.bytes_queued(), - sils_item.oldest_job_start_time(), - sils_item.sleeping_mount(), - sils_item.sleep_time(), - sils_item.disk_system_slept_for(), - sils_item.mount_count() - ); -} - void TextFormatter::printRecycleTapeFileLsHeader() { push_back("HEADER"); push_back( diff --git a/cmdline/CtaAdminTextFormatter.hpp b/cmdline/CtaAdminTextFormatter.hpp index 8298e44ae3..7dd571fdb7 100644 --- a/cmdline/CtaAdminTextFormatter.hpp +++ b/cmdline/CtaAdminTextFormatter.hpp @@ -69,7 +69,6 @@ public: void printVirtualOrganizationLsHeader(); void printVersionHeader(); void printMediaTypeLsHeader(); - void printSchedulingInfoLsHeader(); void printRecycleTapeFileLsHeader(); // Output records @@ -100,7 +99,6 @@ public: void print(const VersionItem & version_item); void print(const MediaTypeLsItem &mtls_item); void print(const RecycleTapeFileLsItem & rtfls_item); - void print(const SchedulingInfoLsItem& sils_item); private: //! Add a line to the buffer diff --git a/cmdline/cta-admin.1cta b/cmdline/cta-admin.1cta index 9b3d5c57b4..9edb0c2ba8 100644 --- a/cmdline/cta-admin.1cta +++ b/cmdline/cta-admin.1cta @@ -103,9 +103,6 @@ progress and display any errors. requestermountrule (rmr) Add, change, remove or list the requester mount rules. .TP -schedulinginfo (si) -List information about scheduling of archive and retrieve jobs. -.TP showqueues (sq) Show the status of all active queues. .TP diff --git a/common/Configuration.hpp b/common/Configuration.hpp index 347de4c91e..07a4b7eef1 100644 --- a/common/Configuration.hpp +++ b/common/Configuration.hpp @@ -152,7 +152,7 @@ namespace cta { namespace common { T value; std::stringstream ss; - ss << strValue.c_str(); + ss << strValue; ss >> value; if(nullptr != log) { @@ -193,7 +193,7 @@ namespace cta { namespace common { T value; std::stringstream ss; - ss << strValue.c_str(); + ss << strValue; ss >> value; if(nullptr != log) { diff --git a/common/dataStructures/Tape.cpp b/common/dataStructures/Tape.cpp index e24154bd47..6ae2849630 100644 --- a/common/dataStructures/Tape.cpp +++ b/common/dataStructures/Tape.cpp @@ -55,7 +55,7 @@ std::string Tape::getAllPossibleStates(){ ret += kv.first + " "; } if(ret.size()) - ret = ret.substr(0,ret.size() - 1); + ret.pop_back(); return ret; } diff --git a/common/dataStructures/TapeDrive.cpp b/common/dataStructures/TapeDrive.cpp index 818918d476..07a01e822b 100644 --- a/common/dataStructures/TapeDrive.cpp +++ b/common/dataStructures/TapeDrive.cpp @@ -69,7 +69,7 @@ std::string TapeDrive::getAllPossibleStates(){ ret += kv.first + " "; } if(ret.size()) - ret = ret.substr(0,ret.size() - 1); + ret.pop_back(); return ret; } diff --git a/continuousintegration/docker/ctafrontend/cc7/doublebuildtree-stage2b-scripts/Dockerfile b/continuousintegration/docker/ctafrontend/cc7/doublebuildtree-stage2b-scripts/Dockerfile index 7f7c497fd4..f707f23878 100644 --- a/continuousintegration/docker/ctafrontend/cc7/doublebuildtree-stage2b-scripts/Dockerfile +++ b/continuousintegration/docker/ctafrontend/cc7/doublebuildtree-stage2b-scripts/Dockerfile @@ -44,7 +44,7 @@ RUN chmod 744 /etc/rc.local ADD ${BASEDIR}/etc/yum/pluginconf.d /etc/yum/pluginconf.d #Install the libs necessary for EOS to run -RUN yum install -y eos-protobuf3 libisa-l libisa-l_crypto-devel xxhash-devel grpc +RUN yum install -y eos-protobuf3 libisa-l libisa-l_crypto libisa-l_crypto-devel xxhash-devel grpc # disable selinux so that systemd works if the container is booted # with an image that already contains selinux diff --git a/continuousintegration/docker/ctafrontend/cc7/etc/yum/pluginconf.d/versionlock.list b/continuousintegration/docker/ctafrontend/cc7/etc/yum/pluginconf.d/versionlock.list index 34bea4a6e9..d9883f3b71 100644 --- a/continuousintegration/docker/ctafrontend/cc7/etc/yum/pluginconf.d/versionlock.list +++ b/continuousintegration/docker/ctafrontend/cc7/etc/yum/pluginconf.d/versionlock.list @@ -17,23 +17,23 @@ 0:eos-xrootd-debuginfo-4.12.8-1.el7.cern.* #### EOS-4-END #### #### EOS-5-START #### -#0:eos-archive-5.0.27-1.el7.cern.x86_64.rpm -#0:eos-cleanup-5.0.27-1.el7.cern.x86_64.rpm -#0:eos-client-5.0.27-1.el7.cern.x86_64.rpm -#0:eos-fuse-5.0.27-1.el7.cern.x86_64.rpm -#0:eos-fuse-core-5.0.27-1.el7.cern.x86_64.rpm -#0:eos-fuse-sysv-5.0.27-1.el7.cern.x86_64.rpm -#0:eos-fusex-5.0.27-1.el7.cern.x86_64.rpm -#0:eos-fusex-core-5.0.27-1.el7.cern.x86_64.rpm -#0:eos-fusex-selinux-5.0.27-1.el7.cern.x86_64.rpm -#0:eos-ns-inspect-5.0.27-1.el7.cern.x86_64.rpm -#0:eos-server-5.0.27-1.el7.cern.x86_64.rpm -#0:eos-srm-5.0.27-1.el7.cern.x86_64.rpm -#0:eos-test-5.0.27-1.el7.cern.x86_64.rpm -#0:eos-testkeytab-5.0.27-1.el7.cern.x86_64.rpm -#0:eos-quarkdb-5.0.27-1.el7.cern.x86_64.rpm -#0:eos-xrootd-5.4.6-1.el7.cern.* -#0:eos-xrootd-debuginfo-5.4.6-1.el7.cern.* +#0:eos-archive-5.1.1-1.el7.cern.x86_64.rpm +#0:eos-cleanup-5.1.1-1.el7.cern.x86_64.rpm +#0:eos-client-5.1.1-1.el7.cern.x86_64.rpm +#0:eos-fuse-5.1.1-1.el7.cern.x86_64.rpm +#0:eos-fuse-core-5.1.1-1.el7.cern.x86_64.rpm +#0:eos-fuse-sysv-5.1.1-1.el7.cern.x86_64.rpm +#0:eos-fusex-5.1.1-1.el7.cern.x86_64.rpm +#0:eos-fusex-core-5.1.1-1.el7.cern.x86_64.rpm +#0:eos-fusex-selinux-5.1.1-1.el7.cern.x86_64.rpm +#0:eos-ns-inspect-5.1.1-1.el7.cern.x86_64.rpm +#0:eos-server-5.1.1-1.el7.cern.x86_64.rpm +#0:eos-srm-5.1.1-1.el7.cern.x86_64.rpm +#0:eos-test-5.1.1-1.el7.cern.x86_64.rpm +#0:eos-testkeytab-5.1.1-1.el7.cern.x86_64.rpm +#0:eos-quarkdb-5.1.1-1.el7.cern.x86_64.rpm +#0:eos-xrootd-5.5.1-1.el7.cern.* +#0:eos-xrootd-debuginfo-5.5.1-1.el7.cern.* #### EOS-5-END #### 0:eos-folly-2019.11.11.00-1.el7.cern.* 0:eos-folly-deps-2019.11.11.00-1.el7.cern.* diff --git a/continuousintegration/orchestration/tests/update_db_test.sh b/continuousintegration/orchestration/tests/update_db_test.sh index 1d2e7858bd..1e8cb81279 100755 --- a/continuousintegration/orchestration/tests/update_db_test.sh +++ b/continuousintegration/orchestration/tests/update_db_test.sh @@ -96,8 +96,10 @@ CURRENT_SCHEMA_VERSION=$(kubectl -n ${NAMESPACE} exec ctafrontend -- cta-catalog | grep -o -E '[0-9]+\.[0-9]') if [ ${CURRENT_SCHEMA_VERSION} == ${NEW_SCHEMA_VERSION} ]; then echo "The current Catalogue Schema Version is: ${CURRENT_SCHEMA_VERSION}" + kubectl -n ${NAMESPACE} logs dbupdatetest &> "../../../pod_logs/${NAMESPACE}/liquibase-update.log" else echo "Error. Unexpected Catalogue Schema Version: ${CURRENT_SCHEMA_VERSION}, it should be: ${NEW_SCHEMA_VERSION}" + kubectl -n ${NAMESPACE} logs dbupdatetest &> "../../../pod_logs/${NAMESPACE}/liquibase-update.log" exit 1 fi diff --git a/cta.spec.in b/cta.spec.in index 26a5786fef..edf6b8fb71 100644 --- a/cta.spec.in +++ b/cta.spec.in @@ -574,6 +574,13 @@ This package contains .repo files, gpg keys and yum-versionlock configuration fo %changelog +* Fri Sep 16 2022 Joao Afonso <joao.afonso@cern.ch> - 4.7.11-1 +- Various fixes and improvements to CTA, see ReleaseNotes.md for details +- Enabled build option without Oracle dependencies + +* Mon Sep 13 2022 Joao Afonso <joao.afonso@cern.ch> - 4.7.10-1 +- Catalogue schema version 12.0 + * Mon Aug 08 2022 Julien Leduc <julien.leduc@cern.ch> - 4.7.9-2 - Fixes packaging issue upgrading from CTA 4.7.8-1 to 4.7.9-1 diff --git a/frontend-grpc/FrontendGRpcSvc.cpp b/frontend-grpc/FrontendGRpcSvc.cpp index 7016e38b16..ed820181f2 100644 --- a/frontend-grpc/FrontendGRpcSvc.cpp +++ b/frontend-grpc/FrontendGRpcSvc.cpp @@ -126,7 +126,7 @@ Status CtaRpcImpl::Archive(::grpc::ServerContext* context, const ::cta::frontend archiveRequest.requester.group = request->md().cli().user().groupname(); archiveRequest.storageClass = storageClass; archiveRequest.srcURL = request->md().transport().dst_url(); - archiveRequest.archiveReportURL = request->md().transport().report_url() + "?archiveid=" + std::to_string(archiveFileId); + archiveRequest.archiveReportURL = request->md().transport().report_url(); archiveRequest.archiveErrorReportURL = request->md().transport().error_report_url(); archiveRequest.creationLog.host = context->peer(); archiveRequest.creationLog.username = instance; diff --git a/rdbms/Login.cpp b/rdbms/Login.cpp index dac6480b73..c81b90b9ec 100644 --- a/rdbms/Login.cpp +++ b/rdbms/Login.cpp @@ -158,7 +158,7 @@ std::list<std::string> Login::readNonEmptyLines(std::istream &inputStream) { { const std::string::size_type newlinePos = line.find("\n"); if (newlinePos != std::string::npos) { - line = line.substr(0, newlinePos); + line.resize(newlinePos); } } diff --git a/scheduler/CMakeLists.txt b/scheduler/CMakeLists.txt index d128e88ae7..87df04ba22 100644 --- a/scheduler/CMakeLists.txt +++ b/scheduler/CMakeLists.txt @@ -34,7 +34,6 @@ set (CTA_SCHEDULER_SRC_FILES Scheduler.cpp SchedulerDatabase.cpp SchedulerDatabaseFactory.cpp - SchedulingInfos.cpp TapeMount.cpp) if(NOT CTA_USE_PGSCHED) diff --git a/scheduler/GenericSchedulerTest.cpp b/scheduler/GenericSchedulerTest.cpp index 1539416200..acdaef77ce 100644 --- a/scheduler/GenericSchedulerTest.cpp +++ b/scheduler/GenericSchedulerTest.cpp @@ -2417,255 +2417,6 @@ TEST_P(SchedulerTest, repackRetrieveRequestsFailToFetchDiskSystem){ ASSERT_EQ(0,1); } -TEST_P(SchedulerTest, getSchedulingInformations) { - //Queue 2 archive requests in two different logical libraries - using namespace cta; - - Scheduler &scheduler = getScheduler(); - auto &catalogue = getCatalogue(); - - setupDefaultCatalogue(); - catalogue.deleteTapeDrive("drive0"); // It's not needed -#ifdef STDOUT_LOGGING - log::StdoutLogger dl("dummy", "unitTest"); -#else - log::DummyLogger dl("", ""); -#endif - log::LogContext lc(dl); - - // Create the environment for the migration to happen (library + tape) - const std::string libraryComment = "Library comment"; - const bool libraryIsDisabled = false; - catalogue.createLogicalLibrary(s_adminOnAdminHost, s_libraryName, - libraryIsDisabled, libraryComment); - { - auto libraries = catalogue.getLogicalLibraries(); - ASSERT_EQ(1, libraries.size()); - ASSERT_EQ(s_libraryName, libraries.front().name); - ASSERT_EQ(libraryComment, libraries.front().comment); - } - - { - auto tape = getDefaultTape(); - catalogue.createTape(s_adminOnAdminHost, tape); - } - - const std::string driveName = "tape_drive"; - - catalogue.tapeLabelled(s_vid, driveName); - - { - // This first initialization is normally done by the dataSession function. - cta::common::dataStructures::DriveInfo driveInfo = { driveName, "myHost", s_libraryName }; - scheduler.reportDriveStatus(driveInfo, cta::common::dataStructures::MountType::NoMount, cta::common::dataStructures::DriveStatus::Down, lc); - scheduler.reportDriveStatus(driveInfo, cta::common::dataStructures::MountType::NoMount, cta::common::dataStructures::DriveStatus::Up, lc); - } - - uint64_t archiveFileId; - - // Queue an archive request. - cta::common::dataStructures::EntryLog creationLog; - creationLog.host="host2"; - creationLog.time=0; - creationLog.username="admin1"; - cta::common::dataStructures::DiskFileInfo diskFileInfo; - diskFileInfo.gid=GROUP_2; - diskFileInfo.owner_uid=CMS_USER; - diskFileInfo.path="path/to/file"; - cta::common::dataStructures::ArchiveRequest request; - request.checksumBlob.insert(cta::checksum::ADLER32, 0x1234abcd); - request.creationLog=creationLog; - request.diskFileInfo=diskFileInfo; - request.diskFileID="diskFileID"; - request.fileSize=100*1000*1000; - cta::common::dataStructures::RequesterIdentity requester; - requester.name = s_userName; - requester.group = "userGroup"; - request.requester = requester; - request.srcURL="srcURL"; - request.storageClass=s_storageClassName; - archiveFileId = scheduler.checkAndGetNextArchiveFileId(s_diskInstance, request.storageClass, request.requester, lc); - scheduler.queueArchiveWithGivenId(archiveFileId, s_diskInstance, request, lc); - - scheduler.waitSchedulerDbSubthreadsComplete(); - - { - auto schedulerInformations = scheduler.getSchedulingInformations(lc); - ASSERT_FALSE(schedulerInformations.empty()); - - auto & schedulerInfo = schedulerInformations.front(); - ASSERT_EQ(s_libraryName,schedulerInfo.getLogicalLibraryName()); - const auto & potentialMounts = schedulerInfo.getPotentialMounts(); - - ASSERT_FALSE(potentialMounts.empty()); - const auto & potentialMount = potentialMounts.front(); - - ASSERT_EQ(request.fileSize,potentialMount.bytesQueued); - ASSERT_EQ(0,potentialMount.capacityInBytes); - ASSERT_EQ("",potentialMount.diskSystemSleptFor); - ASSERT_EQ(1,potentialMount.filesQueued); - ASSERT_EQ(0,potentialMount.mountCount); - ASSERT_EQ(s_minArchiveRequestAge,potentialMount.minRequestAge); - ASSERT_EQ(s_archivePriority,potentialMount.priority); - ASSERT_EQ(0,potentialMount.ratioOfMountQuotaUsed); - ASSERT_EQ(0,potentialMount.sleepTime); - ASSERT_FALSE(potentialMount.sleepingMount); - ASSERT_EQ(s_tapePoolName,potentialMount.tapePool); - ASSERT_EQ(cta::common::dataStructures::MountType::ArchiveForUser,potentialMount.type); - } - - { - std::unique_ptr<cta::TapeMount> mount; - mount.reset(scheduler.getNextMount(s_libraryName, driveName, lc).release()); - ASSERT_NE(nullptr, mount.get()); - std::unique_ptr<cta::ArchiveMount> archiveMount; - archiveMount.reset(dynamic_cast<cta::ArchiveMount*>(mount.release())); - ASSERT_NE(nullptr, archiveMount.get()); - std::list<std::unique_ptr<cta::ArchiveJob>> archiveJobBatch = archiveMount->getNextJobBatch(1,1,lc); - ASSERT_NE(nullptr, archiveJobBatch.front().get()); - std::unique_ptr<ArchiveJob> archiveJob = std::move(archiveJobBatch.front()); - archiveJob->tapeFile.blockId = 1; - archiveJob->tapeFile.fSeq = 1; - archiveJob->tapeFile.checksumBlob.insert(cta::checksum::ADLER32, 0x1234abcd); - archiveJob->tapeFile.fileSize = archiveJob->archiveFile.fileSize; - archiveJob->tapeFile.copyNb = 1; - archiveJob->validate(); - std::queue<std::unique_ptr <cta::ArchiveJob >> sDBarchiveJobBatch; - std::queue<cta::catalogue::TapeItemWritten> sTapeItems; - std::queue<std::unique_ptr <cta::SchedulerDatabase::ArchiveJob >> failedToReportArchiveJobs; - sDBarchiveJobBatch.emplace(std::move(archiveJob)); - archiveMount->reportJobsBatchTransferred(sDBarchiveJobBatch, sTapeItems,failedToReportArchiveJobs, lc); - archiveJobBatch = archiveMount->getNextJobBatch(1,1,lc); - ASSERT_EQ(0, archiveJobBatch.size()); - archiveMount->complete(); - } - - ASSERT_TRUE(scheduler.getSchedulingInformations(lc).empty()); - - //Queue a retrieve request for the archived file - { - cta::common::dataStructures::EntryLog creationLog; - creationLog.host="host2"; - creationLog.time=0; - creationLog.username="admin1"; - cta::common::dataStructures::DiskFileInfo diskFileInfo; - diskFileInfo.gid=GROUP_2; - diskFileInfo.owner_uid=CMS_USER; - diskFileInfo.path="path/to/file"; - cta::common::dataStructures::RetrieveRequest request; - request.archiveFileID = archiveFileId; - request.creationLog = creationLog; - request.diskFileInfo = diskFileInfo; - request.dstURL = "dstURL"; - request.requester.name = s_userName; - request.requester.group = "userGroup"; - scheduler.queueRetrieve(s_diskInstance, request, lc); - scheduler.waitSchedulerDbSubthreadsComplete(); - } - - { - auto schedulerInformations = scheduler.getSchedulingInformations(lc); - ASSERT_FALSE(schedulerInformations.empty()); - - auto & schedulerInfo = schedulerInformations.front(); - ASSERT_EQ(s_libraryName,schedulerInfo.getLogicalLibraryName()); - const auto & potentialMounts = schedulerInfo.getPotentialMounts(); - - ASSERT_FALSE(potentialMounts.empty()); - const auto & potentialMount = potentialMounts.front(); - - ASSERT_EQ(request.fileSize,potentialMount.bytesQueued); - ASSERT_EQ(s_mediaTypeCapacityInBytes,potentialMount.capacityInBytes); - ASSERT_EQ("",potentialMount.diskSystemSleptFor); - ASSERT_EQ(1,potentialMount.filesQueued); - ASSERT_EQ(0,potentialMount.mountCount); - ASSERT_EQ(s_minRetrieveRequestAge,potentialMount.minRequestAge); - ASSERT_EQ(s_retrievePriority,potentialMount.priority); - ASSERT_EQ(0,potentialMount.ratioOfMountQuotaUsed); - ASSERT_EQ(0,potentialMount.sleepTime); - ASSERT_FALSE(potentialMount.sleepingMount); - ASSERT_EQ(s_tapePoolName,potentialMount.tapePool); - ASSERT_EQ(cta::common::dataStructures::MountType::Retrieve,potentialMount.type); - ASSERT_EQ(s_libraryName,potentialMount.logicalLibrary); - ASSERT_EQ(s_vid,potentialMount.vid); - ASSERT_EQ(s_vo,potentialMount.vo); - } - //Now let's queue an Archive request with a high priority - //Modify the mount policy to have an equality between all values - catalogue.modifyMountPolicyArchiveMinRequestAge(s_adminOnAdminHost,s_mountPolicyName,1); - catalogue.modifyMountPolicyArchivePriority(s_adminOnAdminHost,s_mountPolicyName,1); - catalogue.modifyMountPolicyRetrieveMinRequestAge(s_adminOnAdminHost,s_mountPolicyName,1); - catalogue.modifyMountPolicyRetrievePriority(s_adminOnAdminHost,s_mountPolicyName,1); - - { - auto schedulerInformations = scheduler.getSchedulingInformations(lc); - ASSERT_FALSE(schedulerInformations.empty()); - - // Queue an archive request. - cta::common::dataStructures::EntryLog creationLog; - creationLog.host="host2"; - creationLog.time=0; - creationLog.username="admin1"; - cta::common::dataStructures::DiskFileInfo diskFileInfo; - diskFileInfo.gid=GROUP_2; - diskFileInfo.owner_uid=CMS_USER; - diskFileInfo.path="path/to/file2"; - cta::common::dataStructures::ArchiveRequest request; - request.checksumBlob.insert(cta::checksum::ADLER32, 0xabcd1234); - request.creationLog=creationLog; - request.diskFileInfo=diskFileInfo; - request.diskFileID="diskFileID"; - request.fileSize=200*1000*1000; - cta::common::dataStructures::RequesterIdentity requester; - requester.name = s_userName; - requester.group = "userGroup"; - request.requester = requester; - request.srcURL="srcURL2"; - request.storageClass=s_storageClassName; - uint64_t archiveFileId = scheduler.checkAndGetNextArchiveFileId(s_diskInstance, request.storageClass, request.requester, lc); - scheduler.queueArchiveWithGivenId(archiveFileId, s_diskInstance, request, lc); - - scheduler.waitSchedulerDbSubthreadsComplete(); - } - - { - auto schedulingInfos = scheduler.getSchedulingInformations(lc); - ASSERT_FALSE(schedulingInfos.empty()); - //We have only one logical library - EXPECT_EQ(1,schedulingInfos.size()); - const auto & schedulingInfo = schedulingInfos.front(); - //We have two potential mounts - auto potentialMounts = schedulingInfo.getPotentialMounts(); - ASSERT_EQ(2,potentialMounts.size()); - //The first mount should be an Archive and the second one a Retrieve as Archive is more prior than the Retrieve - auto & firstMount = potentialMounts.front(); - ASSERT_EQ(cta::common::dataStructures::MountType::ArchiveForUser,firstMount.type); - potentialMounts.pop_front(); - auto & secondMount = potentialMounts.front(); - ASSERT_EQ(cta::common::dataStructures::MountType::Retrieve,secondMount.type); - } - - //Change the mount policies to have a Retrieve priority higher than the Archive priority - catalogue.modifyMountPolicyRetrievePriority(s_adminOnAdminHost,s_mountPolicyName,10); - - { - auto schedulingInfos = scheduler.getSchedulingInformations(lc); - ASSERT_FALSE(schedulingInfos.empty()); - //We have only one logical library - EXPECT_EQ(1,schedulingInfos.size()); - const auto & schedulingInfo = schedulingInfos.front(); - //We have two potential mounts - auto potentialMounts = schedulingInfo.getPotentialMounts(); - ASSERT_EQ(2,potentialMounts.size()); - //The first mount should be an Archive and the second one a Retrieve as Archive is more prior than the Retrieve - auto & firstMount = potentialMounts.front(); - ASSERT_EQ(cta::common::dataStructures::MountType::Retrieve,firstMount.type); - potentialMounts.pop_front(); - auto & secondMount = potentialMounts.front(); - ASSERT_EQ(cta::common::dataStructures::MountType::ArchiveForUser,secondMount.type); - } -} - TEST_P(SchedulerTest, expandRepackRequestShouldThrowIfUseBufferNotRecallButNoDirectoryCreated){ using namespace cta; unitTests::TempDirectory tempDirectory; diff --git a/scheduler/OStoreDB/OStoreDB.cpp b/scheduler/OStoreDB/OStoreDB.cpp index a1cd9a08e4..88595b5017 100644 --- a/scheduler/OStoreDB/OStoreDB.cpp +++ b/scheduler/OStoreDB/OStoreDB.cpp @@ -483,7 +483,7 @@ void OStoreDB::fetchMountInfo(SchedulerDatabase::TapeMountDecisionInfo& tmdi, Ro // If a next mount exists the drive "counts double", but the corresponding drive // is either about to mount, or about to replace its current mount. double registerFetchTime = 0; - auto driveStates = m_tapeDrivesState->getDriveStates(logContext); + const auto driveStates = m_catalogue.getTapeDrives(); registerFetchTime = t.secs(utils::Timer::resetCounter); using common::dataStructures::DriveStatus; std::set<int> activeDriveStatuses = { diff --git a/scheduler/Scheduler.cpp b/scheduler/Scheduler.cpp index 48d9bb2445..133c180117 100644 --- a/scheduler/Scheduler.cpp +++ b/scheduler/Scheduler.cpp @@ -757,10 +757,10 @@ void Scheduler::RepackReportBatch::report(log::LogContext& lc) { //------------------------------------------------------------------------------ common::dataStructures::DesiredDriveState Scheduler::getDesiredDriveState(const std::string& driveName, log::LogContext & lc) { utils::Timer t; - auto driveStates = m_tapeDrivesState->getDriveStates(lc); - for (auto & driveState : driveStates) { + const auto driveStates = m_catalogue.getTapeDrives(); + for (const auto & driveState : driveStates) { if (driveState.driveName == driveName) { - auto schedulerDbTime = t.secs(); + const auto schedulerDbTime = t.secs(); if (schedulerDbTime > 1) { log::ScopedParamContainer spc(lc); spc.add("drive", driveName) @@ -940,7 +940,7 @@ std::optional<cta::common::dataStructures::TapeDrive> Scheduler::getDriveState(c //------------------------------------------------------------------------------ std::list<common::dataStructures::TapeDrive> Scheduler::getDriveStates(const common::dataStructures::SecurityIdentity &cliIdentity, log::LogContext & lc) const { utils::Timer t; - auto ret = m_tapeDrivesState->getDriveStates(lc); + const auto ret = m_catalogue.getTapeDrives(); auto schedulerDbTime = t.secs(); log::ScopedParamContainer spc(lc); spc.add("schedulerDbTime", schedulerDbTime); @@ -1629,55 +1629,6 @@ auto logicalLibrary = getLogicalLibrary(logicalLibraryName,getLogicalLibrariesTi return std::unique_ptr<TapeMount>(); } -//------------------------------------------------------------------------------ -// getSchedulingInformations -//------------------------------------------------------------------------------ -std::list<SchedulingInfos> Scheduler::getSchedulingInformations(log::LogContext& lc) { - - std::list<SchedulingInfos> ret; - - utils::Timer timer; - double getTapeInfoTime = 0; - double candidateSortingTime = 0; - double getTapeForWriteTime = 0; - - ExistingMountSummaryPerTapepool existingMountsDistinctTypeSummaryPerTapepool; - ExistingMountSummaryPerVo existingMountBasicTypeSummaryPerVo; - std::set<std::string> tapesInUse; - std::list<catalogue::TapeForWriting> tapeList; - - //get all drive informations and sort them by logical library name - cta::common::dataStructures::SecurityIdentity admin; - auto drives = getDriveStates(admin,lc); - - std::map<std::string,std::list<std::string>> logicalLibraryDriveNamesMap; - for(auto & drive : drives){ - logicalLibraryDriveNamesMap[drive.logicalLibrary].push_back(drive.driveName); - } - - for(auto & kv: logicalLibraryDriveNamesMap){ - //get mount informations - std::unique_ptr<SchedulerDatabase::TapeMountDecisionInfo> mountInfo; - mountInfo = m_db.getMountInfoNoLock(cta::SchedulerDatabase::PurposeGetMountInfo::GET_NEXT_MOUNT,lc); - std::string logicalLibrary = kv.first; - cta::SchedulingInfos schedulingInfos(logicalLibrary); - for(auto & driveName: kv.second){ - sortAndGetTapesForMountInfo(mountInfo, logicalLibrary, driveName, timer, - existingMountsDistinctTypeSummaryPerTapepool, existingMountBasicTypeSummaryPerVo, tapesInUse, tapeList, - getTapeInfoTime, candidateSortingTime, getTapeForWriteTime, lc); - //schedulingInfos.addDrivePotentialMount - std::vector<cta::SchedulerDatabase::PotentialMount> potentialMounts = mountInfo->potentialMounts; - for(auto & potentialMount: potentialMounts){ - schedulingInfos.addPotentialMount(potentialMount); - } - } - if(!schedulingInfos.getPotentialMounts().empty()){ - ret.push_back(schedulingInfos); - } - } - return ret; -} - //------------------------------------------------------------------------------ // getQueuesAndMountSummaries //------------------------------------------------------------------------------ diff --git a/scheduler/Scheduler.hpp b/scheduler/Scheduler.hpp index 80ca96dccc..94cbc3ebb1 100644 --- a/scheduler/Scheduler.hpp +++ b/scheduler/Scheduler.hpp @@ -52,7 +52,6 @@ #include "disk/DiskFile.hpp" #include "disk/DiskReporter.hpp" #include "disk/DiskReporterFactory.hpp" -#include "SchedulingInfos.hpp" #include <list> #include <map> @@ -328,13 +327,6 @@ public: */ std::unique_ptr<TapeMount> getNextMount(const std::string &logicalLibraryName, const std::string &driveName, log::LogContext & lc); - /** - * Returns scheduling informations for the cta-admin schedulinginfos ls command - * @param lc the log context - * @return the list of the scheduling informations for the cta-admin schedulinginfos ls command - */ - std::list<SchedulingInfos> getSchedulingInformations(log::LogContext & lc); - /** * A function returning * @param lc diff --git a/scheduler/SchedulerTest.cpp b/scheduler/SchedulerTest.cpp index 5337a767d0..647737d9a7 100644 --- a/scheduler/SchedulerTest.cpp +++ b/scheduler/SchedulerTest.cpp @@ -5307,255 +5307,6 @@ TEST_P(SchedulerTest, repackRetrieveRequestsFailToFetchDiskSystem){ */ } -TEST_P(SchedulerTest, getSchedulingInformations) { - //Queue 2 archive requests in two different logical libraries - using namespace cta; - - Scheduler &scheduler = getScheduler(); - auto &catalogue = getCatalogue(); - - setupDefaultCatalogue(); - catalogue.deleteTapeDrive("drive0"); // It's not needed -#ifdef STDOUT_LOGGING - log::StdoutLogger dl("dummy", "unitTest"); -#else - log::DummyLogger dl("", ""); -#endif - log::LogContext lc(dl); - - // Create the environment for the migration to happen (library + tape) - const std::string libraryComment = "Library comment"; - const bool libraryIsDisabled = false; - catalogue.createLogicalLibrary(s_adminOnAdminHost, s_libraryName, - libraryIsDisabled, libraryComment); - { - auto libraries = catalogue.getLogicalLibraries(); - ASSERT_EQ(1, libraries.size()); - ASSERT_EQ(s_libraryName, libraries.front().name); - ASSERT_EQ(libraryComment, libraries.front().comment); - } - - { - auto tape = getDefaultTape(); - catalogue.createTape(s_adminOnAdminHost, tape); - } - - const std::string driveName = "tape_drive"; - - catalogue.tapeLabelled(s_vid, driveName); - - { - // This first initialization is normally done by the dataSession function. - cta::common::dataStructures::DriveInfo driveInfo = { driveName, "myHost", s_libraryName }; - scheduler.reportDriveStatus(driveInfo, cta::common::dataStructures::MountType::NoMount, cta::common::dataStructures::DriveStatus::Down, lc); - scheduler.reportDriveStatus(driveInfo, cta::common::dataStructures::MountType::NoMount, cta::common::dataStructures::DriveStatus::Up, lc); - } - - uint64_t archiveFileId; - - // Queue an archive request. - cta::common::dataStructures::EntryLog creationLog; - creationLog.host="host2"; - creationLog.time=0; - creationLog.username="admin1"; - cta::common::dataStructures::DiskFileInfo diskFileInfo; - diskFileInfo.gid=GROUP_2; - diskFileInfo.owner_uid=CMS_USER; - diskFileInfo.path="path/to/file"; - cta::common::dataStructures::ArchiveRequest request; - request.checksumBlob.insert(cta::checksum::ADLER32, 0x1234abcd); - request.creationLog=creationLog; - request.diskFileInfo=diskFileInfo; - request.diskFileID="diskFileID"; - request.fileSize=100*1000*1000; - cta::common::dataStructures::RequesterIdentity requester; - requester.name = s_userName; - requester.group = "userGroup"; - request.requester = requester; - request.srcURL="srcURL"; - request.storageClass=s_storageClassName; - archiveFileId = scheduler.checkAndGetNextArchiveFileId(s_diskInstance, request.storageClass, request.requester, lc); - scheduler.queueArchiveWithGivenId(archiveFileId, s_diskInstance, request, lc); - - scheduler.waitSchedulerDbSubthreadsComplete(); - - { - auto schedulerInformations = scheduler.getSchedulingInformations(lc); - ASSERT_FALSE(schedulerInformations.empty()); - - auto & schedulerInfo = schedulerInformations.front(); - ASSERT_EQ(s_libraryName,schedulerInfo.getLogicalLibraryName()); - const auto & potentialMounts = schedulerInfo.getPotentialMounts(); - - ASSERT_FALSE(potentialMounts.empty()); - const auto & potentialMount = potentialMounts.front(); - - ASSERT_EQ(request.fileSize,potentialMount.bytesQueued); - ASSERT_EQ(0,potentialMount.capacityInBytes); - ASSERT_EQ("",potentialMount.diskSystemSleptFor); - ASSERT_EQ(1,potentialMount.filesQueued); - ASSERT_EQ(0,potentialMount.mountCount); - ASSERT_EQ(s_minArchiveRequestAge,potentialMount.minRequestAge); - ASSERT_EQ(s_archivePriority,potentialMount.priority); - ASSERT_EQ(0,potentialMount.ratioOfMountQuotaUsed); - ASSERT_EQ(0,potentialMount.sleepTime); - ASSERT_FALSE(potentialMount.sleepingMount); - ASSERT_EQ(s_tapePoolName,potentialMount.tapePool); - ASSERT_EQ(cta::common::dataStructures::MountType::ArchiveForUser,potentialMount.type); - } - - { - std::unique_ptr<cta::TapeMount> mount; - mount.reset(scheduler.getNextMount(s_libraryName, driveName, lc).release()); - ASSERT_NE(nullptr, mount.get()); - std::unique_ptr<cta::ArchiveMount> archiveMount; - archiveMount.reset(dynamic_cast<cta::ArchiveMount*>(mount.release())); - ASSERT_NE(nullptr, archiveMount.get()); - std::list<std::unique_ptr<cta::ArchiveJob>> archiveJobBatch = archiveMount->getNextJobBatch(1,1,lc); - ASSERT_NE(nullptr, archiveJobBatch.front().get()); - std::unique_ptr<ArchiveJob> archiveJob = std::move(archiveJobBatch.front()); - archiveJob->tapeFile.blockId = 1; - archiveJob->tapeFile.fSeq = 1; - archiveJob->tapeFile.checksumBlob.insert(cta::checksum::ADLER32, 0x1234abcd); - archiveJob->tapeFile.fileSize = archiveJob->archiveFile.fileSize; - archiveJob->tapeFile.copyNb = 1; - archiveJob->validate(); - std::queue<std::unique_ptr <cta::ArchiveJob >> sDBarchiveJobBatch; - std::queue<cta::catalogue::TapeItemWritten> sTapeItems; - std::queue<std::unique_ptr <cta::SchedulerDatabase::ArchiveJob >> failedToReportArchiveJobs; - sDBarchiveJobBatch.emplace(std::move(archiveJob)); - archiveMount->reportJobsBatchTransferred(sDBarchiveJobBatch, sTapeItems,failedToReportArchiveJobs, lc); - archiveJobBatch = archiveMount->getNextJobBatch(1,1,lc); - ASSERT_EQ(0, archiveJobBatch.size()); - archiveMount->complete(); - } - - ASSERT_TRUE(scheduler.getSchedulingInformations(lc).empty()); - - //Queue a retrieve request for the archived file - { - cta::common::dataStructures::EntryLog creationLog; - creationLog.host="host2"; - creationLog.time=0; - creationLog.username="admin1"; - cta::common::dataStructures::DiskFileInfo diskFileInfo; - diskFileInfo.gid=GROUP_2; - diskFileInfo.owner_uid=CMS_USER; - diskFileInfo.path="path/to/file"; - cta::common::dataStructures::RetrieveRequest request; - request.archiveFileID = archiveFileId; - request.creationLog = creationLog; - request.diskFileInfo = diskFileInfo; - request.dstURL = "dstURL"; - request.requester.name = s_userName; - request.requester.group = "userGroup"; - scheduler.queueRetrieve(s_diskInstance, request, lc); - scheduler.waitSchedulerDbSubthreadsComplete(); - } - - { - auto schedulerInformations = scheduler.getSchedulingInformations(lc); - ASSERT_FALSE(schedulerInformations.empty()); - - auto & schedulerInfo = schedulerInformations.front(); - ASSERT_EQ(s_libraryName,schedulerInfo.getLogicalLibraryName()); - const auto & potentialMounts = schedulerInfo.getPotentialMounts(); - - ASSERT_FALSE(potentialMounts.empty()); - const auto & potentialMount = potentialMounts.front(); - - ASSERT_EQ(request.fileSize,potentialMount.bytesQueued); - ASSERT_EQ(s_mediaTypeCapacityInBytes,potentialMount.capacityInBytes); - ASSERT_EQ("",potentialMount.diskSystemSleptFor); - ASSERT_EQ(1,potentialMount.filesQueued); - ASSERT_EQ(0,potentialMount.mountCount); - ASSERT_EQ(s_minRetrieveRequestAge,potentialMount.minRequestAge); - ASSERT_EQ(s_retrievePriority,potentialMount.priority); - ASSERT_EQ(0,potentialMount.ratioOfMountQuotaUsed); - ASSERT_EQ(0,potentialMount.sleepTime); - ASSERT_FALSE(potentialMount.sleepingMount); - ASSERT_EQ(s_tapePoolName,potentialMount.tapePool); - ASSERT_EQ(cta::common::dataStructures::MountType::Retrieve,potentialMount.type); - ASSERT_EQ(s_libraryName,potentialMount.logicalLibrary); - ASSERT_EQ(s_vid,potentialMount.vid); - ASSERT_EQ(s_vo,potentialMount.vo); - } - //Now let's queue an Archive request with a high priority - //Modify the mount policy to have an equality between all values - catalogue.modifyMountPolicyArchiveMinRequestAge(s_adminOnAdminHost,s_mountPolicyName,1); - catalogue.modifyMountPolicyArchivePriority(s_adminOnAdminHost,s_mountPolicyName,1); - catalogue.modifyMountPolicyRetrieveMinRequestAge(s_adminOnAdminHost,s_mountPolicyName,1); - catalogue.modifyMountPolicyRetrievePriority(s_adminOnAdminHost,s_mountPolicyName,1); - - { - auto schedulerInformations = scheduler.getSchedulingInformations(lc); - ASSERT_FALSE(schedulerInformations.empty()); - - // Queue an archive request. - cta::common::dataStructures::EntryLog creationLog; - creationLog.host="host2"; - creationLog.time=0; - creationLog.username="admin1"; - cta::common::dataStructures::DiskFileInfo diskFileInfo; - diskFileInfo.gid=GROUP_2; - diskFileInfo.owner_uid=CMS_USER; - diskFileInfo.path="path/to/file2"; - cta::common::dataStructures::ArchiveRequest request; - request.checksumBlob.insert(cta::checksum::ADLER32, 0xabcd1234); - request.creationLog=creationLog; - request.diskFileInfo=diskFileInfo; - request.diskFileID="diskFileID"; - request.fileSize=200*1000*1000; - cta::common::dataStructures::RequesterIdentity requester; - requester.name = s_userName; - requester.group = "userGroup"; - request.requester = requester; - request.srcURL="srcURL2"; - request.storageClass=s_storageClassName; - uint64_t archiveFileId = scheduler.checkAndGetNextArchiveFileId(s_diskInstance, request.storageClass, request.requester, lc); - scheduler.queueArchiveWithGivenId(archiveFileId, s_diskInstance, request, lc); - - scheduler.waitSchedulerDbSubthreadsComplete(); - } - - { - auto schedulingInfos = scheduler.getSchedulingInformations(lc); - ASSERT_FALSE(schedulingInfos.empty()); - //We have only one logical library - EXPECT_EQ(1,schedulingInfos.size()); - const auto & schedulingInfo = schedulingInfos.front(); - //We have two potential mounts - auto potentialMounts = schedulingInfo.getPotentialMounts(); - ASSERT_EQ(2,potentialMounts.size()); - //The first mount should be an Archive and the second one a Retrieve as Archive is more prior than the Retrieve - auto & firstMount = potentialMounts.front(); - ASSERT_EQ(cta::common::dataStructures::MountType::ArchiveForUser,firstMount.type); - potentialMounts.pop_front(); - auto & secondMount = potentialMounts.front(); - ASSERT_EQ(cta::common::dataStructures::MountType::Retrieve,secondMount.type); - } - - //Change the mount policies to have a Retrieve priority higher than the Archive priority - catalogue.modifyMountPolicyRetrievePriority(s_adminOnAdminHost,s_mountPolicyName,10); - - { - auto schedulingInfos = scheduler.getSchedulingInformations(lc); - ASSERT_FALSE(schedulingInfos.empty()); - //We have only one logical library - EXPECT_EQ(1,schedulingInfos.size()); - const auto & schedulingInfo = schedulingInfos.front(); - //We have two potential mounts - auto potentialMounts = schedulingInfo.getPotentialMounts(); - ASSERT_EQ(2,potentialMounts.size()); - //The first mount should be an Archive and the second one a Retrieve as Archive is more prior than the Retrieve - auto & firstMount = potentialMounts.front(); - ASSERT_EQ(cta::common::dataStructures::MountType::Retrieve,firstMount.type); - potentialMounts.pop_front(); - auto & secondMount = potentialMounts.front(); - ASSERT_EQ(cta::common::dataStructures::MountType::ArchiveForUser,secondMount.type); - } -} - TEST_P(SchedulerTest, expandRepackRequestShouldThrowIfUseBufferNotRecallButNoDirectoryCreated){ using namespace cta; unitTests::TempDirectory tempDirectory; diff --git a/scheduler/SchedulingInfos.cpp b/scheduler/SchedulingInfos.cpp deleted file mode 100644 index dfba450302..0000000000 --- a/scheduler/SchedulingInfos.cpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * @project The CERN Tape Archive (CTA) - * @copyright Copyright © 2021-2022 CERN - * @license This program is free software, distributed under the terms of the GNU General Public - * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can - * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A - * PARTICULAR PURPOSE. See the GNU General Public License for more details. - * - * In applying this licence, CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization or - * submit itself to any jurisdiction. - */ - -#include "SchedulingInfos.hpp" - -namespace cta { - -SchedulingInfos::SchedulingInfos(const std::string & logicalLibraryName):m_logicalLibraryName(logicalLibraryName) { -} - -SchedulingInfos::SchedulingInfos(const SchedulingInfos& other): m_logicalLibraryName(other.m_logicalLibraryName), m_potentialMounts(other.m_potentialMounts) { -} - -SchedulingInfos & SchedulingInfos::operator =(const SchedulingInfos& other){ - m_logicalLibraryName = other.m_logicalLibraryName; - m_potentialMounts = other.m_potentialMounts; - return *this; -} - -void SchedulingInfos::addPotentialMount(const cta::SchedulerDatabase::PotentialMount& potentialMount) { - m_potentialMounts.push_back(potentialMount); -} - -std::string SchedulingInfos::getLogicalLibraryName() const { - return m_logicalLibraryName; -} - -std::list<cta::SchedulerDatabase::PotentialMount> SchedulingInfos::getPotentialMounts() const { - return m_potentialMounts; -} - -SchedulingInfos::~SchedulingInfos() { -} - -} \ No newline at end of file diff --git a/scheduler/SchedulingInfos.hpp b/scheduler/SchedulingInfos.hpp deleted file mode 100644 index 430621b764..0000000000 --- a/scheduler/SchedulingInfos.hpp +++ /dev/null @@ -1,73 +0,0 @@ -/* - * @project The CERN Tape Archive (CTA) - * @copyright Copyright © 2021-2022 CERN - * @license This program is free software, distributed under the terms of the GNU General Public - * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can - * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A - * PARTICULAR PURPOSE. See the GNU General Public License for more details. - * - * In applying this licence, CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization or - * submit itself to any jurisdiction. - */ - -#pragma once - -#include <string> - -#include "SchedulerDatabase.hpp" - - -namespace cta { - -/** - * This class represents a scheduling information for a logical library, it is basically - * a list of potential mounts (cta::SchedulerDatabase::PotentialMount object) for a given logical library. - */ -class SchedulingInfos { -public: - /** - * Constructor - * @param logicalLibraryName the logical library - */ - SchedulingInfos(const std::string & logicalLibraryName); - /** - * Copy constructor - */ - SchedulingInfos(const SchedulingInfos& other); - /** - * Assignment operator - */ - SchedulingInfos & operator=(const SchedulingInfos & other); - /** - * Add a potential mount to this logical library scheduling informations - * @param potentialMount the potential mount to add - */ - void addPotentialMount(const cta::SchedulerDatabase::PotentialMount & potentialMount); - - /** - * Returns the list of potential mounts for this logical library - * @return a list of potential mounts for this logical library - */ - std::list<cta::SchedulerDatabase::PotentialMount> getPotentialMounts() const; - - /** - * Returns the logical library name of this SchedulingInfos instance - * @return the logical library name of this SchedulingInfos instance - */ - std::string getLogicalLibraryName() const; - - /** - * Destructor - */ - virtual ~SchedulingInfos(); -private: - std::string m_logicalLibraryName; - std::list<cta::SchedulerDatabase::PotentialMount> m_potentialMounts; -}; - -} \ No newline at end of file diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp index 2fc5b0cab0..ee0cba412e 100644 --- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp +++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp @@ -42,7 +42,6 @@ using XrdSsiPb::PbException; #include "XrdCtaDiskInstanceSpaceLs.hpp" #include "XrdCtaVirtualOrganizationLs.hpp" #include "XrdCtaVersion.hpp" -#include "XrdCtaSchedulingInfosLs.hpp" #include "XrdCtaRecycleTapeFileLs.hpp" #include "XrdCtaChangeStorageClass.hpp" @@ -315,9 +314,6 @@ void RequestMessage::process(const cta::xrd::Request &request, cta::xrd::Respons case cmd_pair(AdminCmd::CMD_VERSION, AdminCmd::SUBCMD_NONE): processVersion(response, stream); break; - case cmd_pair(AdminCmd::CMD_SCHEDULINGINFOS, AdminCmd::SUBCMD_LS): - processSchedulingInfos_Ls(response,stream); - break; case cmd_pair(AdminCmd::CMD_RECYCLETAPEFILE, AdminCmd::SUBCMD_LS): processRecycleTapeFile_Ls(response,stream); break; @@ -2499,15 +2495,6 @@ void RequestMessage::processVersion(cta::xrd::Response &response, XrdSsiStream * response.set_type(cta::xrd::Response::RSP_SUCCESS); } -void RequestMessage::processSchedulingInfos_Ls(cta::xrd::Response &response, XrdSsiStream * & stream) { - using namespace cta::admin; - - stream = new SchedulingInfosLsStream(*this,m_catalogue,m_scheduler,m_lc); - - response.set_show_header(HeaderType::SCHEDULINGINFOS_LS); - response.set_type(cta::xrd::Response::RSP_SUCCESS); -} - void RequestMessage::processRecycleTapeFile_Ls(cta::xrd::Response &response, XrdSsiStream * & stream) { using namespace cta::admin; diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.hpp b/xroot_plugins/XrdSsiCtaRequestMessage.hpp index 334a0b4383..988b295943 100644 --- a/xroot_plugins/XrdSsiCtaRequestMessage.hpp +++ b/xroot_plugins/XrdSsiCtaRequestMessage.hpp @@ -255,7 +255,6 @@ private: admincmdstream_t processDiskInstanceSpace_Ls; admincmdstream_t processVirtualOrganization_Ls; admincmdstream_t processVersion; - admincmdstream_t processSchedulingInfos_Ls; admincmdstream_t processRecycleTapeFile_Ls; /*! -- GitLab From e8b1874c19b87945c24b3e100a3aaeeaba059b8e Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 7 Oct 2022 08:41:48 +0200 Subject: [PATCH 035/126] Move and remove change-storage-class source files --- .../CtaChangeStorageClassCmdLineArgs.cpp | 108 ----------- .../CtaChangeStorageClassCmdLineArgs.hpp | 82 --------- .../change_storage_class/CtaCmdOptions.hpp | 171 ------------------ .../change_storage_class/CMakeLists.txt | 0 .../CtaChangeStorageClass.cpp | 0 .../CtaChangeStorageClass.hpp | 0 .../CtaChangeStorageClassMain.cpp | 0 7 files changed, 361 deletions(-) delete mode 100644 cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp delete mode 100644 cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp delete mode 100644 cmdline/change_storage_class/CtaCmdOptions.hpp rename cmdline/{ => standalone_cli_tools}/change_storage_class/CMakeLists.txt (100%) rename cmdline/{ => standalone_cli_tools}/change_storage_class/CtaChangeStorageClass.cpp (100%) rename cmdline/{ => standalone_cli_tools}/change_storage_class/CtaChangeStorageClass.hpp (100%) rename cmdline/{ => standalone_cli_tools}/change_storage_class/CtaChangeStorageClassMain.cpp (100%) diff --git a/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp b/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp deleted file mode 100644 index 8a86a0fa66..0000000000 --- a/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.cpp +++ /dev/null @@ -1,108 +0,0 @@ -/* - * @project The CERN Tape Archive (CTA) - * @copyright Copyright © 2021-2022 CERN - * @license This program is free software, distributed under the terms of the GNU General Public - * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can - * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A - * PARTICULAR PURPOSE. See the GNU General Public License for more details. - * - * In applying this licence, CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization or - * submit itself to any jurisdiction. - */ - -#include "cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp" -#include "common/exception/CommandLineNotParsed.hpp" - -#include <getopt.h> -#include <ostream> -#include <string> -#include <limits.h> -#include <fstream> - -namespace cta { -namespace admin{ - -//------------------------------------------------------------------------------ -// constructor -//------------------------------------------------------------------------------ -CtaChangeStorageClassCmdLineArgs::CtaChangeStorageClassCmdLineArgs(const int argc, char *const *const argv): - m_help(false), - m_optionFids("--fid", "-f", true), - m_option_storage_class_name("--storage_class_name", "-scn", true) { - - option_with_multiple_values_map["--fid"] = &m_optionFids; - option_with_multiple_values_map["-f"] = &m_optionFids; - - option_map["--storage_class_name"] = &m_option_storage_class_name; - option_map["-scn"] = &m_option_storage_class_name; - - parseCmd(argc, argv); -} - - -//------------------------------------------------------------------------------ -// printUsage -//------------------------------------------------------------------------------ -const std::string CtaChangeStorageClassCmdLineArgs::getUsageText() const { - return "Usage: cta-update-storage-class [--fid/-f <archive_file_id>] [--storage_class_name/-scn <storage_class_name>]"; -} - -//------------------------------------------------------------------------------ -// isEndOfOption -//------------------------------------------------------------------------------ -bool CtaChangeStorageClassCmdLineArgs::isEndOfOption(int nextOptionOrValueIndex, const char *const *const argv) { - return ( - option_with_multiple_values_map.find(argv[nextOptionOrValueIndex]) != option_with_multiple_values_map.end() || - option_map.find(argv[nextOptionOrValueIndex]) != option_map.end() - ); -} - -//------------------------------------------------------------------------------ -// parseCmd -//------------------------------------------------------------------------------ -void CtaChangeStorageClassCmdLineArgs::parseCmd(const int argc, const char *const *const argv) { - if (argc < 4) { throw std::runtime_error(getUsageText()); } - unsigned int num_cmd_line_enrties_for_option = 2; - for (int i = 1; i < argc; i += num_cmd_line_enrties_for_option){ - num_cmd_line_enrties_for_option = 2; - auto search_option_with_multiple_values_map = option_with_multiple_values_map.find(argv[i]); - if (search_option_with_multiple_values_map != option_with_multiple_values_map.end()) { - auto stringListOption = search_option_with_multiple_values_map->second; - stringListOption->set_present(); - - if (stringListOption->get_name() == "--fid") { - for (int j = 1; j < argc; j += 1) { - int nextOptionOrValueIndex = i + j + 1; - bool end_of_option = isEndOfOption(nextOptionOrValueIndex, argv); - stringListOption->add_value(argv[i + j]); - if (end_of_option || (j + i) == (argc - 1)) { - num_cmd_line_enrties_for_option = j + 1; - break; - } - } - } - } - - auto search_option = option_map.find(argv[i]); - if (search_option != option_map.end()) { - auto stringOption = search_option->second; - stringOption->set_present(); - if (stringOption->get_name() == "--storage_class_name") { - stringOption->set_value(argv[i + 1]); - } - } - - if (search_option_with_multiple_values_map == option_with_multiple_values_map.end() && search_option == option_map.end()) { - std::cout << "Error: Unknown option: " << argv[i] << std::endl; - throw std::runtime_error(getUsageText()); - } - } -} - -} // namespace admin -} // namespace cta diff --git a/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp b/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp deleted file mode 100644 index 3b81f4d5ec..0000000000 --- a/cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * @project The CERN Tape Archive (CTA) - * @copyright Copyright © 2021-2022 CERN - * @license This program is free software, distributed under the terms of the GNU General Public - * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can - * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A - * PARTICULAR PURPOSE. See the GNU General Public License for more details. - * - * In applying this licence, CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization or - * submit itself to any jurisdiction. - */ - -#pragma once - -#include <list> -#include <optional> -#include <map> -#include <iostream> -#include <sstream> - -#include "version.h" -#include "cmdline/CtaCmdOptions.hpp" - - -namespace cta { -namespace admin { - -/** - * class to store the command-line arguments of the command-line tool - * named cta-change-storage-class. - */ -class CtaChangeStorageClassCmdLineArgs { - -public: - /** - * True if the usage message should be printed. - */ - bool m_help; - - /** - * List of archive file ids - */ - StringListOption m_optionFids; - - /** - * The storage name used to replace the names for the files with file id found in m_fids - */ - StringOption m_option_storage_class_name; - - - /** - * Constructor that parses the specified command-line arguments. - * - * @param argc The number of command-line arguments including the name of the - * executable. - * @param argv The vector of command-line arguments. - */ - CtaChangeStorageClassCmdLineArgs(const int argc, char *const *const argv); - - - /** - * Get the text of correct usage of the tool - * - */ - const std::string getUsageText() const; - -private: - bool isEndOfOption(int nextOptionOrValueIndex, const char *const *const argv); - void parseCmd(const int argc, const char *const *const argv); - - std::map<std::string, StringListOption *> option_with_multiple_values_map; - std::map<std::string, StringOption *> option_map; - -}; // class CtaChangeStorageClassCmdLineArgs - -} // namespace admin -} // namespace cta diff --git a/cmdline/change_storage_class/CtaCmdOptions.hpp b/cmdline/change_storage_class/CtaCmdOptions.hpp deleted file mode 100644 index ae4e934f69..0000000000 --- a/cmdline/change_storage_class/CtaCmdOptions.hpp +++ /dev/null @@ -1,171 +0,0 @@ -/* - * @project The CERN Tape Archive (CTA) - * @copyright Copyright © 2021-2022 CERN - * @license This program is free software, distributed under the terms of the GNU General Public - * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can - * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A - * PARTICULAR PURPOSE. See the GNU General Public License for more details. - * - * In applying this licence, CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization or - * submit itself to any jurisdiction. - */ - -#pragma once - -#include <string> -#include <vector> - -/*! -* Simple command line option class for simpler commands like cta-verify-file or cta-send-event -*/ -class StringOption { -public: - /*! - * Constructor - */ - StringOption(const std::string &long_opt, const std::string &short_opt, const bool is_optional) : - m_long_opt(long_opt), - m_short_opt(short_opt), - m_is_optional(is_optional), - m_is_present(false), - m_value("") {} - - /*! - * Check if the supplied key matches the option - */ - bool operator==(const std::string &option) const { - return option == m_short_opt || option == m_long_opt; - } - - /*! - * Return whether the option is optional - */ - bool is_optional() const { - return m_is_optional; - } - - /*! - * Return whether the option is present on the command line - */ - bool is_present() const { - return m_is_present; - } - - /*! - * Sets the option as present on the command line - */ - void set_present() { - m_is_present = true; - } - - /*! - * Return the value of the option given from the command line - */ - const std::string &get_value() const { - return m_value; - } - - /*! - * Return the name of the option (it's long command option) - */ - const std::string &get_name() const { - return m_long_opt; - } - - /*! - * Sets the value of the option from the command line - */ - void set_value(const std::string &mValue) { - m_value = mValue; - } - -private: - - //member variables - const std::string m_long_opt; //!< Long command option - const std::string m_short_opt; //!< Short command option - const bool m_is_optional; //!< Option is optional or compulsory - bool m_is_present; //!< Option is present on the command line - std::string m_value; //!< Option value -}; - - -/*! -* Simple command line option class for simpler commands like cta-verify-file or cta-send-event -*/ -class StringListOption { -public: - /*! - * Constructor - */ - StringListOption(const std::string &long_opt, const std::string &short_opt, const bool is_optional) : - m_long_opt(long_opt), - m_short_opt(short_opt), - m_is_optional(is_optional), - m_is_present(false) {} - - /*! - * Check if the supplied key matches the option - */ - bool operator==(const std::string &option) const { - return option == m_short_opt || option == m_long_opt; - } - - /*! - * Return whether the option is optional - */ - bool is_optional() const { - return m_is_optional; - } - - /*! - * Return whether the option is present on the command line - */ - bool is_present() const { - return m_is_present; - } - - /*! - * Sets the option as present on the command line - */ - void set_present() { - m_is_present = true; - } - - /*! - * Return the value of the option given from the command line - */ - const std::vector<std::string> &get_value() const { - return m_value_list; - } - - /*! - * Return the name of the option (it's long command option) - */ - const std::string &get_name() const { - return m_long_opt; - } - - /*! - * Sets the value of the option from the command line - */ - void add_value(const std::string &mValue) { - m_value_list.push_back(mValue); - } - -private: - - //member variables - const std::string m_long_opt; //!< Long command option - const std::string m_short_opt; //!< Short command option - const bool m_is_optional; //!< Option is optional or compulsory - bool m_is_present; //!< Option is present on the command line - std::vector<std::string> m_value_list; //!< Option value -}; - - diff --git a/cmdline/change_storage_class/CMakeLists.txt b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt similarity index 100% rename from cmdline/change_storage_class/CMakeLists.txt rename to cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt diff --git a/cmdline/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp similarity index 100% rename from cmdline/change_storage_class/CtaChangeStorageClass.cpp rename to cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp diff --git a/cmdline/change_storage_class/CtaChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp similarity index 100% rename from cmdline/change_storage_class/CtaChangeStorageClass.hpp rename to cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp diff --git a/cmdline/change_storage_class/CtaChangeStorageClassMain.cpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClassMain.cpp similarity index 100% rename from cmdline/change_storage_class/CtaChangeStorageClassMain.cpp rename to cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClassMain.cpp -- GitLab From 81394b7760f2451f97bf6b5493848be92930c02f Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 7 Oct 2022 08:47:21 +0200 Subject: [PATCH 036/126] add changes to chage-storage-class CMakefiles --- cmdline/standalone_cli_tools/CMakeLists.txt | 1 + .../standalone_cli_tools/change_storage_class/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmdline/standalone_cli_tools/CMakeLists.txt b/cmdline/standalone_cli_tools/CMakeLists.txt index 1112c75327..18f3c324c4 100644 --- a/cmdline/standalone_cli_tools/CMakeLists.txt +++ b/cmdline/standalone_cli_tools/CMakeLists.txt @@ -15,6 +15,7 @@ cmake_minimum_required (VERSION 3.17) +add_subdirectory (change_storage_class) add_subdirectory (restore_files) find_package(xrootdclient REQUIRED) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt index 00dc829982..c081c25c73 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt +++ b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt @@ -27,7 +27,7 @@ include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include) # Compiled protocol buffers include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS}) -add_executable(cta-change-storage-class CtaChangeStorageClass.cpp CtaChangeStorageClassCmdLineArgs.cpp CtaChangeStorageClassMain.cpp ../restore_files/CmdLineTool.cpp ../CtaAdminCmdParse.cpp CtaCmdOptions.hpp) +add_executable(cta-change-storage-class CtaChangeStorageClass.cpp CtaChangeStorageClassMain.cpp ../common/CmdLineTool.cpp) target_link_libraries(cta-change-storage-class ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient) set_property (TARGET cta-change-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) -- GitLab From b420b2e89280ac34912cb9c2c76e46e5612e0bb3 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 7 Oct 2022 09:25:37 +0200 Subject: [PATCH 037/126] add changes for cta-change-storage-class to work with the common parsing tool --- .../CtaChangeStorageClass.cpp | 16 ++++++---------- .../CtaChangeStorageClass.hpp | 17 +++-------------- .../standalone_cli_tools/common/CmdLineArgs.cpp | 17 +++++++++++++++++ .../standalone_cli_tools/common/CmdLineArgs.hpp | 9 ++++++++- 4 files changed, 34 insertions(+), 25 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp index 11bc076e58..fb59193535 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp @@ -15,12 +15,11 @@ * submit itself to any jurisdiction. */ -#include "cmdline/change_storage_class/CtaChangeStorageClass.hpp" -#include "cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp" +#include "cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp" +#include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" #include "common/utils/utils.hpp" #include "common/checksum/ChecksumBlob.hpp" #include "CtaFrontendApi.hpp" -#include <cmdline/CtaAdminCmdParse.hpp> #include <XrdSsiPbLog.hpp> @@ -59,7 +58,7 @@ void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) namespace cta { -namespace admin { +namespace cliTool { //------------------------------------------------------------------------------ // constructor @@ -72,14 +71,11 @@ CtaChangeStorageClass::CtaChangeStorageClass( const int& argc, char *const *const argv): CmdLineTool(inStream, outStream, errStream), - m_log(log), - m_cmdLineArgs(argc, argv) { - + m_log(log) { - m_archiveFileIds = m_cmdLineArgs.m_optionFids.get_value(); - m_storageClassName = m_cmdLineArgs.m_option_storage_class_name.get_value(); + CmdLineArgs cmdLineArgs(argc, argv, StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS); + m_storageClassName = cmdLineArgs. - if (m_storageClassName == "" || m_archiveFileIds.size() == 0) {throw std::runtime_error(m_cmdLineArgs.getUsageText()); } } diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp index 2fddf2b016..2e2629e901 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp @@ -17,8 +17,8 @@ #pragma once -#include "cmdline/change_storage_class/CtaChangeStorageClassCmdLineArgs.hpp" -#include "cmdline/restore_files/CmdLineTool.hpp" +#include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" +#include "cmdline/standalone_cli_tools/common/CmdLineTool.hpp" #include "eos_grpc_client/GrpcEndpoint.hpp" #include "catalogue/Catalogue.hpp" #include "common/log/StdoutLogger.hpp" @@ -29,7 +29,7 @@ #include "CtaFrontendApi.hpp" namespace cta { -namespace admin { +namespace cliTool { class CtaChangeStorageClass: public CmdLineTool { public: @@ -95,17 +95,6 @@ private: std::string m_storageClassName; - /** - * Disk instance of the files to change - */ - std::optional<std::string> m_diskInstance; - - /** - * Fids of the files to change - */ - std::optional<std::list<uint64_t>> m_eosFids; - - /** *Command line arguments */ diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp index a6523c10aa..01b4746db2 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp @@ -61,16 +61,25 @@ static struct option verifyFileLongOption[] = { {nullptr, 0, nullptr, 0} }; +static struct option changeStorageClassLongOption[] = { + {"id", required_argument, nullptr, 'I'}, + {"storage.class.name", required_argument, nullptr, 'n'}, + {"help", no_argument, nullptr, 'h'}, + {nullptr, 0, nullptr, 0} +}; + std::map<StandaloneCliTool, const option*> longopts = { {StandaloneCliTool::RESTORE_FILES, restoreFilesLongOption}, {StandaloneCliTool::CTA_SEND_EVENT, sendFileLongOption}, {StandaloneCliTool::CTA_VERIFY_FILE, verifyFileLongOption}, + {StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS, changeStorageClassLongOption}, }; std::map<StandaloneCliTool, const char*> shortopts = { {StandaloneCliTool::RESTORE_FILES, "I:i:f:F:v:c:hd:"}, {StandaloneCliTool::CTA_SEND_EVENT, "i:e:u:g:"}, {StandaloneCliTool::CTA_VERIFY_FILE, "I:i:u:g:v:h:"}, + {StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS, "I:n:h:"}, }; //------------------------------------------------------------------------------ @@ -93,6 +102,9 @@ m_help(false), m_debug(false), m_standaloneCliTool{standaloneCliTool} { case StandaloneCliTool::CTA_VERIFY_FILE: opt_index = 2; break; + case StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS: + opt_index = 2; + break; default: opt_index = 3; break; @@ -167,6 +179,11 @@ m_help(false), m_debug(false), m_standaloneCliTool{standaloneCliTool} { m_requestGroup = std::string(optarg); break; } + case 'n': + { + m_storageClassName = std::string(optarg); + break; + } case ':': // Missing parameter { exception::CommandLineNotParsed ex; diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp index a90bf727b1..b8a0ec9215 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp @@ -32,7 +32,8 @@ namespace cliTool { enum class StandaloneCliTool { RESTORE_FILES, CTA_SEND_EVENT, - CTA_VERIFY_FILE + CTA_VERIFY_FILE, + CTA_CHANGE_STORAGE_CLASS }; /** @@ -100,6 +101,12 @@ struct CmdLineArgs { */ StandaloneCliTool m_standaloneCliTool; + /** + * The tool parsing the arguments + */ + std::string m_storageClassName; + + /** * Constructor that parses the specified command-line arguments. * -- GitLab From 8b9f9ac07e6f4d76794f6fa906937df623d7a5c2 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 7 Oct 2022 13:59:31 +0200 Subject: [PATCH 038/126] Merge branch 'main' into 78-create-a-tool-to-change-the-storage-class-of-data-which-has-already-been-written-to-tape --- .gitlab-ci.yml | 4 +-- .gitlab/ci/build.gitlab-ci.yml | 14 +++++----- .gitlab/ci/tagged.gitlab-ci.yml | 2 +- .gitlab/issue_templates/Release.md | 2 +- ReleaseNotes.md | 4 ++- cmake/Findgmock.cmake | 27 ------------------- .../cc7/etc/yum.repos.d/cta-ci.repo | 7 +++++ cta.spec.in | 3 ++- objectstore/AlgorithmsTest.cpp | 11 -------- tests/CMakeLists.txt | 16 +++++------ 10 files changed, 30 insertions(+), 60 deletions(-) delete mode 100644 cmake/Findgmock.cmake diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c861bcfffe..2de6020baa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,8 +58,8 @@ default: - test -n "${CI_COMMIT_TAG}" && export TAG_VERSION=$(echo ${CI_COMMIT_TAG} | sed -e 's/^v//;s/-.*$//') - test -n "${CI_COMMIT_TAG}" && export TAG_RELEASE=$(echo ${CI_COMMIT_TAG} | sed -e 's/^[^-]*-//') - major_version=$(echo ${TAG_VERSION} | cut -d. -f1) - - if [[ ${major_version} == 5 ]]; - then echo "Setting to compile with XRootD version 5"; + - if [[ ${major_version} == 5 ]]; then + echo "Setting to compile with XRootD version 5"; XROOTD_VERSION=5; fi - *prepare-xrootd5 diff --git a/.gitlab/ci/build.gitlab-ci.yml b/.gitlab/ci/build.gitlab-ci.yml index c309b6db0b..48dd423373 100644 --- a/.gitlab/ci/build.gitlab-ci.yml +++ b/.gitlab/ci/build.gitlab-ci.yml @@ -50,7 +50,7 @@ cta_srpm_xrootd4: - .cta_srpm rules: - !reference [.cta_build, rules] - - if: $XROOTD_VERSION == "4" && $SCHED_TYPE == "objectstore" && $ORACLE_SUPPORT == "ON" + - if: ($CI_COMMIT_TAG =~ /^v?4\./ || ($CI_COMMIT_TAG == null && $XROOTD_VERSION == "4")) && $SCHED_TYPE == "objectstore" && $ORACLE_SUPPORT == "ON" cta_rpm_xrootd4: stage: build:rpm @@ -60,7 +60,7 @@ cta_rpm_xrootd4: - !reference [.cta_build, rules] - if: $CI_COMMIT_TAG when: never - - if: $XROOTD_VERSION == "4" && $SCHED_TYPE == "objectstore" && $ORACLE_SUPPORT == "ON" + - if: $XROOTD_VERSION == "4" && $SCHED_TYPE == "objectstore" && $ORACLE_SUPPORT == "ON" cta_srpm_xrootd4_pgsched: stage: build:srpm @@ -68,7 +68,7 @@ cta_srpm_xrootd4_pgsched: - .cta_srpm rules: - !reference [.cta_build, rules] - - if: $XROOTD_VERSION == "4" && $SCHED_TYPE == "pgsched" && $ORACLE_SUPPORT == "OFF" + - if: ($CI_COMMIT_TAG =~ /^v?4\./ || ($CI_COMMIT_TAG == null && $XROOTD_VERSION == "4")) && $SCHED_TYPE == "pgsched" && $ORACLE_SUPPORT == "OFF" cta_rpm_xrootd4_pgsched: stage: build:rpm @@ -86,7 +86,7 @@ cta_srpm_xrootd5: - .cta_srpm rules: - !reference [.cta_build, rules] - - if: $XROOTD_VERSION == "5" && $SCHED_TYPE == "objectstore" && $ORACLE_SUPPORT == "ON" + - if: ($CI_COMMIT_TAG =~ /^v?5\./ || ($CI_COMMIT_TAG == null && $XROOTD_VERSION == "5")) && $SCHED_TYPE == "objectstore" && $ORACLE_SUPPORT == "ON" cta_rpm_xrootd5: stage: build:rpm @@ -96,7 +96,7 @@ cta_rpm_xrootd5: - !reference [.cta_build, rules] - if: $CI_COMMIT_TAG when: never - - if: $XROOTD_VERSION == "5" && $SCHED_TYPE == "objectstore" && $ORACLE_SUPPORT == "ON" + - if: $XROOTD_VERSION == "5" && $SCHED_TYPE == "objectstore" && $ORACLE_SUPPORT == "ON" cta_srpm_no_oracle: stage: build:srpm @@ -104,7 +104,7 @@ cta_srpm_no_oracle: - .cta_srpm rules: - !reference [.cta_build, rules] - - if: $XROOTD_VERSION == "4" && $SCHED_TYPE == "objectstore" && $ORACLE_SUPPORT == "OFF" + - if: ($CI_COMMIT_TAG =~ /^v?4\./ || ($CI_COMMIT_TAG == null && $XROOTD_VERSION == "4")) && $SCHED_TYPE == "objectstore" && $ORACLE_SUPPORT == "OFF" cta_rpm_no_oracle: stage: build:rpm @@ -114,4 +114,4 @@ cta_rpm_no_oracle: - !reference [.cta_build, rules] - if: $CI_COMMIT_TAG when: never - - if: $XROOTD_VERSION == "4" && $SCHED_TYPE == "objectstore" && $ORACLE_SUPPORT == "OFF" + - if: $XROOTD_VERSION == "4" && $SCHED_TYPE == "objectstore" && $ORACLE_SUPPORT == "OFF" diff --git a/.gitlab/ci/tagged.gitlab-ci.yml b/.gitlab/ci/tagged.gitlab-ci.yml index 8b77e15d46..910e7640ac 100644 --- a/.gitlab/ci/tagged.gitlab-ci.yml +++ b/.gitlab/ci/tagged.gitlab-ci.yml @@ -46,6 +46,6 @@ release_public_rpm: allow_failure: true image: gitlab-registry.cern.ch/linuxsupport/cc7-base script: - - EOS_ACCOUNT_USERNAME=${DOCKER_LOGIN_USERNAME} EOS_ACCOUNT_PASSWORD=${DOCKER_LOGIN_PASSWORD} CI_OUTPUT_DIR='build_tagged_rpm/RPM/RPMS/x86_64' EOS_PATH='/eos/user/c/ctareg/www/public/cta-public-repo/7' HOOK='/eos/user/c/ctareg/www/public/update_repos.sh' continuousintegration/ci_helpers/deploy-eos.sh + - EOS_ACCOUNT_USERNAME=${DOCKER_LOGIN_USERNAME} EOS_ACCOUNT_PASSWORD=${DOCKER_LOGIN_PASSWORD} CI_OUTPUT_DIR="build_tagged_rpm/RPM/RPMS/x86_64" EOS_PATH="/eos/user/c/ctareg/www/public/cta-public-repo/cta-$XROOTD_VERSION/el-7/cta" HOOK="/eos/user/c/ctareg/www/public/update_repos.sh" continuousintegration/ci_helpers/deploy-eos.sh tags: - docker \ No newline at end of file diff --git a/.gitlab/issue_templates/Release.md b/.gitlab/issue_templates/Release.md index 8a66c069d9..e6062989d1 100644 --- a/.gitlab/issue_templates/Release.md +++ b/.gitlab/issue_templates/Release.md @@ -4,7 +4,7 @@ ### Initial code references for release -This release should be based on `<commit_id>` from branch `master`. +This release should be based on `<commit_id>` from branch `main`. This release will be tagged as version `vX.Y.Z-R`. diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 5dbbb707f4..2fc8747a30 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -2,7 +2,9 @@ ## Summary ### Features -- cta/CTA#146 - Refactoring of operation tools cmd line parsing +- cta/CTA#146 - Refactoring of operation tools cmd line parsing +### Building and Packaging +- cta/CTA#107 - Check latest version of gtest suite # v4.7.12-1 diff --git a/cmake/Findgmock.cmake b/cmake/Findgmock.cmake deleted file mode 100644 index 5869e3d2bf..0000000000 --- a/cmake/Findgmock.cmake +++ /dev/null @@ -1,27 +0,0 @@ -# @project The CERN Tape Archive (CTA) -# @copyright Copyright © 2015-2022 CERN -# @license This program is free software, distributed under the terms of the GNU General Public -# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can -# redistribute it and/or modify it under the terms of the GPL Version 3, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -# PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# In applying this licence, CERN does not waive the privileges and immunities -# granted to it by virtue of its status as an Intergovernmental Organization or -# submit itself to any jurisdiction. - -# This module will set the following variables: -# GMOCK_SRC - -set(GMOCK_SRC "/usr/src/gmock/gmock-all.cc") - -if(NOT EXISTS ${GMOCK_SRC}) - unset(GMOCK_SRC) -endif(NOT EXISTS ${GMOCK_SRC}) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(gmock DEFAULT_MSG - GMOCK_SRC) diff --git a/continuousintegration/docker/ctafrontend/cc7/etc/yum.repos.d/cta-ci.repo b/continuousintegration/docker/ctafrontend/cc7/etc/yum.repos.d/cta-ci.repo index 9de33fa871..05b95d6fee 100644 --- a/continuousintegration/docker/ctafrontend/cc7/etc/yum.repos.d/cta-ci.repo +++ b/continuousintegration/docker/ctafrontend/cc7/etc/yum.repos.d/cta-ci.repo @@ -39,3 +39,10 @@ baseurl=https://cta-ci-repo.web.cern.ch/cta-ci-repo/xrootd5/ priority=4 gpgcheck=0 enabled=0 + +[cta-ci-dependencies] +name=CTA CI repo dependencies +baseurl=https://cta-ci-repo.web.cern.ch/cta-ci-repo/cta-dependencies/ +priority=4 +gpgcheck=0 +enabled=1 \ No newline at end of file diff --git a/cta.spec.in b/cta.spec.in index c2089a9653..4f372eab5c 100644 --- a/cta.spec.in +++ b/cta.spec.in @@ -52,7 +52,7 @@ BuildRequires: xrootd-server-devel >= %{xrootdVersion} BuildRequires: xrootd-private-devel >= %{xrootdVersion} BuildRequires: librados-devel = %{radosVersion}, libradosstriper-devel = %{radosVersion}, BuildRequires: protobuf3-compiler >= 3.3.1 protobuf3-devel >= 3.3.1 -BuildRequires: gmock-devel >= 1.5.0 gtest-devel >= 1.5.0 +BuildRequires: gtest-devel >= 1.12.0 BuildRequires: sqlite-devel >= 3.6 BuildRequires: libcap-devel >= 2.16 BuildRequires: binutils-devel >= 2.20 @@ -316,6 +316,7 @@ Requires: cta-taped = %{ctaVersion}-%{ctaRelease}%{mydist} Requires: make Requires: moreutils Requires: xrootd-client-libs >= %{xrootdVersion} +Requires: gtest-devel >= 1.12.0 %description -n cta-systemtests CERN Tape Archive: Unit tests and system tests with virtual tape drives diff --git a/objectstore/AlgorithmsTest.cpp b/objectstore/AlgorithmsTest.cpp index 07edd30bc8..dcd239a22a 100644 --- a/objectstore/AlgorithmsTest.cpp +++ b/objectstore/AlgorithmsTest.cpp @@ -32,17 +32,6 @@ #include "RootEntry.hpp" #include "tests/TestsCompileTimeSwitches.hpp" -std::ostream& operator<<(std::ostream& os, const cta::objectstore::ContainerTraits<cta::objectstore::RetrieveQueue, cta::objectstore::RetrieveQueueToTransfer>::PoppedElementsSummary& s) { - os << - "{" - "files=" << s.files << "," - "bytes=" << s.bytes << "," - "diskSystemFull=" << (s.diskSystemFull ? "true" : "false") << "," - "fullDiskSystem=\"" << s.fullDiskSystem << "\"" - "}"; - return os; -} - namespace unitTests { /** diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6fa2ddfe20..a46a965ad4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -14,13 +14,11 @@ # submit itself to any jurisdiction. cmake_minimum_required (VERSION 3.17) -find_package(gmock REQUIRED) find_package(Protobuf3 REQUIRED) include_directories(${PROTOBUF3_INCLUDE_DIRS}) add_executable(cta-unitTests - unit_tests.cpp - ${GMOCK_SRC}) + unit_tests.cpp) set (CTA_UNIT_TEST_LIBS ctacataloguecmdlineunittests @@ -42,6 +40,7 @@ set (CTA_UNIT_TEST_LIBS ctadiskunittests ctatapelabelunittests gtest + gmock pthread ctatapeserverraounittests) @@ -61,8 +60,7 @@ endif (OCCI_SUPPORT) add_executable(cta-rdbmsUnitTests GlobalCatalogueFactoryForUnitTests.cpp RdbmsUnitTestsCmdLineArgs.cpp - rdbmsUnitTests.cpp - ${GMOCK_SRC}) + rdbmsUnitTests.cpp) target_link_libraries(cta-rdbmsUnitTests ctadropschemacmd @@ -72,6 +70,7 @@ target_link_libraries(cta-rdbmsUnitTests ctardbmsunittests ctadisk gtest + gmock pthread ${PROTOBUF3_LIBRARIES}) @@ -81,13 +80,13 @@ if (OCCI_SUPPORT) endif (OCCI_SUPPORT) add_executable(cta-unitTests-multiProcess - unit_tests.cpp - ${GMOCK_SRC}) + unit_tests.cpp) target_link_libraries(cta-unitTests-multiProcess ctadaemonunittests-multiprocess ctacommon gtest + gmock pthread ${PROTOBUF3_LIBRAIRES} sqlite3) @@ -109,8 +108,7 @@ target_link_libraries(systemTestHelperTests ctacommon) add_executable(cta-systemTests - system_tests.cpp - ${GMOCK_SRC}) + system_tests.cpp) set_property (TARGET cta-systemTests APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) -- GitLab From cc6e681be10e1971e374230a2cc60f5c3d189a86 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 7 Oct 2022 14:44:53 +0200 Subject: [PATCH 039/126] Add changes to make the tool compatible with the common parsing tool --- .../change_storage_class/CMakeLists.txt | 2 +- .../CtaChangeStorageClass.cpp | 41 +++++++++++-------- .../CtaChangeStorageClass.hpp | 12 ------ .../CtaChangeStorageClassMain.cpp | 4 +- .../common/CmdLineArgs.hpp | 2 +- 5 files changed, 27 insertions(+), 34 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt index c081c25c73..10add3170b 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt +++ b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt @@ -27,7 +27,7 @@ include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include) # Compiled protocol buffers include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS}) -add_executable(cta-change-storage-class CtaChangeStorageClass.cpp CtaChangeStorageClassMain.cpp ../common/CmdLineTool.cpp) +add_executable(cta-change-storage-class CtaChangeStorageClass.cpp CtaChangeStorageClassMain.cpp ../common/CmdLineTool.cpp ../common/CmdLineArgs.cpp ../../CtaAdminCmdParse.cpp) target_link_libraries(cta-change-storage-class ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient) set_property (TARGET cta-change-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp index fb59193535..b50481bd3d 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp @@ -17,6 +17,7 @@ #include "cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp" #include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" +#include "cmdline/CtaAdminCmdParse.hpp" #include "common/utils/utils.hpp" #include "common/checksum/ChecksumBlob.hpp" #include "CtaFrontendApi.hpp" @@ -74,7 +75,23 @@ CtaChangeStorageClass::CtaChangeStorageClass( m_log(log) { CmdLineArgs cmdLineArgs(argc, argv, StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS); - m_storageClassName = cmdLineArgs. + + if (cmdLineArgs.m_help) { + cmdLineArgs.printUsage(std::cout); + exit(0); + } + + if (!cmdLineArgs.m_storageClassName) { + cmdLineArgs.printUsage(std::cout); + exit(1); + } + + if (!cmdLineArgs.m_archiveFileId && !cmdLineArgs.m_eosFids) { + cmdLineArgs.printUsage(std::cout); + exit(1); + } + + m_storageClassName = cmdLineArgs.m_storageClassName.value(); } @@ -89,18 +106,18 @@ uint64_t CtaChangeStorageClass::changeCatalogue() { request.set_client_cta_version(CTA_VERSION); request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); - admincmd.set_cmd(AdminCmd::CMD_CHANGESTORAGECLASS); - admincmd.set_subcmd(AdminCmd::SUBCMD_CH); + admincmd.set_cmd(cta::admin::AdminCmd::CMD_CHANGESTORAGECLASS); + admincmd.set_subcmd(cta::admin::AdminCmd::SUBCMD_CH); { - auto key = OptionString::STORAGE_CLASS_NAME; + auto key = cta::admin::OptionString::STORAGE_CLASS_NAME; auto new_opt = admincmd.add_option_str(); new_opt->set_key(key); new_opt->set_value(m_storageClassName); } { - auto key = OptionStrList::FILE_ID; + auto key = cta::admin::OptionStrList::FILE_ID; auto new_opt = admincmd.add_option_str_list(); new_opt->set_key(key); for (auto &archiveFileId : m_archiveFileIds) { @@ -111,7 +128,7 @@ uint64_t CtaChangeStorageClass::changeCatalogue() { // Validate the Protocol Buffer try { - validateCmd(request.admincmd()); + cta::admin::validateCmd(admincmd); } catch(std::runtime_error &ex) { throw std::runtime_error("Error: Protocol Buffer validation"); } @@ -140,15 +157,6 @@ uint64_t CtaChangeStorageClass::changeCatalogue() { } -//------------------------------------------------------------------------------ -// printUsage -//------------------------------------------------------------------------------ -void CtaChangeStorageClass::printUsage(std::ostream &os) { - os << "Usage:" << std::endl << - " cta-change-storage-class [--fid/-f <archive_file_id>] [--storage_class_name/-scn <storage_class_name>]" << std::endl; -} - - //------------------------------------------------------------------------------ // readAndSetConfiguration //------------------------------------------------------------------------------ @@ -191,9 +199,6 @@ void CtaChangeStorageClass::readAndSetConfiguration( // exceptionThrowingMain //------------------------------------------------------------------------------ int CtaChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { - if (m_cmdLineArgs.m_help) { - throw std::runtime_error(m_cmdLineArgs.getUsageText()); - } readAndSetConfiguration(getUsername()); diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp index 2e2629e901..afb102490d 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp @@ -69,12 +69,6 @@ public: */ void readAndSetConfiguration(const std::string &userName); - - /** - * Prints the usage message of the command-line tool. - */ - void printUsage(std::ostream &os) override; - private: /** @@ -95,12 +89,6 @@ private: std::string m_storageClassName; - /** - *Command line arguments - */ - CtaChangeStorageClassCmdLineArgs m_cmdLineArgs; - - /** * CTA Frontend service provider */ diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClassMain.cpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClassMain.cpp index 60a6125379..456f0bbf15 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClassMain.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClassMain.cpp @@ -22,7 +22,7 @@ #include <XrdSsiPbLog.hpp> #include <XrdSsiPbIStreamBuffer.hpp> -#include "cmdline/change_storage_class/CtaChangeStorageClass.hpp" +#include "cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp" #include "common/utils/utils.hpp" #include "common/log/StdoutLogger.hpp" @@ -40,7 +40,7 @@ int main(const int argc, char *const *const argv) { } cta::log::StdoutLogger log(hostName, "cta-change-storage-class"); - cta::admin::CtaChangeStorageClass cmd(std::cin, std::cout, std::cerr, log, argc, argv); + cta::cliTool::CtaChangeStorageClass cmd(std::cin, std::cout, std::cerr, log, argc, argv); int ret = cmd.main(argc, argv); // Delete all global objects allocated by libprotobuf google::protobuf::ShutdownProtobufLibrary(); diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp index b8a0ec9215..3c0cb52e96 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp @@ -104,7 +104,7 @@ struct CmdLineArgs { /** * The tool parsing the arguments */ - std::string m_storageClassName; + std::optional<std::string> m_storageClassName; /** -- GitLab From 224c6ed89a951ab7ab0a1755b0be53bd59dcd9ef Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 7 Oct 2022 17:01:53 +0200 Subject: [PATCH 040/126] Add changes to make the tool use the common parsing tool --- .../change_storage_class/CtaChangeStorageClass.cpp | 6 +++++- cmdline/standalone_cli_tools/common/CmdLineArgs.cpp | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp index b50481bd3d..219ae51c97 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp @@ -86,13 +86,17 @@ CtaChangeStorageClass::CtaChangeStorageClass( exit(1); } - if (!cmdLineArgs.m_archiveFileId && !cmdLineArgs.m_eosFids) { + if (!cmdLineArgs.m_archiveFileId) { cmdLineArgs.printUsage(std::cout); exit(1); } m_storageClassName = cmdLineArgs.m_storageClassName.value(); + if (cmdLineArgs.m_archiveFileId) { + m_archiveFileIds.push_back(std::to_string(cmdLineArgs.m_archiveFileId.value())); + } + } diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp index 01b4746db2..d2b5cd3b87 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp @@ -277,6 +277,10 @@ void CmdLineArgs::printUsage(std::ostream &os) const { os << " Usage:" << std::endl << " cta-verify-file --id/-I <archiveFileID> --vid/-v <vid> [--instance/-i <instance>] [--request.user/-u <user>] [request.group/-g <group>]\n" << std::endl; break; + case StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS : + os << " Usage:" << std::endl << + " cta-change-storage-class --id/-I <archiveFileID> --storage.class.name/-n\n" << std::endl; + break; default: break; } -- GitLab From 9e57ba82deefcedc056242af5e9a80f0f5b04972 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Mon, 10 Oct 2022 15:52:50 +0200 Subject: [PATCH 041/126] add changes to usage message --- cmdline/standalone_cli_tools/common/CmdLineArgs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp index d2b5cd3b87..0bef7ac8d6 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp @@ -279,7 +279,7 @@ void CmdLineArgs::printUsage(std::ostream &os) const { break; case StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS : os << " Usage:" << std::endl << - " cta-change-storage-class --id/-I <archiveFileID> --storage.class.name/-n\n" << std::endl; + " cta-change-storage-class --id/-I <archiveFileID> | --filename/-F --storage.class.name/-n <storageClassName>\n" << std::endl; break; default: break; -- GitLab From 7e91211b202a73754055a4d0f78f3e52f6376bf1 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Mon, 10 Oct 2022 16:34:01 +0200 Subject: [PATCH 042/126] change -n to -f --- .../orchestration/tests/common/archive_file.sh | 4 ++-- continuousintegration/orchestration/tests/restore_files.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/continuousintegration/orchestration/tests/common/archive_file.sh b/continuousintegration/orchestration/tests/common/archive_file.sh index bd9a88da51..54e067c4b2 100644 --- a/continuousintegration/orchestration/tests/common/archive_file.sh +++ b/continuousintegration/orchestration/tests/common/archive_file.sh @@ -23,9 +23,9 @@ EOF exit 1 } -while getopts "n:" o; do +while getopts "f:" o; do case "${o}" in - n) + f) TEST_FILE_NAME=${OPTARG} ;; *) diff --git a/continuousintegration/orchestration/tests/restore_files.sh b/continuousintegration/orchestration/tests/restore_files.sh index 3ab6378cf3..cf69296669 100755 --- a/continuousintegration/orchestration/tests/restore_files.sh +++ b/continuousintegration/orchestration/tests/restore_files.sh @@ -64,7 +64,7 @@ echo "Launching restore_files_client.sh on client pod" echo " Archiving file: xrdcp as user1" kubectl -n ${NAMESPACE} cp common/archive_file.sh client:/root/archive_file.sh kubectl -n ${NAMESPACE} cp client_helper.sh client:/root/client_helper.sh -kubectl -n ${NAMESPACE} exec client -- bash /root/archive_file.sh -n ${TEST_FILE_NAME} || exit 1 +kubectl -n ${NAMESPACE} exec client -- bash /root/archive_file.sh -f ${TEST_FILE_NAME} || exit 1 echo METADATA_FILE_PATH=$(mktemp -d).json -- GitLab From 0969b845be4e654efa8d62ea56142edd1c858133 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Mon, 10 Oct 2022 16:44:03 +0200 Subject: [PATCH 043/126] add change-storage-class test --- .../orchestration/tests/changeStorageClass.sh | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100755 continuousintegration/orchestration/tests/changeStorageClass.sh diff --git a/continuousintegration/orchestration/tests/changeStorageClass.sh b/continuousintegration/orchestration/tests/changeStorageClass.sh new file mode 100755 index 0000000000..ebb59af1c8 --- /dev/null +++ b/continuousintegration/orchestration/tests/changeStorageClass.sh @@ -0,0 +1,42 @@ +#bin/bash + +usage() { cat <<EOF 1>&2 +Usage: $0 -n <namespace> +EOF +exit 1 +} + +while getopts "n:" o; do + case "${o}" in + n) + NAMESPACE=${OPTARG} + ;; + *) + usage + ;; + esac +done +shift $((OPTIND-1)) + +if [ -z "${NAMESPACE}" ]; then + usage +fi + +if [ ! -z "${error}" ]; then + echo -e "ERROR:\n${error}" + exit 1 +fi + +NEW_STORAGE_CLASS_NAME=storageClassName +FILE_NAMES=(`uuidgen` `uuidgen` `uuidgen` `uuidgen`) + +for file_name in ${FILE_NAMES[@]}; do + kubectl -n ${NAMESPACE} exec client -- bash /root/archive_file.sh -f ${file_name} || exit 1 +done + +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc add --name ${NEW_STORAGE_CLASS_NAME} --numberofcopies 1 --virtualorganisation vo --comment "comment" +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc ls + +kubectl cp ~/CTA-build/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class ${NAMESPACE}/ctacli:/usr/bin/ +kubectl -n ${NAMESPACE} exec ctacli -- cta-change-storage-class --storage.class.name ${NEW_STORAGE_CLASS_NAME} --id 1 + -- GitLab From 4d7da0eb4225ee4aeb18f584814cf75c7e5f0908 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 11 Oct 2022 08:33:35 +0200 Subject: [PATCH 044/126] add changes to test --- .../orchestration/tests/changeStorageClass.sh | 21 +++++++++++++++---- .../orchestration/tests/restore_files.sh | 1 - 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/continuousintegration/orchestration/tests/changeStorageClass.sh b/continuousintegration/orchestration/tests/changeStorageClass.sh index ebb59af1c8..ca157105eb 100755 --- a/continuousintegration/orchestration/tests/changeStorageClass.sh +++ b/continuousintegration/orchestration/tests/changeStorageClass.sh @@ -28,11 +28,24 @@ if [ ! -z "${error}" ]; then fi NEW_STORAGE_CLASS_NAME=storageClassName -FILE_NAMES=(`uuidgen` `uuidgen` `uuidgen` `uuidgen`) -for file_name in ${FILE_NAMES[@]}; do - kubectl -n ${NAMESPACE} exec client -- bash /root/archive_file.sh -f ${file_name} || exit 1 -done +CHANGE_SC_FILE=`uuidgen` +STATIC_SC_FILE=`uuidgen` +echo +echo "Creating files: ${CHANGE_SC_FILE} ${STATIC_SC_FILE}" + +FRONTEND_IP=$(kubectl -n ${NAMESPACE} get pods ctafrontend -o json | jq .status.podIP | tr -d '"') + +echo +echo "ADD FRONTEND GATEWAY TO EOS" +echo "kubectl -n ${NAMESPACE} exec ctaeos -- bash eos root://${EOSINSTANCE} -r 0 0 vid add gateway ${FRONTEND_IP} grpc" +kubectl -n ${NAMESPACE} exec ctaeos -- eos -r 0 0 vid add gateway ${FRONTEND_IP} grpc +eos vid set map -grpc key:<your key aks any uuid string> vuid:2 vgid:2 + +kubectl -n ${NAMESPACE} cp common/archive_file.sh client:/usr/bin/ +kubectl -n ${NAMESPACE} cp client_helper.sh client:/root/ +kubectl -n ${NAMESPACE} exec client -- bash /usr/bin/archive_file.sh -f ${CHANGE_SC_FILE} || exit 1 +kubectl -n ${NAMESPACE} exec client -- bash /usr/bin/archive_file.sh -f ${STATIC_SC_FILE} || exit 1 kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc add --name ${NEW_STORAGE_CLASS_NAME} --numberofcopies 1 --virtualorganisation vo --comment "comment" kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc ls diff --git a/continuousintegration/orchestration/tests/restore_files.sh b/continuousintegration/orchestration/tests/restore_files.sh index cf69296669..6802e99438 100755 --- a/continuousintegration/orchestration/tests/restore_files.sh +++ b/continuousintegration/orchestration/tests/restore_files.sh @@ -53,7 +53,6 @@ echo echo "ADD FRONTEND GATEWAY TO EOS" echo "kubectl -n ${NAMESPACE} exec ctaeos -- bash eos root://${EOSINSTANCE} -r 0 0 vid add gateway ${FRONTEND_IP} grpc" kubectl -n ${NAMESPACE} exec ctaeos -- eos -r 0 0 vid add gateway ${FRONTEND_IP} grpc -eos vid set map -grpc key:<your key aks any uuid string> vuid:2 vgid:2 echo echo "eos vid ls" -- GitLab From 3ba9f663b645c0a365240fc7d48365df9dcc3362 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 11 Oct 2022 11:06:05 +0200 Subject: [PATCH 045/126] add common eos connection class --- .../common/EosConnection.hpp | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 cmdline/standalone_cli_tools/common/EosConnection.hpp diff --git a/cmdline/standalone_cli_tools/common/EosConnection.hpp b/cmdline/standalone_cli_tools/common/EosConnection.hpp new file mode 100644 index 0000000000..e60b8a7914 --- /dev/null +++ b/cmdline/standalone_cli_tools/common/EosConnection.hpp @@ -0,0 +1,87 @@ +class EosConnection { + +readAndSetConfiguration( + const std::string &userName, + const CmdLineArgs &cmdLineArgs) { + + if (cmdLineArgs.m_debug) { + m_log.setLogMask("DEBUG"); + } else { + m_log.setLogMask("INFO"); + } + + // Set CTA frontend configuration options + const std::string cli_config_file = "/etc/cta/cta-cli.conf"; + XrdSsiPb::Config cliConfig(cli_config_file, "cta"); + cliConfig.set("resource", "/ctafrontend"); + cliConfig.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes + cliConfig.set("request_timeout", DefaultRequestTimeout); // default value = 10s + + // Allow environment variables to override config file + cliConfig.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); + + // If XRDDEBUG=1, switch on all logging + if(getenv("XRDDEBUG")) { + cliConfig.set("log", "all"); + } + // If fine-grained control over log level is required, use XrdSsiPbLogLevel + cliConfig.getEnv("log", "XrdSsiPbLogLevel"); + + // Validate that endpoint was specified in the config file + if(!cliConfig.getOptionValueStr("endpoint").first) { + throw std::runtime_error("Configuration error: cta.endpoint missing from " + cli_config_file); + } + + // If the server is down, we want an immediate failure. Set client retry to a single attempt. + XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); + + m_serviceProviderPtr.reset(new XrdSsiPbServiceType(cliConfig)); + + // Set CTA frontend configuration options to connect to eos + const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf"; + XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); + + // Get the endpoint for namespace queries + auto nsConf = frontendXrootdConfig.getOptionValueStr("ns.config"); + if(nsConf.first) { + setNamespaceMap(nsConf.second); + } else { + throw std::runtime_error("Configuration error: cta.ns.config missing from " + frontend_xrootd_config_file); + } + } + + void setNamespaceMap(const std::string &keytab_file) { + // Open the keytab file for reading + std::ifstream file(keytab_file); + if(!file) { + throw cta::exception::UserError("Failed to open namespace keytab configuration file " + keytab_file); + } + ::eos::client::NamespaceMap_t namespaceMap; + // Parse the keytab line by line + std::string line; + for(int lineno = 0; std::getline(file, line); ++lineno) { + // Strip out comments + auto pos = line.find('#'); + if(pos != std::string::npos) { + line.resize(pos); + } + + // Parse one line + std::stringstream ss(line); + std::string diskInstance; + std::string endpoint; + std::string token; + std::string eol; + ss >> diskInstance >> endpoint >> token >> eol; + + // Ignore blank lines, all other lines must have exactly 3 elements + if(token.empty() || !eol.empty()) { + if(diskInstance.empty() && endpoint.empty() && token.empty()) continue; + throw cta::exception::UserError("Could not parse namespace keytab configuration file line " + std::to_string(lineno) + ": " + line); + } + namespaceMap.insert(std::make_pair(diskInstance, ::eos::client::Namespace(endpoint, token))); + } + m_endpointMapPtr = std::make_unique<::eos::client::EndpointMap>(namespaceMap); + } +} + -- GitLab From a24e4921c6cf3ead8c40bffd903f6c9bbc912eaa Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 11 Oct 2022 17:03:58 +0200 Subject: [PATCH 046/126] delete EosConnection.hpp and add framework for eos update --- .../CtaChangeStorageClass.cpp | 27 ++---- .../CtaChangeStorageClass.hpp | 11 ++- .../common/EosConnection.hpp | 87 ------------------- 3 files changed, 17 insertions(+), 108 deletions(-) delete mode 100644 cmdline/standalone_cli_tools/common/EosConnection.hpp diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp index 219ae51c97..eeefef19f1 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp @@ -99,11 +99,17 @@ CtaChangeStorageClass::CtaChangeStorageClass( } +//------------------------------------------------------------------------------ +// updateStorageClassInEosNamespace +//------------------------------------------------------------------------------ +void CtaChangeStorageClass::updateStorageClassInEosNamespace() { + +} //------------------------------------------------------------------------------ // changeCatalogue //------------------------------------------------------------------------------ -uint64_t CtaChangeStorageClass::changeCatalogue() { +void CtaChangeStorageClass::updateStorageClassInCatalogue() { cta::xrd::Request request; auto &admincmd = *(request.mutable_admincmd()); @@ -156,11 +162,8 @@ uint64_t CtaChangeStorageClass::changeCatalogue() { case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); default: throw XrdSsiPb::PbException("Invalid response type."); } - - return 0; } - //------------------------------------------------------------------------------ // readAndSetConfiguration //------------------------------------------------------------------------------ @@ -205,20 +208,8 @@ void CtaChangeStorageClass::readAndSetConfiguration( int CtaChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { readAndSetConfiguration(getUsername()); - - // ------------ For debugging ------------ // - - std::cout << "Username: " << getUsername() << std::endl; - std::cout << "Hostname: " << getHostname() << std::endl; - - std::cout << "m_storageClassName: " << m_storageClassName << " -- Type: " << typeid(m_storageClassName).name() << std::endl; - for (auto &archiveFileId : m_archiveFileIds) { - std::cout << "archiveFileId: " << archiveFileId << " -- Type: " << typeid(archiveFileId).name() << std::endl; - } - - // --------------------------------------- // - - changeCatalogue(); + updateStorageClassInEosNamespace(); + updateStorageClassInCatalogue(); return 0; } diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp index afb102490d..69d9278f13 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp @@ -105,9 +105,14 @@ private: XROOTD_SSI_PROTOBUF_INTERFACE_VERSION; /** - * Changes the storage class in the catalogue - */ - uint64_t changeCatalogue(); + * Changes the storage class in the EOS namespace + */ + void updateStorageClassInEosNamespace(); + + /** + * Changes the storage class in the catalogue + */ + void updateStorageClassInCatalogue(); } ; // class CtaChangeStorageClass diff --git a/cmdline/standalone_cli_tools/common/EosConnection.hpp b/cmdline/standalone_cli_tools/common/EosConnection.hpp deleted file mode 100644 index e60b8a7914..0000000000 --- a/cmdline/standalone_cli_tools/common/EosConnection.hpp +++ /dev/null @@ -1,87 +0,0 @@ -class EosConnection { - -readAndSetConfiguration( - const std::string &userName, - const CmdLineArgs &cmdLineArgs) { - - if (cmdLineArgs.m_debug) { - m_log.setLogMask("DEBUG"); - } else { - m_log.setLogMask("INFO"); - } - - // Set CTA frontend configuration options - const std::string cli_config_file = "/etc/cta/cta-cli.conf"; - XrdSsiPb::Config cliConfig(cli_config_file, "cta"); - cliConfig.set("resource", "/ctafrontend"); - cliConfig.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes - cliConfig.set("request_timeout", DefaultRequestTimeout); // default value = 10s - - // Allow environment variables to override config file - cliConfig.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); - - // If XRDDEBUG=1, switch on all logging - if(getenv("XRDDEBUG")) { - cliConfig.set("log", "all"); - } - // If fine-grained control over log level is required, use XrdSsiPbLogLevel - cliConfig.getEnv("log", "XrdSsiPbLogLevel"); - - // Validate that endpoint was specified in the config file - if(!cliConfig.getOptionValueStr("endpoint").first) { - throw std::runtime_error("Configuration error: cta.endpoint missing from " + cli_config_file); - } - - // If the server is down, we want an immediate failure. Set client retry to a single attempt. - XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); - - m_serviceProviderPtr.reset(new XrdSsiPbServiceType(cliConfig)); - - // Set CTA frontend configuration options to connect to eos - const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf"; - XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); - - // Get the endpoint for namespace queries - auto nsConf = frontendXrootdConfig.getOptionValueStr("ns.config"); - if(nsConf.first) { - setNamespaceMap(nsConf.second); - } else { - throw std::runtime_error("Configuration error: cta.ns.config missing from " + frontend_xrootd_config_file); - } - } - - void setNamespaceMap(const std::string &keytab_file) { - // Open the keytab file for reading - std::ifstream file(keytab_file); - if(!file) { - throw cta::exception::UserError("Failed to open namespace keytab configuration file " + keytab_file); - } - ::eos::client::NamespaceMap_t namespaceMap; - // Parse the keytab line by line - std::string line; - for(int lineno = 0; std::getline(file, line); ++lineno) { - // Strip out comments - auto pos = line.find('#'); - if(pos != std::string::npos) { - line.resize(pos); - } - - // Parse one line - std::stringstream ss(line); - std::string diskInstance; - std::string endpoint; - std::string token; - std::string eol; - ss >> diskInstance >> endpoint >> token >> eol; - - // Ignore blank lines, all other lines must have exactly 3 elements - if(token.empty() || !eol.empty()) { - if(diskInstance.empty() && endpoint.empty() && token.empty()) continue; - throw cta::exception::UserError("Could not parse namespace keytab configuration file line " + std::to_string(lineno) + ": " + line); - } - namespaceMap.insert(std::make_pair(diskInstance, ::eos::client::Namespace(endpoint, token))); - } - m_endpointMapPtr = std::make_unique<::eos::client::EndpointMap>(namespaceMap); - } -} - -- GitLab From 033c002e9280abbf1c39e9858f15f088c5d3ea47 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 12 Oct 2022 09:09:49 +0200 Subject: [PATCH 047/126] remove unused include --- .../change_storage_class/CtaChangeStorageClassMain.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClassMain.cpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClassMain.cpp index 456f0bbf15..bc965c50ba 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClassMain.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClassMain.cpp @@ -20,7 +20,6 @@ #include <iostream> #include <XrdSsiPbLog.hpp> -#include <XrdSsiPbIStreamBuffer.hpp> #include "cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp" #include "common/utils/utils.hpp" -- GitLab From c335964ce92fc9aceb2ceb05c10adc997abfd030 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 12 Oct 2022 10:09:39 +0200 Subject: [PATCH 048/126] add changes to parsing of multiple ids --- .../CtaChangeStorageClass.cpp | 10 +- .../CtaChangeStorageClass.hpp | 8 +- .../common/CmdLineArgs.cpp | 109 +++++++----------- .../common/CmdLineArgs.hpp | 12 +- .../restore_files/RestoreFilesCmd.cpp | 4 +- .../restore_files/RestoreFilesCmd.hpp | 2 +- 6 files changed, 57 insertions(+), 88 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp index eeefef19f1..edef19544c 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp @@ -33,14 +33,6 @@ #include <iostream> #include <memory> -// GLOBAL VARIABLES : used to pass information between main thread and stream handler thread - -// global synchronisation flag -std::atomic<bool> isHeaderSent(false); - -std::list<cta::admin::RecycleTapeFileLsItem> deletedTapeFiles; -std::list<std::pair<std::string,std::string>> listedTapeFiles; - namespace XrdSsiPb { /*! @@ -94,7 +86,7 @@ CtaChangeStorageClass::CtaChangeStorageClass( m_storageClassName = cmdLineArgs.m_storageClassName.value(); if (cmdLineArgs.m_archiveFileId) { - m_archiveFileIds.push_back(std::to_string(cmdLineArgs.m_archiveFileId.value())); + m_fxIds.push_back(std::to_string(cmdLineArgs.m_archiveFileId.value())); } } diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp index 69d9278f13..3a03027bd6 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp @@ -78,10 +78,14 @@ private: /** - * Archive file id of the files to change + * Fxids of the files to change */ - std::vector<std::string> m_archiveFileIds; + std::vector<std::string> m_fxIds; + /** + * Archive file ids of the files to change + */ + std::vector<std::string> m_archiveFileIds; /** * Archive file id of the files to change diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp index 0bef7ac8d6..afc239d047 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp @@ -35,7 +35,7 @@ static struct option restoreFilesLongOption[] = { {"id", required_argument, nullptr, 'I'}, {"instance", required_argument, nullptr, 'i'}, {"fxid", required_argument, nullptr, 'f'}, - {"fxidfile", required_argument, nullptr, 'F'}, + {"filename", required_argument, nullptr, 'F'}, {"vid", required_argument, nullptr, 'v'}, {"copynb", required_argument, nullptr, 'c'}, {"help", no_argument, nullptr, 'h'}, @@ -62,7 +62,8 @@ static struct option verifyFileLongOption[] = { }; static struct option changeStorageClassLongOption[] = { - {"id", required_argument, nullptr, 'I'}, + {"fxid", required_argument, nullptr, 'I'}, + {"filename", required_argument, nullptr, 'F'}, {"storage.class.name", required_argument, nullptr, 'n'}, {"help", no_argument, nullptr, 'h'}, {nullptr, 0, nullptr, 0} @@ -79,7 +80,7 @@ std::map<StandaloneCliTool, const char*> shortopts = { {StandaloneCliTool::RESTORE_FILES, "I:i:f:F:v:c:hd:"}, {StandaloneCliTool::CTA_SEND_EVENT, "i:e:u:g:"}, {StandaloneCliTool::CTA_VERIFY_FILE, "I:i:u:g:v:h:"}, - {StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS, "I:n:h:"}, + {StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS, "I:F:n:h:"}, }; //------------------------------------------------------------------------------ @@ -112,46 +113,39 @@ m_help(false), m_debug(false), m_standaloneCliTool{standaloneCliTool} { while ((opt = getopt_long(argc, argv, shortopts[m_standaloneCliTool], longopts[m_standaloneCliTool], &opt_index)) != -1) { switch(opt) { - case 'I': + case 'c': { - m_archiveFileId = std::stol(std::string(optarg)); + int64_t copyNumber = std::stol(std::string(optarg)); + if(copyNumber < 0) throw std::out_of_range("copy number value cannot be negative"); + m_copyNumber = copyNumber; break; } - case 'i': + case 'd': { - m_diskInstance = std::string(optarg); + m_debug = true; break; } - case 'f': + case 'e': { - if (! m_eosFids) { - m_eosFids = std::list<uint64_t>(); - } - auto fid = strtoul(optarg, nullptr, 16); - if(fid < 1) { - throw std::runtime_error(std::string(optarg) + " is not a valid file ID"); - } - m_eosFids->push_back(fid); + m_eosEndpoint = std::string(optarg); break; } - case 'F': + case 'f': { - if (! m_eosFids) { - m_eosFids = std::list<uint64_t>(); + if (! m_fxIds) { + m_fxIds = std::list<std::string>(); } - readFidListFromFile(std::string(optarg), m_eosFids.value()); + m_fxIds->push_back(std::string(optarg)); break; } - case 'v': + case 'F': { - m_vid = std::string(optarg); + readIdListFromFile(std::string(optarg)); break; } - case 'c': + case 'g': { - int64_t copyNumber = std::stol(std::string(optarg)); - if(copyNumber < 0) throw std::out_of_range("copy number value cannot be negative"); - m_copyNumber = copyNumber; + m_requestGroup = std::string(optarg); break; } case 'h': @@ -159,29 +153,29 @@ m_help(false), m_debug(false), m_standaloneCliTool{standaloneCliTool} { m_help = true; break; } - case 'd': + case 'I': { - m_debug = true; + m_archiveFileId = std::stol(std::string(optarg)); break; } - case 'e': + case 'i': { - m_eosEndpoint = std::string(optarg); + m_diskInstance = std::string(optarg); break; } - case 'u': + case 'n': { - m_requestUser = std::string(optarg); + m_storageClassName = std::string(optarg); break; } - case 'g': + case 'u': { - m_requestGroup = std::string(optarg); + m_requestUser = std::string(optarg); break; } - case 'n': + case 'v': { - m_storageClassName = std::string(optarg); + m_vid = std::string(optarg); break; } case ':': // Missing parameter @@ -216,7 +210,7 @@ m_help(false), m_debug(false), m_standaloneCliTool{standaloneCliTool} { //------------------------------------------------------------------------------ // readFidListFromFile //------------------------------------------------------------------------------ -void CmdLineArgs::readFidListFromFile(const std::string &filename, std::list<std::uint64_t> &fidList) { +void CmdLineArgs::readIdListFromFile(const std::string &filename) { std::ifstream file(filename); if (file.fail()) { throw std::runtime_error("Unable to open file " + filename); @@ -225,35 +219,20 @@ void CmdLineArgs::readFidListFromFile(const std::string &filename, std::list<std std::string line; while(std::getline(file, line)) { - // Strip out comments - auto pos = line.find('#'); - if(pos != std::string::npos) { - line.resize(pos); - } - - // Extract the list items - std::stringstream ss(line); - while(!ss.eof()) { - std::string item; - ss >> item; - // skip blank lines or lines consisting only of whitespace - if(item.empty()) continue; - - // Special handling for file id lists. The output from "eos find --fid <fid> /path" is: - // path=/path fid=<fid> - // We discard everything except the list of fids. <fid> is a zero-padded hexadecimal number, - // but in the CTA catalogue we store disk IDs as a decimal string, so we need to convert it. - if(item.substr(0, 4) == "fid=") { - auto fid = strtol(item.substr(4).c_str(), nullptr, 16); - if(fid < 1 || fid == LONG_MAX) { - throw std::runtime_error(item + " is not a valid file ID"); - } - fidList.push_back(fid); - } else { - continue; + switch (m_standaloneCliTool) { + case StandaloneCliTool::RESTORE_FILES: + m_fxIds.value().push_back(line); + break; + case StandaloneCliTool::CTA_VERIFY_FILE: + m_fxIds.value().push_back(line); + break; + case StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS: + m_fxIds.value().push_back(line); + break; + default: + break; } - } - } + } } //------------------------------------------------------------------------------ diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp index 3c0cb52e96..780f9beae1 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp @@ -62,14 +62,9 @@ struct CmdLineArgs { std::optional<std::string> m_diskInstance; /** - * Fids of the files to restore + * Fxids of the files to restore */ - std::optional<std::list<uint64_t>> m_eosFids; - - /** - * Fids of the files to restore - */ - std::optional<std::string> m_eosInstance; + std::optional<std::list<std::string>> m_fxIds; /** * Vid of the tape of the files to restore @@ -105,7 +100,6 @@ struct CmdLineArgs { * The tool parsing the arguments */ std::optional<std::string> m_storageClassName; - /** * Constructor that parses the specified command-line arguments. @@ -123,7 +117,7 @@ struct CmdLineArgs { * @param filename The name of the file to read * @param fidList The list of file IDs */ - void readFidListFromFile(const std::string &filename, std::list<uint64_t> &fidList); + void readIdListFromFile(const std::string &filename); /** * Prints the usage message of the command-line tool. diff --git a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp index c50b323dab..ebb5aadaa3 100644 --- a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp +++ b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp @@ -194,7 +194,7 @@ void RestoreFilesCmd::readAndSetConfiguration( m_vid = cmdLineArgs.m_vid; m_diskInstance = cmdLineArgs.m_diskInstance; - m_eosFids = cmdLineArgs.m_eosFids; + m_eosFids = cmdLineArgs.m_fxIds.value(); m_copyNumber = cmdLineArgs.m_copyNumber; m_archiveFileId = cmdLineArgs.m_archiveFileId; @@ -328,7 +328,7 @@ void RestoreFilesCmd::listDeletedFilesCta() const { auto new_opt = admincmd.add_option_str_list(); new_opt->set_key(key); for (auto &fid : m_eosFids.value()) { - new_opt->add_item(std::to_string(fid)); + new_opt->add_item(fid); ss << fid << ","; } auto fids = ss.str(); diff --git a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.hpp b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.hpp index 512397cd1d..d494414173 100644 --- a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.hpp +++ b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.hpp @@ -185,7 +185,7 @@ private: /** * Fids of the files to restore */ - std::optional<std::list<uint64_t>> m_eosFids; + std::optional<std::list<std::string>> m_eosFids; /** * Vid of the tape of the files to restore -- GitLab From a76bc4055e10fef22658a65dd123a7b7cb611164 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 12 Oct 2022 10:11:45 +0200 Subject: [PATCH 049/126] Merge branch 'main' into 78-create-a-tool-to-change-the-storage-class-of-data-which-has-already-been-written-to-tape --- ReleaseNotes.md | 1 + catalogue/CatalogueTest.cpp | 3 +- catalogue/RdbmsCatalogue.cpp | 32 ++++++++++--------- tapeserver/readtp/ReadtpCmdLineArgs.cpp | 2 -- tapeserver/tapelabel/TapeLabelCmdLineArgs.cpp | 2 -- xroot_plugins/XrdSsiCtaRequestMessage.cpp | 2 +- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 8b4b237b20..c06b6458b7 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -2,6 +2,7 @@ ## Summary ### Features +- cta/CTA#41 - Delete verification_status of tape when tape is reclaimed - cta/CTA#153 - Allow verification status to be cleared with cta-admin ### Bug fixes - cta/CTA#161 - Fix bug when using temporary tables with PostgreSQL diff --git a/catalogue/CatalogueTest.cpp b/catalogue/CatalogueTest.cpp index d38b93b715..a9a474d235 100644 --- a/catalogue/CatalogueTest.cpp +++ b/catalogue/CatalogueTest.cpp @@ -6363,7 +6363,7 @@ TEST_P(cta_catalogue_CatalogueTest, modifyTapeVerificationStatus) { const common::dataStructures::Tape tape = tapes.front(); ASSERT_EQ(m_tape1.vid, tape.vid); - ASSERT_EQ(tape.verificationStatus.has_value(), false); + ASSERT_FALSE(tape.verificationStatus); } } @@ -16284,6 +16284,7 @@ TEST_P(cta_catalogue_CatalogueTest, reclaimTape_full_lastFSeq_0_no_tape_files) { ASSERT_EQ(m_vo.name, tape.vo); ASSERT_EQ(m_mediaType.capacityInBytes, tape.capacityInBytes); ASSERT_FALSE(tape.full); + ASSERT_FALSE(tape.verificationStatus); ASSERT_FALSE(tape.isFromCastor); ASSERT_EQ(m_tape1.comment, tape.comment); diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index 940eacb162..820171836c 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -4757,6 +4757,7 @@ void RdbmsCatalogue::resetTapeCounters(rdbms::Conn& conn, const common::dataStru "COPY_NB_GT_1_IN_BYTES = 0," "IS_FULL = '0'," "IS_FROM_CASTOR = '0'," + "VERIFICATION_STATUS = :VERIFICATION_STATUS," "LAST_UPDATE_USER_NAME = :LAST_UPDATE_USER_NAME," "LAST_UPDATE_HOST_NAME = :LAST_UPDATE_HOST_NAME," "LAST_UPDATE_TIME = :LAST_UPDATE_TIME," @@ -4764,6 +4765,7 @@ void RdbmsCatalogue::resetTapeCounters(rdbms::Conn& conn, const common::dataStru "WHERE " "VID = :VID"; auto stmt = conn.createStmt(sql); + stmt.bindString(":VERIFICATION_STATUS", std::nullopt); stmt.bindString(":LAST_UPDATE_USER_NAME", admin.username); stmt.bindString(":LAST_UPDATE_HOST_NAME", admin.host); stmt.bindUint64(":LAST_UPDATE_TIME", now); @@ -4787,29 +4789,29 @@ void RdbmsCatalogue::reclaimTape(const common::dataStructures::SecurityIdentity TapeSearchCriteria searchCriteria; searchCriteria.vid = vid; const auto tapes = getTapes(conn, searchCriteria); - tl.insertAndReset("getTapesTime",t); + tl.insertAndReset("getTapesTime", t); - if(tapes.empty()) { + if (tapes.empty()) { throw exception::UserError(std::string("Cannot reclaim tape ") + vid + " because it does not exist"); } else { - if(!tapes.front().full){ + if (!tapes.front().full) { throw exception::UserError(std::string("Cannot reclaim tape ") + vid + " because it is not FULL"); } } - //The tape exists and is full, we can try to reclaim it - if(this->getNbFilesOnTape(conn,vid) == 0){ - tl.insertAndReset("getNbFilesOnTape",t); - //There is no files on the tape, we can reclaim it : delete the files and reset the counters - deleteFilesFromRecycleLog(conn,vid,lc); - tl.insertAndReset("deleteFileFromRecycleLogTime",t); - resetTapeCounters(conn,admin,vid); - tl.insertAndReset("resetTapeCountersTime",t); + // The tape exists and is full, we can try to reclaim it + if (this->getNbFilesOnTape(conn, vid) == 0) { + tl.insertAndReset("getNbFilesOnTape", t); + // There is no files on the tape, we can reclaim it : delete the files and reset the counters + deleteFilesFromRecycleLog(conn, vid, lc); + tl.insertAndReset("deleteFileFromRecycleLogTime", t); + resetTapeCounters(conn, admin, vid); + tl.insertAndReset("resetTapeCountersTime", t); log::ScopedParamContainer spc(lc); - spc.add("vid",vid); - spc.add("host",admin.host); - spc.add("username",admin.username); + spc.add("vid", vid); + spc.add("host", admin.host); + spc.add("username", admin.username); tl.addToLog(spc); - lc.log(log::INFO,"In RdbmsCatalogue::reclaimTape(), tape reclaimed."); + lc.log(log::INFO, "In RdbmsCatalogue::reclaimTape(), tape reclaimed."); } else { throw exception::UserError(std::string("Cannot reclaim tape ") + vid + " because there is at least one tape" " file in the catalogue that is on the tape"); diff --git a/tapeserver/readtp/ReadtpCmdLineArgs.cpp b/tapeserver/readtp/ReadtpCmdLineArgs.cpp index ac122c526a..82479103f0 100644 --- a/tapeserver/readtp/ReadtpCmdLineArgs.cpp +++ b/tapeserver/readtp/ReadtpCmdLineArgs.cpp @@ -46,8 +46,6 @@ ReadtpCmdLineArgs::ReadtpCmdLineArgs(const int argc, char *const *const argv): throw ex; } m_vid = std::string(argv[1]); - utils::toUpper(m_vid); - m_fSeqRangeList = TapeFileSequenceParser::parse(argv[2]); static struct option longopts[] = { diff --git a/tapeserver/tapelabel/TapeLabelCmdLineArgs.cpp b/tapeserver/tapelabel/TapeLabelCmdLineArgs.cpp index 15c7b840d3..1aca32b5bc 100644 --- a/tapeserver/tapelabel/TapeLabelCmdLineArgs.cpp +++ b/tapeserver/tapelabel/TapeLabelCmdLineArgs.cpp @@ -61,7 +61,6 @@ TapeLabelCmdLineArgs::TapeLabelCmdLineArgs(const int argc, char *const *const ar throw ex; } else { m_vid = std::string(optarg); - utils::toUpper(m_vid); } break; case 'o': @@ -71,7 +70,6 @@ TapeLabelCmdLineArgs::TapeLabelCmdLineArgs(const int argc, char *const *const ar throw ex; } else { m_oldLabel = std::string(optarg); - utils::toUpper(m_oldLabel); } break; case 't': diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp index ee0cba412e..df608b815d 100644 --- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp +++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp @@ -2009,7 +2009,7 @@ void RequestMessage::processTape_Reclaim(cta::xrd::Response &response) auto &vid = getRequired(OptionString::VID); - m_catalogue.reclaimTape(m_cliIdentity, vid,m_lc); + m_catalogue.reclaimTape(m_cliIdentity, vid, m_lc); response.set_type(cta::xrd::Response::RSP_SUCCESS); } -- GitLab From ef7e744b875b818819ea2a9c3049f61fad20980a Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 12 Oct 2022 13:33:07 +0200 Subject: [PATCH 050/126] fix bad optional access --- cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp index ebb5aadaa3..9d334a638b 100644 --- a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp +++ b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp @@ -194,7 +194,7 @@ void RestoreFilesCmd::readAndSetConfiguration( m_vid = cmdLineArgs.m_vid; m_diskInstance = cmdLineArgs.m_diskInstance; - m_eosFids = cmdLineArgs.m_fxIds.value(); + m_eosFids = cmdLineArgs.m_fxIds; m_copyNumber = cmdLineArgs.m_copyNumber; m_archiveFileId = cmdLineArgs.m_archiveFileId; -- GitLab From 45342de80e39c17f7af7fd0964089d2aef07565e Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 12 Oct 2022 13:55:26 +0200 Subject: [PATCH 051/126] add change to db statement --- catalogue/RdbmsCatalogue.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index 820171836c..cdadc8f202 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -1427,7 +1427,7 @@ void RdbmsCatalogue::modifyArchiveFileStorageClassId(const uint64_t& archiveFile try { const char *const sql = "UPDATE ARCHIVE_FILE " - "SET ARCHIVE_FILE.STORAGE_CLASS_ID = (" + "SET STORAGE_CLASS_ID = (" "SELECT STORAGE_CLASS_ID FROM STORAGE_CLASS WHERE STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME " ") " "WHERE " -- GitLab From 4e28f5380285f11ecbd03196bf01f59853b50b19 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 12 Oct 2022 16:37:15 +0200 Subject: [PATCH 052/126] add eosConnection class --- .../common/EosConnection.hpp | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 cmdline/standalone_cli_tools/common/EosConnection.hpp diff --git a/cmdline/standalone_cli_tools/common/EosConnection.hpp b/cmdline/standalone_cli_tools/common/EosConnection.hpp new file mode 100644 index 0000000000..a22f75d346 --- /dev/null +++ b/cmdline/standalone_cli_tools/common/EosConnection.hpp @@ -0,0 +1,125 @@ +#include "eos_grpc_client/GrpcEndpoint.hpp" +#include "common/log/StdoutLogger.hpp" +#include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" + +#include <string> + +namespace cta { +namespace cliTool { + +class EosConnection { + + /** + * The object representing the API of the CTA logging system. + */ + cta::log::StdoutLogger &m_log; + + /** + * Pointer map pointer for communication with EOS + */ + std::unique_ptr<::eos::client::EndpointMap> m_endpointMapPtr; + + EosConnection(cta::log::StdoutLogger log) : m_log{log} {} + + void readAndSetConfiguration( + const std::string &userName, + const CmdLineArgs &cmdLineArgs, + std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr); + + void setNamespaceMap(const std::string &keytab_file); + + const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses + const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by + //!< XRD_REQUESTTIMEOUT environment variable. +}; + +//------------------------------------------------------------------------------ +// readAndSetConfiguration +//------------------------------------------------------------------------------ +void EosConnection::readAndSetConfiguration( + const std::string &userName, + const CmdLineArgs &cmdLineArgs, + std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr) { + + if (cmdLineArgs.m_debug) { + m_log.setLogMask("DEBUG"); + } else { + m_log.setLogMask("INFO"); + } + + // Set CTA frontend configuration options + const std::string cli_config_file = "/etc/cta/cta-cli.conf"; + XrdSsiPb::Config cliConfig(cli_config_file, "cta"); + cliConfig.set("resource", "/ctafrontend"); + cliConfig.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes + cliConfig.set("request_timeout", DefaultRequestTimeout); // default value = 10s + + // Allow environment variables to override config file + cliConfig.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); + + // If XRDDEBUG=1, switch on all logging + if(getenv("XRDDEBUG")) { + cliConfig.set("log", "all"); + } + // If fine-grained control over log level is required, use XrdSsiPbLogLevel + cliConfig.getEnv("log", "XrdSsiPbLogLevel"); + + // Validate that endpoint was specified in the config file + if(!cliConfig.getOptionValueStr("endpoint").first) { + throw std::runtime_error("Configuration error: cta.endpoint missing from " + cli_config_file); + } + + // If the server is down, we want an immediate failure. Set client retry to a single attempt. + XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); + + serviceProviderPtr.reset(new XrdSsiPbServiceType(cliConfig)); + + // Set CTA frontend configuration options to connect to eos + const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf"; + XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); + + // Get the endpoint for namespace queries + auto nsConf = frontendXrootdConfig.getOptionValueStr("ns.config"); + if(nsConf.first) { + setNamespaceMap(nsConf.second); + } else { + throw std::runtime_error("Configuration error: cta.ns.config missing from " + frontend_xrootd_config_file); + } +} + +void EosConnection::setNamespaceMap(const std::string &keytab_file) { + // Open the keytab file for reading + std::ifstream file(keytab_file); + if(!file) { + throw cta::exception::UserError("Failed to open namespace keytab configuration file " + keytab_file); + } + ::eos::client::NamespaceMap_t namespaceMap; + // Parse the keytab line by line + std::string line; + for(int lineno = 0; std::getline(file, line); ++lineno) { + // Strip out comments + auto pos = line.find('#'); + if(pos != std::string::npos) { + line.resize(pos); + } + + // Parse one line + std::stringstream ss(line); + std::string diskInstance; + std::string endpoint; + std::string token; + std::string eol; + ss >> diskInstance >> endpoint >> token >> eol; + + // Ignore blank lines, all other lines must have exactly 3 elements + if(token.empty() || !eol.empty()) { + if(diskInstance.empty() && endpoint.empty() && token.empty()) continue; + throw cta::exception::UserError("Could not parse namespace keytab configuration file line " + std::to_string(lineno) + ": " + line); + } + namespaceMap.insert(std::make_pair(diskInstance, ::eos::client::Namespace(endpoint, token))); + } + m_endpointMapPtr = std::make_unique<::eos::client::EndpointMap>(namespaceMap); +} + +} // namespace cliTool +} // namespace cta \ No newline at end of file -- GitLab From e69fddbdab56ee5a3e9ec86a8e511b989923c6bd Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 12 Oct 2022 16:49:21 +0200 Subject: [PATCH 053/126] add connection configuration file --- ...ection.hpp => ConnectionConfiguration.hpp} | 104 +++++++----------- 1 file changed, 42 insertions(+), 62 deletions(-) rename cmdline/standalone_cli_tools/common/{EosConnection.hpp => ConnectionConfiguration.hpp} (74%) diff --git a/cmdline/standalone_cli_tools/common/EosConnection.hpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp similarity index 74% rename from cmdline/standalone_cli_tools/common/EosConnection.hpp rename to cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp index a22f75d346..8b2b13fcae 100644 --- a/cmdline/standalone_cli_tools/common/EosConnection.hpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp @@ -7,44 +7,58 @@ namespace cta { namespace cliTool { -class EosConnection { - - /** - * The object representing the API of the CTA logging system. - */ - cta::log::StdoutLogger &m_log; - - /** - * Pointer map pointer for communication with EOS - */ - std::unique_ptr<::eos::client::EndpointMap> m_endpointMapPtr; +const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses +const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by + //!< XRD_REQUESTTIMEOUT environment variable. - EosConnection(cta::log::StdoutLogger log) : m_log{log} {} +void setNamespaceMap(const std::string &keytab_file, std::unique_ptr<::eos::client::EndpointMap> &endpointMapPtr) { + // Open the keytab file for reading + std::ifstream file(keytab_file); + if(!file) { + throw cta::exception::UserError("Failed to open namespace keytab configuration file " + keytab_file); + } + ::eos::client::NamespaceMap_t namespaceMap; + // Parse the keytab line by line + std::string line; + for(int lineno = 0; std::getline(file, line); ++lineno) { + // Strip out comments + auto pos = line.find('#'); + if(pos != std::string::npos) { + line.resize(pos); + } - void readAndSetConfiguration( - const std::string &userName, - const CmdLineArgs &cmdLineArgs, - std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr); - - void setNamespaceMap(const std::string &keytab_file); + // Parse one line + std::stringstream ss(line); + std::string diskInstance; + std::string endpoint; + std::string token; + std::string eol; + ss >> diskInstance >> endpoint >> token >> eol; - const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses - const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by - //!< XRD_REQUESTTIMEOUT environment variable. -}; + // Ignore blank lines, all other lines must have exactly 3 elements + if(token.empty() || !eol.empty()) { + if(diskInstance.empty() && endpoint.empty() && token.empty()) continue; + throw cta::exception::UserError("Could not parse namespace keytab configuration file line " + std::to_string(lineno) + ": " + line); + } + namespaceMap.insert(std::make_pair(diskInstance, ::eos::client::Namespace(endpoint, token))); + } + endpointMapPtr = std::make_unique<::eos::client::EndpointMap>(namespaceMap); +} //------------------------------------------------------------------------------ // readAndSetConfiguration //------------------------------------------------------------------------------ -void EosConnection::readAndSetConfiguration( - const std::string &userName, +void readAndSetConfiguration( + cta::log::StdoutLogger &log, + const std::string &userName, const CmdLineArgs &cmdLineArgs, - std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr) { + std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, + std::unique_ptr<::eos::client::EndpointMap> &endpointMapPtr) { if (cmdLineArgs.m_debug) { - m_log.setLogMask("DEBUG"); + log.setLogMask("DEBUG"); } else { - m_log.setLogMask("INFO"); + log.setLogMask("INFO"); } // Set CTA frontend configuration options @@ -81,45 +95,11 @@ void EosConnection::readAndSetConfiguration( // Get the endpoint for namespace queries auto nsConf = frontendXrootdConfig.getOptionValueStr("ns.config"); if(nsConf.first) { - setNamespaceMap(nsConf.second); + setNamespaceMap(nsConf.second, endpointMapPtr); } else { throw std::runtime_error("Configuration error: cta.ns.config missing from " + frontend_xrootd_config_file); } } -void EosConnection::setNamespaceMap(const std::string &keytab_file) { - // Open the keytab file for reading - std::ifstream file(keytab_file); - if(!file) { - throw cta::exception::UserError("Failed to open namespace keytab configuration file " + keytab_file); - } - ::eos::client::NamespaceMap_t namespaceMap; - // Parse the keytab line by line - std::string line; - for(int lineno = 0; std::getline(file, line); ++lineno) { - // Strip out comments - auto pos = line.find('#'); - if(pos != std::string::npos) { - line.resize(pos); - } - - // Parse one line - std::stringstream ss(line); - std::string diskInstance; - std::string endpoint; - std::string token; - std::string eol; - ss >> diskInstance >> endpoint >> token >> eol; - - // Ignore blank lines, all other lines must have exactly 3 elements - if(token.empty() || !eol.empty()) { - if(diskInstance.empty() && endpoint.empty() && token.empty()) continue; - throw cta::exception::UserError("Could not parse namespace keytab configuration file line " + std::to_string(lineno) + ": " + line); - } - namespaceMap.insert(std::make_pair(diskInstance, ::eos::client::Namespace(endpoint, token))); - } - m_endpointMapPtr = std::make_unique<::eos::client::EndpointMap>(namespaceMap); -} - } // namespace cliTool } // namespace cta \ No newline at end of file -- GitLab From 48ed00f97abb2dc8b9b20f2a3276eb93579729c5 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 13 Oct 2022 10:11:45 +0200 Subject: [PATCH 054/126] add include --- .../change_storage_class/CtaChangeStorageClass.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp index edef19544c..ec0961eadd 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp @@ -17,6 +17,7 @@ #include "cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp" #include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" +#include "cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp" #include "cmdline/CtaAdminCmdParse.hpp" #include "common/utils/utils.hpp" #include "common/checksum/ChecksumBlob.hpp" -- GitLab From 0f3e4e6e8c77abb62c2c7868f1c0792a9a5dd2ea Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 14 Oct 2022 08:36:50 +0200 Subject: [PATCH 055/126] add endl to exception message --- cmdline/standalone_cli_tools/common/CmdLineArgs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp index 8657531d33..427dabdcbc 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp @@ -170,7 +170,7 @@ m_help(false), m_debug(false), m_standaloneCliTool{standaloneCliTool} { { exception::CommandLineNotParsed ex; if(0 == optopt) { - ex.getMessage() << "Unknown command-line option"; + ex.getMessage() << "Unknown command-line option" << std::endl; } else { ex.getMessage() << "Unknown command-line option: -" << static_cast<char>(optopt) << std::endl; } -- GitLab From df5d432634c9793d6646296322b276a75ef8c5cc Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 14 Oct 2022 11:55:25 +0200 Subject: [PATCH 056/126] remove to_string() usage --- .../standalone_cli_tools/CtaVerifyFile.cpp | 2 +- .../CtaChangeStorageClass.cpp | 21 +++++++++++++++---- .../common/CmdLineArgs.cpp | 10 ++++----- .../common/CmdLineArgs.hpp | 7 ++++++- .../restore_files/RestoreFilesCmd.cpp | 8 +++---- .../restore_files/RestoreFilesCmd.hpp | 4 ++-- 6 files changed, 35 insertions(+), 17 deletions(-) diff --git a/cmdline/standalone_cli_tools/CtaVerifyFile.cpp b/cmdline/standalone_cli_tools/CtaVerifyFile.cpp index 34d0eb1daf..965a8370e4 100644 --- a/cmdline/standalone_cli_tools/CtaVerifyFile.cpp +++ b/cmdline/standalone_cli_tools/CtaVerifyFile.cpp @@ -81,7 +81,7 @@ void fillNotification(cta::eos::Notification ¬ification, const int argc, char notification.mutable_cli()->mutable_user()->set_groupname(cmdLineArgs.m_requestGroup.value()); } - const std::string archiveFileId(std::to_string(cmdLineArgs.m_archiveFileId.value())); + const std::string archiveFileId(cmdLineArgs.m_archiveFileId.value()); // WF notification.mutable_wf()->set_event(cta::eos::Workflow::PREPARE); diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp index ec0961eadd..f6c82347b5 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp @@ -22,6 +22,7 @@ #include "common/utils/utils.hpp" #include "common/checksum/ChecksumBlob.hpp" #include "CtaFrontendApi.hpp" +#include "common/exception/CommandLineNotParsed.hpp" #include <XrdSsiPbLog.hpp> @@ -87,7 +88,8 @@ CtaChangeStorageClass::CtaChangeStorageClass( m_storageClassName = cmdLineArgs.m_storageClassName.value(); if (cmdLineArgs.m_archiveFileId) { - m_fxIds.push_back(std::to_string(cmdLineArgs.m_archiveFileId.value())); + std::cout << cmdLineArgs.m_archiveFileId.value() << std::endl; + m_archiveFileIds.push_back(cmdLineArgs.m_archiveFileId.value()); } } @@ -124,6 +126,7 @@ void CtaChangeStorageClass::updateStorageClassInCatalogue() { auto new_opt = admincmd.add_option_str_list(); new_opt->set_key(key); for (auto &archiveFileId : m_archiveFileIds) { + std::cout << archiveFileId << std::endl; new_opt->add_item(archiveFileId); } } @@ -200,9 +203,19 @@ void CtaChangeStorageClass::readAndSetConfiguration( //------------------------------------------------------------------------------ int CtaChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { - readAndSetConfiguration(getUsername()); - updateStorageClassInEosNamespace(); - updateStorageClassInCatalogue(); + try { + readAndSetConfiguration(getUsername()); + updateStorageClassInEosNamespace(); + updateStorageClassInCatalogue(); + } + catch (exception::CommandLineNotParsed &e){ + m_log(cta::log::ERR,e.what()); + exit(1); + } + catch (...){ + m_log(cta::log::ERR, "Runtime Error"); + exit(1); + } return 0; } diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp index 427dabdcbc..ac05f7937d 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp @@ -62,7 +62,7 @@ static struct option verifyFileLongOption[] = { }; static struct option changeStorageClassLongOption[] = { - {"fxid", required_argument, nullptr, 'I'}, + {"archiveid", required_argument, nullptr, 'I'}, {"filename", required_argument, nullptr, 'F'}, {"storage.class.name", required_argument, nullptr, 'n'}, {"help", no_argument, nullptr, 'h'}, @@ -117,7 +117,7 @@ m_help(false), m_debug(false), m_standaloneCliTool{standaloneCliTool} { if (! m_fxIds) { m_fxIds = std::list<std::string>(); } - m_fxIds->push_back(std::string(optarg)); + m_fxIds->push_back(optarg); break; } case 'F': @@ -137,7 +137,7 @@ m_help(false), m_debug(false), m_standaloneCliTool{standaloneCliTool} { } case 'I': { - m_archiveFileId = std::stol(std::string(optarg)); + m_archiveFileId = std::string(optarg); break; } case 'i': @@ -203,13 +203,13 @@ void CmdLineArgs::readIdListFromFile(const std::string &filename) { while(std::getline(file, line)) { switch (m_standaloneCliTool) { case StandaloneCliTool::RESTORE_FILES: - m_fxIds.value().push_back(line); + m_archiveFileIds.value().push_back(line); break; case StandaloneCliTool::CTA_VERIFY_FILE: m_fxIds.value().push_back(line); break; case StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS: - m_fxIds.value().push_back(line); + m_archiveFileIds.value().push_back(line); break; default: break; diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp index 780f9beae1..474a8eaa65 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp @@ -54,7 +54,12 @@ struct CmdLineArgs { /** * Archive file id of the files to restore */ - std::optional<uint64_t> m_archiveFileId; + std::optional<std::string> m_archiveFileId; + + /** + * Archive file id of the files to restore + */ + std::optional<std::list<std::string>> m_archiveFileIds; /** * Disk instance of the files to restore diff --git a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp index 9d334a638b..6eea9651bf 100644 --- a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp +++ b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp @@ -194,7 +194,7 @@ void RestoreFilesCmd::readAndSetConfiguration( m_vid = cmdLineArgs.m_vid; m_diskInstance = cmdLineArgs.m_diskInstance; - m_eosFids = cmdLineArgs.m_fxIds; + m_archiveFileIds = cmdLineArgs.m_archiveFileIds; m_copyNumber = cmdLineArgs.m_copyNumber; m_archiveFileId = cmdLineArgs.m_archiveFileId; @@ -313,7 +313,7 @@ void RestoreFilesCmd::listDeletedFilesCta() const { auto key = cta::admin::OptionUInt64::ARCHIVE_FILE_ID; auto new_opt = admincmd.add_option_uint64(); new_opt->set_key(key); - new_opt->set_value(m_archiveFileId.value()); + new_opt->set_value(std::stoi(m_archiveFileId.value())); } if (m_copyNumber) { params.push_back(cta::log::Param("copyNb", m_copyNumber.value())); @@ -322,12 +322,12 @@ void RestoreFilesCmd::listDeletedFilesCta() const { new_opt->set_key(key); new_opt->set_value(m_copyNumber.value()); } - if (m_eosFids) { + if (m_archiveFileIds) { std::stringstream ss; auto key = cta::admin::OptionStrList::FILE_ID; auto new_opt = admincmd.add_option_str_list(); new_opt->set_key(key); - for (auto &fid : m_eosFids.value()) { + for (auto &fid : m_archiveFileIds.value()) { new_opt->add_item(fid); ss << fid << ","; } diff --git a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.hpp b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.hpp index d494414173..85ccf0aa7a 100644 --- a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.hpp +++ b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.hpp @@ -175,7 +175,7 @@ private: /** * Archive file id of the files to restore */ - std::optional<uint64_t> m_archiveFileId; + std::optional<std::string> m_archiveFileId; /** * Disk instance of the files to restore @@ -185,7 +185,7 @@ private: /** * Fids of the files to restore */ - std::optional<std::list<std::string>> m_eosFids; + std::optional<std::list<std::string>> m_archiveFileIds; /** * Vid of the tape of the files to restore -- GitLab From 5eecfc95693d11135dc47c9e1f0245b3ce5686d3 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 14 Oct 2022 14:25:33 +0200 Subject: [PATCH 057/126] add CtaCatalogueFetcherFunctions.hpp --- .../common/CtaCatalogueFetcherFunctions.hpp | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp diff --git a/cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp b/cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp new file mode 100644 index 0000000000..39200c13cc --- /dev/null +++ b/cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp @@ -0,0 +1,64 @@ +#include <string> + +namespace cta { +namespace cliTool { + +//------------------------------------------------------------------------------ +// getFxidFromCTA +//------------------------------------------------------------------------------ +std::pair<std::string,std::string> getInstanceAndFidFromCTA(const string& archiveFileId) { + { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("archiveFileId", file.archive_file_id())); + m_log(cta::log::DEBUG, "getInstanceAndFidFromCTA() ", params); + } + + cta::xrd::Request request; + auto &admincmd = *(request.mutable_admincmd()); + + request.set_client_cta_version(CTA_VERSION); + request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); + admincmd.set_cmd(cta::admin::AdminCmd::CMD_TAPEFILE); + admincmd.set_subcmd(cta::admin::AdminCmd::SUBCMD_LS); + auto new_opt = admincmd.add_option_uint64(); + new_opt->set_key(cta::admin::OptionUInt64::ARCHIVE_FILE_ID); + new_opt->set_value(archive_file_id); + + // Send the Request to the Service and get a Response + cta::xrd::Response response; + auto stream_future = m_serviceProviderPtr->SendAsync(request, response); + + // Handle responses + switch(response.type()) + { + using namespace cta::xrd; + using namespace cta::admin; + case Response::RSP_SUCCESS: + // Print message text + std::cout << response.message_txt(); + // Allow stream processing to commence + isHeaderSent = true; + break; + case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); + case Response::RSP_ERR_USER: throw XrdSsiPb::UserException(response.message_txt()); + case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); + default: throw XrdSsiPb::PbException("Invalid response type."); + } + + // wait until the data stream has been processed before exiting + stream_future.wait(); + if(listedTapeFiles.size() != 1) { + throw std::runtime_error("Unexpected result set: listedTapeFiles size expected=1 received=" + std::to_string(listedTapeFiles.size())); + } + auto listedTapeFile = listedTapeFiles.back(); + listedTapeFiles.clear(); + { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("diskInstance", listedTapeFile.first)); + params.push_back(cta::log::Param("diskFileId", listedTapeFile.second)); + m_log(cta::log::DEBUG, "Obtained file metadata from CTA", params); + } + return listedTapeFile; +} +} // cliTool +} // cta \ No newline at end of file -- GitLab From 1018bcd009a913ff9b6ea6be420926b56238585d Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 14 Oct 2022 15:14:23 +0200 Subject: [PATCH 058/126] move RequestCallback to common folder --- .../CtaChangeStorageClass.cpp | 20 +--- .../common/CtaCatalogueFetcherFunctions.hpp | 94 +++++++++++++++++-- .../restore_files/RestoreFilesCmd.cpp | 67 +------------ 3 files changed, 93 insertions(+), 88 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp index f6c82347b5..8b019a32ad 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp @@ -18,6 +18,7 @@ #include "cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp" #include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" #include "cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp" +#include "cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp" #include "cmdline/CtaAdminCmdParse.hpp" #include "common/utils/utils.hpp" #include "common/checksum/ChecksumBlob.hpp" @@ -35,22 +36,6 @@ #include <iostream> #include <memory> -namespace XrdSsiPb { - -/*! - * Alert callback. - * - * Defines how Alert messages should be logged - */ -template<> -void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) -{ - std::cout << "AlertCallback():" << std::endl; - Log::DumpProtobuf(Log::PROTOBUF, &alert); -} - -} // namespace XrdSsiPb - namespace cta { namespace cliTool { @@ -205,6 +190,9 @@ int CtaChangeStorageClass::exceptionThrowingMain(const int argc, char *const *co try { readAndSetConfiguration(getUsername()); + auto values = cta::cliTool::getInstanceAndFidFromCTA("1", m_serviceProviderPtr, m_log); + std::cout << "fid: " << values.first << std::endl; + std::cout << "instance: " << values.second << std::endl; updateStorageClassInEosNamespace(); updateStorageClassInCatalogue(); } diff --git a/cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp b/cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp index 39200c13cc..51e563add4 100644 --- a/cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp +++ b/cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp @@ -1,16 +1,98 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#pragma once + +#include <XrdSsiPbLog.hpp> +#include <XrdSsiPbIStreamBuffer.hpp> + #include <string> +// GLOBAL VARIABLES : used to pass information between main thread and stream handler thread + +// global synchronisation flag +std::atomic<bool> isHeaderSent(false); + +std::list<cta::admin::RecycleTapeFileLsItem> deletedTapeFiles; +std::list<std::pair<std::string,std::string>> listedTapeFiles; + +namespace XrdSsiPb { + +/*! + * User error exception + */ +class UserException : public std::runtime_error +{ +public: + UserException(const std::string &err_msg) : std::runtime_error(err_msg) {} +}; // class UserException + +/*! + * Alert callback. + * + * Defines how Alert messages should be logged + */ +template<> +void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) +{ + std::cout << "AlertCallback():" << std::endl; + Log::DumpProtobuf(Log::PROTOBUF, &alert); +} + +/*! + * Data/Stream callback. + * + * Defines how incoming records from the stream should be handled + */ +template<> +void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const +{ + using namespace cta::xrd; + using namespace cta::admin; + + // Wait for primary response to be handled before allowing stream response + while(!isHeaderSent) { std::this_thread::yield(); } + + switch(record.data_case()) { + case Data::kTflsItem: + { + auto item = record.tfls_item(); + auto instanceAndFid = std::make_pair(item.df().disk_instance(), item.df().disk_id()); + listedTapeFiles.push_back(instanceAndFid); + } + break; + default: + throw std::runtime_error("Received invalid stream data from CTA Frontend for the cta-restore-deleted-files command."); + } +} + +} // namespace XrdSsiPb + namespace cta { namespace cliTool { //------------------------------------------------------------------------------ // getFxidFromCTA //------------------------------------------------------------------------------ -std::pair<std::string,std::string> getInstanceAndFidFromCTA(const string& archiveFileId) { +std::pair<std::string,std::string> getInstanceAndFidFromCTA(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log) { { std::list<cta::log::Param> params; - params.push_back(cta::log::Param("archiveFileId", file.archive_file_id())); - m_log(cta::log::DEBUG, "getInstanceAndFidFromCTA() ", params); + params.push_back(cta::log::Param("archiveFileId", archiveFileId)); + log(cta::log::DEBUG, "getInstanceAndFidFromCTA() ", params); } cta::xrd::Request request; @@ -22,11 +104,11 @@ std::pair<std::string,std::string> getInstanceAndFidFromCTA(const string& archiv admincmd.set_subcmd(cta::admin::AdminCmd::SUBCMD_LS); auto new_opt = admincmd.add_option_uint64(); new_opt->set_key(cta::admin::OptionUInt64::ARCHIVE_FILE_ID); - new_opt->set_value(archive_file_id); + new_opt->set_value(std::stoi(archiveFileId)); // Send the Request to the Service and get a Response cta::xrd::Response response; - auto stream_future = m_serviceProviderPtr->SendAsync(request, response); + auto stream_future = serviceProviderPtr->SendAsync(request, response); // Handle responses switch(response.type()) @@ -56,7 +138,7 @@ std::pair<std::string,std::string> getInstanceAndFidFromCTA(const string& archiv std::list<cta::log::Param> params; params.push_back(cta::log::Param("diskInstance", listedTapeFile.first)); params.push_back(cta::log::Param("diskFileId", listedTapeFile.second)); - m_log(cta::log::DEBUG, "Obtained file metadata from CTA", params); + log(cta::log::DEBUG, "Obtained file metadata from CTA", params); } return listedTapeFile; } diff --git a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp index 6eea9651bf..4798dc4f8b 100644 --- a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp +++ b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp @@ -16,6 +16,7 @@ */ #include "cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.hpp" +#include "cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp" #include "cmdline/CtaAdminCmdParse.hpp" #include "common/utils/utils.hpp" #include "common/checksum/ChecksumBlob.hpp" @@ -31,72 +32,6 @@ #include <iostream> #include <memory> -// GLOBAL VARIABLES : used to pass information between main thread and stream handler thread - -// global synchronisation flag -std::atomic<bool> isHeaderSent(false); - -std::list<cta::admin::RecycleTapeFileLsItem> deletedTapeFiles; -std::list<std::pair<std::string,std::string>> listedTapeFiles; - -namespace XrdSsiPb { - -/*! - * User error exception - */ -class UserException : public std::runtime_error -{ -public: - UserException(const std::string &err_msg) : std::runtime_error(err_msg) {} -}; // class UserException - -/*! - * Alert callback. - * - * Defines how Alert messages should be logged - */ -template<> -void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) -{ - std::cout << "AlertCallback():" << std::endl; - Log::DumpProtobuf(Log::PROTOBUF, &alert); -} - -/*! - * Data/Stream callback. - * - * Defines how incoming records from the stream should be handled - */ -template<> -void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const -{ - using namespace cta::xrd; - using namespace cta::admin; - - // Wait for primary response to be handled before allowing stream response - while(!isHeaderSent) { std::this_thread::yield(); } - - switch(record.data_case()) { - case Data::kRtflsItem: - { - auto item = record.rtfls_item(); - deletedTapeFiles.push_back(item); - } - break; - case Data::kTflsItem: - { - auto item = record.tfls_item(); - auto instanceAndFid = std::make_pair(item.df().disk_instance(), item.df().disk_id()); - listedTapeFiles.push_back(instanceAndFid); - } - break; - default: - throw std::runtime_error("Received invalid stream data from CTA Frontend for the cta-restore-deleted-files command."); - } -} - -} // namespace XrdSsiPb - namespace cta{ namespace cliTool { -- GitLab From 544dec48f7c24bf5b0f735814b66f27e849fc65f Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 14 Oct 2022 17:09:37 +0200 Subject: [PATCH 059/126] add case to IStreamBuffer --- .../common/CtaCatalogueFetcherFunctions.hpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp b/cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp index 51e563add4..a4fab92ba0 100644 --- a/cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp +++ b/cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp @@ -68,6 +68,12 @@ void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const while(!isHeaderSent) { std::this_thread::yield(); } switch(record.data_case()) { + case Data::kRtflsItem: + { + auto item = record.rtfls_item(); + deletedTapeFiles.push_back(item); + } + break; case Data::kTflsItem: { auto item = record.tfls_item(); @@ -82,6 +88,7 @@ void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const } // namespace XrdSsiPb + namespace cta { namespace cliTool { -- GitLab From ac8814876fcd58670423233d3ac78172bccd77f6 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Mon, 17 Oct 2022 11:17:45 +0200 Subject: [PATCH 060/126] add SetXAttr to GrpcClient --- eos_grpc_client/GrpcClient.cpp | 29 +++++++++++++++++++++++++++++ eos_grpc_client/GrpcClient.hpp | 8 ++++++++ 2 files changed, 37 insertions(+) diff --git a/eos_grpc_client/GrpcClient.cpp b/eos_grpc_client/GrpcClient.cpp index 2d0c0b10bd..8e1a66d1cc 100644 --- a/eos_grpc_client/GrpcClient.cpp +++ b/eos_grpc_client/GrpcClient.cpp @@ -263,4 +263,33 @@ eos::rpc::MDResponse GrpcClient::GetMD(eos::rpc::TYPE type, uint64_t id, const s return response; } +void GrpcClient::SetXAttr( + const uint64_t &id, + const bool &recursive, + const std::map<std::string, std::string> &xattrs, + const std::list<std::string> &keysToDelete, + const bool &create) { + + eos::rpc::NSRequest::SetXAttrRequest request; + + request.mutable_id()->set_id(id); + + for (const auto& [key, value] : xattrs) { + (*request.mutable_xattrs())[key] = value; + } + + for (const auto& key : keysToDelete) { + request.add_keystodelete(key); + } + + request.set_create(create); + + grpc::Status status; + + // Act upon the status of the actual RPC + if(!status.ok()) { + throw std::runtime_error("EOS namespace query failed with error: " + status.error_message()); + } +} + }} // namespace eos::client diff --git a/eos_grpc_client/GrpcClient.hpp b/eos_grpc_client/GrpcClient.hpp index 545e21a17c..5e49ee2a28 100644 --- a/eos_grpc_client/GrpcClient.hpp +++ b/eos_grpc_client/GrpcClient.hpp @@ -49,6 +49,14 @@ public: // Obtain container or file metadata eos::rpc::MDResponse GetMD(eos::rpc::TYPE type, uint64_t id, const std::string &path, bool showJson = false); + void SetXAttr( + const uint64_t &id, + const bool &recursive, + const std::map<std::string, + std::string> &xattrs, + const std::list<std::string> &keysToDelete, + const bool &create); + void set_ssl(bool onoff) { m_SSL = onoff; } -- GitLab From b00dfa9e3c1e444f4bf6a5cbd60238074bddda4c Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 18 Oct 2022 10:34:42 +0200 Subject: [PATCH 061/126] Add exec endpoint --- eos_grpc_client/GrpcClient.cpp | 41 +++++++++++++------------------- eos_grpc_client/GrpcClient.hpp | 8 +------ eos_grpc_client/GrpcEndpoint.cpp | 5 ++++ eos_grpc_client/GrpcEndpoint.hpp | 10 ++++++++ 4 files changed, 33 insertions(+), 31 deletions(-) diff --git a/eos_grpc_client/GrpcClient.cpp b/eos_grpc_client/GrpcClient.cpp index 8e1a66d1cc..bed51c0ec4 100644 --- a/eos_grpc_client/GrpcClient.cpp +++ b/eos_grpc_client/GrpcClient.cpp @@ -263,33 +263,26 @@ eos::rpc::MDResponse GrpcClient::GetMD(eos::rpc::TYPE type, uint64_t id, const s return response; } -void GrpcClient::SetXAttr( - const uint64_t &id, - const bool &recursive, - const std::map<std::string, std::string> &xattrs, - const std::list<std::string> &keysToDelete, - const bool &create) { - eos::rpc::NSRequest::SetXAttrRequest request; - - request.mutable_id()->set_id(id); - - for (const auto& [key, value] : xattrs) { - (*request.mutable_xattrs())[key] = value; - } +int GrpcClient::Exec(const eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply) { + grpc::ClientContext context; + grpc::CompletionQueue cq; + grpc::Status status; + std::unique_ptr<grpc::ClientAsyncResponseReader<eos::rpc::NSResponse> > rpc( + stub_->AsyncExec(&context, request, &cq)); + rpc->Finish(&reply, &status, (void*) 1); + void* got_tag; + bool ok = false; + GPR_ASSERT(cq.Next(&got_tag, &ok)); + GPR_ASSERT(got_tag == (void*) 1); + GPR_ASSERT(ok); - for (const auto& key : keysToDelete) { - request.add_keystodelete(key); + // Act upon the status of the actual RPC. + if (status.ok()) { + return reply.error().code(); + } else { + return -1; } - - request.set_create(create); - - grpc::Status status; - - // Act upon the status of the actual RPC - if(!status.ok()) { - throw std::runtime_error("EOS namespace query failed with error: " + status.error_message()); - } } }} // namespace eos::client diff --git a/eos_grpc_client/GrpcClient.hpp b/eos_grpc_client/GrpcClient.hpp index 5e49ee2a28..bcc309903d 100644 --- a/eos_grpc_client/GrpcClient.hpp +++ b/eos_grpc_client/GrpcClient.hpp @@ -49,13 +49,7 @@ public: // Obtain container or file metadata eos::rpc::MDResponse GetMD(eos::rpc::TYPE type, uint64_t id, const std::string &path, bool showJson = false); - void SetXAttr( - const uint64_t &id, - const bool &recursive, - const std::map<std::string, - std::string> &xattrs, - const std::list<std::string> &keysToDelete, - const bool &create); + int Exec(const eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply); void set_ssl(bool onoff) { m_SSL = onoff; diff --git a/eos_grpc_client/GrpcEndpoint.cpp b/eos_grpc_client/GrpcEndpoint.cpp index 549becb41d..8061fa3506 100644 --- a/eos_grpc_client/GrpcEndpoint.cpp +++ b/eos_grpc_client/GrpcEndpoint.cpp @@ -61,3 +61,8 @@ void eos::client::Endpoint::getCurrentIds(uint64_t &cid, uint64_t &fid) const { eos::rpc::MDResponse eos::client::Endpoint::getMD(eos::rpc::TYPE type, uint64_t id, const std::string &path, bool showJson) const { return m_grpcClient->GetMD(type, id, path, showJson); } + +int eos::client::Endpoint::exec(const eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply) const { + return m_grpcClient->Exec(request, reply); +} + diff --git a/eos_grpc_client/GrpcEndpoint.hpp b/eos_grpc_client/GrpcEndpoint.hpp index 1497df5536..07176faf23 100644 --- a/eos_grpc_client/GrpcEndpoint.hpp +++ b/eos_grpc_client/GrpcEndpoint.hpp @@ -36,6 +36,7 @@ public: ::eos::rpc::InsertReply containerInsert(const ::eos::rpc::ContainerMdProto &container) const; void getCurrentIds(uint64_t &cid, uint64_t &fid) const; ::eos::rpc::MDResponse getMD(::eos::rpc::TYPE type, uint64_t id, const std::string &path, bool showJson) const; + int exec(const eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply) const; private: std::unique_ptr<::eos::client::GrpcClient> m_grpcClient; @@ -97,6 +98,15 @@ public: } } + int Exec(const std::string &diskInstance, const eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply) const { + auto ep_it = m_endpointMap.find(diskInstance); + if(ep_it == m_endpointMap.end()) { + throw cta::exception::UserError("Namespace for disk instance \"" + diskInstance + "\" is not configured in the CTA Frontend"); + } else { + return ep_it->second.exec(request, reply); + } + } + private: std::map<std::string, Endpoint> m_endpointMap; }; -- GitLab From 0ee85254ef046dd36011c48732b6cca4144aedd3 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 18 Oct 2022 11:28:20 +0200 Subject: [PATCH 062/126] add setXattr endpoint --- eos_grpc_client/GrpcClient.cpp | 6 +++++- eos_grpc_client/GrpcClient.hpp | 2 +- eos_grpc_client/GrpcEndpoint.cpp | 8 +++++++- eos_grpc_client/GrpcEndpoint.hpp | 6 +++--- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/eos_grpc_client/GrpcClient.cpp b/eos_grpc_client/GrpcClient.cpp index bed51c0ec4..af9a7c0565 100644 --- a/eos_grpc_client/GrpcClient.cpp +++ b/eos_grpc_client/GrpcClient.cpp @@ -264,13 +264,17 @@ eos::rpc::MDResponse GrpcClient::GetMD(eos::rpc::TYPE type, uint64_t id, const s } -int GrpcClient::Exec(const eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply) { +int GrpcClient::Exec(eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply) const { + + request.set_authkey(token()); + grpc::ClientContext context; grpc::CompletionQueue cq; grpc::Status status; std::unique_ptr<grpc::ClientAsyncResponseReader<eos::rpc::NSResponse> > rpc( stub_->AsyncExec(&context, request, &cq)); rpc->Finish(&reply, &status, (void*) 1); + void* got_tag; bool ok = false; GPR_ASSERT(cq.Next(&got_tag, &ok)); diff --git a/eos_grpc_client/GrpcClient.hpp b/eos_grpc_client/GrpcClient.hpp index bcc309903d..d58290a23e 100644 --- a/eos_grpc_client/GrpcClient.hpp +++ b/eos_grpc_client/GrpcClient.hpp @@ -49,7 +49,7 @@ public: // Obtain container or file metadata eos::rpc::MDResponse GetMD(eos::rpc::TYPE type, uint64_t id, const std::string &path, bool showJson = false); - int Exec(const eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply); + int Exec(eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply) const; void set_ssl(bool onoff) { m_SSL = onoff; diff --git a/eos_grpc_client/GrpcEndpoint.cpp b/eos_grpc_client/GrpcEndpoint.cpp index 8061fa3506..10878ebf9a 100644 --- a/eos_grpc_client/GrpcEndpoint.cpp +++ b/eos_grpc_client/GrpcEndpoint.cpp @@ -62,7 +62,13 @@ eos::rpc::MDResponse eos::client::Endpoint::getMD(eos::rpc::TYPE type, uint64_t return m_grpcClient->GetMD(type, id, path, showJson); } -int eos::client::Endpoint::exec(const eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply) const { +int eos::client::Endpoint::setXAttr(const std::string &path, const std::string &attrKey, const std::string &attrValue) const { + eos::rpc::NSRequest request; + eos::rpc::NSResponse reply; + + request.mutable_xattr()->mutable_id()->set_path(path); + (*(request.mutable_xattr()->mutable_xattrs()))[attrKey] = attrValue; + return m_grpcClient->Exec(request, reply); } diff --git a/eos_grpc_client/GrpcEndpoint.hpp b/eos_grpc_client/GrpcEndpoint.hpp index 07176faf23..e92b63dfb8 100644 --- a/eos_grpc_client/GrpcEndpoint.hpp +++ b/eos_grpc_client/GrpcEndpoint.hpp @@ -36,7 +36,7 @@ public: ::eos::rpc::InsertReply containerInsert(const ::eos::rpc::ContainerMdProto &container) const; void getCurrentIds(uint64_t &cid, uint64_t &fid) const; ::eos::rpc::MDResponse getMD(::eos::rpc::TYPE type, uint64_t id, const std::string &path, bool showJson) const; - int exec(const eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply) const; + int setXAttr(const std::string &path, const std::string &attrKey, const std::string &attrValue) const; private: std::unique_ptr<::eos::client::GrpcClient> m_grpcClient; @@ -98,12 +98,12 @@ public: } } - int Exec(const std::string &diskInstance, const eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply) const { + int setXAttr(const std::string &diskInstance, const std::string &path, const std::string &attrKey, const std::string &attrValue) const { auto ep_it = m_endpointMap.find(diskInstance); if(ep_it == m_endpointMap.end()) { throw cta::exception::UserError("Namespace for disk instance \"" + diskInstance + "\" is not configured in the CTA Frontend"); } else { - return ep_it->second.exec(request, reply); + return ep_it->second.setXAttr(path, attrKey, attrValue); } } -- GitLab From 9c49be1c9ed7353d8aafa221d22052dfcab943e9 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 18 Oct 2022 11:29:11 +0200 Subject: [PATCH 063/126] Add connection configuration in ctaChangeStorageclass.cpp --- .../CtaChangeStorageClass.cpp | 41 +------------------ .../CtaChangeStorageClass.hpp | 9 ---- 2 files changed, 2 insertions(+), 48 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp index 8b019a32ad..e05bc3cb84 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp @@ -77,6 +77,8 @@ CtaChangeStorageClass::CtaChangeStorageClass( m_archiveFileIds.push_back(cmdLineArgs.m_archiveFileId.value()); } + readAndSetConfiguration(log, getUsername(), cmdLineArgs, m_serviceProviderPtr, m_endpointMapPtr); + } //------------------------------------------------------------------------------ @@ -145,51 +147,12 @@ void CtaChangeStorageClass::updateStorageClassInCatalogue() { } } -//------------------------------------------------------------------------------ -// readAndSetConfiguration -//------------------------------------------------------------------------------ -void CtaChangeStorageClass::readAndSetConfiguration( - const std::string &userName) { - - // Set CTA frontend configuration options - const std::string cli_config_file = "/etc/cta/cta-cli.conf"; - XrdSsiPb::Config cliConfig(cli_config_file, "cta"); - cliConfig.set("resource", "/ctafrontend"); - cliConfig.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes - cliConfig.set("request_timeout", DefaultRequestTimeout); // default value = 10s - - // Allow environment variables to override config file - cliConfig.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); - - // If XRDDEBUG=1, switch on all logging - if(getenv("XRDDEBUG")) { - cliConfig.set("log", "all"); - } - // If fine-grained control over log level is required, use XrdSsiPbLogLevel - cliConfig.getEnv("log", "XrdSsiPbLogLevel"); - - // Validate that endpoint was specified in the config file - if(!cliConfig.getOptionValueStr("endpoint").first) { - throw std::runtime_error("Configuration error: cta.endpoint missing from " + cli_config_file); - } - - // If the server is down, we want an immediate failure. Set client retry to a single attempt. - XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); - - m_serviceProviderPtr.reset(new XrdSsiPbServiceType(cliConfig)); - - // Set CTA frontend configuration options to connect to eos - const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf"; - XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); -} - //------------------------------------------------------------------------------ // exceptionThrowingMain //------------------------------------------------------------------------------ int CtaChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { try { - readAndSetConfiguration(getUsername()); auto values = cta::cliTool::getInstanceAndFidFromCTA("1", m_serviceProviderPtr, m_log); std::cout << "fid: " << values.first << std::endl; std::cout << "instance: " << values.second << std::endl; diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp index 3a03027bd6..e168ed3437 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp @@ -60,15 +60,6 @@ public: */ int exceptionThrowingMain(const int argc, char *const *const argv) override; - /** - * Sets internal configuration parameters to be used for reading. - * It reads cta frontend parameters from /etc/cta/cta-cli.conf - * - * @param username The name of the user running the command-line tool. - * @param cmdLineArgs The arguments parsed from the command line. - */ - void readAndSetConfiguration(const std::string &userName); - private: /** -- GitLab From 90071391ff312254c9eafc2c512f7a905717205f Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 18 Oct 2022 11:29:47 +0200 Subject: [PATCH 064/126] add licensing info --- .../common/ConnectionConfiguration.hpp | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp index 8b2b13fcae..2df48f7aab 100644 --- a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp @@ -1,3 +1,22 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#pragma once + #include "eos_grpc_client/GrpcEndpoint.hpp" #include "common/log/StdoutLogger.hpp" #include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" @@ -45,6 +64,16 @@ void setNamespaceMap(const std::string &keytab_file, std::unique_ptr<::eos::clie endpointMapPtr = std::make_unique<::eos::client::EndpointMap>(namespaceMap); } + + /** + * Sets internal configuration parameters to be used for reading. + * It reads cta frontend parameters from /etc/cta/cta-cli.conf + * + * @param username The name of the user running the command-line tool. + * @param cmdLineArgs The arguments parsed from the command line. + * @param serviceProviderPtr Xroot service provider + * @param endpointMapPtr Endpoint for communication with EOS + */ //------------------------------------------------------------------------------ // readAndSetConfiguration //------------------------------------------------------------------------------ -- GitLab From dcbb2b9580382f20bfc3e03c08af98808cc5a57b Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 18 Oct 2022 11:48:37 +0200 Subject: [PATCH 065/126] change --archiveid to --id --- cmdline/standalone_cli_tools/common/CmdLineArgs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp index ac05f7937d..2bfd220910 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp @@ -62,7 +62,7 @@ static struct option verifyFileLongOption[] = { }; static struct option changeStorageClassLongOption[] = { - {"archiveid", required_argument, nullptr, 'I'}, + {"id", required_argument, nullptr, 'I'}, {"filename", required_argument, nullptr, 'F'}, {"storage.class.name", required_argument, nullptr, 'n'}, {"help", no_argument, nullptr, 'h'}, -- GitLab From b2214645338f6a3c012b609750c13d7e80f658b9 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 18 Oct 2022 17:16:49 +0200 Subject: [PATCH 066/126] move helper functions to static class --- .../change_storage_class/CMakeLists.txt | 2 +- .../CtaChangeStorageClass.cpp | 7 +- .../common/ConnectionConfiguration.cpp | 129 ++++++++++++++++++ .../common/ConnectionConfiguration.hpp | 116 ++-------------- 4 files changed, 147 insertions(+), 107 deletions(-) create mode 100644 cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp diff --git a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt index 10add3170b..de3f160578 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt +++ b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt @@ -27,7 +27,7 @@ include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include) # Compiled protocol buffers include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS}) -add_executable(cta-change-storage-class CtaChangeStorageClass.cpp CtaChangeStorageClassMain.cpp ../common/CmdLineTool.cpp ../common/CmdLineArgs.cpp ../../CtaAdminCmdParse.cpp) +add_executable(cta-change-storage-class CtaChangeStorageClass.cpp CtaChangeStorageClassMain.cpp ../common/CmdLineTool.cpp ../common/CmdLineArgs.cpp ../../CtaAdminCmdParse.cpp ../common/ConnectionConfiguration.cpp) target_link_libraries(cta-change-storage-class ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient) set_property (TARGET cta-change-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp index e05bc3cb84..455b25cc76 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp @@ -77,7 +77,7 @@ CtaChangeStorageClass::CtaChangeStorageClass( m_archiveFileIds.push_back(cmdLineArgs.m_archiveFileId.value()); } - readAndSetConfiguration(log, getUsername(), cmdLineArgs, m_serviceProviderPtr, m_endpointMapPtr); + ConnConfiguration::readAndSetConfiguration(log, getUsername(), cmdLineArgs, m_serviceProviderPtr, m_endpointMapPtr); } @@ -85,7 +85,7 @@ CtaChangeStorageClass::CtaChangeStorageClass( // updateStorageClassInEosNamespace //------------------------------------------------------------------------------ void CtaChangeStorageClass::updateStorageClassInEosNamespace() { - + m_endpointMapPtr->setXAttr("ctaeos", "/eos/ctaeos/cta/d5bc5223-4bd2-4d50-8d99-0d058af1d38d", "sys.archive.storage_class", "lasse"); } //------------------------------------------------------------------------------ @@ -158,13 +158,14 @@ int CtaChangeStorageClass::exceptionThrowingMain(const int argc, char *const *co std::cout << "instance: " << values.second << std::endl; updateStorageClassInEosNamespace(); updateStorageClassInCatalogue(); + } catch (exception::CommandLineNotParsed &e){ m_log(cta::log::ERR,e.what()); exit(1); } catch (...){ - m_log(cta::log::ERR, "Runtime Error"); + m_log(cta::log::ERR, "RuntimeError"); exit(1); } diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp new file mode 100644 index 0000000000..2bbf70389e --- /dev/null +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp @@ -0,0 +1,129 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#include "ConnectionConfiguration.hpp" + +#include <string> + +namespace cta { +namespace cliTool { + +void ConnConfiguration::setNamespaceMap(const std::string &keytab_file, std::unique_ptr<::eos::client::EndpointMap> &endpointMapPtr) { + // Open the keytab file for reading + std::ifstream file(keytab_file); + if(!file) { + throw cta::exception::UserError("Failed to open namespace keytab configuration file " + keytab_file); + } + ::eos::client::NamespaceMap_t namespaceMap; + // Parse the keytab line by line + std::string line; + for(int lineno = 0; std::getline(file, line); ++lineno) { + // Strip out comments + auto pos = line.find('#'); + if(pos != std::string::npos) { + line.resize(pos); + } + + // Parse one line + std::stringstream ss(line); + std::string diskInstance; + std::string endpoint; + std::string token; + std::string eol; + ss >> diskInstance >> endpoint >> token >> eol; + + // Ignore blank lines, all other lines must have exactly 3 elements + if(token.empty() || !eol.empty()) { + if(diskInstance.empty() && endpoint.empty() && token.empty()) continue; + throw cta::exception::UserError("Could not parse namespace keytab configuration file line " + std::to_string(lineno) + ": " + line); + } + namespaceMap.insert(std::make_pair(diskInstance, ::eos::client::Namespace(endpoint, token))); + } + endpointMapPtr = std::make_unique<::eos::client::EndpointMap>(namespaceMap); +} + + + /** + * Sets internal configuration parameters to be used for reading. + * It reads cta frontend parameters from /etc/cta/cta-cli.conf + * + * @param username The name of the user running the command-line tool. + * @param cmdLineArgs The arguments parsed from the command line. + * @param serviceProviderPtr Xroot service provider + * @param endpointMapPtr Endpoint for communication with EOS + */ +//------------------------------------------------------------------------------ +// readAndSetConfiguration +//------------------------------------------------------------------------------ +void ConnConfiguration::readAndSetConfiguration( + cta::log::StdoutLogger &log, + const std::string &userName, + const CmdLineArgs &cmdLineArgs, + std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, + std::unique_ptr<::eos::client::EndpointMap> &endpointMapPtr) { + + const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses + const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by + //!< XRD_REQUESTTIMEOUT environment variable. + if (cmdLineArgs.m_debug) { + log.setLogMask("DEBUG"); + } else { + log.setLogMask("INFO"); + } + + // Set CTA frontend configuration options + const std::string cli_config_file = "/etc/cta/cta-cli.conf"; + XrdSsiPb::Config cliConfig(cli_config_file, "cta"); + cliConfig.set("resource", "/ctafrontend"); + cliConfig.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes + cliConfig.set("request_timeout", DefaultRequestTimeout); // default value = 10s + + // Allow environment variables to override config file + cliConfig.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); + + // If XRDDEBUG=1, switch on all logging + if(getenv("XRDDEBUG")) { + cliConfig.set("log", "all"); + } + // If fine-grained control over log level is required, use XrdSsiPbLogLevel + cliConfig.getEnv("log", "XrdSsiPbLogLevel"); + + // Validate that endpoint was specified in the config file + if(!cliConfig.getOptionValueStr("endpoint").first) { + throw std::runtime_error("Configuration error: cta.endpoint missing from " + cli_config_file); + } + + // If the server is down, we want an immediate failure. Set client retry to a single attempt. + XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); + + serviceProviderPtr.reset(new XrdSsiPbServiceType(cliConfig)); + + // Set CTA frontend configuration options to connect to eos + const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf"; + XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); + + // Get the endpoint for namespace queries + auto nsConf = frontendXrootdConfig.getOptionValueStr("ns.config"); + if(nsConf.first) { + setNamespaceMap(nsConf.second, endpointMapPtr); + } else { + throw std::runtime_error("Configuration error: cta.ns.config missing from " + frontend_xrootd_config_file); + } +} + +} // namespace cliTool +} // namespace cta \ No newline at end of file diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp index 2df48f7aab..a2dd3a690a 100644 --- a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp @@ -20,115 +20,25 @@ #include "eos_grpc_client/GrpcEndpoint.hpp" #include "common/log/StdoutLogger.hpp" #include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" +#include "CtaFrontendApi.hpp" + #include <string> namespace cta { namespace cliTool { -const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses -const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by - //!< XRD_REQUESTTIMEOUT environment variable. - -void setNamespaceMap(const std::string &keytab_file, std::unique_ptr<::eos::client::EndpointMap> &endpointMapPtr) { - // Open the keytab file for reading - std::ifstream file(keytab_file); - if(!file) { - throw cta::exception::UserError("Failed to open namespace keytab configuration file " + keytab_file); - } - ::eos::client::NamespaceMap_t namespaceMap; - // Parse the keytab line by line - std::string line; - for(int lineno = 0; std::getline(file, line); ++lineno) { - // Strip out comments - auto pos = line.find('#'); - if(pos != std::string::npos) { - line.resize(pos); - } - - // Parse one line - std::stringstream ss(line); - std::string diskInstance; - std::string endpoint; - std::string token; - std::string eol; - ss >> diskInstance >> endpoint >> token >> eol; - - // Ignore blank lines, all other lines must have exactly 3 elements - if(token.empty() || !eol.empty()) { - if(diskInstance.empty() && endpoint.empty() && token.empty()) continue; - throw cta::exception::UserError("Could not parse namespace keytab configuration file line " + std::to_string(lineno) + ": " + line); - } - namespaceMap.insert(std::make_pair(diskInstance, ::eos::client::Namespace(endpoint, token))); - } - endpointMapPtr = std::make_unique<::eos::client::EndpointMap>(namespaceMap); -} - - - /** - * Sets internal configuration parameters to be used for reading. - * It reads cta frontend parameters from /etc/cta/cta-cli.conf - * - * @param username The name of the user running the command-line tool. - * @param cmdLineArgs The arguments parsed from the command line. - * @param serviceProviderPtr Xroot service provider - * @param endpointMapPtr Endpoint for communication with EOS - */ -//------------------------------------------------------------------------------ -// readAndSetConfiguration -//------------------------------------------------------------------------------ -void readAndSetConfiguration( - cta::log::StdoutLogger &log, - const std::string &userName, - const CmdLineArgs &cmdLineArgs, - std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, - std::unique_ptr<::eos::client::EndpointMap> &endpointMapPtr) { - - if (cmdLineArgs.m_debug) { - log.setLogMask("DEBUG"); - } else { - log.setLogMask("INFO"); - } - - // Set CTA frontend configuration options - const std::string cli_config_file = "/etc/cta/cta-cli.conf"; - XrdSsiPb::Config cliConfig(cli_config_file, "cta"); - cliConfig.set("resource", "/ctafrontend"); - cliConfig.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes - cliConfig.set("request_timeout", DefaultRequestTimeout); // default value = 10s - - // Allow environment variables to override config file - cliConfig.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); - - // If XRDDEBUG=1, switch on all logging - if(getenv("XRDDEBUG")) { - cliConfig.set("log", "all"); - } - // If fine-grained control over log level is required, use XrdSsiPbLogLevel - cliConfig.getEnv("log", "XrdSsiPbLogLevel"); - - // Validate that endpoint was specified in the config file - if(!cliConfig.getOptionValueStr("endpoint").first) { - throw std::runtime_error("Configuration error: cta.endpoint missing from " + cli_config_file); - } - - // If the server is down, we want an immediate failure. Set client retry to a single attempt. - XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); - - serviceProviderPtr.reset(new XrdSsiPbServiceType(cliConfig)); - - // Set CTA frontend configuration options to connect to eos - const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf"; - XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); - - // Get the endpoint for namespace queries - auto nsConf = frontendXrootdConfig.getOptionValueStr("ns.config"); - if(nsConf.first) { - setNamespaceMap(nsConf.second, endpointMapPtr); - } else { - throw std::runtime_error("Configuration error: cta.ns.config missing from " + frontend_xrootd_config_file); - } -} +class ConnConfiguration { +public: + static void setNamespaceMap(const std::string &keytab_file, std::unique_ptr<::eos::client::EndpointMap> &endpointMapPtr); + + static void readAndSetConfiguration( + cta::log::StdoutLogger &log, + const std::string &userName, + const CmdLineArgs &cmdLineArgs, + std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, + std::unique_ptr<::eos::client::EndpointMap> &endpointMapPtr); +}; } // namespace cliTool } // namespace cta \ No newline at end of file -- GitLab From d78904b594bb6c4e53f710611345ecdbd73f7708 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 18 Oct 2022 17:33:07 +0200 Subject: [PATCH 067/126] improve error catching --- .../CtaChangeStorageClass.cpp | 23 ++++--------------- .../orchestration/tests/changeStorageClass.sh | 14 ++++++++--- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp index 455b25cc76..65dd9adcc0 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp @@ -151,24 +151,11 @@ void CtaChangeStorageClass::updateStorageClassInCatalogue() { // exceptionThrowingMain //------------------------------------------------------------------------------ int CtaChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { - - try { - auto values = cta::cliTool::getInstanceAndFidFromCTA("1", m_serviceProviderPtr, m_log); - std::cout << "fid: " << values.first << std::endl; - std::cout << "instance: " << values.second << std::endl; - updateStorageClassInEosNamespace(); - updateStorageClassInCatalogue(); - - } - catch (exception::CommandLineNotParsed &e){ - m_log(cta::log::ERR,e.what()); - exit(1); - } - catch (...){ - m_log(cta::log::ERR, "RuntimeError"); - exit(1); - } - + auto values = cta::cliTool::getInstanceAndFidFromCTA("1", m_serviceProviderPtr, m_log); + std::cout << "fid: " << values.first << std::endl; + std::cout << "instance: " << values.second << std::endl; + updateStorageClassInEosNamespace(); + updateStorageClassInCatalogue(); return 0; } diff --git a/continuousintegration/orchestration/tests/changeStorageClass.sh b/continuousintegration/orchestration/tests/changeStorageClass.sh index ca157105eb..999f51f94a 100755 --- a/continuousintegration/orchestration/tests/changeStorageClass.sh +++ b/continuousintegration/orchestration/tests/changeStorageClass.sh @@ -36,11 +36,15 @@ echo "Creating files: ${CHANGE_SC_FILE} ${STATIC_SC_FILE}" FRONTEND_IP=$(kubectl -n ${NAMESPACE} get pods ctafrontend -o json | jq .status.podIP | tr -d '"') +echo +echo "ENABLE CTAFRONTEND TO EXECUTE CTA ADMIN COMMANDS" +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin admin add --username ctafrontend --comment "for restore files test" +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin admin add --username ctaeos --comment "for restore files test" + echo echo "ADD FRONTEND GATEWAY TO EOS" echo "kubectl -n ${NAMESPACE} exec ctaeos -- bash eos root://${EOSINSTANCE} -r 0 0 vid add gateway ${FRONTEND_IP} grpc" kubectl -n ${NAMESPACE} exec ctaeos -- eos -r 0 0 vid add gateway ${FRONTEND_IP} grpc -eos vid set map -grpc key:<your key aks any uuid string> vuid:2 vgid:2 kubectl -n ${NAMESPACE} cp common/archive_file.sh client:/usr/bin/ kubectl -n ${NAMESPACE} cp client_helper.sh client:/root/ @@ -50,6 +54,10 @@ kubectl -n ${NAMESPACE} exec client -- bash /usr/bin/archive_file.sh -f ${STATIC kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc add --name ${NEW_STORAGE_CLASS_NAME} --numberofcopies 1 --virtualorganisation vo --comment "comment" kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc ls -kubectl cp ~/CTA-build/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class ${NAMESPACE}/ctacli:/usr/bin/ -kubectl -n ${NAMESPACE} exec ctacli -- cta-change-storage-class --storage.class.name ${NEW_STORAGE_CLASS_NAME} --id 1 +kubectl cp ~/CTA-build/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class ${NAMESPACE}/ctafrontend:/usr/bin/ +kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c "XrdSecPROTOCOL=sss XrdSecSSSKT=/etc/cta/eos.sss.keytab cta-change-storage-class --storage.class.name ${NEW_STORAGE_CLASS_NAME} --id 1" + +# Remove authorization +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin admin rm --username ctafrontend +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin admin rm --username ctaeos -- GitLab From 3e77225d45b2611347217015f03ac615e505e0e0 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 19 Oct 2022 08:37:58 +0200 Subject: [PATCH 068/126] Change naming from CtaChangeStorageClass to ChangeStorageClass --- .../change_storage_class/CMakeLists.txt | 2 +- ...taChangeStorageClass.cpp => ChangeStorageClass.cpp} | 10 +++++----- ...taChangeStorageClass.hpp => ChangeStorageClass.hpp} | 4 ++-- ...StorageClassMain.cpp => ChangeStorageClassMain.cpp} | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) rename cmdline/standalone_cli_tools/change_storage_class/{CtaChangeStorageClass.cpp => ChangeStorageClass.cpp} (94%) rename cmdline/standalone_cli_tools/change_storage_class/{CtaChangeStorageClass.hpp => ChangeStorageClass.hpp} (97%) rename cmdline/standalone_cli_tools/change_storage_class/{CtaChangeStorageClassMain.cpp => ChangeStorageClassMain.cpp} (90%) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt index de3f160578..587273912d 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt +++ b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt @@ -27,7 +27,7 @@ include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include) # Compiled protocol buffers include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS}) -add_executable(cta-change-storage-class CtaChangeStorageClass.cpp CtaChangeStorageClassMain.cpp ../common/CmdLineTool.cpp ../common/CmdLineArgs.cpp ../../CtaAdminCmdParse.cpp ../common/ConnectionConfiguration.cpp) +add_executable(cta-change-storage-class ChangeStorageClass.cpp ChangeStorageClassMain.cpp ../common/CmdLineTool.cpp ../common/CmdLineArgs.cpp ../../CtaAdminCmdParse.cpp ../common/ConnectionConfiguration.cpp) target_link_libraries(cta-change-storage-class ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient) set_property (TARGET cta-change-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp similarity index 94% rename from cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp rename to cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 65dd9adcc0..a435b7ad7c 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -15,7 +15,7 @@ * submit itself to any jurisdiction. */ -#include "cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp" +#include "cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp" #include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" #include "cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp" #include "cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp" @@ -43,7 +43,7 @@ namespace cliTool { //------------------------------------------------------------------------------ // constructor //------------------------------------------------------------------------------ -CtaChangeStorageClass::CtaChangeStorageClass( +ChangeStorageClass::ChangeStorageClass( std::istream &inStream, std::ostream &outStream, std::ostream &errStream, @@ -84,14 +84,14 @@ CtaChangeStorageClass::CtaChangeStorageClass( //------------------------------------------------------------------------------ // updateStorageClassInEosNamespace //------------------------------------------------------------------------------ -void CtaChangeStorageClass::updateStorageClassInEosNamespace() { +void ChangeStorageClass::updateStorageClassInEosNamespace() { m_endpointMapPtr->setXAttr("ctaeos", "/eos/ctaeos/cta/d5bc5223-4bd2-4d50-8d99-0d058af1d38d", "sys.archive.storage_class", "lasse"); } //------------------------------------------------------------------------------ // changeCatalogue //------------------------------------------------------------------------------ -void CtaChangeStorageClass::updateStorageClassInCatalogue() { +void ChangeStorageClass::updateStorageClassInCatalogue() { cta::xrd::Request request; auto &admincmd = *(request.mutable_admincmd()); @@ -150,7 +150,7 @@ void CtaChangeStorageClass::updateStorageClassInCatalogue() { //------------------------------------------------------------------------------ // exceptionThrowingMain //------------------------------------------------------------------------------ -int CtaChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { +int ChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { auto values = cta::cliTool::getInstanceAndFidFromCTA("1", m_serviceProviderPtr, m_log); std::cout << "fid: " << values.first << std::endl; std::cout << "instance: " << values.second << std::endl; diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp similarity index 97% rename from cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp rename to cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp index e168ed3437..49a7a68a9b 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp @@ -31,7 +31,7 @@ namespace cta { namespace cliTool { -class CtaChangeStorageClass: public CmdLineTool { +class ChangeStorageClass: public CmdLineTool { public: /** * Constructor. @@ -43,7 +43,7 @@ public: * @param argc * @param argv */ - CtaChangeStorageClass( + ChangeStorageClass( std::istream &inStream, std::ostream &outStream, std::ostream &errStream, diff --git a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClassMain.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClassMain.cpp similarity index 90% rename from cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClassMain.cpp rename to cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClassMain.cpp index bc965c50ba..cdaac9712e 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClassMain.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClassMain.cpp @@ -21,7 +21,7 @@ #include <XrdSsiPbLog.hpp> -#include "cmdline/standalone_cli_tools/change_storage_class/CtaChangeStorageClass.hpp" +#include "cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp" #include "common/utils/utils.hpp" #include "common/log/StdoutLogger.hpp" @@ -39,7 +39,7 @@ int main(const int argc, char *const *const argv) { } cta::log::StdoutLogger log(hostName, "cta-change-storage-class"); - cta::cliTool::CtaChangeStorageClass cmd(std::cin, std::cout, std::cerr, log, argc, argv); + cta::cliTool::ChangeStorageClass cmd(std::cin, std::cout, std::cerr, log, argc, argv); int ret = cmd.main(argc, argv); // Delete all global objects allocated by libprotobuf google::protobuf::ShutdownProtobufLibrary(); -- GitLab From 841d281232165edacaf54b03cc4cc7a60d7282e0 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 19 Oct 2022 09:40:14 +0200 Subject: [PATCH 069/126] add updateStorageClassInEosNamespace --- .../change_storage_class/ChangeStorageClass.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index a435b7ad7c..0a2c46784a 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -73,7 +73,6 @@ ChangeStorageClass::ChangeStorageClass( m_storageClassName = cmdLineArgs.m_storageClassName.value(); if (cmdLineArgs.m_archiveFileId) { - std::cout << cmdLineArgs.m_archiveFileId.value() << std::endl; m_archiveFileIds.push_back(cmdLineArgs.m_archiveFileId.value()); } @@ -85,7 +84,14 @@ ChangeStorageClass::ChangeStorageClass( // updateStorageClassInEosNamespace //------------------------------------------------------------------------------ void ChangeStorageClass::updateStorageClassInEosNamespace() { - m_endpointMapPtr->setXAttr("ctaeos", "/eos/ctaeos/cta/d5bc5223-4bd2-4d50-8d99-0d058af1d38d", "sys.archive.storage_class", "lasse"); + for(auto archiveFileId : m_archiveFileIds){ + auto instanceAndFid = cta::cliTool::getInstanceAndFidFromCTA(archiveFileId, m_serviceProviderPtr, m_log); + std::string diskInstance = instanceAndFid.first; + std::string diskFileId = instanceAndFid.second; + + auto path = m_endpointMapPtr->getPath(diskInstance, diskFileId); + m_endpointMapPtr->setXAttr(diskInstance, path, "sys.archive.storage_class", m_storageClassName); + } } //------------------------------------------------------------------------------ @@ -151,9 +157,6 @@ void ChangeStorageClass::updateStorageClassInCatalogue() { // exceptionThrowingMain //------------------------------------------------------------------------------ int ChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { - auto values = cta::cliTool::getInstanceAndFidFromCTA("1", m_serviceProviderPtr, m_log); - std::cout << "fid: " << values.first << std::endl; - std::cout << "instance: " << values.second << std::endl; updateStorageClassInEosNamespace(); updateStorageClassInCatalogue(); return 0; -- GitLab From aa510ec2b03d2c917aeeba09bed961c53d3b4b4e Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 19 Oct 2022 09:41:04 +0200 Subject: [PATCH 070/126] copy cta-cli.conf to frontend in changeStorageClass test --- .../orchestration/tests/changeStorageClass.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/continuousintegration/orchestration/tests/changeStorageClass.sh b/continuousintegration/orchestration/tests/changeStorageClass.sh index 999f51f94a..1942a37687 100755 --- a/continuousintegration/orchestration/tests/changeStorageClass.sh +++ b/continuousintegration/orchestration/tests/changeStorageClass.sh @@ -46,6 +46,13 @@ echo "ADD FRONTEND GATEWAY TO EOS" echo "kubectl -n ${NAMESPACE} exec ctaeos -- bash eos root://${EOSINSTANCE} -r 0 0 vid add gateway ${FRONTEND_IP} grpc" kubectl -n ${NAMESPACE} exec ctaeos -- eos -r 0 0 vid add gateway ${FRONTEND_IP} grpc +echo +echo "COPY REQUIRED FILES TO FRONTEND POD" +echo "sudo kubectl cp ${NAMESPACE}/ctacli:/etc/cta/cta-cli.conf /etc/cta/cta-cli.conf" +echo "sudo kubectl cp /etc/cta/cta-cli.conf ${NAMESPACE}/ctafrontend:/etc/cta/cta-cli.conf" +sudo kubectl cp ${NAMESPACE}/ctacli:/etc/cta/cta-cli.conf /etc/cta/cta-cli.conf +sudo kubectl cp /etc/cta/cta-cli.conf ${NAMESPACE}/ctafrontend:/etc/cta/cta-cli.conf + kubectl -n ${NAMESPACE} cp common/archive_file.sh client:/usr/bin/ kubectl -n ${NAMESPACE} cp client_helper.sh client:/root/ kubectl -n ${NAMESPACE} exec client -- bash /usr/bin/archive_file.sh -f ${CHANGE_SC_FILE} || exit 1 @@ -55,7 +62,7 @@ kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc add --name ${NEW_STORAGE_CLA kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc ls kubectl cp ~/CTA-build/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class ${NAMESPACE}/ctafrontend:/usr/bin/ -kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c "XrdSecPROTOCOL=sss XrdSecSSSKT=/etc/cta/eos.sss.keytab cta-change-storage-class --storage.class.name ${NEW_STORAGE_CLASS_NAME} --id 1" +kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c "XrdSecPROTOCOL=sss XrdSecSSSKT=/etc/cta/eos.sss.keytab cta-change-storage-class --storage.class.name ${NEW_STORAGE_CLASS_NAME} --id 2" # Remove authorization kubectl -n ${NAMESPACE} exec ctacli -- cta-admin admin rm --username ctafrontend -- GitLab From 8721f9b05009d77054fb86c6377548ca86af378c Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 19 Oct 2022 13:32:53 +0200 Subject: [PATCH 071/126] add changes to updateStorageClassInEosNamespace --- .../change_storage_class/ChangeStorageClass.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 0a2c46784a..122a7cdb92 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -84,11 +84,10 @@ ChangeStorageClass::ChangeStorageClass( // updateStorageClassInEosNamespace //------------------------------------------------------------------------------ void ChangeStorageClass::updateStorageClassInEosNamespace() { - for(auto archiveFileId : m_archiveFileIds){ + for(auto archiveFileId : m_archiveFileIds) { auto instanceAndFid = cta::cliTool::getInstanceAndFidFromCTA(archiveFileId, m_serviceProviderPtr, m_log); std::string diskInstance = instanceAndFid.first; std::string diskFileId = instanceAndFid.second; - auto path = m_endpointMapPtr->getPath(diskInstance, diskFileId); m_endpointMapPtr->setXAttr(diskInstance, path, "sys.archive.storage_class", m_storageClassName); } -- GitLab From aa919caf3e001704a10c5adea33cb93535f73984 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 19 Oct 2022 13:46:58 +0200 Subject: [PATCH 072/126] make setNamespaceMap private --- .../standalone_cli_tools/common/ConnectionConfiguration.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp index a2dd3a690a..2131048627 100644 --- a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp @@ -29,9 +29,9 @@ namespace cta { namespace cliTool { class ConnConfiguration { -public: +private: static void setNamespaceMap(const std::string &keytab_file, std::unique_ptr<::eos::client::EndpointMap> &endpointMapPtr); - +public: static void readAndSetConfiguration( cta::log::StdoutLogger &log, const std::string &userName, -- GitLab From 7845169eefe343321ebff5c53de5cb4507827300 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 19 Oct 2022 13:56:55 +0200 Subject: [PATCH 073/126] remove merge conflict with !69 --- tapeserver/readtp/ReadtpCmdLineArgs.cpp | 2 ++ tapeserver/tapelabel/TapeLabelCmdLineArgs.cpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tapeserver/readtp/ReadtpCmdLineArgs.cpp b/tapeserver/readtp/ReadtpCmdLineArgs.cpp index 82479103f0..6d31f29929 100644 --- a/tapeserver/readtp/ReadtpCmdLineArgs.cpp +++ b/tapeserver/readtp/ReadtpCmdLineArgs.cpp @@ -46,6 +46,8 @@ ReadtpCmdLineArgs::ReadtpCmdLineArgs(const int argc, char *const *const argv): throw ex; } m_vid = std::string(argv[1]); + utils::toUpper(m_vid); + m_fSeqRangeList = TapeFileSequenceParser::parse(argv[2]); static struct option longopts[] = { diff --git a/tapeserver/tapelabel/TapeLabelCmdLineArgs.cpp b/tapeserver/tapelabel/TapeLabelCmdLineArgs.cpp index 1aca32b5bc..c95b29feaf 100644 --- a/tapeserver/tapelabel/TapeLabelCmdLineArgs.cpp +++ b/tapeserver/tapelabel/TapeLabelCmdLineArgs.cpp @@ -61,6 +61,7 @@ TapeLabelCmdLineArgs::TapeLabelCmdLineArgs(const int argc, char *const *const ar throw ex; } else { m_vid = std::string(optarg); + utils::toUpper(m_vid); } break; case 'o': @@ -70,6 +71,7 @@ TapeLabelCmdLineArgs::TapeLabelCmdLineArgs(const int argc, char *const *const ar throw ex; } else { m_oldLabel = std::string(optarg); + utils::toUpper(m_oldLabel); } break; case 't': -- GitLab From 36f13d85d9a275c32b496d5bf53af35eb6d82bb8 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 19 Oct 2022 15:06:43 +0200 Subject: [PATCH 074/126] update CtaAdminCmdParse with correct options --- cmdline/CtaAdminCmdParse.hpp | 8 ++++---- .../change_storage_class/ChangeStorageClass.cpp | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cmdline/CtaAdminCmdParse.hpp b/cmdline/CtaAdminCmdParse.hpp index 7abc1abe35..c84e77ce41 100644 --- a/cmdline/CtaAdminCmdParse.hpp +++ b/cmdline/CtaAdminCmdParse.hpp @@ -344,7 +344,7 @@ const std::map<std::string, OptionString::Key> strOptions = { { "--diskinstancespace", OptionString::DISK_INSTANCE_SPACE }, { "--verificationstatus", OptionString::VERIFICATION_STATUS }, { "--disabledreason", OptionString::DISABLED_REASON }, - { "--storage_class_name", OptionString::STORAGE_CLASS_NAME } + { "--storage.class.name", OptionString::STORAGE_CLASS_NAME } }; @@ -355,7 +355,7 @@ const std::map<std::string, OptionString::Key> strOptions = { const std::map<std::string, OptionStrList::Key> strListOptions = { { "--fxidfile", OptionStrList::FILE_ID }, { "--vidfile", OptionStrList::VID }, - { "--fid", OptionStrList::FILE_ID } + { "--id", OptionStrList::FILE_ID } }; @@ -556,8 +556,8 @@ const Option opt_diskinstancespace_alias { Option::OPT_STR, "--name", const Option opt_verificationstatus { Option::OPT_STR, "--verificationstatus", "--vs", " <verification_status>" }; const Option opt_disabledreason { Option::OPT_STR, "--disabledreason", "--dr", " <disabled_reason>" }; -const Option opt_storage_class_name { Option::OPT_STR, "--storage_class_name", "-scn", " <storage_class_name>" }; -const Option opt_archive_file_ids { Option::OPT_STR_LIST, "--fid", "-f", " <archive_file_id>" }; +const Option opt_storage_class_name { Option::OPT_STR, "--storage.class.name", "-n", " <storage_class_name>" }; +const Option opt_archive_file_ids { Option::OPT_STR_LIST, "--id", "-I", " <archive_file_id>" }; /*! * Map valid options to commands diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 122a7cdb92..95628f15e7 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -123,7 +123,6 @@ void ChangeStorageClass::updateStorageClassInCatalogue() { } } - // Validate the Protocol Buffer try { cta::admin::validateCmd(admincmd); @@ -131,7 +130,6 @@ void ChangeStorageClass::updateStorageClassInCatalogue() { throw std::runtime_error("Error: Protocol Buffer validation"); } - // Send the Request to the Service and get a Response cta::xrd::Response response; m_serviceProviderPtr->Send(request, response); -- GitLab From d4e261a7ac238493051eddde7b0854d23c42cab5 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 19 Oct 2022 15:12:18 +0200 Subject: [PATCH 075/126] edit modifyArchiveFileStorageClassId --- catalogue/RdbmsCatalogue.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index cdadc8f202..452747271a 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -1445,11 +1445,6 @@ void RdbmsCatalogue::modifyArchiveFileStorageClassId(const uint64_t& archiveFile << ":" << newStorageClassName << " does not exist"; throw ue; } - if(0 == stmt.getNbAffectedRows()) { - exception::UserError ue; - ue.getMessage() << "Cannot archive file " << archiveFileId << " because it does not exist"; - throw ue; - } } catch(exception::UserError &) { throw; } catch(exception::Exception &ex) { -- GitLab From 337f7cc3b4057a4d11abed0cb394e21a7799badd Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 19 Oct 2022 15:13:23 +0200 Subject: [PATCH 076/126] remove conflict --- catalogue/RdbmsCatalogue.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index 452747271a..0bc85219f4 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -2242,8 +2242,6 @@ bool RdbmsCatalogue::archiveFileIdExists(rdbms::Conn &conn, const uint64_t archi } } - - //------------------------------------------------------------------------------ // diskFileIdExists //------------------------------------------------------------------------------ -- GitLab From 473ada055953e961d6f9fa01c8f566196ffc9026 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 19 Oct 2022 15:20:29 +0200 Subject: [PATCH 077/126] remove member variables not used in ChangeStorageClass --- .../ChangeStorageClass.cpp | 1 - .../ChangeStorageClass.hpp | 18 +++++------------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 95628f15e7..784c90b643 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -118,7 +118,6 @@ void ChangeStorageClass::updateStorageClassInCatalogue() { auto new_opt = admincmd.add_option_str_list(); new_opt->set_key(key); for (auto &archiveFileId : m_archiveFileIds) { - std::cout << archiveFileId << std::endl; new_opt->add_item(archiveFileId); } } diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp index 49a7a68a9b..acfdf2f066 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp @@ -67,12 +67,6 @@ private: */ cta::log::StdoutLogger &m_log; - - /** - * Fxids of the files to change - */ - std::vector<std::string> m_fxIds; - /** * Archive file ids of the files to change */ @@ -83,29 +77,27 @@ private: */ std::string m_storageClassName; - /** * CTA Frontend service provider */ std::unique_ptr<XrdSsiPbServiceType> m_serviceProviderPtr; + /** + * Endpoint map for interaction with EOS + */ std::unique_ptr<::eos::client::EndpointMap> m_endpointMapPtr; - const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses - const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by - //!< XRD_REQUESTTIMEOUT environment variable. - const std::string m_ctaVersion = CTA_VERSION; //!< The version of CTA const std::string m_xrootdSsiProtobufInterfaceVersion = //!< The xrootd-ssi-protobuf-interface version (=tag) XROOTD_SSI_PROTOBUF_INTERFACE_VERSION; /** - * Changes the storage class in the EOS namespace + * Updates the storage class name in the EOS namespace */ void updateStorageClassInEosNamespace(); /** - * Changes the storage class in the catalogue + * Updates the storage class name in the catalogue */ void updateStorageClassInCatalogue(); -- GitLab From bd23320f45ad486f48a1df0e5a9939065ed5523d Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 19 Oct 2022 17:05:21 +0200 Subject: [PATCH 078/126] change naming to CatalogueFetch --- .../change_storage_class/ChangeStorageClass.cpp | 5 ++--- .../change_storage_class/ChangeStorageClass.hpp | 4 ---- .../{CtaCatalogueFetcherFunctions.hpp => CatalogueFetch.hpp} | 0 .../standalone_cli_tools/restore_files/RestoreFilesCmd.cpp | 2 +- 4 files changed, 3 insertions(+), 8 deletions(-) rename cmdline/standalone_cli_tools/common/{CtaCatalogueFetcherFunctions.hpp => CatalogueFetch.hpp} (100%) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 784c90b643..fe316ef7ab 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -18,7 +18,7 @@ #include "cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp" #include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" #include "cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp" -#include "cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp" +#include "cmdline/standalone_cli_tools/common/CatalogueFetch.hpp" #include "cmdline/CtaAdminCmdParse.hpp" #include "common/utils/utils.hpp" #include "common/checksum/ChecksumBlob.hpp" @@ -36,7 +36,6 @@ #include <iostream> #include <memory> - namespace cta { namespace cliTool { @@ -85,7 +84,7 @@ ChangeStorageClass::ChangeStorageClass( //------------------------------------------------------------------------------ void ChangeStorageClass::updateStorageClassInEosNamespace() { for(auto archiveFileId : m_archiveFileIds) { - auto instanceAndFid = cta::cliTool::getInstanceAndFidFromCTA(archiveFileId, m_serviceProviderPtr, m_log); + auto instanceAndFid = getInstanceAndFidFromCTA(archiveFileId, m_serviceProviderPtr, m_log); std::string diskInstance = instanceAndFid.first; std::string diskFileId = instanceAndFid.second; auto path = m_endpointMapPtr->getPath(diskInstance, diskFileId); diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp index acfdf2f066..f6f585cf08 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp @@ -87,10 +87,6 @@ private: */ std::unique_ptr<::eos::client::EndpointMap> m_endpointMapPtr; - const std::string m_ctaVersion = CTA_VERSION; //!< The version of CTA - const std::string m_xrootdSsiProtobufInterfaceVersion = //!< The xrootd-ssi-protobuf-interface version (=tag) - XROOTD_SSI_PROTOBUF_INTERFACE_VERSION; - /** * Updates the storage class name in the EOS namespace */ diff --git a/cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp similarity index 100% rename from cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp rename to cmdline/standalone_cli_tools/common/CatalogueFetch.hpp diff --git a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp index 4798dc4f8b..684bf5d7d0 100644 --- a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp +++ b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp @@ -16,7 +16,7 @@ */ #include "cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.hpp" -#include "cmdline/standalone_cli_tools/common/CtaCatalogueFetcherFunctions.hpp" +#include "cmdline/standalone_cli_tools/common/CatalogueFetch.hpp" #include "cmdline/CtaAdminCmdParse.hpp" #include "common/utils/utils.hpp" #include "common/checksum/ChecksumBlob.hpp" -- GitLab From 9a1e80e93aef4a61833fb87a0f9e9b82149621c0 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 19 Oct 2022 17:06:36 +0200 Subject: [PATCH 079/126] minor stylistic changes --- .../change_storage_class/ChangeStorageClass.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index fe316ef7ab..dc1537a1a5 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -25,7 +25,6 @@ #include "CtaFrontendApi.hpp" #include "common/exception/CommandLineNotParsed.hpp" - #include <XrdSsiPbLog.hpp> #include <XrdSsiPbIStreamBuffer.hpp> -- GitLab From a3752cf6ccd9ae8148a9fb45a7145e296859579a Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 19 Oct 2022 17:32:51 +0200 Subject: [PATCH 080/126] change CMD_STORAGECLASS to CMD_ARCHIVEFILE --- cmdline/CtaAdminCmdParse.hpp | 2 +- .../change_storage_class/ChangeStorageClass.cpp | 2 +- xroot_plugins/XrdSsiCtaRequestMessage.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmdline/CtaAdminCmdParse.hpp b/cmdline/CtaAdminCmdParse.hpp index c84e77ce41..621c21e049 100644 --- a/cmdline/CtaAdminCmdParse.hpp +++ b/cmdline/CtaAdminCmdParse.hpp @@ -709,7 +709,7 @@ const std::map<cmd_key_t, cmd_val_t> cmdOptions = { {{ AdminCmd::CMD_ACTIVITYMOUNTRULE, AdminCmd::SUBCMD_RM }, { opt_instance, opt_username_alias, opt_activityregex }}, {{ AdminCmd::CMD_ACTIVITYMOUNTRULE, AdminCmd::SUBCMD_LS }, { }}, /*----------------------------------------------------------------------------------------------------*/ - {{ AdminCmd::CMD_CHANGESTORAGECLASS, AdminCmd::SUBCMD_CH }, + {{ AdminCmd::CMD_ARCHIVEFILE, AdminCmd::SUBCMD_CH }, { opt_storage_class_name, opt_archive_file_ids }}, }; diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index dc1537a1a5..1c8c78527c 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -101,7 +101,7 @@ void ChangeStorageClass::updateStorageClassInCatalogue() { request.set_client_cta_version(CTA_VERSION); request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); - admincmd.set_cmd(cta::admin::AdminCmd::CMD_CHANGESTORAGECLASS); + admincmd.set_cmd(cta::admin::AdminCmd::CMD_ARCHIVEFILE); admincmd.set_subcmd(cta::admin::AdminCmd::SUBCMD_CH); { diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp index df608b815d..37c50188d2 100644 --- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp +++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp @@ -320,7 +320,7 @@ void RequestMessage::process(const cta::xrd::Request &request, cta::xrd::Respons case cmd_pair(AdminCmd::CMD_RECYCLETAPEFILE, AdminCmd::SUBCMD_RESTORE): processRecycleTapeFile_Restore(response); break; - case cmd_pair(AdminCmd::CMD_CHANGESTORAGECLASS, AdminCmd::SUBCMD_CH): + case cmd_pair(AdminCmd::CMD_ARCHIVEFILE, AdminCmd::SUBCMD_CH): processChangeStorageClass(response); break; -- GitLab From f82f22f3a972fc845ab831f7834ce72b9515780f Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 20 Oct 2022 14:21:42 +0200 Subject: [PATCH 081/126] add option of using filename as input --- .../ChangeStorageClass.cpp | 19 +++++++++++++------ .../ChangeStorageClass.hpp | 4 ++-- .../common/CmdLineArgs.cpp | 5 ++++- .../restore_files/RestoreFilesCmd.cpp | 1 - 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 1c8c78527c..b95a767fc2 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -15,15 +15,15 @@ * submit itself to any jurisdiction. */ +#include "cmdline/CtaAdminCmdParse.hpp" #include "cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp" +#include "cmdline/standalone_cli_tools/common/CatalogueFetch.hpp" #include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" #include "cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp" -#include "cmdline/standalone_cli_tools/common/CatalogueFetch.hpp" -#include "cmdline/CtaAdminCmdParse.hpp" -#include "common/utils/utils.hpp" #include "common/checksum/ChecksumBlob.hpp" -#include "CtaFrontendApi.hpp" #include "common/exception/CommandLineNotParsed.hpp" +#include "common/utils/utils.hpp" +#include "CtaFrontendApi.hpp" #include <XrdSsiPbLog.hpp> #include <XrdSsiPbIStreamBuffer.hpp> @@ -31,9 +31,9 @@ #include <grpc++/grpc++.h> #include "Rpc.grpc.pb.h" -#include <sys/stat.h> #include <iostream> #include <memory> +#include <sys/stat.h> namespace cta { namespace cliTool { @@ -63,7 +63,7 @@ ChangeStorageClass::ChangeStorageClass( exit(1); } - if (!cmdLineArgs.m_archiveFileId) { + if (!cmdLineArgs.m_archiveFileId && !cmdLineArgs.m_archiveFileIds) { cmdLineArgs.printUsage(std::cout); exit(1); } @@ -74,6 +74,13 @@ ChangeStorageClass::ChangeStorageClass( m_archiveFileIds.push_back(cmdLineArgs.m_archiveFileId.value()); } + if (cmdLineArgs.m_archiveFileIds) { + for ( auto archiveFileId : cmdLineArgs.m_archiveFileIds.value() ) { + std::cout << archiveFileId << std::endl; + m_archiveFileIds.push_back(archiveFileId); + } + } + ConnConfiguration::readAndSetConfiguration(log, getUsername(), cmdLineArgs, m_serviceProviderPtr, m_endpointMapPtr); } diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp index f6f585cf08..2d643f41ad 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp @@ -17,11 +17,11 @@ #pragma once +#include "catalogue/Catalogue.hpp" #include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" #include "cmdline/standalone_cli_tools/common/CmdLineTool.hpp" -#include "eos_grpc_client/GrpcEndpoint.hpp" -#include "catalogue/Catalogue.hpp" #include "common/log/StdoutLogger.hpp" +#include "eos_grpc_client/GrpcEndpoint.hpp" #include <optional> #include <memory> diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp index 2bfd220910..c32bc16536 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp @@ -200,7 +200,7 @@ void CmdLineArgs::readIdListFromFile(const std::string &filename) { std::string line; - while(std::getline(file, line)) { + while(file >> line) { switch (m_standaloneCliTool) { case StandaloneCliTool::RESTORE_FILES: m_archiveFileIds.value().push_back(line); @@ -209,6 +209,9 @@ void CmdLineArgs::readIdListFromFile(const std::string &filename) { m_fxIds.value().push_back(line); break; case StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS: + if (!m_archiveFileIds) { + m_archiveFileIds = std::list<std::string>(); + } m_archiveFileIds.value().push_back(line); break; default: diff --git a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp index 684bf5d7d0..e182a7efd9 100644 --- a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp +++ b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp @@ -64,7 +64,6 @@ RestoreFilesCmd::RestoreFilesCmd(std::istream &inStream, std::ostream &outStream // Default single replica layout id should be 00100012 m_defaultFileLayout = kReplica | kAdler | kStripeSize | kStripeWidth | kBlockChecksum; - } //------------------------------------------------------------------------------ -- GitLab From 806e0ed8f2f9d596568a085f49220a11da35cfdd Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 20 Oct 2022 14:26:26 +0200 Subject: [PATCH 082/126] add filename usage in test --- .../orchestration/tests/changeStorageClass.sh | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/continuousintegration/orchestration/tests/changeStorageClass.sh b/continuousintegration/orchestration/tests/changeStorageClass.sh index 1942a37687..23f63fda16 100755 --- a/continuousintegration/orchestration/tests/changeStorageClass.sh +++ b/continuousintegration/orchestration/tests/changeStorageClass.sh @@ -27,7 +27,10 @@ if [ ! -z "${error}" ]; then exit 1 fi -NEW_STORAGE_CLASS_NAME=storageClassName +EOSINSTANCE=ctaeos + +ORIGINAL_STORAGE_CLASS_NAME=originalStorageClassName +NEW_STORAGE_CLASS_NAME=newStorageClassName CHANGE_SC_FILE=`uuidgen` STATIC_SC_FILE=`uuidgen` @@ -46,6 +49,16 @@ echo "ADD FRONTEND GATEWAY TO EOS" echo "kubectl -n ${NAMESPACE} exec ctaeos -- bash eos root://${EOSINSTANCE} -r 0 0 vid add gateway ${FRONTEND_IP} grpc" kubectl -n ${NAMESPACE} exec ctaeos -- eos -r 0 0 vid add gateway ${FRONTEND_IP} grpc +echo +echo "ADD STORAGE CLASS WITH TWO COPIES ${ORIGINAL_STORAGE_CLASS_NAME}" +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc add --name ${ORIGINAL_STORAGE_CLASS_NAME} --numberofcopies 2 --virtualorganisation vo --comment "comment" +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc ls + +echo +echo "ADD STORAGE CLASS WITH ONE COPIES ${NEW_STORAGE_CLASS_NAME}" +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc add --name ${NEW_STORAGE_CLASS_NAME} --numberofcopies 2 --virtualorganisation vo --comment "comment" +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc ls + echo echo "COPY REQUIRED FILES TO FRONTEND POD" echo "sudo kubectl cp ${NAMESPACE}/ctacli:/etc/cta/cta-cli.conf /etc/cta/cta-cli.conf" @@ -58,11 +71,25 @@ kubectl -n ${NAMESPACE} cp client_helper.sh client:/root/ kubectl -n ${NAMESPACE} exec client -- bash /usr/bin/archive_file.sh -f ${CHANGE_SC_FILE} || exit 1 kubectl -n ${NAMESPACE} exec client -- bash /usr/bin/archive_file.sh -f ${STATIC_SC_FILE} || exit 1 -kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc add --name ${NEW_STORAGE_CLASS_NAME} --numberofcopies 1 --virtualorganisation vo --comment "comment" -kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc ls +EOS_METADATA_PATH=$(mktemp -d).json +echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH}" +touch ${EOS_METADATA_PATH} +kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${CHANGE_SC_FILE} | jq . |& tee ${EOS_METADATA_PATH} +EOS_ARCHIVE_ID=$(jq -r '.xattr | .["sys.archive.file_id"]' ${EOS_METADATA_PATH}) +echo +echo "CREATE FILE WITH IDS" +cd ~ +IDS_FILEPATH=~/archiveFileIds.txt +rm ${IDS_FILEPATH} +touch ${IDS_FILEPATH} +echo ${EOS_ARCHIVE_ID} | tee -a ${IDS_FILEPATH} + +echo kubectl cp ~/CTA-build/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class ${NAMESPACE}/ctafrontend:/usr/bin/ -kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c "XrdSecPROTOCOL=sss XrdSecSSSKT=/etc/cta/eos.sss.keytab cta-change-storage-class --storage.class.name ${NEW_STORAGE_CLASS_NAME} --id 2" +echo "kubectl cp ${IDS_FILEPATH} ${NAMESPACE}/ctafrontend:~/" +kubectl cp ${IDS_FILEPATH} ${NAMESPACE}/ctafrontend:/root/ +kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c "XrdSecPROTOCOL=sss XrdSecSSSKT=/etc/cta/eos.sss.keytab cta-change-storage-class --storage.class.name ${NEW_STORAGE_CLASS_NAME} --filename ${IDS_FILEPATH}" # Remove authorization kubectl -n ${NAMESPACE} exec ctacli -- cta-admin admin rm --username ctafrontend -- GitLab From e14acbf53f06a732c7081d711770e8197706ac70 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 20 Oct 2022 15:46:35 +0200 Subject: [PATCH 083/126] Add new version of xrootd-ssi-protobuf-interface --- xrootd-ssi-protobuf-interface | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xrootd-ssi-protobuf-interface b/xrootd-ssi-protobuf-interface index f05ac4849d..5b175209df 160000 --- a/xrootd-ssi-protobuf-interface +++ b/xrootd-ssi-protobuf-interface @@ -1 +1 @@ -Subproject commit f05ac4849d43590c459fe30bb834ef96eb515d7e +Subproject commit 5b175209dfc5ea6608058214c23eee8a300d69d7 -- GitLab From 1f27de00be82dafb1ad7ecdbbd1678969af7ee9b Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 20 Oct 2022 16:10:40 +0200 Subject: [PATCH 084/126] add changes to cmakefile --- cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt index 587273912d..1187951ce0 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt +++ b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt @@ -33,4 +33,3 @@ set_property (TARGET cta-change-storage-class APPEND PROPERTY INSTALL_RPATH ${PR install(TARGETS cta-change-storage-class DESTINATION usr/bin) -install(FILES cta-change-storage-class.1cta DESTINATION usr/share/man/man1) -- GitLab From 3ec17b5e9ad4cccea77ccefcd716ef25e26aa1fc Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 20 Oct 2022 16:44:20 +0200 Subject: [PATCH 085/126] add cta-change-storage-class to cta.spec.in --- cta.spec.in | 1 + 1 file changed, 1 insertion(+) diff --git a/cta.spec.in b/cta.spec.in index 20d7e54544..19c00d134a 100644 --- a/cta.spec.in +++ b/cta.spec.in @@ -233,6 +233,7 @@ The command line utilities %attr(0755,root,root) %{_bindir}/cta-send-event %attr(0755,root,root) %{_bindir}/cta-send-closew.sh %attr(0755,root,root) %{_bindir}/cta-verify-file +%attr(0755,root,root) %{_bindir}/cta-change-storage-class %attr(0644,root,root) %config(noreplace) %{_sysconfdir}/cta/cta-cli.conf.example %posttrans -n cta-cli -- GitLab From d8600447a4e285820fdf216931e1abbade9588f2 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 20 Oct 2022 17:06:24 +0200 Subject: [PATCH 086/126] add minor changes for efficiency --- xroot_plugins/XrdCtaChangeStorageClass.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xroot_plugins/XrdCtaChangeStorageClass.hpp b/xroot_plugins/XrdCtaChangeStorageClass.hpp index 887d9b6c59..262bcd5e4b 100644 --- a/xroot_plugins/XrdCtaChangeStorageClass.hpp +++ b/xroot_plugins/XrdCtaChangeStorageClass.hpp @@ -29,12 +29,12 @@ private: cta::log::LogContext &m_lc; public: XrdCtaChangeStorageClass(cta::catalogue::Catalogue &catalogue, cta::log::LogContext &lc); - void updateCatalogue(const std::vector<std::basic_string<char>> archiveFileIds, std::string newStorageClassName); + void updateCatalogue(const std::vector<std::basic_string<char>>& archiveFileIds, const std::string& newStorageClassName); }; XrdCtaChangeStorageClass::XrdCtaChangeStorageClass(cta::catalogue::Catalogue &catalogue, cta::log::LogContext &lc): m_catalogue(catalogue), m_lc(lc) {} -void XrdCtaChangeStorageClass::updateCatalogue(const std::vector<std::basic_string<char>> archiveFileIds, const std::string newStorageClassName) { +void XrdCtaChangeStorageClass::updateCatalogue(const std::vector<std::basic_string<char>>& archiveFileIds, const std::string& newStorageClassName) { for (auto & id : archiveFileIds) { const uint64_t archiveFileId = std::stoul(id); -- GitLab From 6d56194f3425b83ff3568c373dde62888b389353 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 21 Oct 2022 11:12:57 +0200 Subject: [PATCH 087/126] prevent in flight files from changing storage class --- .../ChangeStorageClass.cpp | 20 ++++++++++++++++++- .../ChangeStorageClass.hpp | 19 ++++++++++++++---- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index b95a767fc2..909663ccb0 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -76,7 +76,6 @@ ChangeStorageClass::ChangeStorageClass( if (cmdLineArgs.m_archiveFileIds) { for ( auto archiveFileId : cmdLineArgs.m_archiveFileIds.value() ) { - std::cout << archiveFileId << std::endl; m_archiveFileIds.push_back(archiveFileId); } } @@ -85,6 +84,17 @@ ChangeStorageClass::ChangeStorageClass( } +//------------------------------------------------------------------------------ +// fileInFlight +//------------------------------------------------------------------------------ +bool ChangeStorageClass::fileInFlight(const google::protobuf::RepeatedField<unsigned int> &locations) { + for (auto location : locations) { + // file is not in flight if fsid==65535 + if (location == 65535) { return false; } + } + return true; +} + //------------------------------------------------------------------------------ // updateStorageClassInEosNamespace //------------------------------------------------------------------------------ @@ -93,6 +103,14 @@ void ChangeStorageClass::updateStorageClassInEosNamespace() { auto instanceAndFid = getInstanceAndFidFromCTA(archiveFileId, m_serviceProviderPtr, m_log); std::string diskInstance = instanceAndFid.first; std::string diskFileId = instanceAndFid.second; + + // No files in flight should change storage class + auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::FILE, std::stoi(diskFileId), "", false); + if (fileInFlight(md_response.fmd().locations())) { + filesNotChanged.push_back(archiveFileId); + continue; + } + auto path = m_endpointMapPtr->getPath(diskInstance, diskFileId); m_endpointMapPtr->setXAttr(diskInstance, path, "sys.archive.storage_class", m_storageClassName); } diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp index 2d643f41ad..96120dcaa4 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp @@ -87,15 +87,26 @@ private: */ std::unique_ptr<::eos::client::EndpointMap> m_endpointMapPtr; + /** + * Archive file ids which have not been changed because + * they were in flight + */ + std::vector<std::string> filesNotChanged; + /** * Updates the storage class name in the EOS namespace */ void updateStorageClassInEosNamespace(); - /** - * Updates the storage class name in the catalogue - */ - void updateStorageClassInCatalogue(); + /** + * Updates the storage class name in the catalogue + */ + void updateStorageClassInCatalogue(); + + /** + * Checks if a file is in flight + */ + bool fileInFlight(const google::protobuf::RepeatedField<unsigned int> &locations); } ; // class CtaChangeStorageClass -- GitLab From 9c7dcd4e13d639bd7c2ac268ae0b80846ae70f67 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 21 Oct 2022 14:14:34 +0200 Subject: [PATCH 088/126] remove in flight check --- .../ChangeStorageClass.cpp | 18 ------------------ .../ChangeStorageClass.hpp | 11 ----------- 2 files changed, 29 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 909663ccb0..3df55509c8 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -84,17 +84,6 @@ ChangeStorageClass::ChangeStorageClass( } -//------------------------------------------------------------------------------ -// fileInFlight -//------------------------------------------------------------------------------ -bool ChangeStorageClass::fileInFlight(const google::protobuf::RepeatedField<unsigned int> &locations) { - for (auto location : locations) { - // file is not in flight if fsid==65535 - if (location == 65535) { return false; } - } - return true; -} - //------------------------------------------------------------------------------ // updateStorageClassInEosNamespace //------------------------------------------------------------------------------ @@ -103,13 +92,6 @@ void ChangeStorageClass::updateStorageClassInEosNamespace() { auto instanceAndFid = getInstanceAndFidFromCTA(archiveFileId, m_serviceProviderPtr, m_log); std::string diskInstance = instanceAndFid.first; std::string diskFileId = instanceAndFid.second; - - // No files in flight should change storage class - auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::FILE, std::stoi(diskFileId), "", false); - if (fileInFlight(md_response.fmd().locations())) { - filesNotChanged.push_back(archiveFileId); - continue; - } auto path = m_endpointMapPtr->getPath(diskInstance, diskFileId); m_endpointMapPtr->setXAttr(diskInstance, path, "sys.archive.storage_class", m_storageClassName); diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp index 96120dcaa4..dd997cfd60 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp @@ -87,12 +87,6 @@ private: */ std::unique_ptr<::eos::client::EndpointMap> m_endpointMapPtr; - /** - * Archive file ids which have not been changed because - * they were in flight - */ - std::vector<std::string> filesNotChanged; - /** * Updates the storage class name in the EOS namespace */ @@ -103,11 +97,6 @@ private: */ void updateStorageClassInCatalogue(); - /** - * Checks if a file is in flight - */ - bool fileInFlight(const google::protobuf::RepeatedField<unsigned int> &locations); - } ; // class CtaChangeStorageClass } // namespace admin -- GitLab From 1b02106db96cfe00b4361de0e115d1ea34af4adc Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 21 Oct 2022 14:34:31 +0200 Subject: [PATCH 089/126] update documentation --- .../change_storage_class/ChangeStorageClass.cpp | 2 +- cmdline/standalone_cli_tools/common/CatalogueFetch.hpp | 10 +++++++--- .../common/ConnectionConfiguration.cpp | 10 ---------- .../common/ConnectionConfiguration.hpp | 10 ++++++++++ 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 3df55509c8..c77f834a75 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -99,7 +99,7 @@ void ChangeStorageClass::updateStorageClassInEosNamespace() { } //------------------------------------------------------------------------------ -// changeCatalogue +// updateCatalogue //------------------------------------------------------------------------------ void ChangeStorageClass::updateStorageClassInCatalogue() { cta::xrd::Request request; diff --git a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp index a4fab92ba0..5919286bcb 100644 --- a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp +++ b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp @@ -92,9 +92,13 @@ void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const namespace cta { namespace cliTool { -//------------------------------------------------------------------------------ -// getFxidFromCTA -//------------------------------------------------------------------------------ + /** + * Fetches the instance and fid from the CTA catalogue + * + * @param archiveFileId The arhive file id. + * @param serviceProviderPtr Service provider for communication with the catalogue. + * @return a pair with the instance and the fid. + */ std::pair<std::string,std::string> getInstanceAndFidFromCTA(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log) { { std::list<cta::log::Param> params; diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp index 2bbf70389e..18daac0808 100644 --- a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp @@ -56,16 +56,6 @@ void ConnConfiguration::setNamespaceMap(const std::string &keytab_file, std::uni endpointMapPtr = std::make_unique<::eos::client::EndpointMap>(namespaceMap); } - - /** - * Sets internal configuration parameters to be used for reading. - * It reads cta frontend parameters from /etc/cta/cta-cli.conf - * - * @param username The name of the user running the command-line tool. - * @param cmdLineArgs The arguments parsed from the command line. - * @param serviceProviderPtr Xroot service provider - * @param endpointMapPtr Endpoint for communication with EOS - */ //------------------------------------------------------------------------------ // readAndSetConfiguration //------------------------------------------------------------------------------ diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp index 2131048627..6e0e2634d5 100644 --- a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp @@ -32,6 +32,16 @@ class ConnConfiguration { private: static void setNamespaceMap(const std::string &keytab_file, std::unique_ptr<::eos::client::EndpointMap> &endpointMapPtr); public: + + /** + * Sets internal configuration parameters to be used for reading. + * It reads cta frontend parameters from /etc/cta/cta-cli.conf + * + * @param username The name of the user running the command-line tool. + * @param cmdLineArgs The arguments parsed from the command line. + * @param serviceProviderPtr Xroot service provider + * @param endpointMapPtr Endpoint for communication with EOS + */ static void readAndSetConfiguration( cta::log::StdoutLogger &log, const std::string &userName, -- GitLab From 43baaee7866ef1ebc6ef5576e1abd34de29cf3c3 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 21 Oct 2022 16:44:12 +0200 Subject: [PATCH 090/126] remove logging in xroot_plugins/XrdCtaChangeStorageClass.hpp --- cmdline/CMakeLists.txt | 1 - xroot_plugins/XrdCtaChangeStorageClass.hpp | 8 +------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/cmdline/CMakeLists.txt b/cmdline/CMakeLists.txt index c605e9eef3..66d988f3b6 100644 --- a/cmdline/CMakeLists.txt +++ b/cmdline/CMakeLists.txt @@ -36,7 +36,6 @@ add_executable(cta-admin CtaAdminCmd.cpp CtaAdminCmdParse.cpp CtaAdminTextFormat target_link_libraries(cta-admin XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon) set_property (TARGET cta-admin APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) - # # cta-wfe-test archive|retrieve|delete <options> allows testing of the SSI WorkFlow Engine hooks # without invoking EOS. diff --git a/xroot_plugins/XrdCtaChangeStorageClass.hpp b/xroot_plugins/XrdCtaChangeStorageClass.hpp index 262bcd5e4b..63ef0fb553 100644 --- a/xroot_plugins/XrdCtaChangeStorageClass.hpp +++ b/xroot_plugins/XrdCtaChangeStorageClass.hpp @@ -36,13 +36,7 @@ XrdCtaChangeStorageClass::XrdCtaChangeStorageClass(cta::catalogue::Catalogue &ca void XrdCtaChangeStorageClass::updateCatalogue(const std::vector<std::basic_string<char>>& archiveFileIds, const std::string& newStorageClassName) { for (auto & id : archiveFileIds) { - const uint64_t archiveFileId = std::stoul(id); - - m_lc.log(cta::log::INFO, std::to_string(archiveFileId)); - m_lc.log(cta::log::INFO, typeid(archiveFileId).name()); - m_lc.log(cta::log::INFO, newStorageClassName); - m_lc.log(cta::log::INFO, typeid(newStorageClassName).name()); - + const uint64_t archiveFileId = std::stoul(id); m_catalogue.modifyArchiveFileStorageClassId(archiveFileId, newStorageClassName); } } -- GitLab From c8188169b7ef917f6a691c2189c924fb45bb29f7 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 21 Oct 2022 17:17:33 +0200 Subject: [PATCH 091/126] add in flight check --- .../ChangeStorageClass.cpp | 30 ++++++++++++++++++- .../ChangeStorageClass.hpp | 11 +++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index c77f834a75..05eeda555e 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -84,6 +84,18 @@ ChangeStorageClass::ChangeStorageClass( } +//------------------------------------------------------------------------------ +// fileInFlight +//------------------------------------------------------------------------------ +bool ChangeStorageClass::fileInFlight(const google::protobuf::RepeatedField<unsigned int> &locations) { + for (auto location : locations) { + // file is not in flight if fsid==65535 + if (location == 65535) { return false; } + } + return true; +} + + //------------------------------------------------------------------------------ // updateStorageClassInEosNamespace //------------------------------------------------------------------------------ @@ -93,8 +105,24 @@ void ChangeStorageClass::updateStorageClassInEosNamespace() { std::string diskInstance = instanceAndFid.first; std::string diskFileId = instanceAndFid.second; + // No files in flight should change storage class + auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::FILE, std::stoi(diskFileId), "", false); + if (fileInFlight(md_response.fmd().locations())) { + m_archiveIdsNotUpdatedInEos.push_back(archiveFileId); + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("archiveFileId", archiveFileId)); + m_log(cta::log::INFO, "File did not change storage class because the file was in flight", params); + continue; + } + auto path = m_endpointMapPtr->getPath(diskInstance, diskFileId); - m_endpointMapPtr->setXAttr(diskInstance, path, "sys.archive.storage_class", m_storageClassName); + int status = m_endpointMapPtr->setXAttr(diskInstance, path, "sys.archive.storage_class", m_storageClassName); + if (status != 0) { + m_archiveIdsNotUpdatedInEos.push_back(archiveFileId); + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("archiveFileId", archiveFileId)); + m_log(cta::log::INFO, "File did not change storage class because query to EOS failed", params); + } } } diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp index dd997cfd60..57d461bf1a 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp @@ -87,6 +87,11 @@ private: */ std::unique_ptr<::eos::client::EndpointMap> m_endpointMapPtr; + /** + * Endpoint map for interaction with EOS + */ + std::vector<std::string> m_archiveIdsNotUpdatedInEos; + /** * Updates the storage class name in the EOS namespace */ @@ -97,6 +102,12 @@ private: */ void updateStorageClassInCatalogue(); + /** + * Checks if a file is in flight + */ + bool fileInFlight(const google::protobuf::RepeatedField<unsigned int> &locations); + + } ; // class CtaChangeStorageClass } // namespace admin -- GitLab From ceb7869fb5525b6a73a867d80181cf49ac8fbe1e Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Fri, 21 Oct 2022 17:31:58 +0200 Subject: [PATCH 092/126] avoid update to CTA catalogue if EOS was not updated --- .../change_storage_class/ChangeStorageClass.cpp | 5 ++++- .../change_storage_class/ChangeStorageClass.hpp | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 05eeda555e..ce874aa300 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -123,6 +123,9 @@ void ChangeStorageClass::updateStorageClassInEosNamespace() { params.push_back(cta::log::Param("archiveFileId", archiveFileId)); m_log(cta::log::INFO, "File did not change storage class because query to EOS failed", params); } + else { + m_archiveIdsUpdatedInEos.push_back(archiveFileId); + } } } @@ -150,7 +153,7 @@ void ChangeStorageClass::updateStorageClassInCatalogue() { auto key = cta::admin::OptionStrList::FILE_ID; auto new_opt = admincmd.add_option_str_list(); new_opt->set_key(key); - for (auto &archiveFileId : m_archiveFileIds) { + for (auto &archiveFileId : m_archiveIdsUpdatedInEos) { new_opt->add_item(archiveFileId); } } diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp index 57d461bf1a..26d12f8df9 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp @@ -92,6 +92,11 @@ private: */ std::vector<std::string> m_archiveIdsNotUpdatedInEos; + /** + * Endpoint map for interaction with EOS + */ + std::vector<std::string> m_archiveIdsUpdatedInEos; + /** * Updates the storage class name in the EOS namespace */ -- GitLab From c782beb1fa42dc4d072c8dd155b7b9426c023ad1 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Mon, 24 Oct 2022 09:11:38 +0200 Subject: [PATCH 093/126] write ids that did not update the storage class to a txt file --- .../ChangeStorageClass.cpp | 23 +++++++++++++++++++ .../ChangeStorageClass.hpp | 5 ++++ 2 files changed, 28 insertions(+) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index ce874aa300..3885caf8dc 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -31,6 +31,7 @@ #include <grpc++/grpc++.h> #include "Rpc.grpc.pb.h" +#include <fstream> #include <iostream> #include <memory> #include <sys/stat.h> @@ -185,12 +186,34 @@ void ChangeStorageClass::updateStorageClassInCatalogue() { } } +//------------------------------------------------------------------------------ +// writeSkippedArchiveIdsToFile +//------------------------------------------------------------------------------ +void ChangeStorageClass::writeSkippedArchiveIdsToFile() { + std::string filePath = "/var/log/skippedArchiveIds.txt"; + std::ofstream archiveIdFile(filePath); + + if (archiveIdFile.fail()) { + throw std::runtime_error("Unable to open file " + filePath); + } + + if (archiveIdFile.is_open()) { + for (auto archiveId : m_archiveIdsNotUpdatedInEos) { + archiveIdFile << archiveId << std::endl; + } + archiveIdFile.close(); + std::cout << m_archiveIdsNotUpdatedInEos.size() << " files did not update the storage class." << std::endl; + std::cout << "The skipped archive ids can be found here: " << filePath << std::endl; + } +} + //------------------------------------------------------------------------------ // exceptionThrowingMain //------------------------------------------------------------------------------ int ChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { updateStorageClassInEosNamespace(); updateStorageClassInCatalogue(); + writeSkippedArchiveIdsToFile(); return 0; } diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp index 26d12f8df9..6ef82e9d60 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp @@ -107,6 +107,11 @@ private: */ void updateStorageClassInCatalogue(); + /** + * Writes the skipped archive ids to file + */ + void writeSkippedArchiveIdsToFile(); + /** * Checks if a file is in flight */ -- GitLab From 9bdba130585ea5f35fee20f1b704b679e96a74ce Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 25 Oct 2022 13:53:24 +0200 Subject: [PATCH 094/126] create man page for cta-change-storage-class --- .../change_storage_class/CMakeLists.txt | 1 + .../cta-change-storage-class.1cta | 58 +++++++++++++++++++ .../common/CmdLineArgs.cpp | 4 +- 3 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class.1cta diff --git a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt index 1187951ce0..be7e100204 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt +++ b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt @@ -33,3 +33,4 @@ set_property (TARGET cta-change-storage-class APPEND PROPERTY INSTALL_RPATH ${PR install(TARGETS cta-change-storage-class DESTINATION usr/bin) +install(FILES cta-change-storage-class.1cta DESTINATION usr/share/man/man1) \ No newline at end of file diff --git a/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class.1cta b/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class.1cta new file mode 100644 index 0000000000..4cf0ab5b8d --- /dev/null +++ b/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class.1cta @@ -0,0 +1,58 @@ +.\" @project The CERN Tape Archive (CTA) +.\" @copyright Copyright © 2019-2022 CERN +.\" @license This program is free software, distributed under the terms of the GNU General Public +.\" Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can +.\" redistribute it and/or modify it under the terms of the GPL Version 3, or (at your +.\" option) any later version. +.\" +.\" This program is distributed in the hope that it will be useful, but WITHOUT ANY +.\" WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +.\" PARTICULAR PURPOSE. See the GNU General Public License for more details. +.\" +.\" In applying this licence, CERN does not waive the privileges and immunities +.\" granted to it by virtue of its status as an Intergovernmental Organization or +.\" submit itself to any jurisdiction. + +.TH CTA-CHANGE-STORAGE-CLASS "1cta" "2022-25-10" "CTA" "The CERN Tape Archive (CTA)" +.SH NAME +cta-change-storage-class \- Change which storageclass files with a given archive id use + +.SH SYNOPSIS +.HP +\fBcta-change-storage-class\fP --id/-I <archiveFileID> | --filename/-F <filename> --storage.class.name/-n <storageClassName> + +.SH DESCRIPTION +\fBcta-change-storage-class\fP changes the storage class for the files specified by the user. The tool will first change +the storage class on the EOS side, and then change the storage class for the files that were successfully changed in EOS +in the catalogue + +.SH OPTIONS +.TP +\-I, \-\-id \fIarchive_file_id\fR +Archive file id of the files to change. +.TP +\-F, \-\-filename \fIfile_name\fR +Path to a file with a archive file ids. The text file should have one id on each line. +.TP +\-n, \-\-storage.class.name \fIstorageclassname\fR +Path to a file with a archive file ids. The text file should have one id on each line. + +.SH EXIT STATUS +.P +\fBcta-change-storage-class\fP returns 0 on success. + +.SH EXAMPLE +.P +cta-change-storage-class --filename ~/idfile.txt --storage.class.name newStorageClassName + +.SH SEE ALSO +.P +CERN Tape Archive documentation (\fIhttps://eoscta.docs.cern.ch/\fR) + +.SH COPYRIGHT +.P +Copyright © 2022 CERN. License GPLv3+: GNU GPL version 3 or later (\fIhttp://gnu.org/licenses/gpl.html\fR). +This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the +extent permitted by law. In applying this licence, CERN does not waive the privileges and immunities +granted to it by virtue of its status as an Intergovernmental Organization or submit itself to any +jurisdiction. diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp index c32bc16536..0ad9118b2a 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp @@ -239,11 +239,11 @@ void CmdLineArgs::printUsage(std::ostream &os) const { break; case StandaloneCliTool::CTA_VERIFY_FILE : os << " Usage:" << std::endl << - " cta-verify-file --id/-I <archiveFileID> --vid/-v <vid> [--instance/-i <instance>] [--request.user/-u <user>] [request.group/-g <group>]\n" << std::endl; + " cta-verify-file --id/-I <archiveFileID> --vid/-v <vid> [--instance/-i <instance>] [--request.user/-u <user>] [request.group/-g <group>]" << std::endl; break; case StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS : os << " Usage:" << std::endl << - " cta-change-storage-class --id/-I <archiveFileID> | --filename/-F --storage.class.name/-n <storageClassName>\n" << std::endl; + " cta-change-storage-class --id/-I <archiveFileID> | --filename/-F <filename> --storage.class.name/-n <storageClassName>" << std::endl; break; default: break; -- GitLab From 431ecfb767712a9942e85f48d3816a553b81758d Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 25 Oct 2022 14:15:07 +0200 Subject: [PATCH 095/126] remove unused linked libraries --- .../standalone_cli_tools/change_storage_class/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt index be7e100204..d33f4b005f 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt +++ b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt @@ -28,7 +28,7 @@ include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include) include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS}) add_executable(cta-change-storage-class ChangeStorageClass.cpp ChangeStorageClassMain.cpp ../common/CmdLineTool.cpp ../common/CmdLineArgs.cpp ../../CtaAdminCmdParse.cpp ../common/ConnectionConfiguration.cpp) -target_link_libraries(cta-change-storage-class ${PROTOBUF3_LIBRARIES} ${GRPC_LIBRARY} ${GRPC_GRPC++_LIBRARY} XrdSsiPbEosCta XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient) +target_link_libraries(cta-change-storage-class ${PROTOBUF3_LIBRARIES} XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient) set_property (TARGET cta-change-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) -- GitLab From f63370271abe58ff1589ea6950d633dff4212f21 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 25 Oct 2022 14:26:57 +0200 Subject: [PATCH 096/126] improve code quality in ChangeStorageClass.cpp --- .../ChangeStorageClass.cpp | 53 +++++++++---------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 3885caf8dc..61b6cbb382 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -15,6 +15,16 @@ * submit itself to any jurisdiction. */ +#include <fstream> +#include <iostream> +#include <list> +#include <memory> +#include <string> +#include <sys/stat.h> + +#include <XrdSsiPbLog.hpp> +#include <XrdSsiPbIStreamBuffer.hpp> + #include "cmdline/CtaAdminCmdParse.hpp" #include "cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp" #include "cmdline/standalone_cli_tools/common/CatalogueFetch.hpp" @@ -25,17 +35,6 @@ #include "common/utils/utils.hpp" #include "CtaFrontendApi.hpp" -#include <XrdSsiPbLog.hpp> -#include <XrdSsiPbIStreamBuffer.hpp> - -#include <grpc++/grpc++.h> -#include "Rpc.grpc.pb.h" - -#include <fstream> -#include <iostream> -#include <memory> -#include <sys/stat.h> - namespace cta { namespace cliTool { @@ -43,11 +42,11 @@ namespace cliTool { // constructor //------------------------------------------------------------------------------ ChangeStorageClass::ChangeStorageClass( - std::istream &inStream, + std::istream &inStream, std::ostream &outStream, - std::ostream &errStream, + std::ostream &errStream, cta::log::StdoutLogger &log, - const int& argc, + const int& argc, char *const *const argv): CmdLineTool(inStream, outStream, errStream), m_log(log) { @@ -58,7 +57,7 @@ ChangeStorageClass::ChangeStorageClass( cmdLineArgs.printUsage(std::cout); exit(0); } - + if (!cmdLineArgs.m_storageClassName) { cmdLineArgs.printUsage(std::cout); exit(1); @@ -76,7 +75,7 @@ ChangeStorageClass::ChangeStorageClass( } if (cmdLineArgs.m_archiveFileIds) { - for ( auto archiveFileId : cmdLineArgs.m_archiveFileIds.value() ) { + for (const auto archiveFileId : cmdLineArgs.m_archiveFileIds.value() ) { m_archiveFileIds.push_back(archiveFileId); } } @@ -88,7 +87,7 @@ ChangeStorageClass::ChangeStorageClass( //------------------------------------------------------------------------------ // fileInFlight //------------------------------------------------------------------------------ -bool ChangeStorageClass::fileInFlight(const google::protobuf::RepeatedField<unsigned int> &locations) { +bool ChangeStorageClass::fileInFlight(const google::protobuf::RepeatedField<uint32_t> &locations) { for (auto location : locations) { // file is not in flight if fsid==65535 if (location == 65535) { return false; } @@ -101,20 +100,20 @@ bool ChangeStorageClass::fileInFlight(const google::protobuf::RepeatedField<unsi // updateStorageClassInEosNamespace //------------------------------------------------------------------------------ void ChangeStorageClass::updateStorageClassInEosNamespace() { - for(auto archiveFileId : m_archiveFileIds) { + for(const auto archiveFileId : m_archiveFileIds) { auto instanceAndFid = getInstanceAndFidFromCTA(archiveFileId, m_serviceProviderPtr, m_log); std::string diskInstance = instanceAndFid.first; std::string diskFileId = instanceAndFid.second; // No files in flight should change storage class - auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::FILE, std::stoi(diskFileId), "", false); - if (fileInFlight(md_response.fmd().locations())) { + const auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::FILE, std::stoi(diskFileId), "", false); + if (fileInFlight(md_response.fmd().locations())) { m_archiveIdsNotUpdatedInEos.push_back(archiveFileId); std::list<cta::log::Param> params; params.push_back(cta::log::Param("archiveFileId", archiveFileId)); m_log(cta::log::INFO, "File did not change storage class because the file was in flight", params); - continue; - } + continue; + } auto path = m_endpointMapPtr->getPath(diskInstance, diskFileId); int status = m_endpointMapPtr->setXAttr(diskInstance, path, "sys.archive.storage_class", m_storageClassName); @@ -123,7 +122,7 @@ void ChangeStorageClass::updateStorageClassInEosNamespace() { std::list<cta::log::Param> params; params.push_back(cta::log::Param("archiveFileId", archiveFileId)); m_log(cta::log::INFO, "File did not change storage class because query to EOS failed", params); - } + } else { m_archiveIdsUpdatedInEos.push_back(archiveFileId); } @@ -141,7 +140,7 @@ void ChangeStorageClass::updateStorageClassInCatalogue() { request.set_client_cta_version(CTA_VERSION); request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); admincmd.set_cmd(cta::admin::AdminCmd::CMD_ARCHIVEFILE); - admincmd.set_subcmd(cta::admin::AdminCmd::SUBCMD_CH); + admincmd.set_subcmd(cta::admin::AdminCmd::SUBCMD_CH); { auto key = cta::admin::OptionString::STORAGE_CLASS_NAME; @@ -177,7 +176,7 @@ void ChangeStorageClass::updateStorageClassInCatalogue() { using namespace cta::admin; case Response::RSP_SUCCESS: // Print message text - std::cout << response.message_txt(); + std::cout << response.message_txt(); break; case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); case Response::RSP_ERR_USER: throw std::runtime_error(response.message_txt()); @@ -192,11 +191,11 @@ void ChangeStorageClass::updateStorageClassInCatalogue() { void ChangeStorageClass::writeSkippedArchiveIdsToFile() { std::string filePath = "/var/log/skippedArchiveIds.txt"; std::ofstream archiveIdFile(filePath); - + if (archiveIdFile.fail()) { throw std::runtime_error("Unable to open file " + filePath); } - + if (archiveIdFile.is_open()) { for (auto archiveId : m_archiveIdsNotUpdatedInEos) { archiveIdFile << archiveId << std::endl; -- GitLab From 916758046c8495521cf3cfd67ab7f748c91d900b Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 25 Oct 2022 14:58:46 +0200 Subject: [PATCH 097/126] add changes to includes --- .../ChangeStorageClass.cpp | 29 ++++++++------- .../ChangeStorageClass.hpp | 37 +++++++++---------- .../ChangeStorageClassMain.cpp | 1 - .../common/CatalogueFetch.hpp | 2 + 4 files changed, 35 insertions(+), 34 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 61b6cbb382..828ea9e8be 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -29,10 +29,12 @@ #include "cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp" #include "cmdline/standalone_cli_tools/common/CatalogueFetch.hpp" #include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" + #include "cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp" #include "common/checksum/ChecksumBlob.hpp" #include "common/exception/CommandLineNotParsed.hpp" #include "common/utils/utils.hpp" + #include "CtaFrontendApi.hpp" namespace cta { @@ -101,9 +103,9 @@ bool ChangeStorageClass::fileInFlight(const google::protobuf::RepeatedField<uint //------------------------------------------------------------------------------ void ChangeStorageClass::updateStorageClassInEosNamespace() { for(const auto archiveFileId : m_archiveFileIds) { - auto instanceAndFid = getInstanceAndFidFromCTA(archiveFileId, m_serviceProviderPtr, m_log); - std::string diskInstance = instanceAndFid.first; - std::string diskFileId = instanceAndFid.second; + const auto instanceAndFid = getInstanceAndFidFromCTA(archiveFileId, m_serviceProviderPtr, m_log); + const std::string diskInstance = instanceAndFid.first; + const std::string diskFileId = instanceAndFid.second; // No files in flight should change storage class const auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::FILE, std::stoi(diskFileId), "", false); @@ -111,7 +113,7 @@ void ChangeStorageClass::updateStorageClassInEosNamespace() { m_archiveIdsNotUpdatedInEos.push_back(archiveFileId); std::list<cta::log::Param> params; params.push_back(cta::log::Param("archiveFileId", archiveFileId)); - m_log(cta::log::INFO, "File did not change storage class because the file was in flight", params); + m_log(cta::log::WARNING, "File did not change storage class because the file was in flight", params); continue; } @@ -121,7 +123,7 @@ void ChangeStorageClass::updateStorageClassInEosNamespace() { m_archiveIdsNotUpdatedInEos.push_back(archiveFileId); std::list<cta::log::Param> params; params.push_back(cta::log::Param("archiveFileId", archiveFileId)); - m_log(cta::log::INFO, "File did not change storage class because query to EOS failed", params); + m_log(cta::log::WARNING, "File did not change storage class because query to EOS failed", params); } else { m_archiveIdsUpdatedInEos.push_back(archiveFileId); @@ -149,7 +151,7 @@ void ChangeStorageClass::updateStorageClassInCatalogue() { new_opt->set_value(m_storageClassName); } - { + { auto key = cta::admin::OptionStrList::FILE_ID; auto new_opt = admincmd.add_option_str_list(); new_opt->set_key(key); @@ -159,19 +161,18 @@ void ChangeStorageClass::updateStorageClassInCatalogue() { } // Validate the Protocol Buffer - try { - cta::admin::validateCmd(admincmd); - } catch(std::runtime_error &ex) { - throw std::runtime_error("Error: Protocol Buffer validation"); - } + try { + cta::admin::validateCmd(admincmd); + } catch(std::runtime_error &ex) { + throw std::runtime_error("Error: Protocol Buffer validation"); + } // Send the Request to the Service and get a Response cta::xrd::Response response; m_serviceProviderPtr->Send(request, response); // Handle responses - switch(response.type()) - { + switch(response.type()) { using namespace cta::xrd; using namespace cta::admin; case Response::RSP_SUCCESS: @@ -189,7 +190,7 @@ void ChangeStorageClass::updateStorageClassInCatalogue() { // writeSkippedArchiveIdsToFile //------------------------------------------------------------------------------ void ChangeStorageClass::writeSkippedArchiveIdsToFile() { - std::string filePath = "/var/log/skippedArchiveIds.txt"; + const std::string filePath = "/var/log/skippedArchiveIds.txt"; std::ofstream archiveIdFile(filePath); if (archiveIdFile.fail()) { diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp index 6ef82e9d60..72d997617f 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp @@ -17,8 +17,6 @@ #pragma once -#include "catalogue/Catalogue.hpp" -#include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" #include "cmdline/standalone_cli_tools/common/CmdLineTool.hpp" #include "common/log/StdoutLogger.hpp" #include "eos_grpc_client/GrpcEndpoint.hpp" @@ -29,6 +27,7 @@ #include "CtaFrontendApi.hpp" namespace cta { + namespace cliTool { class ChangeStorageClass: public CmdLineTool { @@ -41,25 +40,16 @@ public: * @param errStream Standard error stream. * @param log The object representing the API of the CTA logging system. * @param argc - * @param argv + * @param argv */ ChangeStorageClass( - std::istream &inStream, + std::istream &inStream, std::ostream &outStream, - std::ostream &errStream, + std::ostream &errStream, cta::log::StdoutLogger &log, - const int& argc, + const int& argc, char *const *const argv); - /** - * An exception throwing version of main(). - * - * @param argc The number of command-line arguments including the program name. - * @param argv The command-line arguments. - * @return The exit value of the program. - */ - int exceptionThrowingMain(const int argc, char *const *const argv) override; - private: /** @@ -71,7 +61,7 @@ private: * Archive file ids of the files to change */ std::vector<std::string> m_archiveFileIds; - + /** * Archive file id of the files to change */ @@ -83,17 +73,17 @@ private: std::unique_ptr<XrdSsiPbServiceType> m_serviceProviderPtr; /** - * Endpoint map for interaction with EOS + * Endpoint map for interaction with EOS */ std::unique_ptr<::eos::client::EndpointMap> m_endpointMapPtr; /** - * Endpoint map for interaction with EOS + * Endpoint map for interaction with EOS */ std::vector<std::string> m_archiveIdsNotUpdatedInEos; /** - * Endpoint map for interaction with EOS + * Endpoint map for interaction with EOS */ std::vector<std::string> m_archiveIdsUpdatedInEos; @@ -117,6 +107,15 @@ private: */ bool fileInFlight(const google::protobuf::RepeatedField<unsigned int> &locations); + /** + * An exception throwing version of main(). + * + * @param argc The number of command-line arguments including the program name. + * @param argv The command-line arguments. + * @return The exit value of the program. + */ + int exceptionThrowingMain(const int argc, char *const *const argv) override; + } ; // class CtaChangeStorageClass diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClassMain.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClassMain.cpp index cdaac9712e..d6c91bf74b 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClassMain.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClassMain.cpp @@ -19,7 +19,6 @@ #include <sstream> #include <iostream> -#include <XrdSsiPbLog.hpp> #include "cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp" #include "common/utils/utils.hpp" diff --git a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp index 5919286bcb..961e90936f 100644 --- a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp +++ b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp @@ -20,6 +20,8 @@ #include <XrdSsiPbLog.hpp> #include <XrdSsiPbIStreamBuffer.hpp> +#include "version.h" + #include <string> // GLOBAL VARIABLES : used to pass information between main thread and stream handler thread -- GitLab From 96a4cef779ca56645ccbd9abe5fded25fc49ab11 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 25 Oct 2022 15:16:52 +0200 Subject: [PATCH 098/126] use getenv(""HOSTNAME") --- .../change_storage_class/ChangeStorageClass.cpp | 16 ++++++++-------- .../change_storage_class/ChangeStorageClass.hpp | 1 + .../ChangeStorageClassMain.cpp | 9 +++------ .../common/ConnectionConfiguration.cpp | 2 +- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 828ea9e8be..96cbd1f78b 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -117,8 +117,8 @@ void ChangeStorageClass::updateStorageClassInEosNamespace() { continue; } - auto path = m_endpointMapPtr->getPath(diskInstance, diskFileId); - int status = m_endpointMapPtr->setXAttr(diskInstance, path, "sys.archive.storage_class", m_storageClassName); + const auto path = m_endpointMapPtr->getPath(diskInstance, diskFileId); + const int status = m_endpointMapPtr->setXAttr(diskInstance, path, "sys.archive.storage_class", m_storageClassName); if (status != 0) { m_archiveIdsNotUpdatedInEos.push_back(archiveFileId); std::list<cta::log::Param> params; @@ -137,23 +137,23 @@ void ChangeStorageClass::updateStorageClassInEosNamespace() { void ChangeStorageClass::updateStorageClassInCatalogue() { cta::xrd::Request request; - auto &admincmd = *(request.mutable_admincmd()); + const auto admincmd = request.mutable_admincmd(); request.set_client_cta_version(CTA_VERSION); request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); - admincmd.set_cmd(cta::admin::AdminCmd::CMD_ARCHIVEFILE); - admincmd.set_subcmd(cta::admin::AdminCmd::SUBCMD_CH); + admincmd->set_cmd(cta::admin::AdminCmd::CMD_ARCHIVEFILE); + admincmd->set_subcmd(cta::admin::AdminCmd::SUBCMD_CH); { auto key = cta::admin::OptionString::STORAGE_CLASS_NAME; - auto new_opt = admincmd.add_option_str(); + auto new_opt = admincmd->add_option_str(); new_opt->set_key(key); new_opt->set_value(m_storageClassName); } { auto key = cta::admin::OptionStrList::FILE_ID; - auto new_opt = admincmd.add_option_str_list(); + auto new_opt = admincmd->add_option_str_list(); new_opt->set_key(key); for (auto &archiveFileId : m_archiveIdsUpdatedInEos) { new_opt->add_item(archiveFileId); @@ -162,7 +162,7 @@ void ChangeStorageClass::updateStorageClassInCatalogue() { // Validate the Protocol Buffer try { - cta::admin::validateCmd(admincmd); + cta::admin::validateCmd(*admincmd); } catch(std::runtime_error &ex) { throw std::runtime_error("Error: Protocol Buffer validation"); } diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp index 72d997617f..5da9c64549 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp @@ -23,6 +23,7 @@ #include <optional> #include <memory> +#include <vector> #include "CtaFrontendApi.hpp" diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClassMain.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClassMain.cpp index d6c91bf74b..5175ab2e58 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClassMain.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClassMain.cpp @@ -28,14 +28,11 @@ // main //------------------------------------------------------------------------------ int main(const int argc, char *const *const argv) { - char buf[256]; - std::string hostName; - if(gethostname(buf, sizeof(buf))) { + std::string hostName = std::getenv("HOSTNAME"); + if(hostName.empty()) { hostName = "UNKNOWN"; - } else { - buf[sizeof(buf) - 1] = '\0'; - hostName = buf; } + cta::log::StdoutLogger log(hostName, "cta-change-storage-class"); cta::cliTool::ChangeStorageClass cmd(std::cin, std::cout, std::cerr, log, argc, argv); diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp index 18daac0808..19c1eb385a 100644 --- a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp @@ -31,7 +31,7 @@ void ConnConfiguration::setNamespaceMap(const std::string &keytab_file, std::uni ::eos::client::NamespaceMap_t namespaceMap; // Parse the keytab line by line std::string line; - for(int lineno = 0; std::getline(file, line); ++lineno) { + for(int lineno = 1; std::getline(file, line); ++lineno) { // Strip out comments auto pos = line.find('#'); if(pos != std::string::npos) { -- GitLab From a82e1492c931107d635367bbf929dba034a519eb Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 25 Oct 2022 15:21:50 +0200 Subject: [PATCH 099/126] add includes to CatalogueFetch.hpp --- .../common/CatalogueFetch.hpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp index 961e90936f..9a2ed27149 100644 --- a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp +++ b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp @@ -17,12 +17,16 @@ #pragma once +#include <list> +#include <string> +#include <utility> + #include <XrdSsiPbLog.hpp> #include <XrdSsiPbIStreamBuffer.hpp> #include "version.h" -#include <string> +#include "atomic" // GLOBAL VARIABLES : used to pass information between main thread and stream handler thread @@ -51,7 +55,6 @@ public: template<> void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) { - std::cout << "AlertCallback():" << std::endl; Log::DumpProtobuf(Log::PROTOBUF, &alert); } @@ -109,13 +112,13 @@ std::pair<std::string,std::string> getInstanceAndFidFromCTA(const std::string& a } cta::xrd::Request request; - auto &admincmd = *(request.mutable_admincmd()); + auto admincmd = request.mutable_admincmd(); request.set_client_cta_version(CTA_VERSION); request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); - admincmd.set_cmd(cta::admin::AdminCmd::CMD_TAPEFILE); - admincmd.set_subcmd(cta::admin::AdminCmd::SUBCMD_LS); - auto new_opt = admincmd.add_option_uint64(); + admincmd->set_cmd(cta::admin::AdminCmd::CMD_TAPEFILE); + admincmd->set_subcmd(cta::admin::AdminCmd::SUBCMD_LS); + auto new_opt = admincmd->add_option_uint64(); new_opt->set_key(cta::admin::OptionUInt64::ARCHIVE_FILE_ID); new_opt->set_value(std::stoi(archiveFileId)); @@ -124,8 +127,7 @@ std::pair<std::string,std::string> getInstanceAndFidFromCTA(const std::string& a auto stream_future = serviceProviderPtr->SendAsync(request, response); // Handle responses - switch(response.type()) - { + switch(response.type()) { using namespace cta::xrd; using namespace cta::admin; case Response::RSP_SUCCESS: -- GitLab From 87fcc85bf41b4bbd1fd7aa22b4bc5ec6f5048aee Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 25 Oct 2022 15:24:40 +0200 Subject: [PATCH 100/126] minor code quality changes --- .../common/ConnectionConfiguration.cpp | 5 +++-- .../common/ConnectionConfiguration.hpp | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp index 19c1eb385a..0fac4ccbe4 100644 --- a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp @@ -18,6 +18,7 @@ #include "ConnectionConfiguration.hpp" #include <string> +#include <utility> namespace cta { namespace cliTool { @@ -115,5 +116,5 @@ void ConnConfiguration::readAndSetConfiguration( } } -} // namespace cliTool -} // namespace cta \ No newline at end of file +} // namespace cliTool +} // namespace cta \ No newline at end of file diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp index 6e0e2634d5..fa624f0dc6 100644 --- a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp @@ -29,8 +29,6 @@ namespace cta { namespace cliTool { class ConnConfiguration { -private: - static void setNamespaceMap(const std::string &keytab_file, std::unique_ptr<::eos::client::EndpointMap> &endpointMapPtr); public: /** @@ -48,7 +46,10 @@ public: const CmdLineArgs &cmdLineArgs, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, std::unique_ptr<::eos::client::EndpointMap> &endpointMapPtr); + +private: + static void setNamespaceMap(const std::string &keytab_file, std::unique_ptr<::eos::client::EndpointMap> &endpointMapPtr); }; -} // namespace cliTool -} // namespace cta \ No newline at end of file +} // namespace cliTool +} // namespace cta \ No newline at end of file -- GitLab From 936c878ffae86e261f6a7e947f9a5aa675a51c57 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 25 Oct 2022 15:39:45 +0200 Subject: [PATCH 101/126] remove reference to basic type --- catalogue/Catalogue.hpp | 2 +- catalogue/CatalogueRetryWrapper.cpp | 2 +- catalogue/CatalogueRetryWrapper.hpp | 2 +- catalogue/DummyCatalogue.cpp | 2 +- catalogue/DummyCatalogue.hpp | 2 +- catalogue/RdbmsCatalogue.cpp | 2 +- catalogue/RdbmsCatalogue.hpp | 2 +- cmdline/standalone_cli_tools/common/CatalogueFetch.hpp | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/catalogue/Catalogue.hpp b/catalogue/Catalogue.hpp index 27e98a5baa..805e1cbf73 100644 --- a/catalogue/Catalogue.hpp +++ b/catalogue/Catalogue.hpp @@ -1412,7 +1412,7 @@ public: * @param archiveFileId Id for file found in ARCHIVE_FILE * @param newStorageClassName The name of the storage class */ - virtual void modifyArchiveFileStorageClassId(const uint64_t& archiveFileId, const std::string& newStorageClassName) const = 0; + virtual void modifyArchiveFileStorageClassId(const uint64_t archiveFileId, const std::string& newStorageClassName) const = 0; }; // class Catalogue diff --git a/catalogue/CatalogueRetryWrapper.cpp b/catalogue/CatalogueRetryWrapper.cpp index ff69ecb1ed..0e1010ce86 100644 --- a/catalogue/CatalogueRetryWrapper.cpp +++ b/catalogue/CatalogueRetryWrapper.cpp @@ -763,7 +763,7 @@ void CatalogueRetryWrapper::releaseDiskSpace(const std::string& driveName, const return retryOnLostConnection(m_log, [&]{return m_catalogue->releaseDiskSpace(driveName, mountId, diskSpaceReservation, lc);}, m_maxTriesToConnect); } -void CatalogueRetryWrapper::modifyArchiveFileStorageClassId(const uint64_t& archiveFileId, const std::string& newStorageClassName) const { +void CatalogueRetryWrapper::modifyArchiveFileStorageClassId(const uint64_t archiveFileId, const std::string& newStorageClassName) const { return retryOnLostConnection(m_log, [&]{return m_catalogue->modifyArchiveFileStorageClassId(archiveFileId, newStorageClassName);}, m_maxTriesToConnect); } diff --git a/catalogue/CatalogueRetryWrapper.hpp b/catalogue/CatalogueRetryWrapper.hpp index f83093ecc4..4e1ae122fc 100644 --- a/catalogue/CatalogueRetryWrapper.hpp +++ b/catalogue/CatalogueRetryWrapper.hpp @@ -407,7 +407,7 @@ public: void releaseDiskSpace(const std::string& driveName, const uint64_t mountId, const DiskSpaceReservationRequest& diskSpaceReservation, log::LogContext & lc) override; - void modifyArchiveFileStorageClassId(const uint64_t& archiveFileId, const std::string& newStorageClassName) const override; + void modifyArchiveFileStorageClassId(const uint64_t archiveFileId, const std::string& newStorageClassName) const override; protected: /** * Object representing the API to the CTA logging system. diff --git a/catalogue/DummyCatalogue.cpp b/catalogue/DummyCatalogue.cpp index da428e37c2..7f07160162 100644 --- a/catalogue/DummyCatalogue.cpp +++ b/catalogue/DummyCatalogue.cpp @@ -197,7 +197,7 @@ bool DummyCatalogue::tapePoolExists(const std::string& tapePoolName) const { thr void DummyCatalogue::updateDiskFileId(uint64_t archiveFileId, const std::string &diskInstance, const std::string &diskFileId) { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } void DummyCatalogue::moveArchiveFileToRecycleLog(const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");} -void DummyCatalogue::modifyArchiveFileStorageClassId(const uint64_t &archiveFileId, const std::string &newStorageClassName) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } +void DummyCatalogue::modifyArchiveFileStorageClassId(const uint64_t archiveFileId, const std::string &newStorageClassName) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } // Special functions for unit tests. void DummyCatalogue::addEnabledTape(const std::string & vid) { diff --git a/catalogue/DummyCatalogue.hpp b/catalogue/DummyCatalogue.hpp index 0946f0ab2d..5598d78ed9 100644 --- a/catalogue/DummyCatalogue.hpp +++ b/catalogue/DummyCatalogue.hpp @@ -200,7 +200,7 @@ public: void updateDiskFileId(uint64_t archiveFileId, const std::string &diskInstance, const std::string &diskFileId) override; void moveArchiveFileToRecycleLog(const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override; - void modifyArchiveFileStorageClassId(const uint64_t &archiveFileId, const std::string &newStorageClassName) const override; + void modifyArchiveFileStorageClassId(const uint64_t archiveFileId, const std::string &newStorageClassName) const override; common::dataStructures::VidToTapeMap getTapesByVid(const std::string& vid) const override; diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index 5bdf900e34..842bc82931 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -1450,7 +1450,7 @@ void RdbmsCatalogue::modifyStorageClassName(const common::dataStructures::Securi // ----------------------------------------------------------------------------- // modifyArchiveFileStorageClassId // ----------------------------------------------------------------------------- -void RdbmsCatalogue::modifyArchiveFileStorageClassId(const uint64_t& archiveFileId, const std::string& newStorageClassName) const { +void RdbmsCatalogue::modifyArchiveFileStorageClassId(const uint64_t archiveFileId, const std::string& newStorageClassName) const { try { const char *const sql = "UPDATE ARCHIVE_FILE " diff --git a/catalogue/RdbmsCatalogue.hpp b/catalogue/RdbmsCatalogue.hpp index 63773498de..8cdddbf55d 100644 --- a/catalogue/RdbmsCatalogue.hpp +++ b/catalogue/RdbmsCatalogue.hpp @@ -1270,7 +1270,7 @@ protected: * @param newStorageClassName The name of the storage class * @param archiveFileId Id for file found in ARCHIVE_FILE */ - void modifyArchiveFileStorageClassId(const uint64_t& archiveFileId, const std::string& newStorageClassName) const override; + void modifyArchiveFileStorageClassId(const uint64_t archiveFileId, const std::string& newStorageClassName) const override; /** * Returns true if the specified disk file identifier exists. diff --git a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp index 9a2ed27149..4ffe6b4c63 100644 --- a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp +++ b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp @@ -17,6 +17,7 @@ #pragma once +#include <atomic> #include <list> #include <string> #include <utility> @@ -26,7 +27,6 @@ #include "version.h" -#include "atomic" // GLOBAL VARIABLES : used to pass information between main thread and stream handler thread -- GitLab From bd41d0e68de1e5c74ccde90adae08f0ac5a40720 Mon Sep 17 00:00:00 2001 From: Lasse Wardenaer <lasse.tjernaes.wardenaer@cern.ch> Date: Tue, 25 Oct 2022 17:25:36 +0200 Subject: [PATCH 102/126] add man page to cta.spec.in --- cta.spec.in | 1 + 1 file changed, 1 insertion(+) diff --git a/cta.spec.in b/cta.spec.in index 19c00d134a..5955faf4dd 100644 --- a/cta.spec.in +++ b/cta.spec.in @@ -234,6 +234,7 @@ The command line utilities %attr(0755,root,root) %{_bindir}/cta-send-closew.sh %attr(0755,root,root) %{_bindir}/cta-verify-file %attr(0755,root,root) %{_bindir}/cta-change-storage-class +%attr(0644,root,root) %doc /usr/share/man/man1/cta-change-storage-class.1cta.gz %attr(0644,root,root) %config(noreplace) %{_sysconfdir}/cta/cta-cli.conf.example %posttrans -n cta-cli -- GitLab From c189b9198b4dc209d0a354e750e5853630d7a439 Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 26 Oct 2022 13:47:57 +0200 Subject: [PATCH 103/126] remove trailing spaces --- .../cta-change-storage-class.1cta | 8 ++-- .../common/CmdLineArgs.cpp | 8 ++-- .../common/CmdLineArgs.hpp | 10 ++-- .../common/CmdLineTool.cpp | 2 +- .../restore_files/RestoreFilesCmd.cpp | 46 +++++++++---------- 5 files changed, 37 insertions(+), 37 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class.1cta b/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class.1cta index 4cf0ab5b8d..d7d55b11e5 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class.1cta +++ b/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class.1cta @@ -22,8 +22,8 @@ cta-change-storage-class \- Change which storageclass files with a given archive \fBcta-change-storage-class\fP --id/-I <archiveFileID> | --filename/-F <filename> --storage.class.name/-n <storageClassName> .SH DESCRIPTION -\fBcta-change-storage-class\fP changes the storage class for the files specified by the user. The tool will first change -the storage class on the EOS side, and then change the storage class for the files that were successfully changed in EOS +\fBcta-change-storage-class\fP changes the storage class for the files specified by the user. The tool will first change +the storage class on the EOS side, and then change the storage class for the files that were successfully changed in EOS in the catalogue .SH OPTIONS @@ -32,10 +32,10 @@ in the catalogue Archive file id of the files to change. .TP \-F, \-\-filename \fIfile_name\fR -Path to a file with a archive file ids. The text file should have one id on each line. +Path to a file with a archive file ids. The text file should have one id on each line. .TP \-n, \-\-storage.class.name \fIstorageclassname\fR -Path to a file with a archive file ids. The text file should have one id on each line. +Path to a file with a archive file ids. The text file should have one id on each line. .SH EXIT STATUS .P diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp index 0ad9118b2a..39a3245899 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp @@ -217,7 +217,7 @@ void CmdLineArgs::readIdListFromFile(const std::string &filename) { default: break; } - } + } } //------------------------------------------------------------------------------ @@ -225,16 +225,16 @@ void CmdLineArgs::readIdListFromFile(const std::string &filename) { //------------------------------------------------------------------------------ void CmdLineArgs::printUsage(std::ostream &os) const { switch (m_standaloneCliTool) { - case StandaloneCliTool::RESTORE_FILES: + case StandaloneCliTool::RESTORE_FILES: os << " Usage:" << std::endl << " cta-restore-deleted-files [--id/-I <archive_file_id>] [--instance/-i <disk_instance>]" << std::endl << " [--fxid/-f <eos_fxid>] [--fxidfile/-F <filename>]" << std::endl << - " [--vid/-v <vid>] [--copynb/-c <copy_number>] [--debug/-d]" << std::endl; + " [--vid/-v <vid>] [--copynb/-c <copy_number>] [--debug/-d]" << std::endl; break; case StandaloneCliTool::CTA_SEND_EVENT: os << " Usage:" << std::endl << " eos --json fileinfo /eos/path | cta-send-event CLOSEW|PREPARE " << std::endl << - " -i/--eos.instance <instance> [-e/--eos.endpoint <url>]" << std::endl << + " -i/--eos.instance <instance> [-e/--eos.endpoint <url>]" << std::endl << " -u/--request.user <user> -g/--request.group <group>" << std::endl; break; case StandaloneCliTool::CTA_VERIFY_FILE : diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp index 474a8eaa65..94db0ff922 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp @@ -82,28 +82,28 @@ struct CmdLineArgs { std::optional<uint64_t> m_copyNumber; /** - * Eos endpoint + * Eos endpoint */ std::optional<std::string> m_eosEndpoint; /** * Request user - */ + */ std::optional<std::string> m_requestUser; /** * Request user - */ + */ std::optional<std::string> m_requestGroup; /** * The tool parsing the arguments - */ + */ StandaloneCliTool m_standaloneCliTool; /** * The tool parsing the arguments - */ + */ std::optional<std::string> m_storageClassName; /** diff --git a/cmdline/standalone_cli_tools/common/CmdLineTool.cpp b/cmdline/standalone_cli_tools/common/CmdLineTool.cpp index 5ae584bf85..226d42c958 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineTool.cpp +++ b/cmdline/standalone_cli_tools/common/CmdLineTool.cpp @@ -87,7 +87,7 @@ int CmdLineTool::main(const int argc, char *const *const argv) { } catch(...) { errorMessage = "An unknown exception was thrown"; } - + std::cout << errorMessage << std::endl; return 1; } diff --git a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp index e182a7efd9..29cc6acce1 100644 --- a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp +++ b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp @@ -89,7 +89,7 @@ int RestoreFilesCmd::exceptionThrowingMain(const int argc, char *const *const ar try { if (!fileExistsEos(file.disk_instance(), file.disk_file_id())) { uint64_t new_fid = restoreDeletedFileEos(file); - file.set_disk_file_id(std::to_string(new_fid)); + file.set_disk_file_id(std::to_string(new_fid)); } // archive file exists in CTA, so only need to restore the file copy restoreDeletedFileCopyCta(file); @@ -123,9 +123,9 @@ int RestoreFilesCmd::exceptionThrowingMain(const int argc, char *const *const ar // readAndSetConfiguration //------------------------------------------------------------------------------ void RestoreFilesCmd::readAndSetConfiguration( - const std::string &userName, + const std::string &userName, const CmdLineArgs &cmdLineArgs) { - + m_vid = cmdLineArgs.m_vid; m_diskInstance = cmdLineArgs.m_diskInstance; m_archiveFileIds = cmdLineArgs.m_archiveFileIds; @@ -218,11 +218,11 @@ void RestoreFilesCmd::setNamespaceMap(const std::string &keytab_file) { void RestoreFilesCmd::listDeletedFilesCta() const { std::list<cta::log::Param> params; params.push_back(cta::log::Param("userName", getUsername())); - + cta::xrd::Request request; auto &admincmd = *(request.mutable_admincmd()); - + request.set_client_cta_version(CTA_VERSION); request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); admincmd.set_cmd(cta::admin::AdminCmd::CMD_RECYCLETAPEFILE); @@ -270,7 +270,7 @@ void RestoreFilesCmd::listDeletedFilesCta() const { params.push_back(cta::log::Param("diskFileId", fids)); } - m_log(cta::log::INFO, "Listing deleted file in CTA catalogue", params); + m_log(cta::log::INFO, "Listing deleted file in CTA catalogue", params); // Send the Request to the Service and get a Response cta::xrd::Response response; @@ -297,7 +297,7 @@ void RestoreFilesCmd::listDeletedFilesCta() const { stream_future.wait(); params.push_back(cta::log::Param("nbFiles", deletedTapeFiles.size())); - m_log(cta::log::INFO, "Listed deleted file in CTA catalogue", params); + m_log(cta::log::INFO, "Listed deleted file in CTA catalogue", params); } //------------------------------------------------------------------------------ @@ -312,11 +312,11 @@ void RestoreFilesCmd::restoreDeletedFileCopyCta(const cta::admin::RecycleTapeFil params.push_back(cta::log::Param("archiveFileId", file.archive_file_id())); params.push_back(cta::log::Param("copyNb", file.copy_nb())); params.push_back(cta::log::Param("diskFileId", file.disk_file_id())); - + cta::xrd::Request request; auto &admincmd = *(request.mutable_admincmd()); - + request.set_client_cta_version(CTA_VERSION); request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); admincmd.set_cmd(cta::admin::AdminCmd::CMD_RECYCLETAPEFILE); @@ -362,7 +362,7 @@ void RestoreFilesCmd::restoreDeletedFileCopyCta(const cta::admin::RecycleTapeFil new_opt->set_key(key); new_opt->set_value(ss.str()); } - m_log(cta::log::DEBUG, "Restoring file copy in CTA catalogue", params); + m_log(cta::log::DEBUG, "Restoring file copy in CTA catalogue", params); // This validation will also be done at the server side cta::admin::validateCmd(admincmd); @@ -379,7 +379,7 @@ void RestoreFilesCmd::restoreDeletedFileCopyCta(const cta::admin::RecycleTapeFil case Response::RSP_SUCCESS: // Print message text std::cout << response.message_txt(); - m_log(cta::log::INFO, "Restored file copy in CTA catalogue", params); + m_log(cta::log::INFO, "Restored file copy in CTA catalogue", params); break; case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); case Response::RSP_ERR_USER: throw XrdSsiPb::UserException(response.message_txt()); @@ -413,7 +413,7 @@ uint64_t RestoreFilesCmd::addContainerEos(const std::string &diskInstance, const dir.set_path(path); dir.set_name(cta::utils::getEnclosedName(path)); - // Filemode: filter out S_ISUID, S_ISGID and S_ISVTX because EOS does not follow POSIX semantics for these bits + // Filemode: filter out S_ISUID, S_ISGID and S_ISVTX because EOS does not follow POSIX semantics for these bits uint64_t filemode = (S_IRWXU | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); // 0755 permissions by default filemode &= ~(S_ISUID | S_ISGID | S_ISVTX); dir.set_mode(filemode); @@ -430,7 +430,7 @@ uint64_t RestoreFilesCmd::addContainerEos(const std::string &diskInstance, const auto reply = m_endpointMapPtr->containerInsert(diskInstance, dir); m_log(cta::log::DEBUG, "Inserted container in EOS namespace successfully, querying again for its id", params); - + auto cont_id = containerExistsEos(diskInstance, path); if (!cont_id) { throw RestoreFilesCmdException(std::string("Container ") + path + " does not exist after being inserted in EOS."); @@ -448,7 +448,7 @@ uint64_t RestoreFilesCmd::containerExistsEos(const std::string &diskInstance, co params.push_back(cta::log::Param("path", path)); m_log(cta::log::DEBUG, "Verifying if the container exists in the EOS namespace", params); - + auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::CONTAINER, 0, path, false); auto cid = md_response.cmd().id(); params.push_back(cta::log::Param("containerId", cid)); @@ -472,18 +472,18 @@ bool RestoreFilesCmd::archiveFileExistsCTA(const uint64_t &archiveFileId) const std::list<cta::log::Param> params; params.push_back(cta::log::Param("userName", getUsername())); params.push_back(cta::log::Param("archiveFileId", archiveFileId)); - - m_log(cta::log::DEBUG, "Looking for archive file in the CTA catalogue", params); + + m_log(cta::log::DEBUG, "Looking for archive file in the CTA catalogue", params); cta::xrd::Request request; auto &admincmd = *(request.mutable_admincmd()); - + request.set_client_cta_version(CTA_VERSION); request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); admincmd.set_cmd(cta::admin::AdminCmd::CMD_TAPEFILE); admincmd.set_subcmd(cta::admin::AdminCmd::SUBCMD_LS); - + auto key = cta::admin::OptionUInt64::ARCHIVE_FILE_ID; auto new_opt = admincmd.add_option_uint64(); new_opt->set_key(key); @@ -587,8 +587,8 @@ uint64_t RestoreFilesCmd::restoreDeletedFileEos(const cta::admin::RecycleTapeFil params.push_back(cta::log::Param("archiveFileId", rtfls_item.archive_file_id())); params.push_back(cta::log::Param("diskFileId", rtfls_item.disk_file_id())); params.push_back(cta::log::Param("diskFilePath", rtfls_item.disk_file_path())); - - m_log(cta::log::INFO, "Restoring file in the EOS namespace", params); + + m_log(cta::log::INFO, "Restoring file in the EOS namespace", params); getCurrentEosIds(rtfls_item.disk_instance()); uint64_t file_id = getFileIdEos(rtfls_item.disk_instance(), rtfls_item.disk_file_path()); @@ -598,9 +598,9 @@ uint64_t RestoreFilesCmd::restoreDeletedFileEos(const cta::admin::RecycleTapeFil ::eos::rpc::FileMdProto file; - auto fullPath = rtfls_item.disk_file_path(); + auto fullPath = rtfls_item.disk_file_path(); auto cont_id = addContainerEos(rtfls_item.disk_instance(), cta::utils::getEnclosingPath(fullPath), rtfls_item.storage_class()); - + // We do not set the file id. Since the file was deleted the fid cannot be reused, so EOS will generate a new file id file.set_cont_id(cont_id); file.set_uid(rtfls_item.disk_file_uid()); @@ -664,7 +664,7 @@ uint64_t RestoreFilesCmd::restoreDeletedFileEos(const cta::admin::RecycleTapeFil m_log(cta::log::INFO, "File successfully restored in the EOS namespace", params); m_log(cta::log::DEBUG, "Querying EOS for the new EOS file id", params); - + auto new_fid = getFileIdEos(rtfls_item.disk_instance(), rtfls_item.disk_file_path()); return new_fid; -- GitLab From cd7123cd7ca2c85974a5c045160bacc17ea5ed93 Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 26 Oct 2022 15:10:07 +0200 Subject: [PATCH 104/126] update changeStorageClass test --- .../orchestration/tests/changeStorageClass.sh | 91 +++++++++++++++---- 1 file changed, 74 insertions(+), 17 deletions(-) diff --git a/continuousintegration/orchestration/tests/changeStorageClass.sh b/continuousintegration/orchestration/tests/changeStorageClass.sh index 23f63fda16..49e33db30c 100755 --- a/continuousintegration/orchestration/tests/changeStorageClass.sh +++ b/continuousintegration/orchestration/tests/changeStorageClass.sh @@ -1,4 +1,4 @@ -#bin/bash +#bin/bash usage() { cat <<EOF 1>&2 Usage: $0 -n <namespace> @@ -32,10 +32,10 @@ EOSINSTANCE=ctaeos ORIGINAL_STORAGE_CLASS_NAME=originalStorageClassName NEW_STORAGE_CLASS_NAME=newStorageClassName -CHANGE_SC_FILE=`uuidgen` -STATIC_SC_FILE=`uuidgen` +FILE_1=`uuidgen` +FILE_2=`uuidgen` echo -echo "Creating files: ${CHANGE_SC_FILE} ${STATIC_SC_FILE}" +echo "Creating files: ${FILE_1} ${FILE_2}" FRONTEND_IP=$(kubectl -n ${NAMESPACE} get pods ctafrontend -o json | jq .status.podIP | tr -d '"') @@ -49,17 +49,17 @@ echo "ADD FRONTEND GATEWAY TO EOS" echo "kubectl -n ${NAMESPACE} exec ctaeos -- bash eos root://${EOSINSTANCE} -r 0 0 vid add gateway ${FRONTEND_IP} grpc" kubectl -n ${NAMESPACE} exec ctaeos -- eos -r 0 0 vid add gateway ${FRONTEND_IP} grpc -echo +echo echo "ADD STORAGE CLASS WITH TWO COPIES ${ORIGINAL_STORAGE_CLASS_NAME}" kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc add --name ${ORIGINAL_STORAGE_CLASS_NAME} --numberofcopies 2 --virtualorganisation vo --comment "comment" kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc ls -echo +echo echo "ADD STORAGE CLASS WITH ONE COPIES ${NEW_STORAGE_CLASS_NAME}" kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc add --name ${NEW_STORAGE_CLASS_NAME} --numberofcopies 2 --virtualorganisation vo --comment "comment" kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc ls -echo +echo echo "COPY REQUIRED FILES TO FRONTEND POD" echo "sudo kubectl cp ${NAMESPACE}/ctacli:/etc/cta/cta-cli.conf /etc/cta/cta-cli.conf" echo "sudo kubectl cp /etc/cta/cta-cli.conf ${NAMESPACE}/ctafrontend:/etc/cta/cta-cli.conf" @@ -68,29 +68,86 @@ sudo kubectl cp /etc/cta/cta-cli.conf ${NAMESPACE}/ctafrontend:/etc/cta/cta-cli. kubectl -n ${NAMESPACE} cp common/archive_file.sh client:/usr/bin/ kubectl -n ${NAMESPACE} cp client_helper.sh client:/root/ -kubectl -n ${NAMESPACE} exec client -- bash /usr/bin/archive_file.sh -f ${CHANGE_SC_FILE} || exit 1 -kubectl -n ${NAMESPACE} exec client -- bash /usr/bin/archive_file.sh -f ${STATIC_SC_FILE} || exit 1 +kubectl -n ${NAMESPACE} exec client -- bash /usr/bin/archive_file.sh -f ${FILE_1} || exit 1 +kubectl -n ${NAMESPACE} exec client -- bash /usr/bin/archive_file.sh -f ${FILE_2} || exit 1 -EOS_METADATA_PATH=$(mktemp -d).json +EOS_METADATA_PATH_1=$(mktemp -d).json echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH}" -touch ${EOS_METADATA_PATH} -kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${CHANGE_SC_FILE} | jq . |& tee ${EOS_METADATA_PATH} -EOS_ARCHIVE_ID=$(jq -r '.xattr | .["sys.archive.file_id"]' ${EOS_METADATA_PATH}) +touch ${EOS_METADATA_PATH_1} +kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_1} | jq . |& tee ${EOS_METADATA_PATH_1} +EOS_ARCHIVE_ID_1=$(jq -r '.xattr | .["sys.archive.file_id"]' ${EOS_METADATA_PATH_1}) + +EOS_METADATA_PATH_2=$(mktemp -d).json +echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH_2}" +touch ${EOS_METADATA_PATH_2} +kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_2} | jq . |& tee ${EOS_METADATA_PATH_2} +EOS_ARCHIVE_ID_2=$(jq -r '.xattr | .["sys.archive.file_id"]' ${EOS_METADATA_PATH_2}) -echo -echo "CREATE FILE WITH IDS" +echo +echo "CHANGE FILES WITH IDS" cd ~ IDS_FILEPATH=~/archiveFileIds.txt rm ${IDS_FILEPATH} touch ${IDS_FILEPATH} -echo ${EOS_ARCHIVE_ID} | tee -a ${IDS_FILEPATH} +echo ${EOS_ARCHIVE_ID_1} | tee -a ${IDS_FILEPATH} +echo ${EOS_ARCHIVE_ID_2} | tee -a ${IDS_FILEPATH} -echo +echo kubectl cp ~/CTA-build/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class ${NAMESPACE}/ctafrontend:/usr/bin/ echo "kubectl cp ${IDS_FILEPATH} ${NAMESPACE}/ctafrontend:~/" kubectl cp ${IDS_FILEPATH} ${NAMESPACE}/ctafrontend:/root/ kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c "XrdSecPROTOCOL=sss XrdSecSSSKT=/etc/cta/eos.sss.keytab cta-change-storage-class --storage.class.name ${NEW_STORAGE_CLASS_NAME} --filename ${IDS_FILEPATH}" +EOS_METADATA_PATH_AFTER_CHANGE_1=$(mktemp -d).json +echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH_AFTER_CHANGE_1}" +touch ${EOS_METADATA_PATH_AFTER_CHANGE_1} +kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_1} | jq . |& tee ${EOS_METADATA_PATH_AFTER_CHANGE_1} +EOS_STORAGE_CLASS_1=$(jq -r '.xattr | .["sys.archive.storage_class"]' ${EOS_METADATA_PATH_AFTER_CHANGE_1}) +rm -r ${EOS_METADATA_PATH_AFTER_CHANGE_1} + +EOS_METADATA_PATH_AFTER_CHANGE_2=$(mktemp -d).json +echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH_AFTER_CHANGE_2}" +touch ${EOS_METADATA_PATH_AFTER_CHANGE_2} +kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_2} | jq . |& tee ${EOS_METADATA_PATH_AFTER_CHANGE_2} +EOS_STORAGE_CLASS_2=$(jq -r '.xattr | .["sys.archive.storage_class"]' ${EOS_METADATA_PATH_AFTER_CHANGE_2}) +rm -r ${EOS_METADATA_PATH_AFTER_CHANGE_2} + +CATALOGUE_METADATA_PATH_AFTER_CHANGE_1=$(mktemp -d).json +echo "SEND CATALOGUE METADATA TO JSON FILE: ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1}" +touch ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1} +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tf ls --id ${EOS_ARCHIVE_ID_1} | jq . |& tee ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1} +CATALOGUE_STORAGE_CLASS_1=$(jq . ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1} | jq '.[0]' | jq -r '.af | .["storageClass"]') +rm -r ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1} + +CATALOGUE_METADATA_PATH_AFTER_CHANGE_2=$(mktemp -d).json +echo "SEND CATALOGUE METADATA TO JSON FILE: ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2}" +touch ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2} +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tf ls --id ${EOS_ARCHIVE_ID_2} | jq . |& tee ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2} +CATALOGUE_STORAGE_CLASS_2=$(jq . ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2} | jq '.[0]' | jq -r '.af | .["storageClass"]') +rm -r ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2} + +if test ${EOS_STORAGE_CLASS_1} != ${NEW_STORAGE_CLASS_NAME}; then + echo "ERROR: File ${FILE_1} did not change the storage class in EOS" + exit 1 +fi + +if test ${CATALOGUE_STORAGE_CLASS_1} != ${NEW_STORAGE_CLASS_NAME}; then + echo "ERROR: File ${FILE_1} did not change the storage class in the Catalogue" + exit 1 +fi + +if test ${EOS_STORAGE_CLASS_2} != ${NEW_STORAGE_CLASS_NAME}; then + echo "ERROR: File ${FILE_2} did not change the storage class in EOS" + exit 1 +fi + +if test ${CATALOGUE_STORAGE_CLASS_2} != ${NEW_STORAGE_CLASS_NAME}; then + echo "ERROR: File ${FILE_2} did not change the storage class in the Catalogue" + exit 1 +fi + +echo "All tests passed" + # Remove authorization kubectl -n ${NAMESPACE} exec ctacli -- cta-admin admin rm --username ctafrontend kubectl -n ${NAMESPACE} exec ctacli -- cta-admin admin rm --username ctaeos -- GitLab From 9f6321a108d3791b2844e49655ad6cf45988a13b Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 26 Oct 2022 15:16:13 +0200 Subject: [PATCH 105/126] update changeStorageClass test --- .../orchestration/tests/changeStorageClass.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/continuousintegration/orchestration/tests/changeStorageClass.sh b/continuousintegration/orchestration/tests/changeStorageClass.sh index 49e33db30c..fc44ca2551 100755 --- a/continuousintegration/orchestration/tests/changeStorageClass.sh +++ b/continuousintegration/orchestration/tests/changeStorageClass.sh @@ -28,8 +28,6 @@ if [ ! -z "${error}" ]; then fi EOSINSTANCE=ctaeos - -ORIGINAL_STORAGE_CLASS_NAME=originalStorageClassName NEW_STORAGE_CLASS_NAME=newStorageClassName FILE_1=`uuidgen` @@ -49,11 +47,6 @@ echo "ADD FRONTEND GATEWAY TO EOS" echo "kubectl -n ${NAMESPACE} exec ctaeos -- bash eos root://${EOSINSTANCE} -r 0 0 vid add gateway ${FRONTEND_IP} grpc" kubectl -n ${NAMESPACE} exec ctaeos -- eos -r 0 0 vid add gateway ${FRONTEND_IP} grpc -echo -echo "ADD STORAGE CLASS WITH TWO COPIES ${ORIGINAL_STORAGE_CLASS_NAME}" -kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc add --name ${ORIGINAL_STORAGE_CLASS_NAME} --numberofcopies 2 --virtualorganisation vo --comment "comment" -kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc ls - echo echo "ADD STORAGE CLASS WITH ONE COPIES ${NEW_STORAGE_CLASS_NAME}" kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc add --name ${NEW_STORAGE_CLASS_NAME} --numberofcopies 2 --virtualorganisation vo --comment "comment" @@ -146,6 +139,7 @@ if test ${CATALOGUE_STORAGE_CLASS_2} != ${NEW_STORAGE_CLASS_NAME}; then exit 1 fi +echo echo "All tests passed" # Remove authorization -- GitLab From 56c245e3dc075cf57571bd5c5043454ac80ba277 Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 26 Oct 2022 16:09:13 +0200 Subject: [PATCH 106/126] return the endpoint pointer instead of changeing it by reference --- .../change_storage_class/ChangeStorageClass.cpp | 4 ++-- .../common/ConnectionConfiguration.cpp | 11 +++++------ .../common/ConnectionConfiguration.hpp | 7 +++---- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 96cbd1f78b..e911bd0cd5 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -82,7 +82,7 @@ ChangeStorageClass::ChangeStorageClass( } } - ConnConfiguration::readAndSetConfiguration(log, getUsername(), cmdLineArgs, m_serviceProviderPtr, m_endpointMapPtr); + m_endpointMapPtr = ConnConfiguration::readAndSetConfiguration(log, getUsername(), cmdLineArgs, m_serviceProviderPtr); } @@ -102,7 +102,7 @@ bool ChangeStorageClass::fileInFlight(const google::protobuf::RepeatedField<uint // updateStorageClassInEosNamespace //------------------------------------------------------------------------------ void ChangeStorageClass::updateStorageClassInEosNamespace() { - for(const auto archiveFileId : m_archiveFileIds) { + for(const auto &archiveFileId : m_archiveFileIds) { const auto instanceAndFid = getInstanceAndFidFromCTA(archiveFileId, m_serviceProviderPtr, m_log); const std::string diskInstance = instanceAndFid.first; const std::string diskFileId = instanceAndFid.second; diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp index 0fac4ccbe4..0b0bba1d53 100644 --- a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp @@ -23,7 +23,7 @@ namespace cta { namespace cliTool { -void ConnConfiguration::setNamespaceMap(const std::string &keytab_file, std::unique_ptr<::eos::client::EndpointMap> &endpointMapPtr) { +std::unique_ptr<::eos::client::EndpointMap> ConnConfiguration::setNamespaceMap(const std::string &keytab_file) { // Open the keytab file for reading std::ifstream file(keytab_file); if(!file) { @@ -54,18 +54,17 @@ void ConnConfiguration::setNamespaceMap(const std::string &keytab_file, std::uni } namespaceMap.insert(std::make_pair(diskInstance, ::eos::client::Namespace(endpoint, token))); } - endpointMapPtr = std::make_unique<::eos::client::EndpointMap>(namespaceMap); + return std::make_unique<::eos::client::EndpointMap>(namespaceMap); } //------------------------------------------------------------------------------ // readAndSetConfiguration //------------------------------------------------------------------------------ -void ConnConfiguration::readAndSetConfiguration( +std::unique_ptr<::eos::client::EndpointMap> ConnConfiguration::readAndSetConfiguration( cta::log::StdoutLogger &log, const std::string &userName, const CmdLineArgs &cmdLineArgs, - std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, - std::unique_ptr<::eos::client::EndpointMap> &endpointMapPtr) { + std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr) { const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by @@ -110,7 +109,7 @@ void ConnConfiguration::readAndSetConfiguration( // Get the endpoint for namespace queries auto nsConf = frontendXrootdConfig.getOptionValueStr("ns.config"); if(nsConf.first) { - setNamespaceMap(nsConf.second, endpointMapPtr); + return setNamespaceMap(nsConf.second); } else { throw std::runtime_error("Configuration error: cta.ns.config missing from " + frontend_xrootd_config_file); } diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp index fa624f0dc6..54ce27d723 100644 --- a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp @@ -40,15 +40,14 @@ public: * @param serviceProviderPtr Xroot service provider * @param endpointMapPtr Endpoint for communication with EOS */ - static void readAndSetConfiguration( + static std::unique_ptr<::eos::client::EndpointMap> readAndSetConfiguration( cta::log::StdoutLogger &log, const std::string &userName, const CmdLineArgs &cmdLineArgs, - std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, - std::unique_ptr<::eos::client::EndpointMap> &endpointMapPtr); + std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr); private: - static void setNamespaceMap(const std::string &keytab_file, std::unique_ptr<::eos::client::EndpointMap> &endpointMapPtr); + static std::unique_ptr<::eos::client::EndpointMap> setNamespaceMap(const std::string &keytab_file); }; } // namespace cliTool -- GitLab From 039ec42de7fb510b18f2e1bf5978d2470a3396ee Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 26 Oct 2022 16:11:34 +0200 Subject: [PATCH 107/126] remove whitespaces --- xroot_plugins/XrdCtaChangeStorageClass.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/xroot_plugins/XrdCtaChangeStorageClass.hpp b/xroot_plugins/XrdCtaChangeStorageClass.hpp index 63ef0fb553..1f3f1395ef 100644 --- a/xroot_plugins/XrdCtaChangeStorageClass.hpp +++ b/xroot_plugins/XrdCtaChangeStorageClass.hpp @@ -24,21 +24,21 @@ namespace cta { namespace xrd { class XrdCtaChangeStorageClass { -private: - const cta::catalogue::Catalogue &m_catalogue; - cta::log::LogContext &m_lc; public: XrdCtaChangeStorageClass(cta::catalogue::Catalogue &catalogue, cta::log::LogContext &lc); void updateCatalogue(const std::vector<std::basic_string<char>>& archiveFileIds, const std::string& newStorageClassName); +private: + const cta::catalogue::Catalogue &m_catalogue; + cta::log::LogContext &m_lc; }; XrdCtaChangeStorageClass::XrdCtaChangeStorageClass(cta::catalogue::Catalogue &catalogue, cta::log::LogContext &lc): m_catalogue(catalogue), m_lc(lc) {} void XrdCtaChangeStorageClass::updateCatalogue(const std::vector<std::basic_string<char>>& archiveFileIds, const std::string& newStorageClassName) { for (auto & id : archiveFileIds) { - const uint64_t archiveFileId = std::stoul(id); + const uint64_t archiveFileId = std::stoul(id); m_catalogue.modifyArchiveFileStorageClassId(archiveFileId, newStorageClassName); } -} -} // namespace xrd +} +} // namespace xrd } // namespace cta \ No newline at end of file -- GitLab From 5d74fead0775d1c33a7d444b8a208dbd28292bba Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Wed, 26 Oct 2022 17:27:48 +0200 Subject: [PATCH 108/126] Give user message if storage class is not present when updating Catalogue --- catalogue/RdbmsCatalogue.cpp | 19 ++++++++++--------- .../ChangeStorageClass.cpp | 3 +-- .../ChangeStorageClass.hpp | 2 +- .../common/CatalogueFetch.hpp | 2 +- .../common/CmdLineTool.cpp | 3 +++ .../common/ConnectionConfiguration.cpp | 2 +- xroot_plugins/XrdSsiCtaRequestMessage.cpp | 18 +++++++++++------- 7 files changed, 28 insertions(+), 21 deletions(-) diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index 842bc82931..a78345f751 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -1452,6 +1452,14 @@ void RdbmsCatalogue::modifyStorageClassName(const common::dataStructures::Securi // ----------------------------------------------------------------------------- void RdbmsCatalogue::modifyArchiveFileStorageClassId(const uint64_t archiveFileId, const std::string& newStorageClassName) const { try { + auto conn = m_connPool.getConn(); + if(!storageClassExists(conn, newStorageClassName)) { + exception::UserError ue; + ue.getMessage() << "Cannot modify archive file " << ": " << archiveFileId << " because storage class " + << ":" << newStorageClassName << " does not exist"; + throw ue; + } + const char *const sql = "UPDATE ARCHIVE_FILE " "SET STORAGE_CLASS_ID = (" @@ -1460,20 +1468,13 @@ void RdbmsCatalogue::modifyArchiveFileStorageClassId(const uint64_t archiveFileI "WHERE " "ARCHIVE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID"; - auto conn = m_connPool.getConn(); auto stmt = conn.createStmt(sql); stmt.bindString(":STORAGE_CLASS_NAME", newStorageClassName); stmt.bindUint64(":ARCHIVE_FILE_ID", archiveFileId); auto rset = stmt.executeQuery(); - if(!storageClassExists(conn, newStorageClassName)) { - exception::UserError ue; - ue.getMessage() << "Cannot modify archive file " << ": " << archiveFileId << " because storage class " - << ":" << newStorageClassName << " does not exist"; - throw ue; - } - } catch(exception::UserError &) { - throw; + } catch(exception::UserError &ue) { + throw ue; } catch(exception::Exception &ex) { ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str()); } diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index e911bd0cd5..921f9b7e61 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -29,7 +29,6 @@ #include "cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp" #include "cmdline/standalone_cli_tools/common/CatalogueFetch.hpp" #include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" - #include "cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp" #include "common/checksum/ChecksumBlob.hpp" #include "common/exception/CommandLineNotParsed.hpp" @@ -180,7 +179,7 @@ void ChangeStorageClass::updateStorageClassInCatalogue() { std::cout << response.message_txt(); break; case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); - case Response::RSP_ERR_USER: throw std::runtime_error(response.message_txt()); + case Response::RSP_ERR_USER: throw exception::UserError(response.message_txt()); case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); default: throw XrdSsiPb::PbException("Invalid response type."); } diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp index 5da9c64549..cc20655f74 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp @@ -91,7 +91,7 @@ private: /** * Updates the storage class name in the EOS namespace */ - void updateStorageClassInEosNamespace(); + void updateStorageClassInEosNamespace(); /** * Updates the storage class name in the catalogue diff --git a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp index 4ffe6b4c63..2880ff0f2b 100644 --- a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp +++ b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp @@ -137,7 +137,7 @@ std::pair<std::string,std::string> getInstanceAndFidFromCTA(const std::string& a isHeaderSent = true; break; case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); - case Response::RSP_ERR_USER: throw XrdSsiPb::UserException(response.message_txt()); + case Response::RSP_ERR_USER: throw exception::UserError(response.message_txt()); case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); default: throw XrdSsiPb::PbException("Invalid response type."); } diff --git a/cmdline/standalone_cli_tools/common/CmdLineTool.cpp b/cmdline/standalone_cli_tools/common/CmdLineTool.cpp index 226d42c958..259e312539 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineTool.cpp +++ b/cmdline/standalone_cli_tools/common/CmdLineTool.cpp @@ -17,6 +17,7 @@ #include "cmdline/standalone_cli_tools/common/CmdLineTool.hpp" #include "common/exception/CommandLineNotParsed.hpp" +#include "common/exception/UserError.hpp" #include <iostream> #include <string> @@ -80,6 +81,8 @@ int CmdLineTool::main(const int argc, char *const *const argv) { return exceptionThrowingMain(argc, argv); } catch(exception::CommandLineNotParsed &ue) { errorMessage = ue.getMessage().str(); + } catch(exception::UserError &ue) { + errorMessage = ue.getMessage().str(); } catch(exception::Exception &ex) { errorMessage = ex.getMessage().str(); } catch(std::exception &se) { diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp index 0b0bba1d53..52081abb1c 100644 --- a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp @@ -116,4 +116,4 @@ std::unique_ptr<::eos::client::EndpointMap> ConnConfiguration::readAndSetConfigu } } // namespace cliTool -} // namespace cta \ No newline at end of file +} // namespace cta diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp index 3acd0aabd6..68ba8e5ab1 100644 --- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp +++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp @@ -2541,15 +2541,19 @@ void RequestMessage::processRecycleTapeFile_Restore(cta::xrd::Response& response } void RequestMessage::processChangeStorageClass(cta::xrd::Response& response) { - using namespace cta::admin; - - std::string newStorageClassName = getRequired(OptionString::STORAGE_CLASS_NAME); - auto archiveFileIds = getRequired(OptionStrList::FILE_ID); + try { + using namespace cta::admin; - XrdCtaChangeStorageClass xrdCtaChangeStorageClass(m_catalogue, m_lc); - xrdCtaChangeStorageClass.updateCatalogue(archiveFileIds, newStorageClassName); + std::string newStorageClassName = getRequired(OptionString::STORAGE_CLASS_NAME); + auto archiveFileIds = getRequired(OptionStrList::FILE_ID); - response.set_type(cta::xrd::Response::RSP_SUCCESS); + XrdCtaChangeStorageClass xrdCtaChangeStorageClass(m_catalogue, m_lc); + xrdCtaChangeStorageClass.updateCatalogue(archiveFileIds, newStorageClassName); + response.set_type(cta::xrd::Response::RSP_SUCCESS); + } catch(exception::UserError &ue) { + response.set_message_txt(ue.getMessage().str()); + response.set_type(Response::RSP_ERR_USER); + } } }} // namespace cta::xrd -- GitLab From d6257a24f63f46f380f789c17002f0f46727d8e0 Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 27 Oct 2022 09:46:11 +0200 Subject: [PATCH 109/126] Forward declare StdOutLogger --- .../change_storage_class/ChangeStorageClass.cpp | 1 + .../change_storage_class/ChangeStorageClass.hpp | 5 ++++- cmdline/standalone_cli_tools/common/CatalogueFetch.hpp | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 921f9b7e61..4f08bef290 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -33,6 +33,7 @@ #include "common/checksum/ChecksumBlob.hpp" #include "common/exception/CommandLineNotParsed.hpp" #include "common/utils/utils.hpp" +#include "common/log/StdoutLogger.hpp" #include "CtaFrontendApi.hpp" diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp index cc20655f74..ae3adb9266 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp @@ -18,7 +18,6 @@ #pragma once #include "cmdline/standalone_cli_tools/common/CmdLineTool.hpp" -#include "common/log/StdoutLogger.hpp" #include "eos_grpc_client/GrpcEndpoint.hpp" #include <optional> @@ -29,6 +28,10 @@ namespace cta { +namespace log { +class StdoutLogger; +} + namespace cliTool { class ChangeStorageClass: public CmdLineTool { diff --git a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp index 2880ff0f2b..cdf0fed235 100644 --- a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp +++ b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp @@ -25,6 +25,9 @@ #include <XrdSsiPbLog.hpp> #include <XrdSsiPbIStreamBuffer.hpp> +#include "common/log/StdoutLogger.hpp" +#include "cta_frontend.pb.h" //!< Auto-generated message types from .proto file + #include "version.h" -- GitLab From 8696259098ae10e19de19229c143703c9fb97627 Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 27 Oct 2022 10:41:16 +0200 Subject: [PATCH 110/126] Edit ConnectionConfiguration --- .../change_storage_class/ChangeStorageClass.cpp | 5 +++-- .../common/ConnectionConfiguration.cpp | 11 ++++++----- .../common/ConnectionConfiguration.hpp | 5 ++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 4f08bef290..9af63e7bc1 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -82,8 +82,9 @@ ChangeStorageClass::ChangeStorageClass( } } - m_endpointMapPtr = ConnConfiguration::readAndSetConfiguration(log, getUsername(), cmdLineArgs, m_serviceProviderPtr); - + auto serviceProviderPtrAndEndpointMap = ConnConfiguration::readAndSetConfiguration(log, getUsername(), cmdLineArgs); + m_serviceProviderPtr = std::move(serviceProviderPtrAndEndpointMap.first); + m_endpointMapPtr = std::move(serviceProviderPtrAndEndpointMap.second); } //------------------------------------------------------------------------------ diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp index 52081abb1c..4be6991aa2 100644 --- a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp @@ -60,11 +60,10 @@ std::unique_ptr<::eos::client::EndpointMap> ConnConfiguration::setNamespaceMap(c //------------------------------------------------------------------------------ // readAndSetConfiguration //------------------------------------------------------------------------------ -std::unique_ptr<::eos::client::EndpointMap> ConnConfiguration::readAndSetConfiguration( +std::pair<std::unique_ptr<XrdSsiPbServiceType>, std::unique_ptr<::eos::client::EndpointMap>> ConnConfiguration::readAndSetConfiguration( cta::log::StdoutLogger &log, const std::string &userName, - const CmdLineArgs &cmdLineArgs, - std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr) { + const CmdLineArgs &cmdLineArgs) { const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by @@ -100,7 +99,7 @@ std::unique_ptr<::eos::client::EndpointMap> ConnConfiguration::readAndSetConfigu // If the server is down, we want an immediate failure. Set client retry to a single attempt. XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); - serviceProviderPtr.reset(new XrdSsiPbServiceType(cliConfig)); + std::unique_ptr<XrdSsiPbServiceType> serviceProviderPtr = std::make_unique<XrdSsiPbServiceType>(cliConfig); // Set CTA frontend configuration options to connect to eos const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf"; @@ -109,7 +108,9 @@ std::unique_ptr<::eos::client::EndpointMap> ConnConfiguration::readAndSetConfigu // Get the endpoint for namespace queries auto nsConf = frontendXrootdConfig.getOptionValueStr("ns.config"); if(nsConf.first) { - return setNamespaceMap(nsConf.second); + auto endpointMap = setNamespaceMap(nsConf.second); + std::pair<std::unique_ptr<XrdSsiPbServiceType>, std::unique_ptr<::eos::client::EndpointMap>> serviceProviderPtrAndEndpointMap = std::make_pair(std::move(serviceProviderPtr), std::move(endpointMap)); + return serviceProviderPtrAndEndpointMap; } else { throw std::runtime_error("Configuration error: cta.ns.config missing from " + frontend_xrootd_config_file); } diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp index 54ce27d723..3e5c7e79db 100644 --- a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp @@ -40,11 +40,10 @@ public: * @param serviceProviderPtr Xroot service provider * @param endpointMapPtr Endpoint for communication with EOS */ - static std::unique_ptr<::eos::client::EndpointMap> readAndSetConfiguration( + static std::pair<std::unique_ptr<XrdSsiPbServiceType>, std::unique_ptr<::eos::client::EndpointMap>> readAndSetConfiguration( cta::log::StdoutLogger &log, const std::string &userName, - const CmdLineArgs &cmdLineArgs, - std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr); + const CmdLineArgs &cmdLineArgs); private: static std::unique_ptr<::eos::client::EndpointMap> setNamespaceMap(const std::string &keytab_file); -- GitLab From 6c105f14efb994a88e8787533f420780a2820ce2 Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 27 Oct 2022 10:45:10 +0200 Subject: [PATCH 111/126] fix indentation in modifyArchiveFileStorageClassId --- catalogue/RdbmsCatalogue.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index a78345f751..55c929e164 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -1474,7 +1474,7 @@ void RdbmsCatalogue::modifyArchiveFileStorageClassId(const uint64_t archiveFileI auto rset = stmt.executeQuery(); } catch(exception::UserError &ue) { - throw ue; + throw ue; } catch(exception::Exception &ex) { ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str()); } -- GitLab From 8a0e32f065219d4e80284e8e48dc0730c973ce6d Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 27 Oct 2022 10:53:16 +0200 Subject: [PATCH 112/126] Switch order of storage class update --- .../change_storage_class/ChangeStorageClass.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 9af63e7bc1..7f6eb1c837 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -212,8 +212,8 @@ void ChangeStorageClass::writeSkippedArchiveIdsToFile() { // exceptionThrowingMain //------------------------------------------------------------------------------ int ChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { - updateStorageClassInEosNamespace(); updateStorageClassInCatalogue(); + updateStorageClassInEosNamespace(); writeSkippedArchiveIdsToFile(); return 0; } -- GitLab From 2b6f904566d94276f9634e6cf9e026f4e164d46a Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 27 Oct 2022 11:26:14 +0200 Subject: [PATCH 113/126] Clarify what setCAttr returns --- .../change_storage_class/CMakeLists.txt | 2 +- .../change_storage_class/ChangeStorageClass.cpp | 17 +++++++++-------- eos_grpc_client/GrpcClient.cpp | 8 ++++---- eos_grpc_client/GrpcClient.hpp | 3 ++- eos_grpc_client/GrpcEndpoint.hpp | 3 ++- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt index d33f4b005f..b164b0846a 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt +++ b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt @@ -28,7 +28,7 @@ include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include) include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS}) add_executable(cta-change-storage-class ChangeStorageClass.cpp ChangeStorageClassMain.cpp ../common/CmdLineTool.cpp ../common/CmdLineArgs.cpp ../../CtaAdminCmdParse.cpp ../common/ConnectionConfiguration.cpp) -target_link_libraries(cta-change-storage-class ${PROTOBUF3_LIBRARIES} XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient) +target_link_libraries(cta-change-storage-class XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient) set_property (TARGET cta-change-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 7f6eb1c837..dd0f542f17 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -119,7 +119,7 @@ void ChangeStorageClass::updateStorageClassInEosNamespace() { } const auto path = m_endpointMapPtr->getPath(diskInstance, diskFileId); - const int status = m_endpointMapPtr->setXAttr(diskInstance, path, "sys.archive.storage_class", m_storageClassName); + const auto status = m_endpointMapPtr->setXAttr(diskInstance, path, "sys.archive.storage_class", m_storageClassName); if (status != 0) { m_archiveIdsNotUpdatedInEos.push_back(archiveFileId); std::list<cta::log::Param> params; @@ -146,17 +146,17 @@ void ChangeStorageClass::updateStorageClassInCatalogue() { admincmd->set_subcmd(cta::admin::AdminCmd::SUBCMD_CH); { - auto key = cta::admin::OptionString::STORAGE_CLASS_NAME; - auto new_opt = admincmd->add_option_str(); + const auto key = cta::admin::OptionString::STORAGE_CLASS_NAME; + const auto new_opt = admincmd->add_option_str(); new_opt->set_key(key); new_opt->set_value(m_storageClassName); } { - auto key = cta::admin::OptionStrList::FILE_ID; - auto new_opt = admincmd->add_option_str_list(); + const auto key = cta::admin::OptionStrList::FILE_ID; + const auto new_opt = admincmd->add_option_str_list(); new_opt->set_key(key); - for (auto &archiveFileId : m_archiveIdsUpdatedInEos) { + for (const auto &archiveFileId : m_archiveIdsUpdatedInEos) { new_opt->add_item(archiveFileId); } } @@ -199,7 +199,8 @@ void ChangeStorageClass::writeSkippedArchiveIdsToFile() { } if (archiveIdFile.is_open()) { - for (auto archiveId : m_archiveIdsNotUpdatedInEos) { + archiveIdFile << "Hello" << std::endl; + for (const auto archiveId : m_archiveIdsNotUpdatedInEos) { archiveIdFile << archiveId << std::endl; } archiveIdFile.close(); @@ -212,8 +213,8 @@ void ChangeStorageClass::writeSkippedArchiveIdsToFile() { // exceptionThrowingMain //------------------------------------------------------------------------------ int ChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { - updateStorageClassInCatalogue(); updateStorageClassInEosNamespace(); + updateStorageClassInCatalogue(); writeSkippedArchiveIdsToFile(); return 0; } diff --git a/eos_grpc_client/GrpcClient.cpp b/eos_grpc_client/GrpcClient.cpp index af9a7c0565..ced098e225 100644 --- a/eos_grpc_client/GrpcClient.cpp +++ b/eos_grpc_client/GrpcClient.cpp @@ -263,8 +263,8 @@ eos::rpc::MDResponse GrpcClient::GetMD(eos::rpc::TYPE type, uint64_t id, const s return response; } - -int GrpcClient::Exec(eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply) const { +using QueryStatus = int; +QueryStatus GrpcClient::Exec(eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply) const { request.set_authkey(token()); @@ -274,13 +274,13 @@ int GrpcClient::Exec(eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply) std::unique_ptr<grpc::ClientAsyncResponseReader<eos::rpc::NSResponse> > rpc( stub_->AsyncExec(&context, request, &cq)); rpc->Finish(&reply, &status, (void*) 1); - + void* got_tag; bool ok = false; GPR_ASSERT(cq.Next(&got_tag, &ok)); GPR_ASSERT(got_tag == (void*) 1); GPR_ASSERT(ok); - + // Act upon the status of the actual RPC. if (status.ok()) { return reply.error().code(); diff --git a/eos_grpc_client/GrpcClient.hpp b/eos_grpc_client/GrpcClient.hpp index d58290a23e..79c315181f 100644 --- a/eos_grpc_client/GrpcClient.hpp +++ b/eos_grpc_client/GrpcClient.hpp @@ -49,7 +49,8 @@ public: // Obtain container or file metadata eos::rpc::MDResponse GetMD(eos::rpc::TYPE type, uint64_t id, const std::string &path, bool showJson = false); - int Exec(eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply) const; + using QueryStatus = int; + QueryStatus Exec(eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply) const; void set_ssl(bool onoff) { m_SSL = onoff; diff --git a/eos_grpc_client/GrpcEndpoint.hpp b/eos_grpc_client/GrpcEndpoint.hpp index e92b63dfb8..da8fa0dd1d 100644 --- a/eos_grpc_client/GrpcEndpoint.hpp +++ b/eos_grpc_client/GrpcEndpoint.hpp @@ -98,7 +98,8 @@ public: } } - int setXAttr(const std::string &diskInstance, const std::string &path, const std::string &attrKey, const std::string &attrValue) const { + using QueryStatus = int; + [[nodiscard]] QueryStatus setXAttr(const std::string &diskInstance, const std::string &path, const std::string &attrKey, const std::string &attrValue) const { auto ep_it = m_endpointMap.find(diskInstance); if(ep_it == m_endpointMap.end()) { throw cta::exception::UserError("Namespace for disk instance \"" + diskInstance + "\" is not configured in the CTA Frontend"); -- GitLab From 8f9b145e21b0de168964bc95d38433f352374e61 Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 27 Oct 2022 11:51:51 +0200 Subject: [PATCH 114/126] use filesystem instead of string --- .../standalone_cli_tools/change_storage_class/CMakeLists.txt | 2 +- .../change_storage_class/ChangeStorageClass.cpp | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt index b164b0846a..a9c1aa9cf2 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt +++ b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt @@ -28,7 +28,7 @@ include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include) include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS}) add_executable(cta-change-storage-class ChangeStorageClass.cpp ChangeStorageClassMain.cpp ../common/CmdLineTool.cpp ../common/CmdLineArgs.cpp ../../CtaAdminCmdParse.cpp ../common/ConnectionConfiguration.cpp) -target_link_libraries(cta-change-storage-class XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient) +target_link_libraries(cta-change-storage-class XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient stdc++fs) set_property (TARGET cta-change-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index dd0f542f17..6ce355b529 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -15,6 +15,7 @@ * submit itself to any jurisdiction. */ +#include <filesystem> #include <fstream> #include <iostream> #include <list> @@ -191,11 +192,11 @@ void ChangeStorageClass::updateStorageClassInCatalogue() { // writeSkippedArchiveIdsToFile //------------------------------------------------------------------------------ void ChangeStorageClass::writeSkippedArchiveIdsToFile() { - const std::string filePath = "/var/log/skippedArchiveIds.txt"; + const std::filesystem::path filePath = "/var/log/skippedArchiveIds.txt"; std::ofstream archiveIdFile(filePath); if (archiveIdFile.fail()) { - throw std::runtime_error("Unable to open file " + filePath); + throw std::runtime_error("Unable to open file " + filePath.string()); } if (archiveIdFile.is_open()) { -- GitLab From b261a281eb91c96d3a81aed1263d6cd301071df4 Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 27 Oct 2022 14:33:56 +0200 Subject: [PATCH 115/126] Add feature to check if storage class exists --- .../ChangeStorageClass.cpp | 42 ++++++++++++++++++- .../ChangeStorageClass.hpp | 6 +++ .../common/CatalogueFetch.hpp | 7 ++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 6ce355b529..76e3b9efd0 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -133,6 +133,46 @@ void ChangeStorageClass::updateStorageClassInEosNamespace() { } } +//------------------------------------------------------------------------------ +// storageClassExists +//------------------------------------------------------------------------------ +void ChangeStorageClass::storageClassExists() { + cta::xrd::Request request; + const auto admincmd = request.mutable_admincmd(); + + request.set_client_cta_version(CTA_VERSION); + request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); + admincmd->set_cmd(cta::admin::AdminCmd::CMD_STORAGECLASS); + admincmd->set_subcmd(cta::admin::AdminCmd::SUBCMD_LS); + + cta::xrd::Response response; + auto stream_future = m_serviceProviderPtr->SendAsync(request, response); + + // Handle responses + switch(response.type()) { + using namespace cta::xrd; + using namespace cta::admin; + case Response::RSP_SUCCESS: + // Print message text + std::cout << response.message_txt(); + // Allow stream processing to commence + isHeaderSent = true; + break; + case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); + case Response::RSP_ERR_USER: throw exception::UserError(response.message_txt()); + case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); + default: throw XrdSsiPb::PbException("Invalid response type."); + } + + // wait until the data stream has been processed before exiting + stream_future.wait(); + + std::list<std::string>::iterator storageClassFound = std::find(storageClasses.begin(), storageClasses.end(), m_storageClassName); + if ( storageClasses.end() == storageClassFound ){ + throw(exception::UserError("The storage class provided has not been defined.")); + } +} + //------------------------------------------------------------------------------ // updateCatalogue //------------------------------------------------------------------------------ @@ -200,7 +240,6 @@ void ChangeStorageClass::writeSkippedArchiveIdsToFile() { } if (archiveIdFile.is_open()) { - archiveIdFile << "Hello" << std::endl; for (const auto archiveId : m_archiveIdsNotUpdatedInEos) { archiveIdFile << archiveId << std::endl; } @@ -214,6 +253,7 @@ void ChangeStorageClass::writeSkippedArchiveIdsToFile() { // exceptionThrowingMain //------------------------------------------------------------------------------ int ChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { + storageClassExists(); updateStorageClassInEosNamespace(); updateStorageClassInCatalogue(); writeSkippedArchiveIdsToFile(); diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp index ae3adb9266..2db067be90 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp @@ -96,6 +96,12 @@ private: */ void updateStorageClassInEosNamespace(); + /** + * Checks if the storage class provided to the tool is defined, + * and throws an exception::UserError if it is not found + */ + void storageClassExists(); + /** * Updates the storage class name in the catalogue */ diff --git a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp index cdf0fed235..8a6693ae63 100644 --- a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp +++ b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp @@ -38,6 +38,7 @@ std::atomic<bool> isHeaderSent(false); std::list<cta::admin::RecycleTapeFileLsItem> deletedTapeFiles; std::list<std::pair<std::string,std::string>> listedTapeFiles; +std::list<std::string> storageClasses; namespace XrdSsiPb { @@ -89,6 +90,12 @@ void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const listedTapeFiles.push_back(instanceAndFid); } break; + case Data::kSclsItem: + { + const auto item = record.scls_item(); + storageClasses.push_back(item.name()); + } + break; default: throw std::runtime_error("Received invalid stream data from CTA Frontend for the cta-restore-deleted-files command."); } -- GitLab From 12c78cc6c89350b8e52e8ed7b53744733b70db49 Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 27 Oct 2022 15:45:40 +0200 Subject: [PATCH 116/126] make changes to CatalogueFetch --- .../change_storage_class/CMakeLists.txt | 2 +- .../ChangeStorageClass.cpp | 12 +- .../common/CatalogueFetch.cpp | 160 ++++++++++++++++++ .../common/CatalogueFetch.hpp | 128 +------------- .../common/ConnectionConfiguration.cpp | 2 +- .../common/ConnectionConfiguration.hpp | 7 +- .../restore_files/RestoreFilesCmd.cpp | 55 ++++++ 7 files changed, 233 insertions(+), 133 deletions(-) create mode 100644 cmdline/standalone_cli_tools/common/CatalogueFetch.cpp diff --git a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt index a9c1aa9cf2..6d88dd90e6 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt +++ b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt @@ -27,7 +27,7 @@ include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include) # Compiled protocol buffers include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS}) -add_executable(cta-change-storage-class ChangeStorageClass.cpp ChangeStorageClassMain.cpp ../common/CmdLineTool.cpp ../common/CmdLineArgs.cpp ../../CtaAdminCmdParse.cpp ../common/ConnectionConfiguration.cpp) +add_executable(cta-change-storage-class ChangeStorageClass.cpp ChangeStorageClassMain.cpp ../common/CmdLineTool.cpp ../common/CmdLineArgs.cpp ../../CtaAdminCmdParse.cpp ../common/ConnectionConfiguration.cpp ../common/CatalogueFetch.cpp) target_link_libraries(cta-change-storage-class XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient stdc++fs) set_property (TARGET cta-change-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 76e3b9efd0..17c4992253 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -38,6 +38,12 @@ #include "CtaFrontendApi.hpp" +// GLOBAL VARIABLES : used to pass information between main thread and stream handler thread + +// global synchronisation flag +extern std::atomic<bool> isHeaderSent; +extern std::list<std::string> storageClasses; + namespace cta { namespace cliTool { @@ -83,9 +89,9 @@ ChangeStorageClass::ChangeStorageClass( } } - auto serviceProviderPtrAndEndpointMap = ConnConfiguration::readAndSetConfiguration(log, getUsername(), cmdLineArgs); - m_serviceProviderPtr = std::move(serviceProviderPtrAndEndpointMap.first); - m_endpointMapPtr = std::move(serviceProviderPtrAndEndpointMap.second); + auto [serviceProvider, endpointmap] = ConnConfiguration::readAndSetConfiguration(log, getUsername(), cmdLineArgs); + m_serviceProviderPtr = std::move(serviceProvider); + m_endpointMapPtr = std::move(endpointmap); } //------------------------------------------------------------------------------ diff --git a/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp new file mode 100644 index 0000000000..b812b87bb4 --- /dev/null +++ b/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp @@ -0,0 +1,160 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#include <list> +#include <iostream> +#include <string> +#include <utility> + +#include <XrdSsiPbLog.hpp> +#include <XrdSsiPbIStreamBuffer.hpp> + +#include "cmdline/standalone_cli_tools/common/CatalogueFetch.hpp" +#include "common/exception/UserError.hpp" +#include "common/log/StdoutLogger.hpp" +#include "cta_frontend.pb.h" //!< Auto-generated message types from .proto file + +#include "version.h" + +// GLOBAL VARIABLES : used to pass information between main thread and stream handler thread + +// global synchronisation flag +std::atomic<bool> isHeaderSent = false; + +std::list<cta::admin::RecycleTapeFileLsItem> deletedTapeFiles; +std::list<std::pair<std::string,std::string>> listedTapeFiles; +std::list<std::string> storageClasses; + +namespace XrdSsiPb { + +/*! + * Alert callback. + * + * Defines how Alert messages should be logged + */ +template<> +void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) +{ + Log::DumpProtobuf(Log::PROTOBUF, &alert); +} + +/*! + * Data/Stream callback. + * + * Defines how incoming records from the stream should be handled + */ +template<> +void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const +{ + using namespace cta::xrd; + using namespace cta::admin; + + // Wait for primary response to be handled before allowing stream response + while(!isHeaderSent) { std::this_thread::yield(); } + + switch(record.data_case()) { + case Data::kRtflsItem: + { + auto item = record.rtfls_item(); + deletedTapeFiles.push_back(item); + } + break; + case Data::kTflsItem: + { + auto item = record.tfls_item(); + auto instanceAndFid = std::make_pair(item.df().disk_instance(), item.df().disk_id()); + listedTapeFiles.push_back(instanceAndFid); + } + break; + case Data::kSclsItem: + { + const auto item = record.scls_item(); + storageClasses.push_back(item.name()); + } + break; + default: + throw std::runtime_error("Received invalid stream data from CTA Frontend for the cta-restore-deleted-files command."); + } +} + +} // namespace XrdSsiPb + +namespace cta { +namespace cliTool { + + /** + * Fetches the instance and fid from the CTA catalogue + * + * @param archiveFileId The arhive file id. + * @param serviceProviderPtr Service provider for communication with the catalogue. + * @return a pair with the instance and the fid. + */ +std::pair<std::string,std::string> getInstanceAndFidFromCTA(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log) { + { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("archiveFileId", archiveFileId)); + log(cta::log::DEBUG, "getInstanceAndFidFromCTA() ", params); + } + + cta::xrd::Request request; + auto admincmd = request.mutable_admincmd(); + + request.set_client_cta_version(CTA_VERSION); + request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); + admincmd->set_cmd(cta::admin::AdminCmd::CMD_TAPEFILE); + admincmd->set_subcmd(cta::admin::AdminCmd::SUBCMD_LS); + auto new_opt = admincmd->add_option_uint64(); + new_opt->set_key(cta::admin::OptionUInt64::ARCHIVE_FILE_ID); + new_opt->set_value(std::stoi(archiveFileId)); + + // Send the Request to the Service and get a Response + cta::xrd::Response response; + auto stream_future = serviceProviderPtr->SendAsync(request, response); + + // Handle responses + switch(response.type()) { + using namespace cta::xrd; + using namespace cta::admin; + case Response::RSP_SUCCESS: + // Print message text + std::cout << response.message_txt(); + // Allow stream processing to commence + isHeaderSent = true; + break; + case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); + case Response::RSP_ERR_USER: throw exception::UserError(response.message_txt()); + case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); + default: throw XrdSsiPb::PbException("Invalid response type."); + } + + // wait until the data stream has been processed before exiting + stream_future.wait(); + if(listedTapeFiles.size() != 1) { + throw std::runtime_error("Unexpected result set: listedTapeFiles size expected=1 received=" + std::to_string(listedTapeFiles.size())); + } + auto listedTapeFile = listedTapeFiles.back(); + listedTapeFiles.clear(); + { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("diskInstance", listedTapeFile.first)); + params.push_back(cta::log::Param("diskFileId", listedTapeFile.second)); + log(cta::log::DEBUG, "Obtained file metadata from CTA", params); + } + return listedTapeFile; +} +} // cliTool +} // cta \ No newline at end of file diff --git a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp index 8a6693ae63..0b53e4d3d0 100644 --- a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp +++ b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp @@ -26,84 +26,11 @@ #include <XrdSsiPbIStreamBuffer.hpp> #include "common/log/StdoutLogger.hpp" +#include "xrootd-ssi-protobuf-interface/eos_cta/include/CtaFrontendApi.hpp" #include "cta_frontend.pb.h" //!< Auto-generated message types from .proto file #include "version.h" - -// GLOBAL VARIABLES : used to pass information between main thread and stream handler thread - -// global synchronisation flag -std::atomic<bool> isHeaderSent(false); - -std::list<cta::admin::RecycleTapeFileLsItem> deletedTapeFiles; -std::list<std::pair<std::string,std::string>> listedTapeFiles; -std::list<std::string> storageClasses; - -namespace XrdSsiPb { - -/*! - * User error exception - */ -class UserException : public std::runtime_error -{ -public: - UserException(const std::string &err_msg) : std::runtime_error(err_msg) {} -}; // class UserException - -/*! - * Alert callback. - * - * Defines how Alert messages should be logged - */ -template<> -void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) -{ - Log::DumpProtobuf(Log::PROTOBUF, &alert); -} - -/*! - * Data/Stream callback. - * - * Defines how incoming records from the stream should be handled - */ -template<> -void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const -{ - using namespace cta::xrd; - using namespace cta::admin; - - // Wait for primary response to be handled before allowing stream response - while(!isHeaderSent) { std::this_thread::yield(); } - - switch(record.data_case()) { - case Data::kRtflsItem: - { - auto item = record.rtfls_item(); - deletedTapeFiles.push_back(item); - } - break; - case Data::kTflsItem: - { - auto item = record.tfls_item(); - auto instanceAndFid = std::make_pair(item.df().disk_instance(), item.df().disk_id()); - listedTapeFiles.push_back(instanceAndFid); - } - break; - case Data::kSclsItem: - { - const auto item = record.scls_item(); - storageClasses.push_back(item.name()); - } - break; - default: - throw std::runtime_error("Received invalid stream data from CTA Frontend for the cta-restore-deleted-files command."); - } -} - -} // namespace XrdSsiPb - - namespace cta { namespace cliTool { @@ -114,58 +41,7 @@ namespace cliTool { * @param serviceProviderPtr Service provider for communication with the catalogue. * @return a pair with the instance and the fid. */ -std::pair<std::string,std::string> getInstanceAndFidFromCTA(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log) { - { - std::list<cta::log::Param> params; - params.push_back(cta::log::Param("archiveFileId", archiveFileId)); - log(cta::log::DEBUG, "getInstanceAndFidFromCTA() ", params); - } - - cta::xrd::Request request; - auto admincmd = request.mutable_admincmd(); - - request.set_client_cta_version(CTA_VERSION); - request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); - admincmd->set_cmd(cta::admin::AdminCmd::CMD_TAPEFILE); - admincmd->set_subcmd(cta::admin::AdminCmd::SUBCMD_LS); - auto new_opt = admincmd->add_option_uint64(); - new_opt->set_key(cta::admin::OptionUInt64::ARCHIVE_FILE_ID); - new_opt->set_value(std::stoi(archiveFileId)); - - // Send the Request to the Service and get a Response - cta::xrd::Response response; - auto stream_future = serviceProviderPtr->SendAsync(request, response); - - // Handle responses - switch(response.type()) { - using namespace cta::xrd; - using namespace cta::admin; - case Response::RSP_SUCCESS: - // Print message text - std::cout << response.message_txt(); - // Allow stream processing to commence - isHeaderSent = true; - break; - case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); - case Response::RSP_ERR_USER: throw exception::UserError(response.message_txt()); - case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); - default: throw XrdSsiPb::PbException("Invalid response type."); - } +std::pair<std::string,std::string> getInstanceAndFidFromCTA(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log); - // wait until the data stream has been processed before exiting - stream_future.wait(); - if(listedTapeFiles.size() != 1) { - throw std::runtime_error("Unexpected result set: listedTapeFiles size expected=1 received=" + std::to_string(listedTapeFiles.size())); - } - auto listedTapeFile = listedTapeFiles.back(); - listedTapeFiles.clear(); - { - std::list<cta::log::Param> params; - params.push_back(cta::log::Param("diskInstance", listedTapeFile.first)); - params.push_back(cta::log::Param("diskFileId", listedTapeFile.second)); - log(cta::log::DEBUG, "Obtained file metadata from CTA", params); - } - return listedTapeFile; -} } // cliTool } // cta \ No newline at end of file diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp index 4be6991aa2..d820d10d06 100644 --- a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp @@ -60,7 +60,7 @@ std::unique_ptr<::eos::client::EndpointMap> ConnConfiguration::setNamespaceMap(c //------------------------------------------------------------------------------ // readAndSetConfiguration //------------------------------------------------------------------------------ -std::pair<std::unique_ptr<XrdSsiPbServiceType>, std::unique_ptr<::eos::client::EndpointMap>> ConnConfiguration::readAndSetConfiguration( +std::tuple<std::unique_ptr<XrdSsiPbServiceType>, std::unique_ptr<::eos::client::EndpointMap>> ConnConfiguration::readAndSetConfiguration( cta::log::StdoutLogger &log, const std::string &userName, const CmdLineArgs &cmdLineArgs) { diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp index 3e5c7e79db..151849571e 100644 --- a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp @@ -24,6 +24,7 @@ #include <string> +#include <tuple> namespace cta { namespace cliTool { @@ -40,13 +41,15 @@ public: * @param serviceProviderPtr Xroot service provider * @param endpointMapPtr Endpoint for communication with EOS */ - static std::pair<std::unique_ptr<XrdSsiPbServiceType>, std::unique_ptr<::eos::client::EndpointMap>> readAndSetConfiguration( + using XrdSsiPbServiceTypePtr = std::unique_ptr<XrdSsiPbServiceType>; + using EndpointMapPtr = std::unique_ptr<::eos::client::EndpointMap>; + static std::tuple<XrdSsiPbServiceTypePtr, EndpointMapPtr> readAndSetConfiguration( cta::log::StdoutLogger &log, const std::string &userName, const CmdLineArgs &cmdLineArgs); private: - static std::unique_ptr<::eos::client::EndpointMap> setNamespaceMap(const std::string &keytab_file); + static EndpointMapPtr setNamespaceMap(const std::string &keytab_file); }; } // namespace cliTool diff --git a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp index 29cc6acce1..100e06aec4 100644 --- a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp +++ b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp @@ -32,6 +32,61 @@ #include <iostream> #include <memory> +// GLOBAL VARIABLES : used to pass information between main thread and stream handler thread + +// global synchronisation flag +std::atomic<bool> isHeaderSent; +std::list<cta::admin::RecycleTapeFileLsItem> deletedTapeFiles; +std::list<std::pair<std::string,std::string>> listedTapeFiles; + +namespace XrdSsiPb { +/*! + * User error exception + */ +class UserException : public std::runtime_error +{ +public: + UserException(const std::string &err_msg) : std::runtime_error(err_msg) {} +}; // class UserException + +/*! + * Alert callback. + * + * Defines how Alert messages should be logged + */ +template<> +void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) +{ + Log::DumpProtobuf(Log::PROTOBUF, &alert); +} + +/*! + * Data/Stream callback. + * + * Defines how incoming records from the stream should be handled + */ +template<> +void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const +{ + using namespace cta::xrd; + using namespace cta::admin; + + // Wait for primary response to be handled before allowing stream response + while(!isHeaderSent) { std::this_thread::yield(); } + + switch(record.data_case()) { + case Data::kRtflsItem: + { + auto item = record.rtfls_item(); + deletedTapeFiles.push_back(item); + } + break; + default: + throw std::runtime_error("Received invalid stream data from CTA Frontend for the cta-restore-deleted-files command."); + } +} + +} namespace cta{ namespace cliTool { -- GitLab From 1f187c3e58ec27953fdf110439364dbd33b9d1a0 Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 27 Oct 2022 16:45:01 +0200 Subject: [PATCH 117/126] add option for restricting the amount of requests to EOS per second --- .../change_storage_class/ChangeStorageClass.cpp | 14 ++++++++++++++ .../change_storage_class/ChangeStorageClass.hpp | 5 +++++ .../cta-change-storage-class.1cta | 3 +++ .../standalone_cli_tools/common/CmdLineArgs.cpp | 10 ++++++++-- .../standalone_cli_tools/common/CmdLineArgs.hpp | 5 +++++ eos_grpc_client/GrpcEndpoint.hpp | 3 ++- 6 files changed, 37 insertions(+), 3 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index 17c4992253..d9093bb7e5 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -22,6 +22,7 @@ #include <memory> #include <string> #include <sys/stat.h> +#include <unistd.h> #include <XrdSsiPbLog.hpp> #include <XrdSsiPbIStreamBuffer.hpp> @@ -89,6 +90,12 @@ ChangeStorageClass::ChangeStorageClass( } } + if (cmdLineArgs.m_frequency) { + m_eosUpdateFrequency = cmdLineArgs.m_frequency.value(); + } else { + m_eosUpdateFrequency = 100; + } + auto [serviceProvider, endpointmap] = ConnConfiguration::readAndSetConfiguration(log, getUsername(), cmdLineArgs); m_serviceProviderPtr = std::move(serviceProvider); m_endpointMapPtr = std::move(endpointmap); @@ -110,7 +117,14 @@ bool ChangeStorageClass::fileInFlight(const google::protobuf::RepeatedField<uint // updateStorageClassInEosNamespace //------------------------------------------------------------------------------ void ChangeStorageClass::updateStorageClassInEosNamespace() { + uint64_t requestCounter = 0; for(const auto &archiveFileId : m_archiveFileIds) { + requestCounter++; + if(requestCounter >= m_eosUpdateFrequency) { + requestCounter = 0; + sleep(1); + } + const auto instanceAndFid = getInstanceAndFidFromCTA(archiveFileId, m_serviceProviderPtr, m_log); const std::string diskInstance = instanceAndFid.first; const std::string diskFileId = instanceAndFid.second; diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp index 2db067be90..87a5b6187a 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp @@ -91,6 +91,11 @@ private: */ std::vector<std::string> m_archiveIdsUpdatedInEos; + /** + * Endpoint map for interaction with EOS + */ + uint64_t m_eosUpdateFrequency; + /** * Updates the storage class name in the EOS namespace */ diff --git a/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class.1cta b/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class.1cta index d7d55b11e5..b442be2366 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class.1cta +++ b/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class.1cta @@ -36,6 +36,9 @@ Path to a file with a archive file ids. The text file should have one id on each .TP \-n, \-\-storage.class.name \fIstorageclassname\fR Path to a file with a archive file ids. The text file should have one id on each line. +.TP +\-t, \-\-frequenzy \feosRequestFrequency\fR +After the eosRequestFrequency amount of requests to EOS the tool will sleep for 1 second. .SH EXIT STATUS .P diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp index 39a3245899..22a036c30b 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp @@ -65,6 +65,7 @@ static struct option changeStorageClassLongOption[] = { {"id", required_argument, nullptr, 'I'}, {"filename", required_argument, nullptr, 'F'}, {"storage.class.name", required_argument, nullptr, 'n'}, + {"frequenzy", required_argument, nullptr, 't'}, {"help", no_argument, nullptr, 'h'}, {nullptr, 0, nullptr, 0} }; @@ -80,7 +81,7 @@ std::map<StandaloneCliTool, const char*> shortopts = { {StandaloneCliTool::RESTORE_FILES, "I:i:f:F:v:c:hd:"}, {StandaloneCliTool::CTA_SEND_EVENT, "i:e:u:g:"}, {StandaloneCliTool::CTA_VERIFY_FILE, "I:i:u:g:v:h:"}, - {StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS, "I:F:n:h:"}, + {StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS, "I:F:n:t:h:"}, }; //------------------------------------------------------------------------------ @@ -150,6 +151,11 @@ m_help(false), m_debug(false), m_standaloneCliTool{standaloneCliTool} { m_storageClassName = std::string(optarg); break; } + case 't': + { + m_frequency = std::stol(std::string(optarg)); + break; + } case 'u': { m_requestUser = std::string(optarg); @@ -243,7 +249,7 @@ void CmdLineArgs::printUsage(std::ostream &os) const { break; case StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS : os << " Usage:" << std::endl << - " cta-change-storage-class --id/-I <archiveFileID> | --filename/-F <filename> --storage.class.name/-n <storageClassName>" << std::endl; + " cta-change-storage-class --id/-I <archiveFileID> | --filename/-F <filename> --storage.class.name/-n <storageClassName> [--frequenzy/-t <eosRequestFrequency>]" << std::endl; break; default: break; diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp index 94db0ff922..5602f1dada 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp @@ -106,6 +106,11 @@ struct CmdLineArgs { */ std::optional<std::string> m_storageClassName; + /** + * Frequency + */ + std::optional<uint64_t> m_frequency; + /** * Constructor that parses the specified command-line arguments. * diff --git a/eos_grpc_client/GrpcEndpoint.hpp b/eos_grpc_client/GrpcEndpoint.hpp index da8fa0dd1d..88a2263d8f 100644 --- a/eos_grpc_client/GrpcEndpoint.hpp +++ b/eos_grpc_client/GrpcEndpoint.hpp @@ -36,7 +36,8 @@ public: ::eos::rpc::InsertReply containerInsert(const ::eos::rpc::ContainerMdProto &container) const; void getCurrentIds(uint64_t &cid, uint64_t &fid) const; ::eos::rpc::MDResponse getMD(::eos::rpc::TYPE type, uint64_t id, const std::string &path, bool showJson) const; - int setXAttr(const std::string &path, const std::string &attrKey, const std::string &attrValue) const; + using QueryStatus = int; + [[nodiscard]] QueryStatus setXAttr(const std::string &path, const std::string &attrKey, const std::string &attrValue) const; private: std::unique_ptr<::eos::client::GrpcClient> m_grpcClient; -- GitLab From 7affa48dabc5ccd5c54b13f4c70f9b8761592c96 Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 27 Oct 2022 16:52:27 +0200 Subject: [PATCH 118/126] move getInstanceAndFidFromCTA to static class --- .../change_storage_class/ChangeStorageClass.cpp | 2 +- cmdline/standalone_cli_tools/common/CatalogueFetch.cpp | 2 +- cmdline/standalone_cli_tools/common/CatalogueFetch.hpp | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index d9093bb7e5..c2438094ef 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -125,7 +125,7 @@ void ChangeStorageClass::updateStorageClassInEosNamespace() { sleep(1); } - const auto instanceAndFid = getInstanceAndFidFromCTA(archiveFileId, m_serviceProviderPtr, m_log); + const auto instanceAndFid = CatalogueFetch::getInstanceAndFidFromCTA(archiveFileId, m_serviceProviderPtr, m_log); const std::string diskInstance = instanceAndFid.first; const std::string diskFileId = instanceAndFid.second; diff --git a/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp index b812b87bb4..736f2a8819 100644 --- a/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp +++ b/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp @@ -103,7 +103,7 @@ namespace cliTool { * @param serviceProviderPtr Service provider for communication with the catalogue. * @return a pair with the instance and the fid. */ -std::pair<std::string,std::string> getInstanceAndFidFromCTA(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log) { +std::pair<std::string,std::string> CatalogueFetch::getInstanceAndFidFromCTA(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log) { { std::list<cta::log::Param> params; params.push_back(cta::log::Param("archiveFileId", archiveFileId)); diff --git a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp index 0b53e4d3d0..16aa2a352a 100644 --- a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp +++ b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp @@ -34,6 +34,9 @@ namespace cta { namespace cliTool { + +class CatalogueFetch { +public: /** * Fetches the instance and fid from the CTA catalogue * @@ -41,7 +44,9 @@ namespace cliTool { * @param serviceProviderPtr Service provider for communication with the catalogue. * @return a pair with the instance and the fid. */ -std::pair<std::string,std::string> getInstanceAndFidFromCTA(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log); +static std::pair<std::string,std::string> getInstanceAndFidFromCTA(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log); + +}; } // cliTool } // cta \ No newline at end of file -- GitLab From 004da2c2d5e0c6e790426eb11c721b1abd367d7c Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 27 Oct 2022 16:55:55 +0200 Subject: [PATCH 119/126] add forward declarations --- .../common/ConnectionConfiguration.cpp | 2 ++ .../common/ConnectionConfiguration.hpp | 10 +++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp index d820d10d06..da6cda335e 100644 --- a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp @@ -15,6 +15,8 @@ * submit itself to any jurisdiction. */ +#include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" +#include "common/log/StdoutLogger.hpp" #include "ConnectionConfiguration.hpp" #include <string> diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp index 151849571e..d81d911626 100644 --- a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp @@ -18,17 +18,21 @@ #pragma once #include "eos_grpc_client/GrpcEndpoint.hpp" -#include "common/log/StdoutLogger.hpp" -#include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" #include "CtaFrontendApi.hpp" - #include <string> #include <tuple> namespace cta { + +namespace log { +class StdoutLogger; +} + namespace cliTool { +class CmdLineArgs; + class ConnConfiguration { public: -- GitLab From 2d70944b811c28107efdc085ff7009ad18e7a30c Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 27 Oct 2022 17:00:22 +0200 Subject: [PATCH 120/126] include cleanup XrdSsiCtaRequestMessage --- xroot_plugins/XrdSsiCtaRequestMessage.cpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp index 68ba8e5ab1..fb2ba5b6d3 100644 --- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp +++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp @@ -31,31 +31,27 @@ using XrdSsiPb::PbException; #include "XrdCtaActivityMountRuleLs.hpp" #include "XrdCtaAdminLs.hpp" #include "XrdCtaArchiveRouteLs.hpp" +#include "XrdCtaChangeStorageClass.hpp" +#include "XrdCtaDiskInstanceLs.hpp" +#include "XrdCtaDiskInstanceSpaceLs.hpp" +#include "XrdCtaDiskSystemLs.hpp" #include "XrdCtaDriveLs.hpp" #include "XrdCtaFailedRequestLs.hpp" #include "XrdCtaGroupMountRuleLs.hpp" #include "XrdCtaLogicalLibraryLs.hpp" #include "XrdCtaMediaTypeLs.hpp" #include "XrdCtaMountPolicyLs.hpp" +#include "XrdCtaRecycleTapeFileLs.hpp" #include "XrdCtaRepackLs.hpp" -#include "XrdSsiCtaRequestMessage.hpp" #include "XrdCtaRequesterMountRuleLs.hpp" #include "XrdCtaShowQueues.hpp" -#include "XrdCtaDiskInstanceLs.hpp" -#include "XrdCtaDiskSystemLs.hpp" #include "XrdCtaStorageClassLs.hpp" #include "XrdCtaTapeFileLs.hpp" #include "XrdCtaTapeLs.hpp" #include "XrdCtaTapePoolLs.hpp" -#include "XrdCtaDiskInstanceSpaceLs.hpp" -#include "XrdCtaRecycleTapeFileLs.hpp" -#include "XrdCtaChangeStorageClass.hpp" - -#include <limits> -#include <sstream> -#include <string> #include "XrdCtaVersion.hpp" #include "XrdCtaVirtualOrganizationLs.hpp" +#include "XrdSsiCtaRequestMessage.hpp" namespace cta { namespace xrd { -- GitLab From 5fa193d858db3c94eec6fa017b04c1a1942df405 Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Thu, 27 Oct 2022 17:03:50 +0200 Subject: [PATCH 121/126] include cleanup XrdSsiCtaRequestMessage --- xroot_plugins/XrdSsiCtaRequestMessage.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp index fb2ba5b6d3..afebbb6073 100644 --- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp +++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp @@ -31,27 +31,27 @@ using XrdSsiPb::PbException; #include "XrdCtaActivityMountRuleLs.hpp" #include "XrdCtaAdminLs.hpp" #include "XrdCtaArchiveRouteLs.hpp" -#include "XrdCtaChangeStorageClass.hpp" -#include "XrdCtaDiskInstanceLs.hpp" -#include "XrdCtaDiskInstanceSpaceLs.hpp" -#include "XrdCtaDiskSystemLs.hpp" #include "XrdCtaDriveLs.hpp" #include "XrdCtaFailedRequestLs.hpp" #include "XrdCtaGroupMountRuleLs.hpp" #include "XrdCtaLogicalLibraryLs.hpp" #include "XrdCtaMediaTypeLs.hpp" #include "XrdCtaMountPolicyLs.hpp" -#include "XrdCtaRecycleTapeFileLs.hpp" #include "XrdCtaRepackLs.hpp" +#include "XrdSsiCtaRequestMessage.hpp" #include "XrdCtaRequesterMountRuleLs.hpp" #include "XrdCtaShowQueues.hpp" +#include "XrdCtaDiskInstanceLs.hpp" +#include "XrdCtaDiskSystemLs.hpp" #include "XrdCtaStorageClassLs.hpp" #include "XrdCtaTapeFileLs.hpp" #include "XrdCtaTapeLs.hpp" #include "XrdCtaTapePoolLs.hpp" +#include "XrdCtaDiskInstanceSpaceLs.hpp" +#include "XrdCtaRecycleTapeFileLs.hpp" +#include "XrdCtaChangeStorageClass.hpp" #include "XrdCtaVersion.hpp" #include "XrdCtaVirtualOrganizationLs.hpp" -#include "XrdSsiCtaRequestMessage.hpp" namespace cta { namespace xrd { -- GitLab From ba54a3595551b5a2c7305a6f414b6b71fd33aba6 Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Mon, 31 Oct 2022 08:54:44 +0100 Subject: [PATCH 122/126] add minor changes to change-storage-class test --- .../orchestration/tests/changeStorageClass.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/continuousintegration/orchestration/tests/changeStorageClass.sh b/continuousintegration/orchestration/tests/changeStorageClass.sh index fc44ca2551..217428feb8 100755 --- a/continuousintegration/orchestration/tests/changeStorageClass.sh +++ b/continuousintegration/orchestration/tests/changeStorageClass.sh @@ -67,13 +67,13 @@ kubectl -n ${NAMESPACE} exec client -- bash /usr/bin/archive_file.sh -f ${FILE_2 EOS_METADATA_PATH_1=$(mktemp -d).json echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH}" touch ${EOS_METADATA_PATH_1} -kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_1} | jq . |& tee ${EOS_METADATA_PATH_1} +kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_1} | jq . | tee ${EOS_METADATA_PATH_1} EOS_ARCHIVE_ID_1=$(jq -r '.xattr | .["sys.archive.file_id"]' ${EOS_METADATA_PATH_1}) EOS_METADATA_PATH_2=$(mktemp -d).json echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH_2}" touch ${EOS_METADATA_PATH_2} -kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_2} | jq . |& tee ${EOS_METADATA_PATH_2} +kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_2} | jq . | tee ${EOS_METADATA_PATH_2} EOS_ARCHIVE_ID_2=$(jq -r '.xattr | .["sys.archive.file_id"]' ${EOS_METADATA_PATH_2}) echo @@ -89,33 +89,34 @@ echo kubectl cp ~/CTA-build/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class ${NAMESPACE}/ctafrontend:/usr/bin/ echo "kubectl cp ${IDS_FILEPATH} ${NAMESPACE}/ctafrontend:~/" kubectl cp ${IDS_FILEPATH} ${NAMESPACE}/ctafrontend:/root/ -kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c "XrdSecPROTOCOL=sss XrdSecSSSKT=/etc/cta/eos.sss.keytab cta-change-storage-class --storage.class.name ${NEW_STORAGE_CLASS_NAME} --filename ${IDS_FILEPATH}" +echo "kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c XrdSecPROTOCOL=sss XrdSecSSSKT=/etc/cta/eos.sss.keytab cta-change-storage-class --storage.class.name ${NEW_STORAGE_CLASS_NAME} --filename ${IDS_FILEPATH}" +kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c "XrdSecPROTOCOL=sss XrdSecSSSKT=/etc/cta/eos.sss.keytab cta-change-storage-class --storage.class.name ${NEW_STORAGE_CLASS_NAME} --filename ${IDS_FILEPATH} -t 1" EOS_METADATA_PATH_AFTER_CHANGE_1=$(mktemp -d).json echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH_AFTER_CHANGE_1}" touch ${EOS_METADATA_PATH_AFTER_CHANGE_1} -kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_1} | jq . |& tee ${EOS_METADATA_PATH_AFTER_CHANGE_1} +kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_1} | jq . | tee ${EOS_METADATA_PATH_AFTER_CHANGE_1} EOS_STORAGE_CLASS_1=$(jq -r '.xattr | .["sys.archive.storage_class"]' ${EOS_METADATA_PATH_AFTER_CHANGE_1}) rm -r ${EOS_METADATA_PATH_AFTER_CHANGE_1} EOS_METADATA_PATH_AFTER_CHANGE_2=$(mktemp -d).json echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH_AFTER_CHANGE_2}" touch ${EOS_METADATA_PATH_AFTER_CHANGE_2} -kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_2} | jq . |& tee ${EOS_METADATA_PATH_AFTER_CHANGE_2} +kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_2} | jq . | tee ${EOS_METADATA_PATH_AFTER_CHANGE_2} EOS_STORAGE_CLASS_2=$(jq -r '.xattr | .["sys.archive.storage_class"]' ${EOS_METADATA_PATH_AFTER_CHANGE_2}) rm -r ${EOS_METADATA_PATH_AFTER_CHANGE_2} CATALOGUE_METADATA_PATH_AFTER_CHANGE_1=$(mktemp -d).json echo "SEND CATALOGUE METADATA TO JSON FILE: ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1}" touch ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1} -kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tf ls --id ${EOS_ARCHIVE_ID_1} | jq . |& tee ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1} +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tf ls --id ${EOS_ARCHIVE_ID_1} | jq . | tee ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1} CATALOGUE_STORAGE_CLASS_1=$(jq . ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1} | jq '.[0]' | jq -r '.af | .["storageClass"]') rm -r ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1} CATALOGUE_METADATA_PATH_AFTER_CHANGE_2=$(mktemp -d).json echo "SEND CATALOGUE METADATA TO JSON FILE: ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2}" touch ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2} -kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tf ls --id ${EOS_ARCHIVE_ID_2} | jq . |& tee ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2} +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tf ls --id ${EOS_ARCHIVE_ID_2} | jq . | tee ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2} CATALOGUE_STORAGE_CLASS_2=$(jq . ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2} | jq '.[0]' | jq -r '.af | .["storageClass"]') rm -r ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2} -- GitLab From 745a0e9007cd0b47d4cf01ad77553df602d13938 Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Mon, 31 Oct 2022 09:57:23 +0100 Subject: [PATCH 123/126] Add helper function to CatalogueFetch --- .../ChangeStorageClass.cpp | 8 ++--- .../common/CatalogueFetch.cpp | 36 +++++++++++-------- .../common/CatalogueFetch.hpp | 20 ++++++----- 3 files changed, 36 insertions(+), 28 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index c2438094ef..a9efc1e2c6 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -43,7 +43,7 @@ // global synchronisation flag extern std::atomic<bool> isHeaderSent; -extern std::list<std::string> storageClasses; +extern std::list<std::string> g_storageClasses; namespace cta { namespace cliTool { @@ -125,7 +125,7 @@ void ChangeStorageClass::updateStorageClassInEosNamespace() { sleep(1); } - const auto instanceAndFid = CatalogueFetch::getInstanceAndFidFromCTA(archiveFileId, m_serviceProviderPtr, m_log); + const auto instanceAndFid = CatalogueFetch::getInstanceAndFid(archiveFileId, m_serviceProviderPtr, m_log); const std::string diskInstance = instanceAndFid.first; const std::string diskFileId = instanceAndFid.second; @@ -187,8 +187,8 @@ void ChangeStorageClass::storageClassExists() { // wait until the data stream has been processed before exiting stream_future.wait(); - std::list<std::string>::iterator storageClassFound = std::find(storageClasses.begin(), storageClasses.end(), m_storageClassName); - if ( storageClasses.end() == storageClassFound ){ + std::list<std::string>::iterator storageClassFound = std::find(g_storageClasses.begin(), g_storageClasses.end(), m_storageClassName); + if ( g_storageClasses.end() == storageClassFound ){ throw(exception::UserError("The storage class provided has not been defined.")); } } diff --git a/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp index 736f2a8819..01d7ca2f30 100644 --- a/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp +++ b/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp @@ -37,7 +37,7 @@ std::atomic<bool> isHeaderSent = false; std::list<cta::admin::RecycleTapeFileLsItem> deletedTapeFiles; std::list<std::pair<std::string,std::string>> listedTapeFiles; -std::list<std::string> storageClasses; +std::list<std::string> g_storageClasses; namespace XrdSsiPb { @@ -83,7 +83,7 @@ void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const case Data::kSclsItem: { const auto item = record.scls_item(); - storageClasses.push_back(item.name()); + g_storageClasses.push_back(item.name()); } break; default: @@ -103,7 +103,7 @@ namespace cliTool { * @param serviceProviderPtr Service provider for communication with the catalogue. * @return a pair with the instance and the fid. */ -std::pair<std::string,std::string> CatalogueFetch::getInstanceAndFidFromCTA(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log) { +std::pair<std::string,std::string> CatalogueFetch::getInstanceAndFid(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log) { { std::list<cta::log::Param> params; params.push_back(cta::log::Param("archiveFileId", archiveFileId)); @@ -121,6 +121,23 @@ std::pair<std::string,std::string> CatalogueFetch::getInstanceAndFidFromCTA(cons new_opt->set_key(cta::admin::OptionUInt64::ARCHIVE_FILE_ID); new_opt->set_value(std::stoi(archiveFileId)); + handleResponse(request, serviceProviderPtr); + + if(listedTapeFiles.size() != 1) { + throw std::runtime_error("Unexpected result set: listedTapeFiles size expected=1 received=" + std::to_string(listedTapeFiles.size())); + } + auto listedTapeFile = listedTapeFiles.back(); + listedTapeFiles.clear(); + { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("diskInstance", listedTapeFile.first)); + params.push_back(cta::log::Param("diskFileId", listedTapeFile.second)); + log(cta::log::DEBUG, "Obtained file metadata from CTA", params); + } + return listedTapeFile; +} + +void CatalogueFetch::handleResponse(const cta::xrd::Request &request, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr) { // Send the Request to the Service and get a Response cta::xrd::Response response; auto stream_future = serviceProviderPtr->SendAsync(request, response); @@ -143,18 +160,7 @@ std::pair<std::string,std::string> CatalogueFetch::getInstanceAndFidFromCTA(cons // wait until the data stream has been processed before exiting stream_future.wait(); - if(listedTapeFiles.size() != 1) { - throw std::runtime_error("Unexpected result set: listedTapeFiles size expected=1 received=" + std::to_string(listedTapeFiles.size())); - } - auto listedTapeFile = listedTapeFiles.back(); - listedTapeFiles.clear(); - { - std::list<cta::log::Param> params; - params.push_back(cta::log::Param("diskInstance", listedTapeFile.first)); - params.push_back(cta::log::Param("diskFileId", listedTapeFile.second)); - log(cta::log::DEBUG, "Obtained file metadata from CTA", params); - } - return listedTapeFile; } + } // cliTool } // cta \ No newline at end of file diff --git a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp index 16aa2a352a..d2580cb87b 100644 --- a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp +++ b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp @@ -37,15 +37,17 @@ namespace cliTool { class CatalogueFetch { public: - /** - * Fetches the instance and fid from the CTA catalogue - * - * @param archiveFileId The arhive file id. - * @param serviceProviderPtr Service provider for communication with the catalogue. - * @return a pair with the instance and the fid. - */ -static std::pair<std::string,std::string> getInstanceAndFidFromCTA(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log); - + /** + * Fetches the instance and fid from the CTA catalogue + * + * @param archiveFileId The arhive file id. + * @param serviceProviderPtr Service provider for communication with the catalogue. + * @return a pair with the instance and the fid. + */ + static std::pair<std::string,std::string> getInstanceAndFid(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log); + +private: + static void handleResponse(const cta::xrd::Request &request, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr); }; } // cliTool -- GitLab From 7d919c956818e39c0e01a3914aea64cd276e64d8 Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Mon, 31 Oct 2022 11:27:32 +0100 Subject: [PATCH 124/126] use tuple instead of pair in CatalogueFetch --- .../change_storage_class/ChangeStorageClass.cpp | 4 +--- cmdline/standalone_cli_tools/common/CatalogueFetch.cpp | 2 +- cmdline/standalone_cli_tools/common/CatalogueFetch.hpp | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp index a9efc1e2c6..f57d441ef8 100644 --- a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -125,9 +125,7 @@ void ChangeStorageClass::updateStorageClassInEosNamespace() { sleep(1); } - const auto instanceAndFid = CatalogueFetch::getInstanceAndFid(archiveFileId, m_serviceProviderPtr, m_log); - const std::string diskInstance = instanceAndFid.first; - const std::string diskFileId = instanceAndFid.second; + const auto [diskInstance, diskFileId] = CatalogueFetch::getInstanceAndFid(archiveFileId, m_serviceProviderPtr, m_log); // No files in flight should change storage class const auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::FILE, std::stoi(diskFileId), "", false); diff --git a/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp index 01d7ca2f30..f9c87a8b22 100644 --- a/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp +++ b/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp @@ -103,7 +103,7 @@ namespace cliTool { * @param serviceProviderPtr Service provider for communication with the catalogue. * @return a pair with the instance and the fid. */ -std::pair<std::string,std::string> CatalogueFetch::getInstanceAndFid(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log) { +std::tuple<std::string,std::string> CatalogueFetch::getInstanceAndFid(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log) { { std::list<cta::log::Param> params; params.push_back(cta::log::Param("archiveFileId", archiveFileId)); diff --git a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp index d2580cb87b..bfd7d6fdd2 100644 --- a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp +++ b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp @@ -44,7 +44,7 @@ public: * @param serviceProviderPtr Service provider for communication with the catalogue. * @return a pair with the instance and the fid. */ - static std::pair<std::string,std::string> getInstanceAndFid(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log); + static std::tuple<std::string,std::string> getInstanceAndFid(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log); private: static void handleResponse(const cta::xrd::Request &request, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr); -- GitLab From 74c423255dc984cdfd3cea0c23b7dccbc8d6b89a Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Mon, 31 Oct 2022 11:36:28 +0100 Subject: [PATCH 125/126] update release notes --- ReleaseNotes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ReleaseNotes.md b/ReleaseNotes.md index c39bf109e1..03f4349a61 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -1,10 +1,10 @@ # v4.NEXT -### Features -- cta/CTA#16 - Add option for a user config file ## Summary ### Features +- cta/CTA#16 - Add option for a user config file - cta/CTA#23 - Change Owner Identifier String and System Code of Creating System values in tape labels - cta/CTA#41 - Delete verification_status of tape when tape is reclaimed +- cta/CTA#78 - Tool to update the storage class - cta/CTA#153 - Allow verification status to be cleared with cta-admin - cta/CTA#173 - Update release notes and small changes to refactoring of operation tools cmd line parsing - Compatible with operations 0.4-95 or later ### Continuous Integration -- GitLab From d04d5a1ef3ba588aa5221b3c1a1602e9218c0eed Mon Sep 17 00:00:00 2001 From: lwardena <lasse.tjernaes.wardenaer@cern.ch> Date: Mon, 31 Oct 2022 11:39:03 +0100 Subject: [PATCH 126/126] add missing include after merge with main --- xroot_plugins/XrdSsiCtaRequestMessage.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp index d94ae7fe82..7409eae874 100644 --- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp +++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp @@ -31,6 +31,7 @@ #include "xroot_plugins/XrdCtaActivityMountRuleLs.hpp" #include "xroot_plugins/XrdCtaAdminLs.hpp" #include "xroot_plugins/XrdCtaArchiveRouteLs.hpp" +#include "xroot_plugins/XrdCtaChangeStorageClass.hpp" #include "xroot_plugins/XrdCtaDiskInstanceLs.hpp" #include "xroot_plugins/XrdCtaDiskInstanceSpaceLs.hpp" #include "xroot_plugins/XrdCtaDiskSystemLs.hpp" -- GitLab