diff --git a/ReleaseNotes.md b/ReleaseNotes.md index c39bf109e18500aad92d98ba55535977d7190551..03f4349a614590bb6de2ae7d5be8ca0cebdd598b 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -1,10 +1,10 @@ # v4.NEXT -### Features -- cta/CTA#16 - Add option for a user config file ## Summary ### Features +- cta/CTA#16 - Add option for a user config file - cta/CTA#23 - Change Owner Identifier String and System Code of Creating System values in tape labels - cta/CTA#41 - Delete verification_status of tape when tape is reclaimed +- cta/CTA#78 - Tool to update the storage class - cta/CTA#153 - Allow verification status to be cleared with cta-admin - cta/CTA#173 - Update release notes and small changes to refactoring of operation tools cmd line parsing - Compatible with operations 0.4-95 or later ### Continuous Integration diff --git a/catalogue/Catalogue.hpp b/catalogue/Catalogue.hpp index bb6b296c351c27a16cd31b458f4240e82cdb005a..805e1cbf73f46ff5837f04838725289dae3cd92e 100644 --- a/catalogue/Catalogue.hpp +++ b/catalogue/Catalogue.hpp @@ -1406,7 +1406,16 @@ public: * If the amount released exceeds the current reservation, the reservation will be reduced to zero. */ virtual void releaseDiskSpace(const std::string& driveName, const uint64_t mountId, const DiskSpaceReservationRequest& diskSpaceReservation, log::LogContext & lc) = 0; -}; // class Catalogue -} // namespace catalogue -} // namespace cta + /** + * Changes the name of hte storage class + * @param archiveFileId Id for file found in ARCHIVE_FILE + * @param newStorageClassName The name of the storage class + */ + virtual void modifyArchiveFileStorageClassId(const uint64_t archiveFileId, const std::string& newStorageClassName) const = 0; + +}; // class Catalogue + +} // namespace catalogue +} // namespace cta + diff --git a/catalogue/CatalogueRetryWrapper.cpp b/catalogue/CatalogueRetryWrapper.cpp index f2c009281427dda6f5a26f15405773680e0b42c1..0e1010ce869fd7f760440cca089de135145a2a65 100644 --- a/catalogue/CatalogueRetryWrapper.cpp +++ b/catalogue/CatalogueRetryWrapper.cpp @@ -763,5 +763,10 @@ void CatalogueRetryWrapper::releaseDiskSpace(const std::string& driveName, const return retryOnLostConnection(m_log, [&]{return m_catalogue->releaseDiskSpace(driveName, mountId, diskSpaceReservation, lc);}, m_maxTriesToConnect); } +void CatalogueRetryWrapper::modifyArchiveFileStorageClassId(const uint64_t archiveFileId, const std::string& newStorageClassName) const { + return retryOnLostConnection(m_log, [&]{return m_catalogue->modifyArchiveFileStorageClassId(archiveFileId, newStorageClassName);}, m_maxTriesToConnect); +} + + } // namespace catalogue } // namespace cta diff --git a/catalogue/CatalogueRetryWrapper.hpp b/catalogue/CatalogueRetryWrapper.hpp index f53daf15f0bf662890f7a6f0949bddabc14d5fde..4e1ae122fc3a80b4c733fc7ab4b51d9d27c884a4 100644 --- a/catalogue/CatalogueRetryWrapper.hpp +++ b/catalogue/CatalogueRetryWrapper.hpp @@ -407,6 +407,7 @@ public: void releaseDiskSpace(const std::string& driveName, const uint64_t mountId, const DiskSpaceReservationRequest& diskSpaceReservation, log::LogContext & lc) override; + void modifyArchiveFileStorageClassId(const uint64_t archiveFileId, const std::string& newStorageClassName) const override; protected: /** * Object representing the API to the CTA logging system. diff --git a/catalogue/DummyCatalogue.cpp b/catalogue/DummyCatalogue.cpp index ed7c365e2585f440e265f0129e5aaf7e2a2d850b..7f07160162f4ddf0f2080c266185772e29f9b62a 100644 --- a/catalogue/DummyCatalogue.cpp +++ b/catalogue/DummyCatalogue.cpp @@ -197,6 +197,7 @@ bool DummyCatalogue::tapePoolExists(const std::string& tapePoolName) const { thr void DummyCatalogue::updateDiskFileId(uint64_t archiveFileId, const std::string &diskInstance, const std::string &diskFileId) { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } void DummyCatalogue::moveArchiveFileToRecycleLog(const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) {throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented");} +void DummyCatalogue::modifyArchiveFileStorageClassId(const uint64_t archiveFileId, const std::string &newStorageClassName) const { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } // Special functions for unit tests. void DummyCatalogue::addEnabledTape(const std::string & vid) { diff --git a/catalogue/DummyCatalogue.hpp b/catalogue/DummyCatalogue.hpp index 0d6fd342cc1a35531a07130b54206ea387eecec9..5598d78ed9aac87cb887d31aaf12bfaa0d43093a 100644 --- a/catalogue/DummyCatalogue.hpp +++ b/catalogue/DummyCatalogue.hpp @@ -200,6 +200,7 @@ public: void updateDiskFileId(uint64_t archiveFileId, const std::string &diskInstance, const std::string &diskFileId) override; void moveArchiveFileToRecycleLog(const common::dataStructures::DeleteArchiveRequest &request, log::LogContext & lc) override; + void modifyArchiveFileStorageClassId(const uint64_t archiveFileId, const std::string &newStorageClassName) const override; common::dataStructures::VidToTapeMap getTapesByVid(const std::string& vid) const override; diff --git a/catalogue/RdbmsCatalogue.cpp b/catalogue/RdbmsCatalogue.cpp index 90fa70a8367391e973c8fc7e4f2cd15856b2fdc0..55c929e1643f88db01947cfdaa23835c99921240 100644 --- a/catalogue/RdbmsCatalogue.cpp +++ b/catalogue/RdbmsCatalogue.cpp @@ -1447,6 +1447,39 @@ void RdbmsCatalogue::modifyStorageClassName(const common::dataStructures::Securi } } +// ----------------------------------------------------------------------------- +// modifyArchiveFileStorageClassId +// ----------------------------------------------------------------------------- +void RdbmsCatalogue::modifyArchiveFileStorageClassId(const uint64_t archiveFileId, const std::string& newStorageClassName) const { + try { + auto conn = m_connPool.getConn(); + if(!storageClassExists(conn, newStorageClassName)) { + exception::UserError ue; + ue.getMessage() << "Cannot modify archive file " << ": " << archiveFileId << " because storage class " + << ":" << newStorageClassName << " does not exist"; + throw ue; + } + + const char *const sql = + "UPDATE ARCHIVE_FILE " + "SET STORAGE_CLASS_ID = (" + "SELECT STORAGE_CLASS_ID FROM STORAGE_CLASS WHERE STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME " + ") " + "WHERE " + "ARCHIVE_FILE.ARCHIVE_FILE_ID = :ARCHIVE_FILE_ID"; + + auto stmt = conn.createStmt(sql); + stmt.bindString(":STORAGE_CLASS_NAME", newStorageClassName); + stmt.bindUint64(":ARCHIVE_FILE_ID", archiveFileId); + auto rset = stmt.executeQuery(); + + } catch(exception::UserError &ue) { + throw ue; + } catch(exception::Exception &ex) { + ex.getMessage().str(std::string(__FUNCTION__) + ": " + ex.getMessage().str()); + } +} + //------------------------------------------------------------------------------ // createMediaType //------------------------------------------------------------------------------ diff --git a/catalogue/RdbmsCatalogue.hpp b/catalogue/RdbmsCatalogue.hpp index f0f9e6a73737a622418c5ca836cd145d27a731e7..8cdddbf55dbb1b96bb4c27c96862130633d52010 100644 --- a/catalogue/RdbmsCatalogue.hpp +++ b/catalogue/RdbmsCatalogue.hpp @@ -1267,6 +1267,12 @@ protected: bool archiveFileIdExists(rdbms::Conn &conn, const uint64_t archiveFileId) const; /** + * @param newStorageClassName The name of the storage class + * @param archiveFileId Id for file found in ARCHIVE_FILE + */ + void modifyArchiveFileStorageClassId(const uint64_t archiveFileId, const std::string& newStorageClassName) const override; + +/** * Returns true if the specified disk file identifier exists. * * @param conn The database connection. diff --git a/cmdline/CtaAdminCmdParse.hpp b/cmdline/CtaAdminCmdParse.hpp index ed15d5f9feda33f6903611fe500ab5e0d653643b..621c21e049a0f2b6ff1f6f082d0899a5f7ebd2b9 100644 --- a/cmdline/CtaAdminCmdParse.hpp +++ b/cmdline/CtaAdminCmdParse.hpp @@ -344,6 +344,7 @@ const std::map<std::string, OptionString::Key> strOptions = { { "--diskinstancespace", OptionString::DISK_INSTANCE_SPACE }, { "--verificationstatus", OptionString::VERIFICATION_STATUS }, { "--disabledreason", OptionString::DISABLED_REASON }, + { "--storage.class.name", OptionString::STORAGE_CLASS_NAME } }; @@ -353,7 +354,8 @@ const std::map<std::string, OptionString::Key> strOptions = { */ const std::map<std::string, OptionStrList::Key> strListOptions = { { "--fxidfile", OptionStrList::FILE_ID }, - { "--vidfile", OptionStrList::VID } + { "--vidfile", OptionStrList::VID }, + { "--id", OptionStrList::FILE_ID } }; @@ -554,6 +556,9 @@ const Option opt_diskinstancespace_alias { Option::OPT_STR, "--name", const Option opt_verificationstatus { Option::OPT_STR, "--verificationstatus", "--vs", " <verification_status>" }; const Option opt_disabledreason { Option::OPT_STR, "--disabledreason", "--dr", " <disabled_reason>" }; +const Option opt_storage_class_name { Option::OPT_STR, "--storage.class.name", "-n", " <storage_class_name>" }; +const Option opt_archive_file_ids { Option::OPT_STR_LIST, "--id", "-I", " <archive_file_id>" }; + /*! * Map valid options to commands */ @@ -703,6 +708,9 @@ const std::map<cmd_key_t, cmd_val_t> cmdOptions = { { opt_instance, opt_username_alias, opt_activityregex, opt_mountpolicy.optional(), opt_comment.optional() }}, {{ AdminCmd::CMD_ACTIVITYMOUNTRULE, AdminCmd::SUBCMD_RM }, { opt_instance, opt_username_alias, opt_activityregex }}, {{ AdminCmd::CMD_ACTIVITYMOUNTRULE, AdminCmd::SUBCMD_LS }, { }}, + /*----------------------------------------------------------------------------------------------------*/ + {{ AdminCmd::CMD_ARCHIVEFILE, AdminCmd::SUBCMD_CH }, + { opt_storage_class_name, opt_archive_file_ids }}, }; @@ -711,6 +719,7 @@ const std::map<cmd_key_t, cmd_val_t> cmdOptions = { * Validate that all required command line options are present * * Throws a std::runtime_error if the command is invalid + * This function is used on the server side */ void validateCmd(const cta::admin::AdminCmd &admincmd); }} // namespace cta::admin diff --git a/cmdline/standalone_cli_tools/CMakeLists.txt b/cmdline/standalone_cli_tools/CMakeLists.txt index 1112c75327f990ad650cbd3d31d02441a9ab8da8..18f3c324c43063d6a35a3ccb21de036c8150b334 100644 --- a/cmdline/standalone_cli_tools/CMakeLists.txt +++ b/cmdline/standalone_cli_tools/CMakeLists.txt @@ -15,6 +15,7 @@ cmake_minimum_required (VERSION 3.17) +add_subdirectory (change_storage_class) add_subdirectory (restore_files) find_package(xrootdclient REQUIRED) diff --git a/cmdline/standalone_cli_tools/CtaVerifyFile.cpp b/cmdline/standalone_cli_tools/CtaVerifyFile.cpp index 34d0eb1dafd3fa569ef442d7ebfb5d17c1281415..965a8370e4852c59909ad391aa0e931206149bfb 100644 --- a/cmdline/standalone_cli_tools/CtaVerifyFile.cpp +++ b/cmdline/standalone_cli_tools/CtaVerifyFile.cpp @@ -81,7 +81,7 @@ void fillNotification(cta::eos::Notification ¬ification, const int argc, char notification.mutable_cli()->mutable_user()->set_groupname(cmdLineArgs.m_requestGroup.value()); } - const std::string archiveFileId(std::to_string(cmdLineArgs.m_archiveFileId.value())); + const std::string archiveFileId(cmdLineArgs.m_archiveFileId.value()); // WF notification.mutable_wf()->set_event(cta::eos::Workflow::PREPARE); diff --git a/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..6d88dd90e6d8b12fa7ef02be182302c7360d2a8a --- /dev/null +++ b/cmdline/standalone_cli_tools/change_storage_class/CMakeLists.txt @@ -0,0 +1,36 @@ +# @project The CERN Tape Archive (CTA) +# @copyright Copyright © 2015-2022 CERN +# @license This program is free software, distributed under the terms of the GNU General Public +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can +# redistribute it and/or modify it under the terms of the GPL Version 3, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# In applying this licence, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +cmake_minimum_required (VERSION 3.17) + +find_package(xrootdclient REQUIRED) +find_package(Protobuf3 REQUIRED) + +# XRootD SSI +include_directories(${XROOTD_INCLUDE_DIR} ${XROOTD_INCLUDE_DIR}/private ) + +# XRootD SSI Protocol Buffer bindings +include_directories(${XRD_SSI_PB_DIR}/include ${XRD_SSI_PB_DIR}/eos_cta/include) + +# Compiled protocol buffers +include_directories(${CMAKE_BINARY_DIR}/eos_cta ${PROTOBUF3_INCLUDE_DIRS}) + +add_executable(cta-change-storage-class ChangeStorageClass.cpp ChangeStorageClassMain.cpp ../common/CmdLineTool.cpp ../common/CmdLineArgs.cpp ../../CtaAdminCmdParse.cpp ../common/ConnectionConfiguration.cpp ../common/CatalogueFetch.cpp) +target_link_libraries(cta-change-storage-class XrdSsiLib XrdUtils ctacommon EosCtaGrpc EosGrpcClient stdc++fs) +set_property (TARGET cta-change-storage-class APPEND PROPERTY INSTALL_RPATH ${PROTOBUF3_RPATH}) + + +install(TARGETS cta-change-storage-class DESTINATION usr/bin) +install(FILES cta-change-storage-class.1cta DESTINATION usr/share/man/man1) \ No newline at end of file diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f57d441ef8d1c70a1d040815d2f9ca97c6e0ea44 --- /dev/null +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.cpp @@ -0,0 +1,283 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#include <filesystem> +#include <fstream> +#include <iostream> +#include <list> +#include <memory> +#include <string> +#include <sys/stat.h> +#include <unistd.h> + +#include <XrdSsiPbLog.hpp> +#include <XrdSsiPbIStreamBuffer.hpp> + +#include "cmdline/CtaAdminCmdParse.hpp" +#include "cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp" +#include "cmdline/standalone_cli_tools/common/CatalogueFetch.hpp" +#include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" +#include "cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp" +#include "common/checksum/ChecksumBlob.hpp" +#include "common/exception/CommandLineNotParsed.hpp" +#include "common/utils/utils.hpp" +#include "common/log/StdoutLogger.hpp" + +#include "CtaFrontendApi.hpp" + +// GLOBAL VARIABLES : used to pass information between main thread and stream handler thread + +// global synchronisation flag +extern std::atomic<bool> isHeaderSent; +extern std::list<std::string> g_storageClasses; + +namespace cta { +namespace cliTool { + +//------------------------------------------------------------------------------ +// constructor +//------------------------------------------------------------------------------ +ChangeStorageClass::ChangeStorageClass( + std::istream &inStream, + std::ostream &outStream, + std::ostream &errStream, + cta::log::StdoutLogger &log, + const int& argc, + char *const *const argv): + CmdLineTool(inStream, outStream, errStream), + m_log(log) { + + CmdLineArgs cmdLineArgs(argc, argv, StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS); + + if (cmdLineArgs.m_help) { + cmdLineArgs.printUsage(std::cout); + exit(0); + } + + if (!cmdLineArgs.m_storageClassName) { + cmdLineArgs.printUsage(std::cout); + exit(1); + } + + if (!cmdLineArgs.m_archiveFileId && !cmdLineArgs.m_archiveFileIds) { + cmdLineArgs.printUsage(std::cout); + exit(1); + } + + m_storageClassName = cmdLineArgs.m_storageClassName.value(); + + if (cmdLineArgs.m_archiveFileId) { + m_archiveFileIds.push_back(cmdLineArgs.m_archiveFileId.value()); + } + + if (cmdLineArgs.m_archiveFileIds) { + for (const auto archiveFileId : cmdLineArgs.m_archiveFileIds.value() ) { + m_archiveFileIds.push_back(archiveFileId); + } + } + + if (cmdLineArgs.m_frequency) { + m_eosUpdateFrequency = cmdLineArgs.m_frequency.value(); + } else { + m_eosUpdateFrequency = 100; + } + + auto [serviceProvider, endpointmap] = ConnConfiguration::readAndSetConfiguration(log, getUsername(), cmdLineArgs); + m_serviceProviderPtr = std::move(serviceProvider); + m_endpointMapPtr = std::move(endpointmap); +} + +//------------------------------------------------------------------------------ +// fileInFlight +//------------------------------------------------------------------------------ +bool ChangeStorageClass::fileInFlight(const google::protobuf::RepeatedField<uint32_t> &locations) { + for (auto location : locations) { + // file is not in flight if fsid==65535 + if (location == 65535) { return false; } + } + return true; +} + + +//------------------------------------------------------------------------------ +// updateStorageClassInEosNamespace +//------------------------------------------------------------------------------ +void ChangeStorageClass::updateStorageClassInEosNamespace() { + uint64_t requestCounter = 0; + for(const auto &archiveFileId : m_archiveFileIds) { + requestCounter++; + if(requestCounter >= m_eosUpdateFrequency) { + requestCounter = 0; + sleep(1); + } + + const auto [diskInstance, diskFileId] = CatalogueFetch::getInstanceAndFid(archiveFileId, m_serviceProviderPtr, m_log); + + // No files in flight should change storage class + const auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::FILE, std::stoi(diskFileId), "", false); + if (fileInFlight(md_response.fmd().locations())) { + m_archiveIdsNotUpdatedInEos.push_back(archiveFileId); + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("archiveFileId", archiveFileId)); + m_log(cta::log::WARNING, "File did not change storage class because the file was in flight", params); + continue; + } + + const auto path = m_endpointMapPtr->getPath(diskInstance, diskFileId); + const auto status = m_endpointMapPtr->setXAttr(diskInstance, path, "sys.archive.storage_class", m_storageClassName); + if (status != 0) { + m_archiveIdsNotUpdatedInEos.push_back(archiveFileId); + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("archiveFileId", archiveFileId)); + m_log(cta::log::WARNING, "File did not change storage class because query to EOS failed", params); + } + else { + m_archiveIdsUpdatedInEos.push_back(archiveFileId); + } + } +} + +//------------------------------------------------------------------------------ +// storageClassExists +//------------------------------------------------------------------------------ +void ChangeStorageClass::storageClassExists() { + cta::xrd::Request request; + const auto admincmd = request.mutable_admincmd(); + + request.set_client_cta_version(CTA_VERSION); + request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); + admincmd->set_cmd(cta::admin::AdminCmd::CMD_STORAGECLASS); + admincmd->set_subcmd(cta::admin::AdminCmd::SUBCMD_LS); + + cta::xrd::Response response; + auto stream_future = m_serviceProviderPtr->SendAsync(request, response); + + // Handle responses + switch(response.type()) { + using namespace cta::xrd; + using namespace cta::admin; + case Response::RSP_SUCCESS: + // Print message text + std::cout << response.message_txt(); + // Allow stream processing to commence + isHeaderSent = true; + break; + case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); + case Response::RSP_ERR_USER: throw exception::UserError(response.message_txt()); + case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); + default: throw XrdSsiPb::PbException("Invalid response type."); + } + + // wait until the data stream has been processed before exiting + stream_future.wait(); + + std::list<std::string>::iterator storageClassFound = std::find(g_storageClasses.begin(), g_storageClasses.end(), m_storageClassName); + if ( g_storageClasses.end() == storageClassFound ){ + throw(exception::UserError("The storage class provided has not been defined.")); + } +} + +//------------------------------------------------------------------------------ +// updateCatalogue +//------------------------------------------------------------------------------ +void ChangeStorageClass::updateStorageClassInCatalogue() { + cta::xrd::Request request; + + const auto admincmd = request.mutable_admincmd(); + + request.set_client_cta_version(CTA_VERSION); + request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); + admincmd->set_cmd(cta::admin::AdminCmd::CMD_ARCHIVEFILE); + admincmd->set_subcmd(cta::admin::AdminCmd::SUBCMD_CH); + + { + const auto key = cta::admin::OptionString::STORAGE_CLASS_NAME; + const auto new_opt = admincmd->add_option_str(); + new_opt->set_key(key); + new_opt->set_value(m_storageClassName); + } + + { + const auto key = cta::admin::OptionStrList::FILE_ID; + const auto new_opt = admincmd->add_option_str_list(); + new_opt->set_key(key); + for (const auto &archiveFileId : m_archiveIdsUpdatedInEos) { + new_opt->add_item(archiveFileId); + } + } + + // Validate the Protocol Buffer + try { + cta::admin::validateCmd(*admincmd); + } catch(std::runtime_error &ex) { + throw std::runtime_error("Error: Protocol Buffer validation"); + } + + // Send the Request to the Service and get a Response + cta::xrd::Response response; + m_serviceProviderPtr->Send(request, response); + + // Handle responses + switch(response.type()) { + using namespace cta::xrd; + using namespace cta::admin; + case Response::RSP_SUCCESS: + // Print message text + std::cout << response.message_txt(); + break; + case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); + case Response::RSP_ERR_USER: throw exception::UserError(response.message_txt()); + case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); + default: throw XrdSsiPb::PbException("Invalid response type."); + } +} + +//------------------------------------------------------------------------------ +// writeSkippedArchiveIdsToFile +//------------------------------------------------------------------------------ +void ChangeStorageClass::writeSkippedArchiveIdsToFile() { + const std::filesystem::path filePath = "/var/log/skippedArchiveIds.txt"; + std::ofstream archiveIdFile(filePath); + + if (archiveIdFile.fail()) { + throw std::runtime_error("Unable to open file " + filePath.string()); + } + + if (archiveIdFile.is_open()) { + for (const auto archiveId : m_archiveIdsNotUpdatedInEos) { + archiveIdFile << archiveId << std::endl; + } + archiveIdFile.close(); + std::cout << m_archiveIdsNotUpdatedInEos.size() << " files did not update the storage class." << std::endl; + std::cout << "The skipped archive ids can be found here: " << filePath << std::endl; + } +} + +//------------------------------------------------------------------------------ +// exceptionThrowingMain +//------------------------------------------------------------------------------ +int ChangeStorageClass::exceptionThrowingMain(const int argc, char *const *const argv) { + storageClassExists(); + updateStorageClassInEosNamespace(); + updateStorageClassInCatalogue(); + writeSkippedArchiveIdsToFile(); + return 0; +} + + +} // namespace admin +} // namespace cta diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp new file mode 100644 index 0000000000000000000000000000000000000000..87a5b6187ad73a831b9d9accf4931be5dcf8e5f6 --- /dev/null +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp @@ -0,0 +1,138 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#pragma once + +#include "cmdline/standalone_cli_tools/common/CmdLineTool.hpp" +#include "eos_grpc_client/GrpcEndpoint.hpp" + +#include <optional> +#include <memory> +#include <vector> + +#include "CtaFrontendApi.hpp" + +namespace cta { + +namespace log { +class StdoutLogger; +} + +namespace cliTool { + +class ChangeStorageClass: public CmdLineTool { +public: + /** + * Constructor. + * + * @param inStream Standard input stream. + * @param outStream Standard output stream. + * @param errStream Standard error stream. + * @param log The object representing the API of the CTA logging system. + * @param argc + * @param argv + */ + ChangeStorageClass( + std::istream &inStream, + std::ostream &outStream, + std::ostream &errStream, + cta::log::StdoutLogger &log, + const int& argc, + char *const *const argv); + +private: + + /** + * The object representing the API of the CTA logging system. + */ + cta::log::StdoutLogger &m_log; + + /** + * Archive file ids of the files to change + */ + std::vector<std::string> m_archiveFileIds; + + /** + * Archive file id of the files to change + */ + std::string m_storageClassName; + + /** + * CTA Frontend service provider + */ + std::unique_ptr<XrdSsiPbServiceType> m_serviceProviderPtr; + + /** + * Endpoint map for interaction with EOS + */ + std::unique_ptr<::eos::client::EndpointMap> m_endpointMapPtr; + + /** + * Endpoint map for interaction with EOS + */ + std::vector<std::string> m_archiveIdsNotUpdatedInEos; + + /** + * Endpoint map for interaction with EOS + */ + std::vector<std::string> m_archiveIdsUpdatedInEos; + + /** + * Endpoint map for interaction with EOS + */ + uint64_t m_eosUpdateFrequency; + + /** + * Updates the storage class name in the EOS namespace + */ + void updateStorageClassInEosNamespace(); + + /** + * Checks if the storage class provided to the tool is defined, + * and throws an exception::UserError if it is not found + */ + void storageClassExists(); + + /** + * Updates the storage class name in the catalogue + */ + void updateStorageClassInCatalogue(); + + /** + * Writes the skipped archive ids to file + */ + void writeSkippedArchiveIdsToFile(); + + /** + * Checks if a file is in flight + */ + bool fileInFlight(const google::protobuf::RepeatedField<unsigned int> &locations); + + /** + * An exception throwing version of main(). + * + * @param argc The number of command-line arguments including the program name. + * @param argv The command-line arguments. + * @return The exit value of the program. + */ + int exceptionThrowingMain(const int argc, char *const *const argv) override; + + +} ; // class CtaChangeStorageClass + +} // namespace admin +} // namespace cta diff --git a/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClassMain.cpp b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClassMain.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5175ab2e58891f811d4aa74395d46903ca208062 --- /dev/null +++ b/cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClassMain.cpp @@ -0,0 +1,43 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + + +#include <sstream> +#include <iostream> + + +#include "cmdline/standalone_cli_tools/change_storage_class/ChangeStorageClass.hpp" +#include "common/utils/utils.hpp" +#include "common/log/StdoutLogger.hpp" + +//------------------------------------------------------------------------------ +// main +//------------------------------------------------------------------------------ +int main(const int argc, char *const *const argv) { + std::string hostName = std::getenv("HOSTNAME"); + if(hostName.empty()) { + hostName = "UNKNOWN"; + } + + cta::log::StdoutLogger log(hostName, "cta-change-storage-class"); + + cta::cliTool::ChangeStorageClass cmd(std::cin, std::cout, std::cerr, log, argc, argv); + int ret = cmd.main(argc, argv); + // Delete all global objects allocated by libprotobuf + google::protobuf::ShutdownProtobufLibrary(); + return ret; +} diff --git a/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class.1cta b/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class.1cta new file mode 100644 index 0000000000000000000000000000000000000000..b442be2366a7bda7e2bd903d98684172ecbad6c7 --- /dev/null +++ b/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class.1cta @@ -0,0 +1,61 @@ +.\" @project The CERN Tape Archive (CTA) +.\" @copyright Copyright © 2019-2022 CERN +.\" @license This program is free software, distributed under the terms of the GNU General Public +.\" Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can +.\" redistribute it and/or modify it under the terms of the GPL Version 3, or (at your +.\" option) any later version. +.\" +.\" This program is distributed in the hope that it will be useful, but WITHOUT ANY +.\" WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +.\" PARTICULAR PURPOSE. See the GNU General Public License for more details. +.\" +.\" In applying this licence, CERN does not waive the privileges and immunities +.\" granted to it by virtue of its status as an Intergovernmental Organization or +.\" submit itself to any jurisdiction. + +.TH CTA-CHANGE-STORAGE-CLASS "1cta" "2022-25-10" "CTA" "The CERN Tape Archive (CTA)" +.SH NAME +cta-change-storage-class \- Change which storageclass files with a given archive id use + +.SH SYNOPSIS +.HP +\fBcta-change-storage-class\fP --id/-I <archiveFileID> | --filename/-F <filename> --storage.class.name/-n <storageClassName> + +.SH DESCRIPTION +\fBcta-change-storage-class\fP changes the storage class for the files specified by the user. The tool will first change +the storage class on the EOS side, and then change the storage class for the files that were successfully changed in EOS +in the catalogue + +.SH OPTIONS +.TP +\-I, \-\-id \fIarchive_file_id\fR +Archive file id of the files to change. +.TP +\-F, \-\-filename \fIfile_name\fR +Path to a file with a archive file ids. The text file should have one id on each line. +.TP +\-n, \-\-storage.class.name \fIstorageclassname\fR +Path to a file with a archive file ids. The text file should have one id on each line. +.TP +\-t, \-\-frequenzy \feosRequestFrequency\fR +After the eosRequestFrequency amount of requests to EOS the tool will sleep for 1 second. + +.SH EXIT STATUS +.P +\fBcta-change-storage-class\fP returns 0 on success. + +.SH EXAMPLE +.P +cta-change-storage-class --filename ~/idfile.txt --storage.class.name newStorageClassName + +.SH SEE ALSO +.P +CERN Tape Archive documentation (\fIhttps://eoscta.docs.cern.ch/\fR) + +.SH COPYRIGHT +.P +Copyright © 2022 CERN. License GPLv3+: GNU GPL version 3 or later (\fIhttp://gnu.org/licenses/gpl.html\fR). +This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the +extent permitted by law. In applying this licence, CERN does not waive the privileges and immunities +granted to it by virtue of its status as an Intergovernmental Organization or submit itself to any +jurisdiction. diff --git a/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f9c87a8b22a57d9c5e551b07e08d1898a5b93b30 --- /dev/null +++ b/cmdline/standalone_cli_tools/common/CatalogueFetch.cpp @@ -0,0 +1,166 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#include <list> +#include <iostream> +#include <string> +#include <utility> + +#include <XrdSsiPbLog.hpp> +#include <XrdSsiPbIStreamBuffer.hpp> + +#include "cmdline/standalone_cli_tools/common/CatalogueFetch.hpp" +#include "common/exception/UserError.hpp" +#include "common/log/StdoutLogger.hpp" +#include "cta_frontend.pb.h" //!< Auto-generated message types from .proto file + +#include "version.h" + +// GLOBAL VARIABLES : used to pass information between main thread and stream handler thread + +// global synchronisation flag +std::atomic<bool> isHeaderSent = false; + +std::list<cta::admin::RecycleTapeFileLsItem> deletedTapeFiles; +std::list<std::pair<std::string,std::string>> listedTapeFiles; +std::list<std::string> g_storageClasses; + +namespace XrdSsiPb { + +/*! + * Alert callback. + * + * Defines how Alert messages should be logged + */ +template<> +void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) +{ + Log::DumpProtobuf(Log::PROTOBUF, &alert); +} + +/*! + * Data/Stream callback. + * + * Defines how incoming records from the stream should be handled + */ +template<> +void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const +{ + using namespace cta::xrd; + using namespace cta::admin; + + // Wait for primary response to be handled before allowing stream response + while(!isHeaderSent) { std::this_thread::yield(); } + + switch(record.data_case()) { + case Data::kRtflsItem: + { + auto item = record.rtfls_item(); + deletedTapeFiles.push_back(item); + } + break; + case Data::kTflsItem: + { + auto item = record.tfls_item(); + auto instanceAndFid = std::make_pair(item.df().disk_instance(), item.df().disk_id()); + listedTapeFiles.push_back(instanceAndFid); + } + break; + case Data::kSclsItem: + { + const auto item = record.scls_item(); + g_storageClasses.push_back(item.name()); + } + break; + default: + throw std::runtime_error("Received invalid stream data from CTA Frontend for the cta-restore-deleted-files command."); + } +} + +} // namespace XrdSsiPb + +namespace cta { +namespace cliTool { + + /** + * Fetches the instance and fid from the CTA catalogue + * + * @param archiveFileId The arhive file id. + * @param serviceProviderPtr Service provider for communication with the catalogue. + * @return a pair with the instance and the fid. + */ +std::tuple<std::string,std::string> CatalogueFetch::getInstanceAndFid(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log) { + { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("archiveFileId", archiveFileId)); + log(cta::log::DEBUG, "getInstanceAndFidFromCTA() ", params); + } + + cta::xrd::Request request; + auto admincmd = request.mutable_admincmd(); + + request.set_client_cta_version(CTA_VERSION); + request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); + admincmd->set_cmd(cta::admin::AdminCmd::CMD_TAPEFILE); + admincmd->set_subcmd(cta::admin::AdminCmd::SUBCMD_LS); + auto new_opt = admincmd->add_option_uint64(); + new_opt->set_key(cta::admin::OptionUInt64::ARCHIVE_FILE_ID); + new_opt->set_value(std::stoi(archiveFileId)); + + handleResponse(request, serviceProviderPtr); + + if(listedTapeFiles.size() != 1) { + throw std::runtime_error("Unexpected result set: listedTapeFiles size expected=1 received=" + std::to_string(listedTapeFiles.size())); + } + auto listedTapeFile = listedTapeFiles.back(); + listedTapeFiles.clear(); + { + std::list<cta::log::Param> params; + params.push_back(cta::log::Param("diskInstance", listedTapeFile.first)); + params.push_back(cta::log::Param("diskFileId", listedTapeFile.second)); + log(cta::log::DEBUG, "Obtained file metadata from CTA", params); + } + return listedTapeFile; +} + +void CatalogueFetch::handleResponse(const cta::xrd::Request &request, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr) { + // Send the Request to the Service and get a Response + cta::xrd::Response response; + auto stream_future = serviceProviderPtr->SendAsync(request, response); + + // Handle responses + switch(response.type()) { + using namespace cta::xrd; + using namespace cta::admin; + case Response::RSP_SUCCESS: + // Print message text + std::cout << response.message_txt(); + // Allow stream processing to commence + isHeaderSent = true; + break; + case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); + case Response::RSP_ERR_USER: throw exception::UserError(response.message_txt()); + case Response::RSP_ERR_CTA: throw std::runtime_error(response.message_txt()); + default: throw XrdSsiPb::PbException("Invalid response type."); + } + + // wait until the data stream has been processed before exiting + stream_future.wait(); +} + +} // cliTool +} // cta \ No newline at end of file diff --git a/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp new file mode 100644 index 0000000000000000000000000000000000000000..bfd7d6fdd2513ac5e94559fdfec8813eb979fcc8 --- /dev/null +++ b/cmdline/standalone_cli_tools/common/CatalogueFetch.hpp @@ -0,0 +1,54 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#pragma once + +#include <atomic> +#include <list> +#include <string> +#include <utility> + +#include <XrdSsiPbLog.hpp> +#include <XrdSsiPbIStreamBuffer.hpp> + +#include "common/log/StdoutLogger.hpp" +#include "xrootd-ssi-protobuf-interface/eos_cta/include/CtaFrontendApi.hpp" +#include "cta_frontend.pb.h" //!< Auto-generated message types from .proto file + +#include "version.h" + +namespace cta { +namespace cliTool { + + +class CatalogueFetch { +public: + /** + * Fetches the instance and fid from the CTA catalogue + * + * @param archiveFileId The arhive file id. + * @param serviceProviderPtr Service provider for communication with the catalogue. + * @return a pair with the instance and the fid. + */ + static std::tuple<std::string,std::string> getInstanceAndFid(const std::string& archiveFileId, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr, cta::log::StdoutLogger &log); + +private: + static void handleResponse(const cta::xrd::Request &request, std::unique_ptr<XrdSsiPbServiceType> &serviceProviderPtr); +}; + +} // cliTool +} // cta \ No newline at end of file diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp index 62149e2b44349fce292be96524f43af92a2e3e35..22a036c30bd286532f23ec4b79ced762a0f1f18c 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.cpp @@ -35,7 +35,7 @@ static struct option restoreFilesLongOption[] = { {"id", required_argument, nullptr, 'I'}, {"instance", required_argument, nullptr, 'i'}, {"fxid", required_argument, nullptr, 'f'}, - {"fxidfile", required_argument, nullptr, 'F'}, + {"filename", required_argument, nullptr, 'F'}, {"vid", required_argument, nullptr, 'v'}, {"copynb", required_argument, nullptr, 'c'}, {"help", no_argument, nullptr, 'h'}, @@ -61,16 +61,27 @@ static struct option verifyFileLongOption[] = { {nullptr, 0, nullptr, 0} }; +static struct option changeStorageClassLongOption[] = { + {"id", required_argument, nullptr, 'I'}, + {"filename", required_argument, nullptr, 'F'}, + {"storage.class.name", required_argument, nullptr, 'n'}, + {"frequenzy", required_argument, nullptr, 't'}, + {"help", no_argument, nullptr, 'h'}, + {nullptr, 0, nullptr, 0} +}; + std::map<StandaloneCliTool, const option*> longopts = { {StandaloneCliTool::RESTORE_FILES, restoreFilesLongOption}, {StandaloneCliTool::CTA_SEND_EVENT, sendFileLongOption}, {StandaloneCliTool::CTA_VERIFY_FILE, verifyFileLongOption}, + {StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS, changeStorageClassLongOption}, }; std::map<StandaloneCliTool, const char*> shortopts = { {StandaloneCliTool::RESTORE_FILES, "I:i:f:F:v:c:hd:"}, {StandaloneCliTool::CTA_SEND_EVENT, "i:e:u:g:"}, {StandaloneCliTool::CTA_VERIFY_FILE, "I:i:u:g:v:h:"}, + {StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS, "I:F:n:t:h:"}, }; //------------------------------------------------------------------------------ @@ -85,46 +96,39 @@ m_help(false), m_debug(false), m_standaloneCliTool{standaloneCliTool} { while ((opt = getopt_long(argc, argv, shortopts[m_standaloneCliTool], longopts[m_standaloneCliTool], &opt_index)) != -1) { switch(opt) { - case 'I': + case 'c': { - m_archiveFileId = std::stol(std::string(optarg)); + int64_t copyNumber = std::stol(std::string(optarg)); + if(copyNumber < 0) throw std::out_of_range("copy number value cannot be negative"); + m_copyNumber = copyNumber; break; } - case 'i': + case 'd': { - m_diskInstance = std::string(optarg); + m_debug = true; break; } - case 'f': + case 'e': { - if (! m_eosFids) { - m_eosFids = std::list<uint64_t>(); - } - auto fid = strtoul(optarg, nullptr, 16); - if(fid < 1) { - throw std::runtime_error(std::string(optarg) + " is not a valid file ID"); - } - m_eosFids->push_back(fid); + m_eosEndpoint = std::string(optarg); break; } - case 'F': + case 'f': { - if (! m_eosFids) { - m_eosFids = std::list<uint64_t>(); + if (! m_fxIds) { + m_fxIds = std::list<std::string>(); } - readFidListFromFile(std::string(optarg), m_eosFids.value()); + m_fxIds->push_back(optarg); break; } - case 'v': + case 'F': { - m_vid = std::string(optarg); + readIdListFromFile(std::string(optarg)); break; } - case 'c': + case 'g': { - int64_t copyNumber = std::stol(std::string(optarg)); - if(copyNumber < 0) throw std::out_of_range("copy number value cannot be negative"); - m_copyNumber = copyNumber; + m_requestGroup = std::string(optarg); break; } case 'h': @@ -132,14 +136,24 @@ m_help(false), m_debug(false), m_standaloneCliTool{standaloneCliTool} { m_help = true; break; } - case 'd': + case 'I': { - m_debug = true; + m_archiveFileId = std::string(optarg); break; } - case 'e': + case 'i': { - m_eosEndpoint = std::string(optarg); + m_diskInstance = std::string(optarg); + break; + } + case 'n': + { + m_storageClassName = std::string(optarg); + break; + } + case 't': + { + m_frequency = std::stol(std::string(optarg)); break; } case 'u': @@ -147,9 +161,9 @@ m_help(false), m_debug(false), m_standaloneCliTool{standaloneCliTool} { m_requestUser = std::string(optarg); break; } - case 'g': + case 'v': { - m_requestGroup = std::string(optarg); + m_vid = std::string(optarg); break; } case ':': // Missing parameter @@ -162,7 +176,7 @@ m_help(false), m_debug(false), m_standaloneCliTool{standaloneCliTool} { { exception::CommandLineNotParsed ex; if(0 == optopt) { - ex.getMessage() << "Unknown command-line option"; + ex.getMessage() << "Unknown command-line option" << std::endl; } else { ex.getMessage() << "Unknown command-line option: -" << static_cast<char>(optopt) << std::endl; } @@ -184,7 +198,7 @@ m_help(false), m_debug(false), m_standaloneCliTool{standaloneCliTool} { //------------------------------------------------------------------------------ // readFidListFromFile //------------------------------------------------------------------------------ -void CmdLineArgs::readFidListFromFile(const std::string &filename, std::list<std::uint64_t> &fidList) { +void CmdLineArgs::readIdListFromFile(const std::string &filename) { std::ifstream file(filename); if (file.fail()) { throw std::runtime_error("Unable to open file " + filename); @@ -192,35 +206,23 @@ void CmdLineArgs::readFidListFromFile(const std::string &filename, std::list<std std::string line; - while(std::getline(file, line)) { - // Strip out comments - auto pos = line.find('#'); - if(pos != std::string::npos) { - line.resize(pos); - } - - // Extract the list items - std::stringstream ss(line); - while(!ss.eof()) { - std::string item; - ss >> item; - // skip blank lines or lines consisting only of whitespace - if(item.empty()) continue; - - // Special handling for file id lists. The output from "eos find --fid <fid> /path" is: - // path=/path fid=<fid> - // We discard everything except the list of fids. <fid> is a zero-padded hexadecimal number, - // but in the CTA catalogue we store disk IDs as a decimal string, so we need to convert it. - if(item.substr(0, 4) == "fid=") { - auto fid = strtol(item.substr(4).c_str(), nullptr, 16); - if(fid < 1 || fid == LONG_MAX) { - throw std::runtime_error(item + " is not a valid file ID"); + while(file >> line) { + switch (m_standaloneCliTool) { + case StandaloneCliTool::RESTORE_FILES: + m_archiveFileIds.value().push_back(line); + break; + case StandaloneCliTool::CTA_VERIFY_FILE: + m_fxIds.value().push_back(line); + break; + case StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS: + if (!m_archiveFileIds) { + m_archiveFileIds = std::list<std::string>(); } - fidList.push_back(fid); - } else { - continue; + m_archiveFileIds.value().push_back(line); + break; + default: + break; } - } } } @@ -229,21 +231,25 @@ void CmdLineArgs::readFidListFromFile(const std::string &filename, std::list<std //------------------------------------------------------------------------------ void CmdLineArgs::printUsage(std::ostream &os) const { switch (m_standaloneCliTool) { - case StandaloneCliTool::RESTORE_FILES: + case StandaloneCliTool::RESTORE_FILES: os << " Usage:" << std::endl << " cta-restore-deleted-files [--id/-I <archive_file_id>] [--instance/-i <disk_instance>]" << std::endl << " [--fxid/-f <eos_fxid>] [--fxidfile/-F <filename>]" << std::endl << - " [--vid/-v <vid>] [--copynb/-c <copy_number>] [--debug/-d]" << std::endl; + " [--vid/-v <vid>] [--copynb/-c <copy_number>] [--debug/-d]" << std::endl; break; case StandaloneCliTool::CTA_SEND_EVENT: os << " Usage:" << std::endl << " eos --json fileinfo /eos/path | cta-send-event CLOSEW|PREPARE " << std::endl << - " -i/--eos.instance <instance> [-e/--eos.endpoint <url>]" << std::endl << + " -i/--eos.instance <instance> [-e/--eos.endpoint <url>]" << std::endl << " -u/--request.user <user> -g/--request.group <group>" << std::endl; break; case StandaloneCliTool::CTA_VERIFY_FILE : os << " Usage:" << std::endl << - " cta-verify-file --id/-I <archiveFileID> --vid/-v <vid> [--instance/-i <instance>] [--request.user/-u <user>] [request.group/-g <group>]\n" << std::endl; + " cta-verify-file --id/-I <archiveFileID> --vid/-v <vid> [--instance/-i <instance>] [--request.user/-u <user>] [request.group/-g <group>]" << std::endl; + break; + case StandaloneCliTool::CTA_CHANGE_STORAGE_CLASS : + os << " Usage:" << std::endl << + " cta-change-storage-class --id/-I <archiveFileID> | --filename/-F <filename> --storage.class.name/-n <storageClassName> [--frequenzy/-t <eosRequestFrequency>]" << std::endl; break; default: break; diff --git a/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp b/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp index a90bf727b1998c8ca4ba23d790f34fe35ed2ce5c..5602f1dada07202d61fee652ffedf05c087f7d47 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp +++ b/cmdline/standalone_cli_tools/common/CmdLineArgs.hpp @@ -32,7 +32,8 @@ namespace cliTool { enum class StandaloneCliTool { RESTORE_FILES, CTA_SEND_EVENT, - CTA_VERIFY_FILE + CTA_VERIFY_FILE, + CTA_CHANGE_STORAGE_CLASS }; /** @@ -53,22 +54,22 @@ struct CmdLineArgs { /** * Archive file id of the files to restore */ - std::optional<uint64_t> m_archiveFileId; + std::optional<std::string> m_archiveFileId; /** - * Disk instance of the files to restore + * Archive file id of the files to restore */ - std::optional<std::string> m_diskInstance; + std::optional<std::list<std::string>> m_archiveFileIds; /** - * Fids of the files to restore + * Disk instance of the files to restore */ - std::optional<std::list<uint64_t>> m_eosFids; + std::optional<std::string> m_diskInstance; - /** - * Fids of the files to restore + /** + * Fxids of the files to restore */ - std::optional<std::string> m_eosInstance; + std::optional<std::list<std::string>> m_fxIds; /** * Vid of the tape of the files to restore @@ -81,25 +82,35 @@ struct CmdLineArgs { std::optional<uint64_t> m_copyNumber; /** - * Eos endpoint + * Eos endpoint */ std::optional<std::string> m_eosEndpoint; /** * Request user - */ + */ std::optional<std::string> m_requestUser; /** * Request user - */ + */ std::optional<std::string> m_requestGroup; /** * The tool parsing the arguments - */ + */ StandaloneCliTool m_standaloneCliTool; + /** + * The tool parsing the arguments + */ + std::optional<std::string> m_storageClassName; + + /** + * Frequency + */ + std::optional<uint64_t> m_frequency; + /** * Constructor that parses the specified command-line arguments. * @@ -116,7 +127,7 @@ struct CmdLineArgs { * @param filename The name of the file to read * @param fidList The list of file IDs */ - void readFidListFromFile(const std::string &filename, std::list<uint64_t> &fidList); + void readIdListFromFile(const std::string &filename); /** * Prints the usage message of the command-line tool. diff --git a/cmdline/standalone_cli_tools/common/CmdLineTool.cpp b/cmdline/standalone_cli_tools/common/CmdLineTool.cpp index 5ae584bf85b878f49d8dd069bfa52704dba6bb56..259e3125397cc3d62dee3b1c1d9c85d69863adb9 100644 --- a/cmdline/standalone_cli_tools/common/CmdLineTool.cpp +++ b/cmdline/standalone_cli_tools/common/CmdLineTool.cpp @@ -17,6 +17,7 @@ #include "cmdline/standalone_cli_tools/common/CmdLineTool.hpp" #include "common/exception/CommandLineNotParsed.hpp" +#include "common/exception/UserError.hpp" #include <iostream> #include <string> @@ -80,6 +81,8 @@ int CmdLineTool::main(const int argc, char *const *const argv) { return exceptionThrowingMain(argc, argv); } catch(exception::CommandLineNotParsed &ue) { errorMessage = ue.getMessage().str(); + } catch(exception::UserError &ue) { + errorMessage = ue.getMessage().str(); } catch(exception::Exception &ex) { errorMessage = ex.getMessage().str(); } catch(std::exception &se) { @@ -87,7 +90,7 @@ int CmdLineTool::main(const int argc, char *const *const argv) { } catch(...) { errorMessage = "An unknown exception was thrown"; } - + std::cout << errorMessage << std::endl; return 1; } diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp new file mode 100644 index 0000000000000000000000000000000000000000..da6cda335ed81effda6a01b4010709543b300d3b --- /dev/null +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.cpp @@ -0,0 +1,122 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#include "cmdline/standalone_cli_tools/common/CmdLineArgs.hpp" +#include "common/log/StdoutLogger.hpp" +#include "ConnectionConfiguration.hpp" + +#include <string> +#include <utility> + +namespace cta { +namespace cliTool { + +std::unique_ptr<::eos::client::EndpointMap> ConnConfiguration::setNamespaceMap(const std::string &keytab_file) { + // Open the keytab file for reading + std::ifstream file(keytab_file); + if(!file) { + throw cta::exception::UserError("Failed to open namespace keytab configuration file " + keytab_file); + } + ::eos::client::NamespaceMap_t namespaceMap; + // Parse the keytab line by line + std::string line; + for(int lineno = 1; std::getline(file, line); ++lineno) { + // Strip out comments + auto pos = line.find('#'); + if(pos != std::string::npos) { + line.resize(pos); + } + + // Parse one line + std::stringstream ss(line); + std::string diskInstance; + std::string endpoint; + std::string token; + std::string eol; + ss >> diskInstance >> endpoint >> token >> eol; + + // Ignore blank lines, all other lines must have exactly 3 elements + if(token.empty() || !eol.empty()) { + if(diskInstance.empty() && endpoint.empty() && token.empty()) continue; + throw cta::exception::UserError("Could not parse namespace keytab configuration file line " + std::to_string(lineno) + ": " + line); + } + namespaceMap.insert(std::make_pair(diskInstance, ::eos::client::Namespace(endpoint, token))); + } + return std::make_unique<::eos::client::EndpointMap>(namespaceMap); +} + +//------------------------------------------------------------------------------ +// readAndSetConfiguration +//------------------------------------------------------------------------------ +std::tuple<std::unique_ptr<XrdSsiPbServiceType>, std::unique_ptr<::eos::client::EndpointMap>> ConnConfiguration::readAndSetConfiguration( + cta::log::StdoutLogger &log, + const std::string &userName, + const CmdLineArgs &cmdLineArgs) { + + const std::string StreamBufferSize = "1024"; //!< Buffer size for Data/Stream Responses + const std::string DefaultRequestTimeout = "10"; //!< Default Request Timeout. Can be overridden by + //!< XRD_REQUESTTIMEOUT environment variable. + if (cmdLineArgs.m_debug) { + log.setLogMask("DEBUG"); + } else { + log.setLogMask("INFO"); + } + + // Set CTA frontend configuration options + const std::string cli_config_file = "/etc/cta/cta-cli.conf"; + XrdSsiPb::Config cliConfig(cli_config_file, "cta"); + cliConfig.set("resource", "/ctafrontend"); + cliConfig.set("response_bufsize", StreamBufferSize); // default value = 1024 bytes + cliConfig.set("request_timeout", DefaultRequestTimeout); // default value = 10s + + // Allow environment variables to override config file + cliConfig.getEnv("request_timeout", "XRD_REQUESTTIMEOUT"); + + // If XRDDEBUG=1, switch on all logging + if(getenv("XRDDEBUG")) { + cliConfig.set("log", "all"); + } + // If fine-grained control over log level is required, use XrdSsiPbLogLevel + cliConfig.getEnv("log", "XrdSsiPbLogLevel"); + + // Validate that endpoint was specified in the config file + if(!cliConfig.getOptionValueStr("endpoint").first) { + throw std::runtime_error("Configuration error: cta.endpoint missing from " + cli_config_file); + } + + // If the server is down, we want an immediate failure. Set client retry to a single attempt. + XrdSsiProviderClient->SetTimeout(XrdSsiProvider::connect_N, 1); + + std::unique_ptr<XrdSsiPbServiceType> serviceProviderPtr = std::make_unique<XrdSsiPbServiceType>(cliConfig); + + // Set CTA frontend configuration options to connect to eos + const std::string frontend_xrootd_config_file = "/etc/cta/cta-frontend-xrootd.conf"; + XrdSsiPb::Config frontendXrootdConfig(frontend_xrootd_config_file, "cta"); + + // Get the endpoint for namespace queries + auto nsConf = frontendXrootdConfig.getOptionValueStr("ns.config"); + if(nsConf.first) { + auto endpointMap = setNamespaceMap(nsConf.second); + std::pair<std::unique_ptr<XrdSsiPbServiceType>, std::unique_ptr<::eos::client::EndpointMap>> serviceProviderPtrAndEndpointMap = std::make_pair(std::move(serviceProviderPtr), std::move(endpointMap)); + return serviceProviderPtrAndEndpointMap; + } else { + throw std::runtime_error("Configuration error: cta.ns.config missing from " + frontend_xrootd_config_file); + } +} + +} // namespace cliTool +} // namespace cta diff --git a/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp new file mode 100644 index 0000000000000000000000000000000000000000..d81d9116265d74e87946f9f3b7f9b0d9a41d4981 --- /dev/null +++ b/cmdline/standalone_cli_tools/common/ConnectionConfiguration.hpp @@ -0,0 +1,60 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#pragma once + +#include "eos_grpc_client/GrpcEndpoint.hpp" +#include "CtaFrontendApi.hpp" + +#include <string> +#include <tuple> + +namespace cta { + +namespace log { +class StdoutLogger; +} + +namespace cliTool { + +class CmdLineArgs; + +class ConnConfiguration { +public: + + /** + * Sets internal configuration parameters to be used for reading. + * It reads cta frontend parameters from /etc/cta/cta-cli.conf + * + * @param username The name of the user running the command-line tool. + * @param cmdLineArgs The arguments parsed from the command line. + * @param serviceProviderPtr Xroot service provider + * @param endpointMapPtr Endpoint for communication with EOS + */ + using XrdSsiPbServiceTypePtr = std::unique_ptr<XrdSsiPbServiceType>; + using EndpointMapPtr = std::unique_ptr<::eos::client::EndpointMap>; + static std::tuple<XrdSsiPbServiceTypePtr, EndpointMapPtr> readAndSetConfiguration( + cta::log::StdoutLogger &log, + const std::string &userName, + const CmdLineArgs &cmdLineArgs); + +private: + static EndpointMapPtr setNamespaceMap(const std::string &keytab_file); +}; + +} // namespace cliTool +} // namespace cta \ No newline at end of file diff --git a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp index c50b323dab4de34d6f4fc8193f5a91d4a9d1db44..100e06aec469d14f4b178e4c8f8c6c0693186fee 100644 --- a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp +++ b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.cpp @@ -16,6 +16,7 @@ */ #include "cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.hpp" +#include "cmdline/standalone_cli_tools/common/CatalogueFetch.hpp" #include "cmdline/CtaAdminCmdParse.hpp" #include "common/utils/utils.hpp" #include "common/checksum/ChecksumBlob.hpp" @@ -34,13 +35,11 @@ // GLOBAL VARIABLES : used to pass information between main thread and stream handler thread // global synchronisation flag -std::atomic<bool> isHeaderSent(false); - +std::atomic<bool> isHeaderSent; std::list<cta::admin::RecycleTapeFileLsItem> deletedTapeFiles; std::list<std::pair<std::string,std::string>> listedTapeFiles; namespace XrdSsiPb { - /*! * User error exception */ @@ -58,7 +57,6 @@ public: template<> void RequestCallback<cta::xrd::Alert>::operator()(const cta::xrd::Alert &alert) { - std::cout << "AlertCallback():" << std::endl; Log::DumpProtobuf(Log::PROTOBUF, &alert); } @@ -83,20 +81,12 @@ void IStreamBuffer<cta::xrd::Data>::DataCallback(cta::xrd::Data record) const deletedTapeFiles.push_back(item); } break; - case Data::kTflsItem: - { - auto item = record.tfls_item(); - auto instanceAndFid = std::make_pair(item.df().disk_instance(), item.df().disk_id()); - listedTapeFiles.push_back(instanceAndFid); - } - break; default: throw std::runtime_error("Received invalid stream data from CTA Frontend for the cta-restore-deleted-files command."); } } -} // namespace XrdSsiPb - +} namespace cta{ namespace cliTool { @@ -129,7 +119,6 @@ RestoreFilesCmd::RestoreFilesCmd(std::istream &inStream, std::ostream &outStream // Default single replica layout id should be 00100012 m_defaultFileLayout = kReplica | kAdler | kStripeSize | kStripeWidth | kBlockChecksum; - } //------------------------------------------------------------------------------ @@ -155,7 +144,7 @@ int RestoreFilesCmd::exceptionThrowingMain(const int argc, char *const *const ar try { if (!fileExistsEos(file.disk_instance(), file.disk_file_id())) { uint64_t new_fid = restoreDeletedFileEos(file); - file.set_disk_file_id(std::to_string(new_fid)); + file.set_disk_file_id(std::to_string(new_fid)); } // archive file exists in CTA, so only need to restore the file copy restoreDeletedFileCopyCta(file); @@ -189,12 +178,12 @@ int RestoreFilesCmd::exceptionThrowingMain(const int argc, char *const *const ar // readAndSetConfiguration //------------------------------------------------------------------------------ void RestoreFilesCmd::readAndSetConfiguration( - const std::string &userName, + const std::string &userName, const CmdLineArgs &cmdLineArgs) { - + m_vid = cmdLineArgs.m_vid; m_diskInstance = cmdLineArgs.m_diskInstance; - m_eosFids = cmdLineArgs.m_eosFids; + m_archiveFileIds = cmdLineArgs.m_archiveFileIds; m_copyNumber = cmdLineArgs.m_copyNumber; m_archiveFileId = cmdLineArgs.m_archiveFileId; @@ -284,11 +273,11 @@ void RestoreFilesCmd::setNamespaceMap(const std::string &keytab_file) { void RestoreFilesCmd::listDeletedFilesCta() const { std::list<cta::log::Param> params; params.push_back(cta::log::Param("userName", getUsername())); - + cta::xrd::Request request; auto &admincmd = *(request.mutable_admincmd()); - + request.set_client_cta_version(CTA_VERSION); request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); admincmd.set_cmd(cta::admin::AdminCmd::CMD_RECYCLETAPEFILE); @@ -313,7 +302,7 @@ void RestoreFilesCmd::listDeletedFilesCta() const { auto key = cta::admin::OptionUInt64::ARCHIVE_FILE_ID; auto new_opt = admincmd.add_option_uint64(); new_opt->set_key(key); - new_opt->set_value(m_archiveFileId.value()); + new_opt->set_value(std::stoi(m_archiveFileId.value())); } if (m_copyNumber) { params.push_back(cta::log::Param("copyNb", m_copyNumber.value())); @@ -322,13 +311,13 @@ void RestoreFilesCmd::listDeletedFilesCta() const { new_opt->set_key(key); new_opt->set_value(m_copyNumber.value()); } - if (m_eosFids) { + if (m_archiveFileIds) { std::stringstream ss; auto key = cta::admin::OptionStrList::FILE_ID; auto new_opt = admincmd.add_option_str_list(); new_opt->set_key(key); - for (auto &fid : m_eosFids.value()) { - new_opt->add_item(std::to_string(fid)); + for (auto &fid : m_archiveFileIds.value()) { + new_opt->add_item(fid); ss << fid << ","; } auto fids = ss.str(); @@ -336,7 +325,7 @@ void RestoreFilesCmd::listDeletedFilesCta() const { params.push_back(cta::log::Param("diskFileId", fids)); } - m_log(cta::log::INFO, "Listing deleted file in CTA catalogue", params); + m_log(cta::log::INFO, "Listing deleted file in CTA catalogue", params); // Send the Request to the Service and get a Response cta::xrd::Response response; @@ -363,7 +352,7 @@ void RestoreFilesCmd::listDeletedFilesCta() const { stream_future.wait(); params.push_back(cta::log::Param("nbFiles", deletedTapeFiles.size())); - m_log(cta::log::INFO, "Listed deleted file in CTA catalogue", params); + m_log(cta::log::INFO, "Listed deleted file in CTA catalogue", params); } //------------------------------------------------------------------------------ @@ -378,11 +367,11 @@ void RestoreFilesCmd::restoreDeletedFileCopyCta(const cta::admin::RecycleTapeFil params.push_back(cta::log::Param("archiveFileId", file.archive_file_id())); params.push_back(cta::log::Param("copyNb", file.copy_nb())); params.push_back(cta::log::Param("diskFileId", file.disk_file_id())); - + cta::xrd::Request request; auto &admincmd = *(request.mutable_admincmd()); - + request.set_client_cta_version(CTA_VERSION); request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); admincmd.set_cmd(cta::admin::AdminCmd::CMD_RECYCLETAPEFILE); @@ -428,7 +417,7 @@ void RestoreFilesCmd::restoreDeletedFileCopyCta(const cta::admin::RecycleTapeFil new_opt->set_key(key); new_opt->set_value(ss.str()); } - m_log(cta::log::DEBUG, "Restoring file copy in CTA catalogue", params); + m_log(cta::log::DEBUG, "Restoring file copy in CTA catalogue", params); // This validation will also be done at the server side cta::admin::validateCmd(admincmd); @@ -445,7 +434,7 @@ void RestoreFilesCmd::restoreDeletedFileCopyCta(const cta::admin::RecycleTapeFil case Response::RSP_SUCCESS: // Print message text std::cout << response.message_txt(); - m_log(cta::log::INFO, "Restored file copy in CTA catalogue", params); + m_log(cta::log::INFO, "Restored file copy in CTA catalogue", params); break; case Response::RSP_ERR_PROTOBUF: throw XrdSsiPb::PbException(response.message_txt()); case Response::RSP_ERR_USER: throw XrdSsiPb::UserException(response.message_txt()); @@ -479,7 +468,7 @@ uint64_t RestoreFilesCmd::addContainerEos(const std::string &diskInstance, const dir.set_path(path); dir.set_name(cta::utils::getEnclosedName(path)); - // Filemode: filter out S_ISUID, S_ISGID and S_ISVTX because EOS does not follow POSIX semantics for these bits + // Filemode: filter out S_ISUID, S_ISGID and S_ISVTX because EOS does not follow POSIX semantics for these bits uint64_t filemode = (S_IRWXU | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); // 0755 permissions by default filemode &= ~(S_ISUID | S_ISGID | S_ISVTX); dir.set_mode(filemode); @@ -496,7 +485,7 @@ uint64_t RestoreFilesCmd::addContainerEos(const std::string &diskInstance, const auto reply = m_endpointMapPtr->containerInsert(diskInstance, dir); m_log(cta::log::DEBUG, "Inserted container in EOS namespace successfully, querying again for its id", params); - + auto cont_id = containerExistsEos(diskInstance, path); if (!cont_id) { throw RestoreFilesCmdException(std::string("Container ") + path + " does not exist after being inserted in EOS."); @@ -514,7 +503,7 @@ uint64_t RestoreFilesCmd::containerExistsEos(const std::string &diskInstance, co params.push_back(cta::log::Param("path", path)); m_log(cta::log::DEBUG, "Verifying if the container exists in the EOS namespace", params); - + auto md_response = m_endpointMapPtr->getMD(diskInstance, ::eos::rpc::CONTAINER, 0, path, false); auto cid = md_response.cmd().id(); params.push_back(cta::log::Param("containerId", cid)); @@ -538,18 +527,18 @@ bool RestoreFilesCmd::archiveFileExistsCTA(const uint64_t &archiveFileId) const std::list<cta::log::Param> params; params.push_back(cta::log::Param("userName", getUsername())); params.push_back(cta::log::Param("archiveFileId", archiveFileId)); - - m_log(cta::log::DEBUG, "Looking for archive file in the CTA catalogue", params); + + m_log(cta::log::DEBUG, "Looking for archive file in the CTA catalogue", params); cta::xrd::Request request; auto &admincmd = *(request.mutable_admincmd()); - + request.set_client_cta_version(CTA_VERSION); request.set_client_xrootd_ssi_protobuf_interface_version(XROOTD_SSI_PROTOBUF_INTERFACE_VERSION); admincmd.set_cmd(cta::admin::AdminCmd::CMD_TAPEFILE); admincmd.set_subcmd(cta::admin::AdminCmd::SUBCMD_LS); - + auto key = cta::admin::OptionUInt64::ARCHIVE_FILE_ID; auto new_opt = admincmd.add_option_uint64(); new_opt->set_key(key); @@ -653,8 +642,8 @@ uint64_t RestoreFilesCmd::restoreDeletedFileEos(const cta::admin::RecycleTapeFil params.push_back(cta::log::Param("archiveFileId", rtfls_item.archive_file_id())); params.push_back(cta::log::Param("diskFileId", rtfls_item.disk_file_id())); params.push_back(cta::log::Param("diskFilePath", rtfls_item.disk_file_path())); - - m_log(cta::log::INFO, "Restoring file in the EOS namespace", params); + + m_log(cta::log::INFO, "Restoring file in the EOS namespace", params); getCurrentEosIds(rtfls_item.disk_instance()); uint64_t file_id = getFileIdEos(rtfls_item.disk_instance(), rtfls_item.disk_file_path()); @@ -664,9 +653,9 @@ uint64_t RestoreFilesCmd::restoreDeletedFileEos(const cta::admin::RecycleTapeFil ::eos::rpc::FileMdProto file; - auto fullPath = rtfls_item.disk_file_path(); + auto fullPath = rtfls_item.disk_file_path(); auto cont_id = addContainerEos(rtfls_item.disk_instance(), cta::utils::getEnclosingPath(fullPath), rtfls_item.storage_class()); - + // We do not set the file id. Since the file was deleted the fid cannot be reused, so EOS will generate a new file id file.set_cont_id(cont_id); file.set_uid(rtfls_item.disk_file_uid()); @@ -730,7 +719,7 @@ uint64_t RestoreFilesCmd::restoreDeletedFileEos(const cta::admin::RecycleTapeFil m_log(cta::log::INFO, "File successfully restored in the EOS namespace", params); m_log(cta::log::DEBUG, "Querying EOS for the new EOS file id", params); - + auto new_fid = getFileIdEos(rtfls_item.disk_instance(), rtfls_item.disk_file_path()); return new_fid; diff --git a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.hpp b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.hpp index bfa9a8acb41451ea948c432bcdfd4d96d8cf8a9b..e88a6a4019eb2e215460130beb3bb4b88f65c143 100644 --- a/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.hpp +++ b/cmdline/standalone_cli_tools/restore_files/RestoreFilesCmd.hpp @@ -174,7 +174,7 @@ private: /** * Archive file id of the files to restore */ - std::optional<uint64_t> m_archiveFileId; + std::optional<std::string> m_archiveFileId; /** * Disk instance of the files to restore @@ -184,7 +184,7 @@ private: /** * Fids of the files to restore */ - std::optional<std::list<uint64_t>> m_eosFids; + std::optional<std::list<std::string>> m_archiveFileIds; /** * Vid of the tape of the files to restore diff --git a/continuousintegration/orchestration/tests/changeStorageClass.sh b/continuousintegration/orchestration/tests/changeStorageClass.sh new file mode 100755 index 0000000000000000000000000000000000000000..217428feb848a6615043be8742a07b44eb97d5df --- /dev/null +++ b/continuousintegration/orchestration/tests/changeStorageClass.sh @@ -0,0 +1,149 @@ +#bin/bash + +usage() { cat <<EOF 1>&2 +Usage: $0 -n <namespace> +EOF +exit 1 +} + +while getopts "n:" o; do + case "${o}" in + n) + NAMESPACE=${OPTARG} + ;; + *) + usage + ;; + esac +done +shift $((OPTIND-1)) + +if [ -z "${NAMESPACE}" ]; then + usage +fi + +if [ ! -z "${error}" ]; then + echo -e "ERROR:\n${error}" + exit 1 +fi + +EOSINSTANCE=ctaeos +NEW_STORAGE_CLASS_NAME=newStorageClassName + +FILE_1=`uuidgen` +FILE_2=`uuidgen` +echo +echo "Creating files: ${FILE_1} ${FILE_2}" + +FRONTEND_IP=$(kubectl -n ${NAMESPACE} get pods ctafrontend -o json | jq .status.podIP | tr -d '"') + +echo +echo "ENABLE CTAFRONTEND TO EXECUTE CTA ADMIN COMMANDS" +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin admin add --username ctafrontend --comment "for restore files test" +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin admin add --username ctaeos --comment "for restore files test" + +echo +echo "ADD FRONTEND GATEWAY TO EOS" +echo "kubectl -n ${NAMESPACE} exec ctaeos -- bash eos root://${EOSINSTANCE} -r 0 0 vid add gateway ${FRONTEND_IP} grpc" +kubectl -n ${NAMESPACE} exec ctaeos -- eos -r 0 0 vid add gateway ${FRONTEND_IP} grpc + +echo +echo "ADD STORAGE CLASS WITH ONE COPIES ${NEW_STORAGE_CLASS_NAME}" +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc add --name ${NEW_STORAGE_CLASS_NAME} --numberofcopies 2 --virtualorganisation vo --comment "comment" +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin sc ls + +echo +echo "COPY REQUIRED FILES TO FRONTEND POD" +echo "sudo kubectl cp ${NAMESPACE}/ctacli:/etc/cta/cta-cli.conf /etc/cta/cta-cli.conf" +echo "sudo kubectl cp /etc/cta/cta-cli.conf ${NAMESPACE}/ctafrontend:/etc/cta/cta-cli.conf" +sudo kubectl cp ${NAMESPACE}/ctacli:/etc/cta/cta-cli.conf /etc/cta/cta-cli.conf +sudo kubectl cp /etc/cta/cta-cli.conf ${NAMESPACE}/ctafrontend:/etc/cta/cta-cli.conf + +kubectl -n ${NAMESPACE} cp common/archive_file.sh client:/usr/bin/ +kubectl -n ${NAMESPACE} cp client_helper.sh client:/root/ +kubectl -n ${NAMESPACE} exec client -- bash /usr/bin/archive_file.sh -f ${FILE_1} || exit 1 +kubectl -n ${NAMESPACE} exec client -- bash /usr/bin/archive_file.sh -f ${FILE_2} || exit 1 + +EOS_METADATA_PATH_1=$(mktemp -d).json +echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH}" +touch ${EOS_METADATA_PATH_1} +kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_1} | jq . | tee ${EOS_METADATA_PATH_1} +EOS_ARCHIVE_ID_1=$(jq -r '.xattr | .["sys.archive.file_id"]' ${EOS_METADATA_PATH_1}) + +EOS_METADATA_PATH_2=$(mktemp -d).json +echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH_2}" +touch ${EOS_METADATA_PATH_2} +kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_2} | jq . | tee ${EOS_METADATA_PATH_2} +EOS_ARCHIVE_ID_2=$(jq -r '.xattr | .["sys.archive.file_id"]' ${EOS_METADATA_PATH_2}) + +echo +echo "CHANGE FILES WITH IDS" +cd ~ +IDS_FILEPATH=~/archiveFileIds.txt +rm ${IDS_FILEPATH} +touch ${IDS_FILEPATH} +echo ${EOS_ARCHIVE_ID_1} | tee -a ${IDS_FILEPATH} +echo ${EOS_ARCHIVE_ID_2} | tee -a ${IDS_FILEPATH} + +echo +kubectl cp ~/CTA-build/cmdline/standalone_cli_tools/change_storage_class/cta-change-storage-class ${NAMESPACE}/ctafrontend:/usr/bin/ +echo "kubectl cp ${IDS_FILEPATH} ${NAMESPACE}/ctafrontend:~/" +kubectl cp ${IDS_FILEPATH} ${NAMESPACE}/ctafrontend:/root/ +echo "kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c XrdSecPROTOCOL=sss XrdSecSSSKT=/etc/cta/eos.sss.keytab cta-change-storage-class --storage.class.name ${NEW_STORAGE_CLASS_NAME} --filename ${IDS_FILEPATH}" +kubectl -n ${NAMESPACE} exec ctafrontend -- bash -c "XrdSecPROTOCOL=sss XrdSecSSSKT=/etc/cta/eos.sss.keytab cta-change-storage-class --storage.class.name ${NEW_STORAGE_CLASS_NAME} --filename ${IDS_FILEPATH} -t 1" + +EOS_METADATA_PATH_AFTER_CHANGE_1=$(mktemp -d).json +echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH_AFTER_CHANGE_1}" +touch ${EOS_METADATA_PATH_AFTER_CHANGE_1} +kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_1} | jq . | tee ${EOS_METADATA_PATH_AFTER_CHANGE_1} +EOS_STORAGE_CLASS_1=$(jq -r '.xattr | .["sys.archive.storage_class"]' ${EOS_METADATA_PATH_AFTER_CHANGE_1}) +rm -r ${EOS_METADATA_PATH_AFTER_CHANGE_1} + +EOS_METADATA_PATH_AFTER_CHANGE_2=$(mktemp -d).json +echo "SEND EOS METADATA TO JSON FILE: ${EOS_METADATA_PATH_AFTER_CHANGE_2}" +touch ${EOS_METADATA_PATH_AFTER_CHANGE_2} +kubectl -n ${NAMESPACE} exec client -- eos -j root://${EOSINSTANCE} file info /eos/ctaeos/cta/${FILE_2} | jq . | tee ${EOS_METADATA_PATH_AFTER_CHANGE_2} +EOS_STORAGE_CLASS_2=$(jq -r '.xattr | .["sys.archive.storage_class"]' ${EOS_METADATA_PATH_AFTER_CHANGE_2}) +rm -r ${EOS_METADATA_PATH_AFTER_CHANGE_2} + +CATALOGUE_METADATA_PATH_AFTER_CHANGE_1=$(mktemp -d).json +echo "SEND CATALOGUE METADATA TO JSON FILE: ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1}" +touch ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1} +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tf ls --id ${EOS_ARCHIVE_ID_1} | jq . | tee ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1} +CATALOGUE_STORAGE_CLASS_1=$(jq . ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1} | jq '.[0]' | jq -r '.af | .["storageClass"]') +rm -r ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_1} + +CATALOGUE_METADATA_PATH_AFTER_CHANGE_2=$(mktemp -d).json +echo "SEND CATALOGUE METADATA TO JSON FILE: ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2}" +touch ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2} +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tf ls --id ${EOS_ARCHIVE_ID_2} | jq . | tee ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2} +CATALOGUE_STORAGE_CLASS_2=$(jq . ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2} | jq '.[0]' | jq -r '.af | .["storageClass"]') +rm -r ${CATALOGUE_METADATA_PATH_AFTER_CHANGE_2} + +if test ${EOS_STORAGE_CLASS_1} != ${NEW_STORAGE_CLASS_NAME}; then + echo "ERROR: File ${FILE_1} did not change the storage class in EOS" + exit 1 +fi + +if test ${CATALOGUE_STORAGE_CLASS_1} != ${NEW_STORAGE_CLASS_NAME}; then + echo "ERROR: File ${FILE_1} did not change the storage class in the Catalogue" + exit 1 +fi + +if test ${EOS_STORAGE_CLASS_2} != ${NEW_STORAGE_CLASS_NAME}; then + echo "ERROR: File ${FILE_2} did not change the storage class in EOS" + exit 1 +fi + +if test ${CATALOGUE_STORAGE_CLASS_2} != ${NEW_STORAGE_CLASS_NAME}; then + echo "ERROR: File ${FILE_2} did not change the storage class in the Catalogue" + exit 1 +fi + +echo +echo "All tests passed" + +# Remove authorization +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin admin rm --username ctafrontend +kubectl -n ${NAMESPACE} exec ctacli -- cta-admin admin rm --username ctaeos + diff --git a/continuousintegration/orchestration/tests/common/archive_file.sh b/continuousintegration/orchestration/tests/common/archive_file.sh index bd9a88da5199527897f4725b7be7c7fe3d492cbb..54e067c4b2e44936eabbbc0b7444f1e663b83f04 100644 --- a/continuousintegration/orchestration/tests/common/archive_file.sh +++ b/continuousintegration/orchestration/tests/common/archive_file.sh @@ -23,9 +23,9 @@ EOF exit 1 } -while getopts "n:" o; do +while getopts "f:" o; do case "${o}" in - n) + f) TEST_FILE_NAME=${OPTARG} ;; *) diff --git a/continuousintegration/orchestration/tests/restore_files.sh b/continuousintegration/orchestration/tests/restore_files.sh index 3ab6378cf3342e5aa35d177a666245a13ef5ca0e..6802e994387b33ca034ddd700e4b0a3656a42cb5 100755 --- a/continuousintegration/orchestration/tests/restore_files.sh +++ b/continuousintegration/orchestration/tests/restore_files.sh @@ -53,7 +53,6 @@ echo echo "ADD FRONTEND GATEWAY TO EOS" echo "kubectl -n ${NAMESPACE} exec ctaeos -- bash eos root://${EOSINSTANCE} -r 0 0 vid add gateway ${FRONTEND_IP} grpc" kubectl -n ${NAMESPACE} exec ctaeos -- eos -r 0 0 vid add gateway ${FRONTEND_IP} grpc -eos vid set map -grpc key:<your key aks any uuid string> vuid:2 vgid:2 echo echo "eos vid ls" @@ -64,7 +63,7 @@ echo "Launching restore_files_client.sh on client pod" echo " Archiving file: xrdcp as user1" kubectl -n ${NAMESPACE} cp common/archive_file.sh client:/root/archive_file.sh kubectl -n ${NAMESPACE} cp client_helper.sh client:/root/client_helper.sh -kubectl -n ${NAMESPACE} exec client -- bash /root/archive_file.sh -n ${TEST_FILE_NAME} || exit 1 +kubectl -n ${NAMESPACE} exec client -- bash /root/archive_file.sh -f ${TEST_FILE_NAME} || exit 1 echo METADATA_FILE_PATH=$(mktemp -d).json diff --git a/cta.spec.in b/cta.spec.in index 20d7e54544bb6f11f35c047a5ef2282bec0bb20e..5955faf4ddc2182b97932b175e0a3ab0b623a0f4 100644 --- a/cta.spec.in +++ b/cta.spec.in @@ -233,6 +233,8 @@ The command line utilities %attr(0755,root,root) %{_bindir}/cta-send-event %attr(0755,root,root) %{_bindir}/cta-send-closew.sh %attr(0755,root,root) %{_bindir}/cta-verify-file +%attr(0755,root,root) %{_bindir}/cta-change-storage-class +%attr(0644,root,root) %doc /usr/share/man/man1/cta-change-storage-class.1cta.gz %attr(0644,root,root) %config(noreplace) %{_sysconfdir}/cta/cta-cli.conf.example %posttrans -n cta-cli diff --git a/eos_grpc_client/GrpcClient.cpp b/eos_grpc_client/GrpcClient.cpp index 2d0c0b10bddd67e095f88b3d5e9c29f634474804..ced098e225ce29b5e2bb17fa7bb3d8463faebda1 100644 --- a/eos_grpc_client/GrpcClient.cpp +++ b/eos_grpc_client/GrpcClient.cpp @@ -263,4 +263,30 @@ eos::rpc::MDResponse GrpcClient::GetMD(eos::rpc::TYPE type, uint64_t id, const s return response; } +using QueryStatus = int; +QueryStatus GrpcClient::Exec(eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply) const { + + request.set_authkey(token()); + + grpc::ClientContext context; + grpc::CompletionQueue cq; + grpc::Status status; + std::unique_ptr<grpc::ClientAsyncResponseReader<eos::rpc::NSResponse> > rpc( + stub_->AsyncExec(&context, request, &cq)); + rpc->Finish(&reply, &status, (void*) 1); + + void* got_tag; + bool ok = false; + GPR_ASSERT(cq.Next(&got_tag, &ok)); + GPR_ASSERT(got_tag == (void*) 1); + GPR_ASSERT(ok); + + // Act upon the status of the actual RPC. + if (status.ok()) { + return reply.error().code(); + } else { + return -1; + } +} + }} // namespace eos::client diff --git a/eos_grpc_client/GrpcClient.hpp b/eos_grpc_client/GrpcClient.hpp index 545e21a17c835b7159774b97714d09f6b00416a6..79c315181fbddc3ebdca95d6fb9a970a42cd0ae8 100644 --- a/eos_grpc_client/GrpcClient.hpp +++ b/eos_grpc_client/GrpcClient.hpp @@ -49,6 +49,9 @@ public: // Obtain container or file metadata eos::rpc::MDResponse GetMD(eos::rpc::TYPE type, uint64_t id, const std::string &path, bool showJson = false); + using QueryStatus = int; + QueryStatus Exec(eos::rpc::NSRequest& request, eos::rpc::NSResponse& reply) const; + void set_ssl(bool onoff) { m_SSL = onoff; } diff --git a/eos_grpc_client/GrpcEndpoint.cpp b/eos_grpc_client/GrpcEndpoint.cpp index 549becb41d5d977abe41c0af47524c868d68781a..10878ebf9aa908ea5679940284ab6c99149ca561 100644 --- a/eos_grpc_client/GrpcEndpoint.cpp +++ b/eos_grpc_client/GrpcEndpoint.cpp @@ -61,3 +61,14 @@ void eos::client::Endpoint::getCurrentIds(uint64_t &cid, uint64_t &fid) const { eos::rpc::MDResponse eos::client::Endpoint::getMD(eos::rpc::TYPE type, uint64_t id, const std::string &path, bool showJson) const { return m_grpcClient->GetMD(type, id, path, showJson); } + +int eos::client::Endpoint::setXAttr(const std::string &path, const std::string &attrKey, const std::string &attrValue) const { + eos::rpc::NSRequest request; + eos::rpc::NSResponse reply; + + request.mutable_xattr()->mutable_id()->set_path(path); + (*(request.mutable_xattr()->mutable_xattrs()))[attrKey] = attrValue; + + return m_grpcClient->Exec(request, reply); +} + diff --git a/eos_grpc_client/GrpcEndpoint.hpp b/eos_grpc_client/GrpcEndpoint.hpp index 1497df55367a9ac92567aa32d9653d88673cc350..88a2263d8fc73fa88f5c80a4b53879f099a2421d 100644 --- a/eos_grpc_client/GrpcEndpoint.hpp +++ b/eos_grpc_client/GrpcEndpoint.hpp @@ -36,6 +36,8 @@ public: ::eos::rpc::InsertReply containerInsert(const ::eos::rpc::ContainerMdProto &container) const; void getCurrentIds(uint64_t &cid, uint64_t &fid) const; ::eos::rpc::MDResponse getMD(::eos::rpc::TYPE type, uint64_t id, const std::string &path, bool showJson) const; + using QueryStatus = int; + [[nodiscard]] QueryStatus setXAttr(const std::string &path, const std::string &attrKey, const std::string &attrValue) const; private: std::unique_ptr<::eos::client::GrpcClient> m_grpcClient; @@ -97,6 +99,16 @@ public: } } + using QueryStatus = int; + [[nodiscard]] QueryStatus setXAttr(const std::string &diskInstance, const std::string &path, const std::string &attrKey, const std::string &attrValue) const { + auto ep_it = m_endpointMap.find(diskInstance); + if(ep_it == m_endpointMap.end()) { + throw cta::exception::UserError("Namespace for disk instance \"" + diskInstance + "\" is not configured in the CTA Frontend"); + } else { + return ep_it->second.setXAttr(path, attrKey, attrValue); + } + } + private: std::map<std::string, Endpoint> m_endpointMap; }; diff --git a/tapeserver/readtp/ReadtpCmdLineArgs.cpp b/tapeserver/readtp/ReadtpCmdLineArgs.cpp index ac122c526a375000475a25ea6a26772911abaf10..6d31f299292ea4146b728c6f626f0e1997df7ad6 100644 --- a/tapeserver/readtp/ReadtpCmdLineArgs.cpp +++ b/tapeserver/readtp/ReadtpCmdLineArgs.cpp @@ -47,7 +47,7 @@ ReadtpCmdLineArgs::ReadtpCmdLineArgs(const int argc, char *const *const argv): } m_vid = std::string(argv[1]); utils::toUpper(m_vid); - + m_fSeqRangeList = TapeFileSequenceParser::parse(argv[2]); static struct option longopts[] = { diff --git a/tapeserver/tapelabel/TapeLabelCmdLineArgs.cpp b/tapeserver/tapelabel/TapeLabelCmdLineArgs.cpp index 15c7b840d329d121cb4cf1ebf030a1f64c0ea7bc..c95b29feafc9feb4779e1658009f416966c610a6 100644 --- a/tapeserver/tapelabel/TapeLabelCmdLineArgs.cpp +++ b/tapeserver/tapelabel/TapeLabelCmdLineArgs.cpp @@ -71,7 +71,7 @@ TapeLabelCmdLineArgs::TapeLabelCmdLineArgs(const int argc, char *const *const ar throw ex; } else { m_oldLabel = std::string(optarg); - utils::toUpper(m_oldLabel); + utils::toUpper(m_oldLabel); } break; case 't': diff --git a/xroot_plugins/XrdCtaChangeStorageClass.hpp b/xroot_plugins/XrdCtaChangeStorageClass.hpp new file mode 100644 index 0000000000000000000000000000000000000000..1f3f1395efcfad187fba58dcb66d4694d343a394 --- /dev/null +++ b/xroot_plugins/XrdCtaChangeStorageClass.hpp @@ -0,0 +1,44 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#pragma once + +#include <xroot_plugins/XrdCtaStream.hpp> +#include <xroot_plugins/XrdSsiCtaRequestMessage.hpp> + + +namespace cta { namespace xrd { + +class XrdCtaChangeStorageClass { +public: + XrdCtaChangeStorageClass(cta::catalogue::Catalogue &catalogue, cta::log::LogContext &lc); + void updateCatalogue(const std::vector<std::basic_string<char>>& archiveFileIds, const std::string& newStorageClassName); +private: + const cta::catalogue::Catalogue &m_catalogue; + cta::log::LogContext &m_lc; +}; + +XrdCtaChangeStorageClass::XrdCtaChangeStorageClass(cta::catalogue::Catalogue &catalogue, cta::log::LogContext &lc): m_catalogue(catalogue), m_lc(lc) {} + +void XrdCtaChangeStorageClass::updateCatalogue(const std::vector<std::basic_string<char>>& archiveFileIds, const std::string& newStorageClassName) { + for (auto & id : archiveFileIds) { + const uint64_t archiveFileId = std::stoul(id); + m_catalogue.modifyArchiveFileStorageClassId(archiveFileId, newStorageClassName); + } +} +} // namespace xrd +} // namespace cta \ No newline at end of file diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.cpp b/xroot_plugins/XrdSsiCtaRequestMessage.cpp index 98a4eee8ade0cc5a972c806b0f7243e7e71371a0..7409eae874a8d2f35108094b7c8233c2211f756f 100644 --- a/xroot_plugins/XrdSsiCtaRequestMessage.cpp +++ b/xroot_plugins/XrdSsiCtaRequestMessage.cpp @@ -31,6 +31,7 @@ #include "xroot_plugins/XrdCtaActivityMountRuleLs.hpp" #include "xroot_plugins/XrdCtaAdminLs.hpp" #include "xroot_plugins/XrdCtaArchiveRouteLs.hpp" +#include "xroot_plugins/XrdCtaChangeStorageClass.hpp" #include "xroot_plugins/XrdCtaDiskInstanceLs.hpp" #include "xroot_plugins/XrdCtaDiskInstanceSpaceLs.hpp" #include "xroot_plugins/XrdCtaDiskSystemLs.hpp" @@ -323,6 +324,9 @@ void RequestMessage::process(const cta::xrd::Request &request, cta::xrd::Respons case cmd_pair(AdminCmd::CMD_RECYCLETAPEFILE, AdminCmd::SUBCMD_RESTORE): processRecycleTapeFile_Restore(response); break; + case cmd_pair(AdminCmd::CMD_ARCHIVEFILE, AdminCmd::SUBCMD_CH): + processChangeStorageClass(response); + break; default: throw XrdSsiPb::PbException("Admin command pair <" + @@ -2532,4 +2536,20 @@ void RequestMessage::processRecycleTapeFile_Restore(cta::xrd::Response& response response.set_type(cta::xrd::Response::RSP_SUCCESS); } +void RequestMessage::processChangeStorageClass(cta::xrd::Response& response) { + try { + using namespace cta::admin; + + std::string newStorageClassName = getRequired(OptionString::STORAGE_CLASS_NAME); + auto archiveFileIds = getRequired(OptionStrList::FILE_ID); + + XrdCtaChangeStorageClass xrdCtaChangeStorageClass(m_catalogue, m_lc); + xrdCtaChangeStorageClass.updateCatalogue(archiveFileIds, newStorageClassName); + response.set_type(cta::xrd::Response::RSP_SUCCESS); + } catch(exception::UserError &ue) { + response.set_message_txt(ue.getMessage().str()); + response.set_type(Response::RSP_ERR_USER); + } +} + }} // namespace cta::xrd diff --git a/xroot_plugins/XrdSsiCtaRequestMessage.hpp b/xroot_plugins/XrdSsiCtaRequestMessage.hpp index 3c8674d326e13d6e9b60e305fbd0dab532b9d5d7..988b295943029057fc0a057863516c7c268088a7 100644 --- a/xroot_plugins/XrdSsiCtaRequestMessage.hpp +++ b/xroot_plugins/XrdSsiCtaRequestMessage.hpp @@ -223,6 +223,7 @@ private: void processVirtualOrganization_Ch(cta::xrd::Response &response); void processVirtualOrganization_Rm(cta::xrd::Response &response); void processRecycleTapeFile_Restore(cta::xrd::Response &response); + void processChangeStorageClass (cta::xrd::Response &response); /*! * Process AdminCmd events which can return a stream response diff --git a/xrootd-ssi-protobuf-interface b/xrootd-ssi-protobuf-interface index f05ac4849d43590c459fe30bb834ef96eb515d7e..5b175209dfc5ea6608058214c23eee8a300d69d7 160000 --- a/xrootd-ssi-protobuf-interface +++ b/xrootd-ssi-protobuf-interface @@ -1 +1 @@ -Subproject commit f05ac4849d43590c459fe30bb834ef96eb515d7e +Subproject commit 5b175209dfc5ea6608058214c23eee8a300d69d7