diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 56df1221e131a373b8a13a36e8fcb777a0015826..a1c8f64d7e2907baa24eec753ac0b5123f56fa85 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -8,6 +8,7 @@ - cta/CTA#771 - Remove `cta-versionlock` script and extract `cta-checkpkg` script. Added strict version checking for XRootD - cta/CTA#696 - Added a completely new README.md - cta/CTA#782 - Added functionality to allow for the automatic generation of the CHANGELOG.md +- cta/CTA#738 - Add new support for repack archive routes ### Bug Fixes - cta/CTA#715 - Fix CTA compatibility with multiple catalogue versions diff --git a/catalogue/dummy/DummyArchiveRouteCatalogue.cpp b/catalogue/dummy/DummyArchiveRouteCatalogue.cpp index a6b6e42839d200aab57c1034222ac0a51cffec1c..42639c3e2fc7da5a1f7eb90a7e8a3276a82b0d40 100644 --- a/catalogue/dummy/DummyArchiveRouteCatalogue.cpp +++ b/catalogue/dummy/DummyArchiveRouteCatalogue.cpp @@ -21,11 +21,11 @@ namespace cta::catalogue { void DummyArchiveRouteCatalogue::createArchiveRoute(const common::dataStructures::SecurityIdentity &admin, const std::string &storageClassName, - const uint32_t copyNb, const std::string &tapePoolName, const std::string &comment) { + const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType, const std::string &tapePoolName, const std::string &comment) { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } -void DummyArchiveRouteCatalogue::deleteArchiveRoute(const std::string &storageClassName, const uint32_t copyNb) { +void DummyArchiveRouteCatalogue::deleteArchiveRoute(const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType) { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } @@ -39,12 +39,14 @@ std::list<common::dataStructures::ArchiveRoute> DummyArchiveRouteCatalogue::getA } void DummyArchiveRouteCatalogue::modifyArchiveRouteTapePoolName(const common::dataStructures::SecurityIdentity &admin, - const std::string &storageClassName, const uint32_t copyNb, const std::string &tapePoolName) { + const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType, + const std::string &tapePoolName) { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } void DummyArchiveRouteCatalogue::modifyArchiveRouteComment(const common::dataStructures::SecurityIdentity &admin, - const std::string &storageClassName, const uint32_t copyNb, const std::string &comment) { + const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType, + const std::string &comment) { throw exception::Exception(std::string("In ")+__PRETTY_FUNCTION__+": not implemented"); } diff --git a/catalogue/dummy/DummyArchiveRouteCatalogue.hpp b/catalogue/dummy/DummyArchiveRouteCatalogue.hpp index ab333a3b2443140567a6afb878b0a28a13096aaf..8c50020832286c7e243ed56aefe9c1af56882f08 100644 --- a/catalogue/dummy/DummyArchiveRouteCatalogue.hpp +++ b/catalogue/dummy/DummyArchiveRouteCatalogue.hpp @@ -27,9 +27,9 @@ public: ~DummyArchiveRouteCatalogue() override = default; void createArchiveRoute(const common::dataStructures::SecurityIdentity &admin, const std::string &storageClassName, - const uint32_t copyNb, const std::string &tapePoolName, const std::string &comment) override; + const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType, const std::string &tapePoolName, const std::string &comment) override; - void deleteArchiveRoute(const std::string &storageClassName, const uint32_t copyNb) override; + void deleteArchiveRoute(const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType) override; std::list<common::dataStructures::ArchiveRoute> getArchiveRoutes() const override; @@ -37,10 +37,11 @@ public: const std::string &tapePoolName) const override; void modifyArchiveRouteTapePoolName(const common::dataStructures::SecurityIdentity &admin, - const std::string &storageClassName, const uint32_t copyNb, const std::string &tapePoolName) override; + const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType, const std::string &tapePoolName) override; void modifyArchiveRouteComment(const common::dataStructures::SecurityIdentity &admin, - const std::string &storageClassName, const uint32_t copyNb, const std::string &comment) override; + const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType, + const std::string &comment) override; }; } // namespace cta::catalogue diff --git a/catalogue/interfaces/ArchiveRouteCatalogue.hpp b/catalogue/interfaces/ArchiveRouteCatalogue.hpp index 4c76bcdac97b2d9f73bb40d9f367c87416e5f927..e2e08ba33f5a9f85b44b1c62f41ef015514eaed9 100644 --- a/catalogue/interfaces/ArchiveRouteCatalogue.hpp +++ b/catalogue/interfaces/ArchiveRouteCatalogue.hpp @@ -22,6 +22,7 @@ #include <string> #include <list> +#include "common/dataStructures/ArchiveRoute.hpp" #include "common/exception/UserError.hpp" namespace cta { @@ -44,6 +45,7 @@ public: const common::dataStructures::SecurityIdentity &admin, const std::string &storageClassName, const uint32_t copyNb, + const common::dataStructures::ArchiveRouteType &archiveRouteType, const std::string &tapePoolName, const std::string &comment) = 0; @@ -55,7 +57,8 @@ public: */ virtual void deleteArchiveRoute( const std::string &storageClassName, - const uint32_t copyNb) = 0; + const uint32_t copyNb, + const common::dataStructures::ArchiveRouteType &archiveRouteType) = 0; virtual std::list<common::dataStructures::ArchiveRoute> getArchiveRoutes() const = 0; @@ -85,10 +88,12 @@ public: * non-existent tape pool. */ virtual void modifyArchiveRouteTapePoolName(const common::dataStructures::SecurityIdentity &admin, - const std::string &storageClassName, const uint32_t copyNb, const std::string &tapePoolName) = 0; + const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType, + const std::string &tapePoolName) = 0; virtual void modifyArchiveRouteComment(const common::dataStructures::SecurityIdentity &admin, - const std::string &storageClassName, const uint32_t copyNb, const std::string &comment) = 0; + const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType, + const std::string &comment) = 0; }; }} // namespace cta::catalogue diff --git a/catalogue/rdbms/RdbmsArchiveFileCatalogue.cpp b/catalogue/rdbms/RdbmsArchiveFileCatalogue.cpp index 93f2109b241a9033c5d2c125ed849c1eb153ace3..aa44b076679efd2ef0c105866bf8dd59f05989be 100644 --- a/catalogue/rdbms/RdbmsArchiveFileCatalogue.cpp +++ b/catalogue/rdbms/RdbmsArchiveFileCatalogue.cpp @@ -59,8 +59,8 @@ uint64_t RdbmsArchiveFileCatalogue::checkAndGetNextArchiveFileId(const std::stri const std::string &storageClassName, const common::dataStructures::RequesterIdentity &user) { try { const auto storageClass = catalogue::StorageClass(storageClassName); - const auto copyToPoolMap = getCachedTapeCopyToPoolMap(storageClass); - const auto expectedNbRoutes = getCachedExpectedNbArchiveRoutes(storageClass); + const auto copyToPoolMap = getCachedTapeCopyToPoolMap(storageClass, false); + const auto expectedNbRoutes = getCachedExpectedNbArchiveRoutes(storageClass, false); // Check that the number of archive routes is correct if(copyToPoolMap.empty()) { @@ -115,8 +115,8 @@ common::dataStructures::ArchiveFileQueueCriteria RdbmsArchiveFileCatalogue::getA const std::string &diskInstanceName, const std::string &storageClassName, const common::dataStructures::RequesterIdentity &user) { const auto storageClass = catalogue::StorageClass(storageClassName); - const common::dataStructures::TapeCopyToPoolMap copyToPoolMap = getCachedTapeCopyToPoolMap(storageClass); - const uint64_t expectedNbRoutes = getCachedExpectedNbArchiveRoutes(storageClass); + const common::dataStructures::TapeCopyToPoolMap copyToPoolMap = getCachedTapeCopyToPoolMap(storageClass, false); + const uint64_t expectedNbRoutes = getCachedExpectedNbArchiveRoutes(storageClass, false); // Check that the number of archive routes is correct if(copyToPoolMap.empty()) { @@ -504,19 +504,19 @@ std::unique_ptr<common::dataStructures::ArchiveFile> RdbmsArchiveFileCatalogue:: } common::dataStructures::TapeCopyToPoolMap RdbmsArchiveFileCatalogue::getCachedTapeCopyToPoolMap( - const catalogue::StorageClass &storageClass) const { - auto l_getNonCachedValue = [this,&storageClass] { + const catalogue::StorageClass &storageClass, bool useRepackArchiveRoute) const { + auto l_getNonCachedValue = [this,&storageClass,&useRepackArchiveRoute] { auto conn = m_connPool->getConn(); - return getTapeCopyToPoolMap(conn, storageClass); + return getTapeCopyToPoolMap(conn, storageClass, useRepackArchiveRoute); }; return m_tapeCopyToPoolCache.getCachedValue(storageClass, l_getNonCachedValue).value; } uint64_t RdbmsArchiveFileCatalogue::getCachedExpectedNbArchiveRoutes( - const catalogue::StorageClass &storageClass) const { - auto l_getNonCachedValue = [this,&storageClass] { + const catalogue::StorageClass &storageClass, bool useRepackArchiveRoute) const { + auto l_getNonCachedValue = [this,&storageClass,&useRepackArchiveRoute] { auto conn = m_connPool->getConn(); - return getExpectedNbArchiveRoutes(conn, storageClass); + return getExpectedNbArchiveRoutes(conn, storageClass, useRepackArchiveRoute); }; return m_expectedNbArchiveRoutesCache.getCachedValue(storageClass, l_getNonCachedValue).value; } @@ -598,11 +598,12 @@ ArchiveFileItor RdbmsArchiveFileCatalogue::getTapeContentsItor(const std::string } common::dataStructures::TapeCopyToPoolMap RdbmsArchiveFileCatalogue::getTapeCopyToPoolMap(rdbms::Conn &conn, - const catalogue::StorageClass &storageClass) const { + const catalogue::StorageClass &storageClass, bool useRepackArchiveRoute) const { common::dataStructures::TapeCopyToPoolMap copyToPoolMap; - const char* const sql = R"SQL( + std::string sql = R"SQL( SELECT ARCHIVE_ROUTE.COPY_NB AS COPY_NB, + ARCHIVE_ROUTE.ARCHIVE_ROUTE_TYPE AS ARCHIVE_ROUTE_TYPE, TAPE_POOL.TAPE_POOL_NAME AS TAPE_POOL_NAME FROM ARCHIVE_ROUTE @@ -613,12 +614,22 @@ common::dataStructures::TapeCopyToPoolMap RdbmsArchiveFileCatalogue::getTapeCopy WHERE STORAGE_CLASS.STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME )SQL"; + if (!useRepackArchiveRoute) { + sql += " AND ARCHIVE_ROUTE.ARCHIVE_ROUTE_TYPE != '" + + common::dataStructures::toString(common::dataStructures::ArchiveRouteType::REPACK) + "'"; + } auto stmt = conn.createStmt(sql); stmt.bindString(":STORAGE_CLASS_NAME", storageClass.storageClassName); auto rset = stmt.executeQuery(); while (rset.next()) { const auto copyNb = static_cast<uint32_t>(rset.columnUint64("COPY_NB")); const std::string tapePoolName = rset.columnString("TAPE_POOL_NAME"); + auto archiveRouteTypeStr = rset.columnString("ARCHIVE_ROUTE_TYPE"); + auto archiveRouteType = common::dataStructures::strToArchiveRouteType(archiveRouteTypeStr); + if (archiveRouteType == common::dataStructures::ArchiveRouteType::DEFAULT && copyToPoolMap.count(copyNb)) { + // A DEFAULT archive route type should not override a previously found value + continue; + } copyToPoolMap[copyNb] = tapePoolName; } @@ -626,17 +637,26 @@ common::dataStructures::TapeCopyToPoolMap RdbmsArchiveFileCatalogue::getTapeCopy } uint64_t RdbmsArchiveFileCatalogue::getExpectedNbArchiveRoutes(rdbms::Conn &conn, - const catalogue::StorageClass &storageClass) const { - const char* const sql = R"SQL( - SELECT - COUNT(*) AS NB_ROUTES - FROM - ARCHIVE_ROUTE - INNER JOIN STORAGE_CLASS ON - ARCHIVE_ROUTE.STORAGE_CLASS_ID = STORAGE_CLASS.STORAGE_CLASS_ID - WHERE - STORAGE_CLASS.STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME + const catalogue::StorageClass &storageClass, bool useRepackArchiveRoute) const { + std::string sql = R"SQL( + SELECT + COUNT(*) AS NB_ROUTES + FROM ( + SELECT DISTINCT + ARCHIVE_ROUTE.STORAGE_CLASS_ID, + ARCHIVE_ROUTE.COPY_NB + FROM + ARCHIVE_ROUTE + INNER JOIN STORAGE_CLASS ON + ARCHIVE_ROUTE.STORAGE_CLASS_ID = STORAGE_CLASS.STORAGE_CLASS_ID + WHERE + STORAGE_CLASS.STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME )SQL"; + if (!useRepackArchiveRoute) { + sql += " AND ARCHIVE_ROUTE.ARCHIVE_ROUTE_TYPE != '" + + common::dataStructures::toString(common::dataStructures::ArchiveRouteType::REPACK) + "'"; + } + sql += ") ROUTES"; auto stmt = conn.createStmt(sql); stmt.bindString(":STORAGE_CLASS_NAME", storageClass.storageClassName); auto rset = stmt.executeQuery(); diff --git a/catalogue/rdbms/RdbmsArchiveFileCatalogue.hpp b/catalogue/rdbms/RdbmsArchiveFileCatalogue.hpp index 0f60d7a11f932ca3a130444b508adb02b0895efd..707e804d63eff2b9f7f1ab48a294073b722e9ffe 100644 --- a/catalogue/rdbms/RdbmsArchiveFileCatalogue.hpp +++ b/catalogue/rdbms/RdbmsArchiveFileCatalogue.hpp @@ -164,11 +164,12 @@ private: * * @param storageClass The fully qualified storage class, in other words the * name of the disk instance and the name of the storage class. + * @param useRepackArchiveRoute True if looking for repack archive routes. * @return The mapping from tape copy to tape pool for the specified storage * class. */ common::dataStructures::TapeCopyToPoolMap getCachedTapeCopyToPoolMap( - const catalogue::StorageClass &storageClass) const; + const catalogue::StorageClass &storageClass, bool useRepackArchiveRoute) const; /** * Returns a cached version of the expected number of archive routes for the @@ -180,9 +181,10 @@ private: * * @param storageClass The fully qualified storage class, in other words the * name of the disk instance and the name of the storage class. + * @param useRepackArchiveRoute True if looking for repack archive routes. * @return The expected number of archive routes. */ - uint64_t getCachedExpectedNbArchiveRoutes(const catalogue::StorageClass &storageClass) const; + uint64_t getCachedExpectedNbArchiveRoutes(const catalogue::StorageClass &storageClass, bool useRepackArchiveRoute) const; /** * Returns a cached version of the specified requester mount-policy or std::nullopt @@ -251,12 +253,13 @@ private: * * @param conn The database connection. * @param storageClass The fully qualified storage class, in other words the + * @param isRepack True if we are mapping copies to repack tape pools * name of the disk instance and the name of the storage class. * @return The mapping from tape copy to tape pool for the specified storage * class. */ common::dataStructures::TapeCopyToPoolMap getTapeCopyToPoolMap(rdbms::Conn &conn, - const catalogue::StorageClass &storageClass) const; + const catalogue::StorageClass &storageClass, bool isRepack) const; /** * Returns the expected number of archive routes for the specified storage @@ -269,7 +272,7 @@ private: * name of the disk instance and the name of the storage class. * @return The expected number of archive routes. */ - uint64_t getExpectedNbArchiveRoutes(rdbms::Conn &conn, const catalogue::StorageClass &storageClass) const; + uint64_t getExpectedNbArchiveRoutes(rdbms::Conn &conn, const catalogue::StorageClass &storageClass, bool useRepackArchiveRoute) const; /** * Throws an exception if the delete request passed in parameter is not consistent diff --git a/catalogue/rdbms/RdbmsArchiveRouteCatalogue.cpp b/catalogue/rdbms/RdbmsArchiveRouteCatalogue.cpp index e753576481f767826551f34e18a8833070264b8b..b288ef5f887f0d6091ad425da9806b77e15d717a 100644 --- a/catalogue/rdbms/RdbmsArchiveRouteCatalogue.cpp +++ b/catalogue/rdbms/RdbmsArchiveRouteCatalogue.cpp @@ -36,8 +36,8 @@ RdbmsArchiveRouteCatalogue::RdbmsArchiveRouteCatalogue(log::Logger &log, std::sh m_log(log), m_connPool(connPool) {} void RdbmsArchiveRouteCatalogue::createArchiveRoute(const common::dataStructures::SecurityIdentity &admin, - const std::string &storageClassName, const uint32_t copyNb, const std::string &tapePoolName, - const std::string &comment) { + const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType, + const std::string &tapePoolName, const std::string &comment) { if(storageClassName.empty()) { throw UserSpecifiedAnEmptyStringStorageClassName("Cannot create archive route because storage class name is an" " empty string"); @@ -56,7 +56,7 @@ void RdbmsArchiveRouteCatalogue::createArchiveRoute(const common::dataStructures const time_t now = time(nullptr); auto conn = m_connPool->getConn(); - if(RdbmsCatalogueUtils::archiveRouteExists(conn, storageClassName, copyNb)) { + if(archiveRouteExists(conn, storageClassName, copyNb, archiveRouteType)) { exception::UserError ue; ue.getMessage() << "Cannot create archive route " << ": " << storageClassName << "," << copyNb << "->" << tapePoolName << " because it already exists"; @@ -66,7 +66,7 @@ void RdbmsArchiveRouteCatalogue::createArchiveRoute(const common::dataStructures if(const auto routes = getArchiveRoutes(conn, storageClassName, tapePoolName); !routes.empty()) { exception::UserError ue; ue.getMessage() << "Cannot create archive route " << ": " << storageClassName << "," << copyNb - << "->" << tapePoolName << " because a route already exists for this storage class and tape pool"; + << "->" << tapePoolName << " because a route already exists for this storage class, tape pool and type"; throw ue; } if(!RdbmsCatalogueUtils::storageClassExists(conn, storageClassName)) { @@ -87,6 +87,7 @@ void RdbmsArchiveRouteCatalogue::createArchiveRoute(const common::dataStructures INSERT INTO ARCHIVE_ROUTE( STORAGE_CLASS_ID, COPY_NB, + ARCHIVE_ROUTE_TYPE, TAPE_POOL_ID, USER_COMMENT, @@ -101,6 +102,7 @@ void RdbmsArchiveRouteCatalogue::createArchiveRoute(const common::dataStructures SELECT STORAGE_CLASS_ID, :COPY_NB, + :ARCHIVE_ROUTE_TYPE, (SELECT TAPE_POOL_ID FROM TAPE_POOL WHERE TAPE_POOL_NAME = :TAPE_POOL_NAME) AS TAPE_POOL_ID, :USER_COMMENT, @@ -122,6 +124,7 @@ void RdbmsArchiveRouteCatalogue::createArchiveRoute(const common::dataStructures stmt.bindString(":STORAGE_CLASS_NAME", storageClassName); stmt.bindUint64(":COPY_NB", copyNb); stmt.bindString(":TAPE_POOL_NAME", tapePoolName); + stmt.bindString(":ARCHIVE_ROUTE_TYPE", cta::common::dataStructures::toString(archiveRouteType)); stmt.bindString(":USER_COMMENT", trimmedComment); @@ -136,7 +139,7 @@ void RdbmsArchiveRouteCatalogue::createArchiveRoute(const common::dataStructures stmt.executeNonQuery(); } -void RdbmsArchiveRouteCatalogue::deleteArchiveRoute(const std::string &storageClassName, const uint32_t copyNb) { +void RdbmsArchiveRouteCatalogue::deleteArchiveRoute(const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType) { const char* const sql = R"SQL( DELETE FROM ARCHIVE_ROUTE @@ -147,18 +150,19 @@ void RdbmsArchiveRouteCatalogue::deleteArchiveRoute(const std::string &storageCl FROM STORAGE_CLASS WHERE - STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME) AND COPY_NB = :COPY_NB + STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME) AND COPY_NB = :COPY_NB AND ARCHIVE_ROUTE_TYPE = :ARCHIVE_ROUTE_TYPE )SQL"; auto conn = m_connPool->getConn(); auto stmt = conn.createStmt(sql); stmt.bindString(":STORAGE_CLASS_NAME", storageClassName); stmt.bindUint64(":COPY_NB", copyNb); + stmt.bindString(":ARCHIVE_ROUTE_TYPE", common::dataStructures::toString(archiveRouteType)); stmt.executeNonQuery(); if(0 == stmt.getNbAffectedRows()) { exception::UserError ue; - ue.getMessage() << "Cannot delete archive route for storage-class " << ":" + storageClassName + - " and copy number " << copyNb << " because it does not exist"; + ue.getMessage() << "Cannot delete archive route for storage-class " << storageClassName + + ", copy number " << copyNb << " and type " << common::dataStructures::toString(archiveRouteType) << " because it does not exist"; throw ue; } } @@ -169,6 +173,7 @@ std::list<common::dataStructures::ArchiveRoute> RdbmsArchiveRouteCatalogue::getA SELECT STORAGE_CLASS.STORAGE_CLASS_NAME AS STORAGE_CLASS_NAME, ARCHIVE_ROUTE.COPY_NB AS COPY_NB, + ARCHIVE_ROUTE.ARCHIVE_ROUTE_TYPE AS ARCHIVE_ROUTE_TYPE, TAPE_POOL.TAPE_POOL_NAME AS TAPE_POOL_NAME, ARCHIVE_ROUTE.USER_COMMENT AS USER_COMMENT, @@ -187,7 +192,7 @@ std::list<common::dataStructures::ArchiveRoute> RdbmsArchiveRouteCatalogue::getA INNER JOIN TAPE_POOL ON ARCHIVE_ROUTE.TAPE_POOL_ID = TAPE_POOL.TAPE_POOL_ID ORDER BY - STORAGE_CLASS_NAME, COPY_NB + STORAGE_CLASS_NAME, COPY_NB, ARCHIVE_ROUTE_TYPE )SQL"; auto conn = m_connPool->getConn(); auto stmt = conn.createStmt(sql); @@ -197,6 +202,7 @@ std::list<common::dataStructures::ArchiveRoute> RdbmsArchiveRouteCatalogue::getA route.storageClassName = rset.columnString("STORAGE_CLASS_NAME"); route.copyNb = static_cast<uint8_t>(rset.columnUint64("COPY_NB")); + route.type = common::dataStructures::strToArchiveRouteType(rset.columnString("ARCHIVE_ROUTE_TYPE")); route.tapePoolName = rset.columnString("TAPE_POOL_NAME"); route.comment = rset.columnString("USER_COMMENT"); route.creationLog.username = rset.columnString("CREATION_LOG_USER_NAME"); @@ -224,7 +230,8 @@ std::list<common::dataStructures::ArchiveRoute> RdbmsArchiveRouteCatalogue::getA const char* const sql = R"SQL( SELECT STORAGE_CLASS.STORAGE_CLASS_NAME AS STORAGE_CLASS_NAME, - ARCHIVE_ROUTE.COPY_NB AS COPY_NB, + ARCHIVE_ROUTE.COPY_NB AS COPY_NB, + ARCHIVE_ROUTE.ARCHIVE_ROUTE_TYPE AS ARCHIVE_ROUTE_TYPE, TAPE_POOL.TAPE_POOL_NAME AS TAPE_POOL_NAME, ARCHIVE_ROUTE.USER_COMMENT AS USER_COMMENT, @@ -246,7 +253,7 @@ std::list<common::dataStructures::ArchiveRoute> RdbmsArchiveRouteCatalogue::getA STORAGE_CLASS.STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME AND TAPE_POOL.TAPE_POOL_NAME = :TAPE_POOL_NAME ORDER BY - STORAGE_CLASS_NAME, COPY_NB + STORAGE_CLASS_NAME, COPY_NB, ARCHIVE_ROUTE_TYPE )SQL"; auto stmt = conn.createStmt(sql); stmt.bindString(":STORAGE_CLASS_NAME", storageClassName); @@ -257,6 +264,7 @@ std::list<common::dataStructures::ArchiveRoute> RdbmsArchiveRouteCatalogue::getA route.storageClassName = rset.columnString("STORAGE_CLASS_NAME"); route.copyNb = static_cast<uint8_t>(rset.columnUint64("COPY_NB")); + route.type = common::dataStructures::strToArchiveRouteType(rset.columnString("ARCHIVE_ROUTE_TYPE")); route.tapePoolName = rset.columnString("TAPE_POOL_NAME"); route.comment = rset.columnString("USER_COMMENT"); route.creationLog.username = rset.columnString("CREATION_LOG_USER_NAME"); @@ -273,7 +281,8 @@ std::list<common::dataStructures::ArchiveRoute> RdbmsArchiveRouteCatalogue::getA } void RdbmsArchiveRouteCatalogue::modifyArchiveRouteTapePoolName(const common::dataStructures::SecurityIdentity &admin, - const std::string &storageClassName, const uint32_t copyNb, const std::string &tapePoolName) { + const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType, + const std::string &tapePoolName) { try { const time_t now = time(nullptr); const char* const sql = R"SQL( @@ -289,11 +298,11 @@ void RdbmsArchiveRouteCatalogue::modifyArchiveRouteTapePoolName(const common::da FROM STORAGE_CLASS WHERE - STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME) AND COPY_NB = :COPY_NB + STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME) AND COPY_NB = :COPY_NB AND ARCHIVE_ROUTE_TYPE = :ARCHIVE_ROUTE_TYPE )SQL"; auto conn = m_connPool->getConn(); - if(!RdbmsCatalogueUtils::archiveRouteExists(conn, storageClassName, copyNb)) { + if(!archiveRouteExists(conn, storageClassName, copyNb, archiveRouteType)) { throw UserSpecifiedANonExistentArchiveRoute("Archive route does not exist"); } @@ -308,6 +317,7 @@ void RdbmsArchiveRouteCatalogue::modifyArchiveRouteTapePoolName(const common::da stmt.bindUint64(":LAST_UPDATE_TIME", now); stmt.bindString(":STORAGE_CLASS_NAME", storageClassName); stmt.bindUint64(":COPY_NB", copyNb); + stmt.bindString(":ARCHIVE_ROUTE_TYPE", common::dataStructures::toString(archiveRouteType)); stmt.executeNonQuery(); if(0 == stmt.getNbAffectedRows()) { @@ -323,7 +333,8 @@ void RdbmsArchiveRouteCatalogue::modifyArchiveRouteTapePoolName(const common::da } void RdbmsArchiveRouteCatalogue::modifyArchiveRouteComment(const common::dataStructures::SecurityIdentity &admin, - const std::string &storageClassName, const uint32_t copyNb, const std::string &comment) { + const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType & archiveRouteType, + const std::string &comment) { const auto trimmedComment = RdbmsCatalogueUtils::checkCommentOrReasonMaxLength(comment, &m_log); const time_t now = time(nullptr); const char* const sql = R"SQL( @@ -339,7 +350,7 @@ void RdbmsArchiveRouteCatalogue::modifyArchiveRouteComment(const common::dataStr FROM STORAGE_CLASS WHERE - STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME) AND COPY_NB = :COPY_NB + STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME) AND COPY_NB = :COPY_NB AND ARCHIVE_ROUTE_TYPE = :ARCHIVE_ROUTE_TYPE )SQL"; auto conn = m_connPool->getConn(); auto stmt = conn.createStmt(sql); @@ -349,6 +360,7 @@ void RdbmsArchiveRouteCatalogue::modifyArchiveRouteComment(const common::dataStr stmt.bindUint64(":LAST_UPDATE_TIME", now); stmt.bindString(":STORAGE_CLASS_NAME", storageClassName); stmt.bindUint64(":COPY_NB", copyNb); + stmt.bindString(":ARCHIVE_ROUTE_TYPE", common::dataStructures::toString(archiveRouteType)); stmt.executeNonQuery(); if(0 == stmt.getNbAffectedRows()) { @@ -359,4 +371,27 @@ void RdbmsArchiveRouteCatalogue::modifyArchiveRouteComment(const common::dataStr } } +bool RdbmsArchiveRouteCatalogue::archiveRouteExists(rdbms::Conn &conn, const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType & archiveRouteType) { + const char* const sql = R"SQL( + SELECT + ARCHIVE_ROUTE.STORAGE_CLASS_ID AS STORAGE_CLASS_ID, + ARCHIVE_ROUTE.COPY_NB AS COPY_NB, + ARCHIVE_ROUTE.ARCHIVE_ROUTE_TYPE AS ARCHIVE_ROUTE_TYPE + FROM + ARCHIVE_ROUTE + INNER JOIN STORAGE_CLASS ON + ARCHIVE_ROUTE.STORAGE_CLASS_ID = STORAGE_CLASS.STORAGE_CLASS_ID + WHERE + STORAGE_CLASS.STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME AND + ARCHIVE_ROUTE.COPY_NB = :COPY_NB AND + ARCHIVE_ROUTE.ARCHIVE_ROUTE_TYPE = :ARCHIVE_ROUTE_TYPE + )SQL"; + auto stmt = conn.createStmt(sql); + stmt.bindString(":STORAGE_CLASS_NAME", storageClassName); + stmt.bindUint64(":COPY_NB", copyNb); + stmt.bindString(":ARCHIVE_ROUTE_TYPE", common::dataStructures::toString(archiveRouteType)); + auto rset = stmt.executeQuery(); + return rset.next(); +} + } // namespace cta::catalogue diff --git a/catalogue/rdbms/RdbmsArchiveRouteCatalogue.hpp b/catalogue/rdbms/RdbmsArchiveRouteCatalogue.hpp index 80aa51607f0e78d2aab66f016860a833f26e2da2..3e16a2318492bd4fec6a17b62e926e8c22e6d5de 100644 --- a/catalogue/rdbms/RdbmsArchiveRouteCatalogue.hpp +++ b/catalogue/rdbms/RdbmsArchiveRouteCatalogue.hpp @@ -37,9 +37,10 @@ public: ~RdbmsArchiveRouteCatalogue() override = default; void createArchiveRoute(const common::dataStructures::SecurityIdentity &admin, const std::string &storageClassName, - const uint32_t copyNb, const std::string &tapePoolName, const std::string &comment) override; + const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType, + const std::string &tapePoolName, const std::string &comment) override; - void deleteArchiveRoute(const std::string &storageClassName, const uint32_t copyNb) override; + void deleteArchiveRoute(const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType) override; std::list<common::dataStructures::ArchiveRoute> getArchiveRoutes() const override; @@ -47,12 +48,18 @@ public: const std::string &tapePoolName) const override; void modifyArchiveRouteTapePoolName(const common::dataStructures::SecurityIdentity &admin, - const std::string &storageClassName, const uint32_t copyNb, const std::string &tapePoolName) override; + const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType, + const std::string &tapePoolName) override; void modifyArchiveRouteComment(const common::dataStructures::SecurityIdentity &admin, - const std::string &storageClassName, const uint32_t copyNb, const std::string &comment) override; + const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType & archiveRouteType, + const std::string &comment) override; private: + + bool archiveRouteExists(rdbms::Conn &conn, const std::string &storageClassName, const uint32_t copyNb, + const common::dataStructures::ArchiveRouteType & archiveRouteType); + log::Logger &m_log; std::shared_ptr<rdbms::ConnPool> m_connPool; diff --git a/catalogue/rdbms/RdbmsCatalogueUtils.cpp b/catalogue/rdbms/RdbmsCatalogueUtils.cpp index b8264e93822d79dbd60efc52cf33bd322d68d1bc..1127e6804becd4cefcc46249c70d30a674909704 100644 --- a/catalogue/rdbms/RdbmsCatalogueUtils.cpp +++ b/catalogue/rdbms/RdbmsCatalogueUtils.cpp @@ -21,6 +21,7 @@ #include "catalogue/rdbms/RdbmsCatalogueUtils.hpp" #include "common/dataStructures/SecurityIdentity.hpp" +#include "common/dataStructures/ArchiveRoute.hpp" #include "common/exception/Exception.hpp" #include "common/exception/UserError.hpp" #include "common/log/LogContext.hpp" @@ -213,27 +214,6 @@ bool RdbmsCatalogueUtils::mountPolicyExists(rdbms::Conn &conn, const std::string return rset.next(); } -bool RdbmsCatalogueUtils::archiveRouteExists(rdbms::Conn &conn, const std::string &storageClassName, - const uint32_t copyNb) { - const char* const sql = R"SQL( - SELECT - ARCHIVE_ROUTE.STORAGE_CLASS_ID AS STORAGE_CLASS_ID, - ARCHIVE_ROUTE.COPY_NB AS COPY_NB - FROM - ARCHIVE_ROUTE - INNER JOIN STORAGE_CLASS ON - ARCHIVE_ROUTE.STORAGE_CLASS_ID = STORAGE_CLASS.STORAGE_CLASS_ID - WHERE - STORAGE_CLASS.STORAGE_CLASS_NAME = :STORAGE_CLASS_NAME AND - ARCHIVE_ROUTE.COPY_NB = :COPY_NB - )SQL"; - auto stmt = conn.createStmt(sql); - stmt.bindString(":STORAGE_CLASS_NAME", storageClassName); - stmt.bindUint64(":COPY_NB", copyNb); - auto rset = stmt.executeQuery(); - return rset.next(); -} - bool RdbmsCatalogueUtils::requesterActivityMountRuleExists(rdbms::Conn &conn, const std::string &diskInstanceName, const std::string &requesterName, const std::string &activityRegex) { const char* const sql = R"SQL( diff --git a/catalogue/rdbms/RdbmsCatalogueUtils.hpp b/catalogue/rdbms/RdbmsCatalogueUtils.hpp index df2341598e201df85f7b2dac657e4019711ac6a9..95960ef5074ab540fea781633afbf6c56214c3fe 100644 --- a/catalogue/rdbms/RdbmsCatalogueUtils.hpp +++ b/catalogue/rdbms/RdbmsCatalogueUtils.hpp @@ -57,7 +57,6 @@ public: static bool tapeExists(rdbms::Conn &conn, const std::string &vid); static bool archiveFileIdExists(rdbms::Conn &conn, const uint64_t archiveFileId); static bool mountPolicyExists(rdbms::Conn &conn, const std::string &mountPolicyName); - static bool archiveRouteExists(rdbms::Conn &conn, const std::string &storageClassName, const uint32_t copyNb); static bool requesterActivityMountRuleExists(rdbms::Conn &conn, const std::string &diskInstanceName, const std::string &requesterName, const std::string &activityRegex); static bool diskFileIdExists(rdbms::Conn &conn, const std::string &diskInstanceName, const std::string &diskFileId); diff --git a/catalogue/retrywrappers/ArchiveRouteCatalogueRetryWrapper.cpp b/catalogue/retrywrappers/ArchiveRouteCatalogueRetryWrapper.cpp index c07ff7af870867af151466c3d5f81c31e8ec49ed..e990f482dc151c205a3b06659fbdffafd1442279 100644 --- a/catalogue/retrywrappers/ArchiveRouteCatalogueRetryWrapper.cpp +++ b/catalogue/retrywrappers/ArchiveRouteCatalogueRetryWrapper.cpp @@ -30,16 +30,16 @@ ArchiveRouteCatalogueRetryWrapper::ArchiveRouteCatalogueRetryWrapper(const std:: : m_catalogue(catalogue), m_log(log), m_maxTriesToConnect(maxTriesToConnect) {} void ArchiveRouteCatalogueRetryWrapper::createArchiveRoute(const common::dataStructures::SecurityIdentity &admin, - const std::string &storageClassName, const uint32_t copyNb, const std::string &tapePoolName, - const std::string &comment) { - return retryOnLostConnection(m_log, [this,&admin,&storageClassName,©Nb,&tapePoolName,&comment] { - return m_catalogue->ArchiveRoute()->createArchiveRoute(admin, storageClassName, copyNb, tapePoolName, comment); + const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType, + const std::string &tapePoolName, const std::string &comment) { + return retryOnLostConnection(m_log, [this,&admin,&storageClassName,©Nb,&archiveRouteType,&tapePoolName,&comment] { + return m_catalogue->ArchiveRoute()->createArchiveRoute(admin, storageClassName, copyNb, archiveRouteType, tapePoolName, comment); }, m_maxTriesToConnect); } -void ArchiveRouteCatalogueRetryWrapper::deleteArchiveRoute(const std::string &storageClassName, const uint32_t copyNb) { - return retryOnLostConnection(m_log, [this,&storageClassName,©Nb] { - return m_catalogue->ArchiveRoute()->deleteArchiveRoute(storageClassName, copyNb); +void ArchiveRouteCatalogueRetryWrapper::deleteArchiveRoute(const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType) { + return retryOnLostConnection(m_log, [this,&storageClassName,©Nb,&archiveRouteType] { + return m_catalogue->ArchiveRoute()->deleteArchiveRoute(storageClassName, copyNb, archiveRouteType); }, m_maxTriesToConnect); } @@ -58,16 +58,16 @@ std::list<common::dataStructures::ArchiveRoute> ArchiveRouteCatalogueRetryWrappe void ArchiveRouteCatalogueRetryWrapper::modifyArchiveRouteTapePoolName( const common::dataStructures::SecurityIdentity &admin, const std::string &storageClassName, const uint32_t copyNb, - const std::string &tapePoolName) { - return retryOnLostConnection(m_log, [this,&admin,&storageClassName,©Nb,&tapePoolName] { - return m_catalogue->ArchiveRoute()->modifyArchiveRouteTapePoolName(admin, storageClassName, copyNb, tapePoolName); + const common::dataStructures::ArchiveRouteType &archiveRouteType, const std::string &tapePoolName) { + return retryOnLostConnection(m_log, [this,&admin,&storageClassName,©Nb,&archiveRouteType,&tapePoolName] { + return m_catalogue->ArchiveRoute()->modifyArchiveRouteTapePoolName(admin, storageClassName, copyNb, archiveRouteType, tapePoolName); }, m_maxTriesToConnect); } void ArchiveRouteCatalogueRetryWrapper::modifyArchiveRouteComment(const common::dataStructures::SecurityIdentity &admin, - const std::string &storageClassName, const uint32_t copyNb, const std::string &comment) { - return retryOnLostConnection(m_log, [this,&admin,&storageClassName,©Nb,&comment] { - return m_catalogue->ArchiveRoute()->modifyArchiveRouteComment(admin, storageClassName, copyNb, comment); + const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType, const std::string &comment) { + return retryOnLostConnection(m_log, [this,&admin,&storageClassName,©Nb,&archiveRouteType,&comment] { + return m_catalogue->ArchiveRoute()->modifyArchiveRouteComment(admin, storageClassName, copyNb, archiveRouteType, comment); }, m_maxTriesToConnect); } diff --git a/catalogue/retrywrappers/ArchiveRouteCatalogueRetryWrapper.hpp b/catalogue/retrywrappers/ArchiveRouteCatalogueRetryWrapper.hpp index 70ce4a8ca40edeb4cbda4a5fb182021c1e113a36..4658dc7bede098d9eb933d603cdae0fb310df486 100644 --- a/catalogue/retrywrappers/ArchiveRouteCatalogueRetryWrapper.hpp +++ b/catalogue/retrywrappers/ArchiveRouteCatalogueRetryWrapper.hpp @@ -38,9 +38,10 @@ public: ~ArchiveRouteCatalogueRetryWrapper() override = default; void createArchiveRoute(const common::dataStructures::SecurityIdentity &admin, const std::string &storageClassName, - const uint32_t copyNb, const std::string &tapePoolName, const std::string &comment) override; + const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType, + const std::string &tapePoolName, const std::string &comment) override; - void deleteArchiveRoute(const std::string &storageClassName, const uint32_t copyNb) override; + void deleteArchiveRoute(const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType) override; std::list<common::dataStructures::ArchiveRoute> getArchiveRoutes() const override; @@ -48,10 +49,12 @@ public: const std::string &tapePoolName) const override; void modifyArchiveRouteTapePoolName(const common::dataStructures::SecurityIdentity &admin, - const std::string &storageClassName, const uint32_t copyNb, const std::string &tapePoolName) override; + const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType, + const std::string &tapePoolName) override; void modifyArchiveRouteComment(const common::dataStructures::SecurityIdentity &admin, - const std::string &storageClassName, const uint32_t copyNb, const std::string &comment) override; + const std::string &storageClassName, const uint32_t copyNb, const common::dataStructures::ArchiveRouteType &archiveRouteType, + const std::string &comment) override; private: const std::unique_ptr<Catalogue>& m_catalogue; diff --git a/catalogue/tests/CatalogueTestUtils.cpp b/catalogue/tests/CatalogueTestUtils.cpp index b94b33b59ca8cf43405849fc5799942c21b9e273..73f4718483fe6c7fc711a76f387828ff443a1129 100644 --- a/catalogue/tests/CatalogueTestUtils.cpp +++ b/catalogue/tests/CatalogueTestUtils.cpp @@ -81,7 +81,7 @@ void CatalogueTestUtils::wipeDatabase(cta::catalogue::Catalogue *catalogue, cta: const auto archiveRoutes = catalogue->ArchiveRoute()->getArchiveRoutes(); for(auto &archiveRoute: archiveRoutes) { catalogue->ArchiveRoute()->deleteArchiveRoute(archiveRoute.storageClassName, - archiveRoute.copyNb); + archiveRoute.copyNb, archiveRoute.type); } const auto amRules = catalogue->RequesterActivityMountRule()->getRequesterActivityMountRules(); for(const auto &rule: amRules) { diff --git a/catalogue/tests/modules/ArchiveFileCatalogueTest.cpp b/catalogue/tests/modules/ArchiveFileCatalogueTest.cpp index b621f796b9d0f32dffcc7fa6a6b18de4eea9eb7c..60639351231bc40b5d9dde4097a88130db1f057d 100644 --- a/catalogue/tests/modules/ArchiveFileCatalogueTest.cpp +++ b/catalogue/tests/modules/ArchiveFileCatalogueTest.cpp @@ -117,7 +117,7 @@ TEST_P(cta_catalogue_ArchiveFileTest, checkAndGetNextArchiveFileId_no_mount_rule const uint32_t copyNb = 1; const std::string archiveRouteComment = "Create archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, tapePoolName, archiveRouteComment); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, tapePoolName, archiveRouteComment); const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); @@ -183,7 +183,7 @@ TEST_P(cta_catalogue_ArchiveFileTest, checkAndGetNextArchiveFileId_after_cached_ const uint32_t copyNb = 1; const std::string archiveRouteComment = "Create archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, tapePoolName, archiveRouteComment); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, tapePoolName, archiveRouteComment); const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); @@ -260,7 +260,7 @@ TEST_P(cta_catalogue_ArchiveFileTest, checkAndGetNextArchiveFileId_requester_mou const uint32_t copyNb = 1; const std::string archiveRouteComment = "Create archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, tapePoolName, archiveRouteComment); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, tapePoolName, archiveRouteComment); const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); @@ -330,7 +330,7 @@ TEST_P(cta_catalogue_ArchiveFileTest, checkAndGetNextArchiveFileId_requester_gro const uint32_t copyNb = 1; const std::string archiveRouteComment = "Create archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, tapePoolName, archiveRouteComment); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, tapePoolName, archiveRouteComment); const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); @@ -400,7 +400,7 @@ TEST_P(cta_catalogue_ArchiveFileTest, checkAndGetNextArchiveFileId_after_cached_ const uint32_t copyNb = 1; const std::string archiveRouteComment = "Create archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, tapePoolName, archiveRouteComment); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, tapePoolName, archiveRouteComment); const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); @@ -497,7 +497,7 @@ TEST_P(cta_catalogue_ArchiveFileTest, checkAndGetNextArchiveFileId_requester_mou const uint32_t copyNb = 1; const std::string archiveRouteComment = "Create archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, tapePoolName, archiveRouteComment); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, tapePoolName, archiveRouteComment); const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); @@ -603,7 +603,7 @@ TEST_P(cta_catalogue_ArchiveFileTest, getArchiveFileQueueCriteria_requester_moun const uint32_t copyNb = 1; const std::string archiveRouteComment = "Create archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, tapePoolName, archiveRouteComment); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, tapePoolName, archiveRouteComment); const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); @@ -666,7 +666,7 @@ TEST_P(cta_catalogue_ArchiveFileTest, getArchiveFileQueueCriteria_requester_grou const uint32_t copyNb = 1; const std::string archiveRouteComment = "Create archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, tapePoolName, archiveRouteComment); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, tapePoolName, archiveRouteComment); const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); @@ -745,7 +745,7 @@ TEST_P(cta_catalogue_ArchiveFileTest, getArchiveFileQueueCriteria_requester_moun const uint32_t copyNb = 1; const std::string archiveRouteComment = "Create archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, tapePoolName, archiveRouteComment); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, tapePoolName, archiveRouteComment); const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); @@ -4906,5 +4906,85 @@ TEST_P(cta_catalogue_ArchiveFileTest, deleteArchiveFile_by_archive_file_id_non_e m_catalogue->ArchiveFile()->DO_NOT_USE_deleteArchiveFile_DO_NOT_USE("disk_instance", 12345678, dummyLc); } +TEST_P(cta_catalogue_ArchiveFileTest, getArchiveFileQueueCriteria_ignore_repack_archiveroutes) { + + auto mountPolicyToAdd = CatalogueTestUtils::getMountPolicy1(); + std::string mountPolicyName = mountPolicyToAdd.name; + m_catalogue->MountPolicy()->createMountPolicy(m_admin,mountPolicyToAdd); + m_catalogue->DiskInstance()->createDiskInstance(m_admin, m_diskInstance.name, m_diskInstance.comment); + + const std::string comment = "Create mount rule for requester group"; + const std::string diskInstanceName = m_diskInstance.name; + const std::string requesterGroupName = "requester_group"; + m_catalogue->RequesterGroupMountRule()->createRequesterGroupMountRule(m_admin, mountPolicyName, diskInstanceName, + requesterGroupName, comment); + + m_catalogue->VO()->createVirtualOrganization(m_admin, m_vo); + m_catalogue->StorageClass()->createStorageClass(m_admin, m_storageClassSingleCopy); + + // Setup archive routes, 2 default and 1 for repack + + const std::string tapePoolName_default_1 = "tape_pool_default_1"; + const std::string tapePoolName_repack_1 = "tape_pool_repack_1"; + const std::string tapePoolName_default_2 = "tape_pool_default_2"; + const uint64_t nbPartialTapes = 2; + const bool isEncrypted = true; + const std::optional<std::string> supply("value for the supply pool mechanism"); + m_catalogue->TapePool()->createTapePool(m_admin, tapePoolName_default_1, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool for default 1"); + m_catalogue->TapePool()->createTapePool(m_admin, tapePoolName_repack_1, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool for repack 1"); + m_catalogue->TapePool()->createTapePool(m_admin, tapePoolName_default_2, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool for default 2"); + + ASSERT_TRUE(m_catalogue->ArchiveRoute()->getArchiveRoutes().empty()); + + const uint32_t copyNb_1 = 1; + const uint32_t copyNb_2 = 2; + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb_1, cta::common::dataStructures::ArchiveRouteType::DEFAULT, tapePoolName_default_1, "Create archive route for default 1"); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb_1, cta::common::dataStructures::ArchiveRouteType::REPACK, tapePoolName_repack_1, "Create archive route for repack 1"); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb_2, cta::common::dataStructures::ArchiveRouteType::DEFAULT, tapePoolName_default_2, "Create archive route for default 2"); + + const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); + + ASSERT_EQ(3, routes.size()); + + // Archive routes will be ordered by "storage class -> copy nr -> type" + + auto routes_it = routes.begin(); + + ASSERT_EQ(m_storageClassSingleCopy.name, routes_it->storageClassName); + ASSERT_EQ(copyNb_1, routes_it->copyNb); + ASSERT_EQ(tapePoolName_default_1, routes_it->tapePoolName); + ASSERT_EQ(cta::common::dataStructures::ArchiveRouteType::DEFAULT, routes_it->type); + + routes_it++; + + ASSERT_EQ(m_storageClassSingleCopy.name, routes_it->storageClassName); + ASSERT_EQ(copyNb_1, routes_it->copyNb); + ASSERT_EQ(tapePoolName_repack_1, routes_it->tapePoolName); + ASSERT_EQ(cta::common::dataStructures::ArchiveRouteType::REPACK, routes_it->type); + + routes_it++; + + ASSERT_EQ(m_storageClassSingleCopy.name, routes_it->storageClassName); + ASSERT_EQ(copyNb_2, routes_it->copyNb); + ASSERT_EQ(tapePoolName_default_2, routes_it->tapePoolName); + ASSERT_EQ(cta::common::dataStructures::ArchiveRouteType::DEFAULT, routes_it->type); + + // Check copyToPoolMap, which should contain only the archive routes for default/non-repack + + cta::common::dataStructures::RequesterIdentity requesterIdentity; + requesterIdentity.name = "username"; + requesterIdentity.group = requesterGroupName; + auto archiveFileQueueCriteria = m_catalogue->ArchiveFile()->getArchiveFileQueueCriteria(diskInstanceName, m_storageClassSingleCopy.name, requesterIdentity); + + ASSERT_EQ(2, archiveFileQueueCriteria.copyToPoolMap.size()); + + auto copyToPoolMap_it = archiveFileQueueCriteria.copyToPoolMap.begin(); + ASSERT_EQ(copyNb_1, copyToPoolMap_it->first); + ASSERT_EQ(tapePoolName_default_1, copyToPoolMap_it->second); + + copyToPoolMap_it++; + ASSERT_EQ(copyNb_2, copyToPoolMap_it->first); + ASSERT_EQ(tapePoolName_default_2, copyToPoolMap_it->second); +} } // namespace unitTests diff --git a/catalogue/tests/modules/ArchiveRouteCatalogueTest.cpp b/catalogue/tests/modules/ArchiveRouteCatalogueTest.cpp index 31785650e6fd4965dd9436817ca0f3fa422f47f9..9d7df4d1a4a4cb7f9bad91090fe6fb0b7f5d013f 100644 --- a/catalogue/tests/modules/ArchiveRouteCatalogueTest.cpp +++ b/catalogue/tests/modules/ArchiveRouteCatalogueTest.cpp @@ -60,7 +60,7 @@ TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute) { const uint32_t copyNb = 1; const std::string comment = "Create archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, tapePoolName, comment); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, tapePoolName, comment); { const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); @@ -118,8 +118,8 @@ TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute_emptyStringStorageClas const std::string storageClassName = ""; const uint32_t copyNb = 1; const std::string comment = "Create archive route"; - ASSERT_THROW(m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, storageClassName, copyNb, - tapePoolName, comment), cta::catalogue::UserSpecifiedAnEmptyStringStorageClassName); + ASSERT_THROW(m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, storageClassName, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, + tapePoolName, comment), cta::catalogue::UserSpecifiedAnEmptyStringStorageClassName); } TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute_zeroCopyNb) { @@ -134,8 +134,8 @@ TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute_zeroCopyNb) { const uint32_t copyNb = 0; const std::string comment = "Create archive route"; - ASSERT_THROW(m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, - m_tape1.tapePoolName, comment), cta::catalogue::UserSpecifiedAZeroCopyNb); + ASSERT_THROW(m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, + m_tape1.tapePoolName, comment), cta::catalogue::UserSpecifiedAZeroCopyNb); } TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute_emptyStringTapePoolName) { @@ -147,7 +147,7 @@ TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute_emptyStringTapePoolNam const std::string tapePoolName = ""; const uint32_t copyNb = 1; const std::string comment = "Create archive route"; - ASSERT_THROW(m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, tapePoolName, comment), cta::catalogue::UserSpecifiedAnEmptyStringTapePoolName); + ASSERT_THROW(m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, tapePoolName, comment), cta::catalogue::UserSpecifiedAnEmptyStringTapePoolName); } TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute_emptyStringComment) { @@ -162,8 +162,8 @@ TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute_emptyStringComment) { const uint32_t copyNb = 1; const std::string comment = ""; - ASSERT_THROW(m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, - m_tape1.tapePoolName, comment), cta::catalogue::UserSpecifiedAnEmptyStringComment); + ASSERT_THROW(m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, + m_tape1.tapePoolName, comment), cta::catalogue::UserSpecifiedAnEmptyStringComment); } TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute_non_existent_storage_class) { @@ -178,7 +178,7 @@ TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute_non_existent_storage_c const uint32_t copyNb = 1; const std::string comment = "Create archive route"; - ASSERT_THROW(m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, storageClassName, copyNb, m_tape1.tapePoolName, comment), cta::exception::UserError); + ASSERT_THROW(m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, storageClassName, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, m_tape1.tapePoolName, comment), cta::exception::UserError); } TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute_non_existent_tape_pool) { @@ -191,7 +191,7 @@ TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute_non_existent_tape_pool const uint32_t copyNb = 1; const std::string comment = "Create archive route"; - ASSERT_THROW(m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, tapePoolName, comment), cta::exception::UserError); + ASSERT_THROW(m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, tapePoolName, comment), cta::exception::UserError); } TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute_same_twice) { @@ -206,8 +206,8 @@ TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute_same_twice) { const uint32_t copyNb = 1; const std::string comment = "Create archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, m_tape1.tapePoolName, comment); - ASSERT_THROW(m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, m_tape1.tapePoolName, comment), cta::exception::Exception); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, m_tape1.tapePoolName, comment); + ASSERT_THROW(m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, m_tape1.tapePoolName, comment), cta::exception::Exception); } TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute_two_routes_same_pool) { @@ -222,11 +222,11 @@ TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute_two_routes_same_pool) const uint32_t copyNb1 = 1; const std::string comment1 = "Create archive route for copy 1"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb1, m_tape1.tapePoolName, comment1); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb1, cta::common::dataStructures::ArchiveRouteType::DEFAULT, m_tape1.tapePoolName, comment1); const uint32_t copyNb2 = 2; const std::string comment2 = "Create archive route for copy 2"; - ASSERT_THROW(m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb2, m_tape1.tapePoolName, comment2), cta::exception::UserError); + ASSERT_THROW(m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb2, cta::common::dataStructures::ArchiveRouteType::DEFAULT, m_tape1.tapePoolName, comment2), cta::exception::UserError); } TEST_P(cta_catalogue_ArchiveRouteTest, deleteArchiveRoute) { @@ -241,7 +241,7 @@ TEST_P(cta_catalogue_ArchiveRouteTest, deleteArchiveRoute) { const uint32_t copyNb = 1; const std::string comment = "Create archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, m_tape1.tapePoolName, comment); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, m_tape1.tapePoolName, comment); const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); @@ -260,13 +260,13 @@ TEST_P(cta_catalogue_ArchiveRouteTest, deleteArchiveRoute) { const cta::common::dataStructures::EntryLog lastModificationLog = route.lastModificationLog; ASSERT_EQ(creationLog, lastModificationLog); - m_catalogue->ArchiveRoute()->deleteArchiveRoute(m_storageClassSingleCopy.name, copyNb); + m_catalogue->ArchiveRoute()->deleteArchiveRoute(m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT); ASSERT_TRUE(m_catalogue->ArchiveRoute()->getArchiveRoutes().empty()); } TEST_P(cta_catalogue_ArchiveRouteTest, deleteArchiveRoute_non_existent) { - ASSERT_THROW(m_catalogue->ArchiveRoute()->deleteArchiveRoute("non_existent_storage_class", 1234), cta::exception::UserError); + ASSERT_THROW(m_catalogue->ArchiveRoute()->deleteArchiveRoute("non_existent_storage_class", 1234, cta::common::dataStructures::ArchiveRouteType::DEFAULT), cta::exception::UserError); } TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute_deleteStorageClass) { @@ -281,7 +281,7 @@ TEST_P(cta_catalogue_ArchiveRouteTest, createArchiveRoute_deleteStorageClass) { const uint32_t copyNb = 1; const std::string comment = "Create archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, m_tape1.tapePoolName, comment); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, m_tape1.tapePoolName, comment); const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); @@ -319,7 +319,7 @@ TEST_P(cta_catalogue_ArchiveRouteTest, modifyArchiveRouteTapePoolName) { const uint32_t copyNb = 1; const std::string comment = "Create archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, m_tape1.tapePoolName, comment); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, m_tape1.tapePoolName, comment); { const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); @@ -340,7 +340,7 @@ TEST_P(cta_catalogue_ArchiveRouteTest, modifyArchiveRouteTapePoolName) { ASSERT_EQ(creationLog, lastModificationLog); } - m_catalogue->ArchiveRoute()->modifyArchiveRouteTapePoolName(m_admin, m_storageClassSingleCopy.name, copyNb, anotherTapePoolName); + m_catalogue->ArchiveRoute()->modifyArchiveRouteTapePoolName(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, anotherTapePoolName); { const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); @@ -374,7 +374,7 @@ TEST_P(cta_catalogue_ArchiveRouteTest, modifyArchiveRouteTapePoolName_nonExisten const uint32_t copyNb = 1; const std::string comment = "Create archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, m_tape1.tapePoolName, comment); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, m_tape1.tapePoolName, comment); { const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); @@ -395,7 +395,7 @@ TEST_P(cta_catalogue_ArchiveRouteTest, modifyArchiveRouteTapePoolName_nonExisten ASSERT_EQ(creationLog, lastModificationLog); } - ASSERT_THROW(m_catalogue->ArchiveRoute()->modifyArchiveRouteTapePoolName(m_admin, m_storageClassSingleCopy.name, copyNb, "non_existent_tape_pool"), cta::catalogue::UserSpecifiedANonExistentTapePool); + ASSERT_THROW(m_catalogue->ArchiveRoute()->modifyArchiveRouteTapePoolName(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, "non_existent_tape_pool"), cta::catalogue::UserSpecifiedANonExistentTapePool); } TEST_P(cta_catalogue_ArchiveRouteTest, modifyArchiveRouteTapePoolName_nonExistentArchiveRoute) { @@ -409,7 +409,7 @@ TEST_P(cta_catalogue_ArchiveRouteTest, modifyArchiveRouteTapePoolName_nonExisten m_catalogue->TapePool()->createTapePool(m_admin, m_tape1.tapePoolName, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool"); const uint32_t copyNb = 1; - ASSERT_THROW(m_catalogue->ArchiveRoute()->modifyArchiveRouteTapePoolName(m_admin, m_storageClassSingleCopy.name, copyNb, m_tape1.tapePoolName), cta::catalogue::UserSpecifiedANonExistentArchiveRoute); + ASSERT_THROW(m_catalogue->ArchiveRoute()->modifyArchiveRouteTapePoolName(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, m_tape1.tapePoolName), cta::catalogue::UserSpecifiedANonExistentArchiveRoute); } TEST_P(cta_catalogue_ArchiveRouteTest, modifyArchiveRouteComment) { @@ -424,7 +424,7 @@ TEST_P(cta_catalogue_ArchiveRouteTest, modifyArchiveRouteComment) { const uint32_t copyNb = 1; const std::string comment = "Create archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, m_tape1.tapePoolName, comment); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, m_tape1.tapePoolName, comment); { const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); @@ -446,7 +446,7 @@ TEST_P(cta_catalogue_ArchiveRouteTest, modifyArchiveRouteComment) { } const std::string modifiedComment = "Modified comment"; - m_catalogue->ArchiveRoute()->modifyArchiveRouteComment(m_admin, m_storageClassSingleCopy.name, copyNb, modifiedComment); + m_catalogue->ArchiveRoute()->modifyArchiveRouteComment(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, modifiedComment); { const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); @@ -477,7 +477,7 @@ TEST_P(cta_catalogue_ArchiveRouteTest, modifyArchiveRouteComment_nonExistentArch const uint32_t copyNb = 1; const std::string comment = "Comment"; - ASSERT_THROW(m_catalogue->ArchiveRoute()->modifyArchiveRouteComment(m_admin, m_storageClassSingleCopy.name, copyNb, comment), cta::exception::UserError); + ASSERT_THROW(m_catalogue->ArchiveRoute()->modifyArchiveRouteComment(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, comment), cta::exception::UserError); } } // namespace unitTests \ No newline at end of file diff --git a/catalogue/tests/modules/TapePoolCatalogueTest.cpp b/catalogue/tests/modules/TapePoolCatalogueTest.cpp index 50975a9c94216c6724957876bd0835767332aae0..a246b6f819f709bdbcaa199701145a03bd58f82d 100644 --- a/catalogue/tests/modules/TapePoolCatalogueTest.cpp +++ b/catalogue/tests/modules/TapePoolCatalogueTest.cpp @@ -393,7 +393,7 @@ TEST_P(cta_catalogue_TapePoolTest, deleteTapePool) { anotherTapePoolIsEncrypted, supply, anotherTapePoolComment); const uint32_t copyNb = 1; const std::string comment = "Create a separate archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_anotherStorageClass.name, copyNb, anotherTapePoolName, comment); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_anotherStorageClass.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, anotherTapePoolName, comment); } { @@ -587,7 +587,7 @@ TEST_P(cta_catalogue_TapePoolTest, deleteTapePool_used_in_an_archive_route) { const uint32_t copyNb = 1; const std::string comment = "Create archive route"; - m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, tapePoolName, comment); + m_catalogue->ArchiveRoute()->createArchiveRoute(m_admin, m_storageClassSingleCopy.name, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, tapePoolName, comment); { const std::list<cta::common::dataStructures::ArchiveRoute> routes = m_catalogue->ArchiveRoute()->getArchiveRoutes(); diff --git a/cmdline/CtaAdminCmdParse.hpp b/cmdline/CtaAdminCmdParse.hpp index 21ea0e08ed587e5f60503b1c8b947a6d1dfecb9d..c991cf06fa75e83c4e23854d3fe3a3d729373799 100644 --- a/cmdline/CtaAdminCmdParse.hpp +++ b/cmdline/CtaAdminCmdParse.hpp @@ -345,7 +345,7 @@ const std::map<std::string, OptionString::Key> strOptions = { { "--freespacequeryurl", OptionString::FREE_SPACE_QUERY_URL }, { "--reason", OptionString::REASON }, { "--state", OptionString::STATE }, - { "--activityregex", OptionString::ACTIVITY_REGEX}, + { "--activityregex", OptionString::ACTIVITY_REGEX}, { "--diskinstance", OptionString::DISK_INSTANCE }, { "--diskinstancespace", OptionString::DISK_INSTANCE_SPACE }, { "--verificationstatus", OptionString::VERIFICATION_STATUS }, @@ -357,7 +357,8 @@ const std::map<std::string, OptionString::Key> strOptions = { { "--type", OptionString::LIBRARY_TYPE }, { "--guiurl", OptionString::GUI_URL }, { "--webcamurl", OptionString::WEBCAM_URL }, - { "--location", OptionString::LIBRARY_LOCATION } + { "--location", OptionString::LIBRARY_LOCATION }, + { "--archiveroutetype", OptionString::ARCHIVE_ROUTE_TYPE } }; /*! @@ -505,6 +506,7 @@ const Option opt_isrepackvo {Option::OPT_BOOL, "--isrepackvo", "--irvo", R"( <"t const Option opt_max_files_to_select {Option::OPT_UINT, "--maxfilestoselect", "--mfts", " <max_files_to_select>"}; const Option opt_log_unixtime_min {Option::OPT_UINT, "--logunixtimemin", "--ltmin", " <min_recycle_log_unixtime>"}; const Option opt_log_unixtime_max {Option::OPT_UINT, "--logunixtimemax", "--ltmax", " <max_recycle_log_unixtime>"}; +const Option opt_archive_route_type {Option::OPT_STR, "--archiveroutetype", "--art", " <archive_route_type>"}; /*! * Subset of commands that return streaming output @@ -586,10 +588,11 @@ archiveroute (ar) : Add, change, remove or list the archive routes. Archive routes are the policies linking namespace entries to tape pools. */ - {{AdminCmd::CMD_ARCHIVEROUTE, AdminCmd::SUBCMD_ADD}, {opt_storageclass, opt_copynb, opt_tapepool, opt_comment} }, + {{AdminCmd::CMD_ARCHIVEROUTE, AdminCmd::SUBCMD_ADD}, + {opt_storageclass, opt_copynb, opt_archive_route_type.optional(), opt_tapepool, opt_comment} }, {{AdminCmd::CMD_ARCHIVEROUTE, AdminCmd::SUBCMD_CH}, - {opt_storageclass, opt_copynb, opt_tapepool.optional(), opt_comment.optional()} }, - {{AdminCmd::CMD_ARCHIVEROUTE, AdminCmd::SUBCMD_RM}, {opt_storageclass, opt_copynb} }, + {opt_storageclass, opt_copynb, opt_archive_route_type, opt_tapepool.optional(), opt_comment.optional()} }, + {{AdminCmd::CMD_ARCHIVEROUTE, AdminCmd::SUBCMD_RM}, {opt_storageclass, opt_copynb, opt_archive_route_type} }, {{AdminCmd::CMD_ARCHIVEROUTE, AdminCmd::SUBCMD_LS}, {} }, /**md diff --git a/cmdline/CtaAdminTextFormatter.cpp b/cmdline/CtaAdminTextFormatter.cpp index 19a90ea111b07ea00d98012a960d61d51c1318f7..822472776bbb4bf169367439a4c6fef5d39bc740 100644 --- a/cmdline/CtaAdminTextFormatter.cpp +++ b/cmdline/CtaAdminTextFormatter.cpp @@ -21,6 +21,7 @@ #include <common/checksum/ChecksumBlobSerDeser.hpp> #include <common/dataStructures/DriveStatusSerDeser.hpp> #include <common/dataStructures/MountTypeSerDeser.hpp> +#include <common/dataStructures/ArchiveRouteType.hpp> #include "common/utils/utils.hpp" @@ -227,6 +228,7 @@ void TextFormatter::printArchiveRouteLsHeader() { push_back( "storage class", "copy number", + "type", "tapepool", "c.user", "c.host", @@ -242,6 +244,7 @@ void TextFormatter::print(const ArchiveRouteLsItem &arls_item) { push_back( arls_item.storage_class(), arls_item.copy_number(), + arls_item.archive_route_type(), arls_item.tapepool(), arls_item.creation_log().username(), arls_item.creation_log().host(), diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index cdbd6ca12f693c57d412ea607baedc04dbbd6031..af049f2bd877413de354c7c11885d5ef2aeb4bdf 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -46,6 +46,7 @@ set (COMMON_LIB_SRC_FILES dataStructures/ArchiveJob.cpp dataStructures/ArchiveRequest.cpp dataStructures/ArchiveRoute.cpp + dataStructures/ArchiveRouteType.cpp dataStructures/CancelRetrieveRequest.cpp dataStructures/DiskSpaceReservationRequest.cpp dataStructures/DeleteArchiveRequest.cpp diff --git a/common/dataStructures/ArchiveRoute.cpp b/common/dataStructures/ArchiveRoute.cpp index a214df40d100cd461d91ddf7880154cad48bff42..4eb0fcefca5a6bc31fdad136610a035d61f55611 100644 --- a/common/dataStructures/ArchiveRoute.cpp +++ b/common/dataStructures/ArchiveRoute.cpp @@ -16,8 +16,11 @@ */ #include "common/dataStructures/ArchiveRoute.hpp" +#include "common/dataStructures/ArchiveRouteType.hpp" #include "common/dataStructures/utils.hpp" #include "common/exception/Exception.hpp" +#include "common/exception/UserError.hpp" +#include "common/utils/utils.hpp" namespace cta::common::dataStructures { @@ -25,7 +28,7 @@ namespace cta::common::dataStructures { // constructor //------------------------------------------------------------------------------ ArchiveRoute::ArchiveRoute(): - copyNb(0) {} + copyNb(0), type(ArchiveRouteType::DEFAULT) {} //------------------------------------------------------------------------------ // operator== @@ -33,6 +36,7 @@ ArchiveRoute::ArchiveRoute(): bool ArchiveRoute::operator==(const ArchiveRoute &rhs) const { return storageClassName==rhs.storageClassName && copyNb==rhs.copyNb + && type==rhs.type && tapePoolName==rhs.tapePoolName && creationLog==rhs.creationLog && lastModificationLog==rhs.lastModificationLog @@ -52,6 +56,7 @@ bool ArchiveRoute::operator!=(const ArchiveRoute &rhs) const { std::ostream &operator<<(std::ostream &os, const ArchiveRoute &obj) { os << "(storageClassName=" << obj.storageClassName << " copyNb=" << obj.copyNb + << " archiveRouteType=" << obj.type << " tapePoolName=" << obj.tapePoolName << " creationLog=" << obj.creationLog << " lastModificationLog=" << obj.lastModificationLog diff --git a/common/dataStructures/ArchiveRoute.hpp b/common/dataStructures/ArchiveRoute.hpp index 599f9063ee279647171e19a13fd07095157dd629..5299fbf08910219c602cc2497027f3db6b4a1225 100644 --- a/common/dataStructures/ArchiveRoute.hpp +++ b/common/dataStructures/ArchiveRoute.hpp @@ -22,6 +22,7 @@ #include <stdint.h> #include <string> +#include "common/dataStructures/ArchiveRouteType.hpp" #include "common/dataStructures/EntryLog.hpp" namespace cta::common::dataStructures { @@ -45,10 +46,15 @@ struct ArchiveRoute { std::string storageClassName; /** - * The cipy number of the tape file. + * The copy number of the tape file. */ uint8_t copyNb; + /** + * The type of the archive route. + */ + ArchiveRouteType type; + std::string tapePoolName; EntryLog creationLog; EntryLog lastModificationLog; diff --git a/common/dataStructures/ArchiveRouteType.cpp b/common/dataStructures/ArchiveRouteType.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fcc284f9cff1469a0b79a812c449253c390d0dbf --- /dev/null +++ b/common/dataStructures/ArchiveRouteType.cpp @@ -0,0 +1,50 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#include "common/dataStructures/ArchiveRouteType.hpp" +#include "common/exception/Exception.hpp" +#include "common/exception/UserError.hpp" +#include "common/utils/utils.hpp" + +#include <sstream> + +namespace cta::common::dataStructures { + +std::string toString(ArchiveRouteType type) { + switch (type) { + case ArchiveRouteType::DEFAULT: + return "DEFAULT"; + case ArchiveRouteType::REPACK: + return "REPACK"; + default: + throw cta::exception::Exception(std::string("The type given (") + std::to_string(type) + ") does not exist. Possible values are DEFAULT and REPACK."); + } +} + +ArchiveRouteType strToArchiveRouteType(const std::string &archiveRouteTypeStr) { + std::string typeUpperCase = archiveRouteTypeStr; + cta::utils::toUpper(typeUpperCase); + if (typeUpperCase == "DEFAULT") { + return ArchiveRouteType::DEFAULT; + } else if (typeUpperCase == "REPACK") { + return ArchiveRouteType::REPACK; + } else { + throw cta::exception::UserError(std::string("The type given (") + typeUpperCase + ") does not exist. Possible values are DEFAULT and REPACK."); + } +} + +} \ No newline at end of file diff --git a/common/dataStructures/ArchiveRouteType.hpp b/common/dataStructures/ArchiveRouteType.hpp new file mode 100644 index 0000000000000000000000000000000000000000..a16f68febffcc611475e21e4b69b779b5fba4a8f --- /dev/null +++ b/common/dataStructures/ArchiveRouteType.hpp @@ -0,0 +1,39 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#pragma once + +#include <string> + +namespace cta::common::dataStructures { + +enum ArchiveRouteType { + DEFAULT = 1, + REPACK = 2, +}; + +/** + * Convert enum to string for storage in DB and logging + */ +std::string toString(ArchiveRouteType type); + +/** + * Convert string to enum. Needed to get values from DB. + */ +ArchiveRouteType strToArchiveRouteType(const std::string& archiveRouteTypeStr); + +} // namespace cta::common::dataStructures diff --git a/common/dataStructures/ArchiveRouteTypeSerDeser.hpp b/common/dataStructures/ArchiveRouteTypeSerDeser.hpp new file mode 100644 index 0000000000000000000000000000000000000000..f7b16109832ac35815f15b9a3210b0782b4830d1 --- /dev/null +++ b/common/dataStructures/ArchiveRouteTypeSerDeser.hpp @@ -0,0 +1,48 @@ +/* + * @project The CERN Tape Archive (CTA) + * @copyright Copyright © 2021-2022 CERN + * @license This program is free software, distributed under the terms of the GNU General Public + * Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can + * redistribute it and/or modify it under the terms of the GPL Version 3, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * In applying this licence, CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization or + * submit itself to any jurisdiction. + */ + +#pragma once + +#include <common/dataStructures/ArchiveRouteType.hpp> +#include "cta_admin.pb.h" + +namespace cta::admin { + +ArchiveRouteType ProtobufToArchiveRouteTypeFormat(ArchiveRouteLsItem::ArchiveRouteType archiveRouteType) { + + using namespace common::dataStructures; + + switch (archiveRouteType) { + case ArchiveRouteLsItem::DEFAULT: return ArchiveRouteType::DEFAULT; + case ArchiveRouteLsItem::REPACK: return ArchiveRouteType::REPACK; + default: + throw std::runtime_error("In ProtobufToArchiveRouteTypeFormat(): unknown archive route type " + std::to_string(archiveRouteType)); + } +} + +ArchiveRouteLsItem::ArchiveRouteType ArchiveRouteTypeFormatToProtobuf(ArchiveRouteType archiveRouteType) { + using namespace common::dataStructures; + + switch (archiveRouteType) { + case ArchiveRouteType::DEFAULT: return ArchiveRouteLsItem::DEFAULT; + case ArchiveRouteType::REPACK: return ArchiveRouteLsItem::REPACK; + default: + throw std::runtime_error("In ArchiveRouteTypeFormatToProtobuf(): unknown archive route type " + std::to_string(archiveRouteType)); + } +} + +} // namespace cta::admin diff --git a/continuousintegration/orchestration/tests/cta_admin.sh b/continuousintegration/orchestration/tests/cta_admin.sh index f8a7ca34a0f6a5af0e0e9edcdfbf7eadcc7cd1b2..165234c867e085127d0a460d08fa096ae57cef7e 100755 --- a/continuousintegration/orchestration/tests/cta_admin.sh +++ b/continuousintegration/orchestration/tests/cta_admin.sh @@ -462,14 +462,14 @@ test_assert || exit 1 # Archive Route (ar) -test_start "archvie route" "ar" -test_and_check_cmd "Adding archive route" "${command}" "add" "-s ctaStorageClass -c 2 -t ctasystest_A -m 'cta-admin systest add'"\ +test_start "archive route" "ar" +test_and_check_cmd "Adding archive route" "${command}" "add" "-s ctaStorageClass -c 2 --art DEFAULT -t ctasystest_A -m 'cta-admin systest add'"\ 'select(.storageClass=="ctaStorageClass" and .copyNumber==2 and .tapepool=="ctasystest_A" and .comment=="cta-admin systest add") | .storageClass'\ "1" "adding archive route" || exit 1 -test_and_check_cmd "Changing archive route" "${command}" "ch" "-s ctaStorageClass -c 2 -t ctasystest_B -m 'cta-admin systest ch'"\ +test_and_check_cmd "Changing archive route" "${command}" "ch" "-s ctaStorageClass -c 2 --art DEFAULT -t ctasystest_B -m 'cta-admin systest ch'"\ 'select(.storageClass=="ctaStorageClass" and .copyNumber==2 and .tapepool=="ctasystest_B" and .comment=="cta-admin systest ch") | .storageClass'\ "1" "changing archive route" || exit 1 -test_command "Removing archive route" "${command}" "rm" "-s ctaStorageClass -c 2" || exit 1 +test_command "Removing archive route" "${command}" "rm" "-s ctaStorageClass -c 2 --art DEFAULT" || exit 1 test_assert || exit 1 diff --git a/continuousintegration/orchestration/tests/prepare_tests.sh b/continuousintegration/orchestration/tests/prepare_tests.sh index 87e2af6b9aaf5302b44144178a6adb760348b5af..32769a553bb59c1ad7c2cdc9f4e6260be91728ad 100755 --- a/continuousintegration/orchestration/tests/prepare_tests.sh +++ b/continuousintegration/orchestration/tests/prepare_tests.sh @@ -107,7 +107,7 @@ kubectl --namespace ${NAMESPACE} exec ctacli -- cta-admin --json version | jq cta-admin tape rm -v {} kubectl --namespace ${NAMESPACE} exec ctacli -- cta-admin --json archiveroute ls | \ - jq '.[] | " -s " + .storageClass + " -c " + (.copyNumber|tostring)' | \ + jq '.[] | " -s " + .storageClass + " -c " + (.copyNumber|tostring) + " --art " + .archiveRouteType' | \ xargs -I{} bash -c "kubectl --namespace ${NAMESPACE} exec ctacli -- cta-admin archiveroute rm {}" kubectl --namespace ${NAMESPACE} exec ctacli -- cta-admin --json tapepool ls | \ diff --git a/continuousintegration/orchestration/tests/repack_helper.sh b/continuousintegration/orchestration/tests/repack_helper.sh index 528addf8a33f17d60748693c629763dc048de41d..287a43646943549c28fe3aed5d7c15fd5e136b77 100755 --- a/continuousintegration/orchestration/tests/repack_helper.sh +++ b/continuousintegration/orchestration/tests/repack_helper.sh @@ -48,7 +48,7 @@ executeReclaim() { } getFirstVidContainingFiles() { - vidToRepack=$(kubectl -n ${NAMESPACE} exec ctacli -ti -- cta-admin --json ta ls --all | jq -r '[.[] | select(.occupancy != "0") | select(.lastFseq != 0) | .vid] | .[0]') + vidToRepack=$(kubectl -n ${NAMESPACE} exec ctacli -ti -- cta-admin --json ta ls --all | jq -r '[.[] | select(.occupancy != "0") | select(.lastFseq != "0") | .vid] | .[0]') echo $vidToRepack } diff --git a/continuousintegration/orchestration/tests/repack_systemtest.sh b/continuousintegration/orchestration/tests/repack_systemtest.sh index 22bd651964adc4e58d28f51d8fb55d7c75b1f25d..398ec7f55d65a8aaa2ff4b9cadd586f237c06b5b 100755 --- a/continuousintegration/orchestration/tests/repack_systemtest.sh +++ b/continuousintegration/orchestration/tests/repack_systemtest.sh @@ -264,10 +264,20 @@ fi amountArchivedFiles=`admin_cta --json repack ls --vid ${VID_TO_REPACK} | jq -r ". [0] | .archivedFiles"` amountRecyleTapeFilesNew=`admin_cta --json recycletf ls --vid ${VID_TO_REPACK} | jq "length"` amountRecyleTapeFiles=$((amountRecyleTapeFilesNew-$amountRecyleTapeFilesPrev)) +filesLeftToRetrieve=`admin_cta --json repack ls --vid ${VID_TO_REPACK} | jq -r ". [0] | .filesLeftToRetrieve"` +filesLeftToArchive=`admin_cta --json repack ls --vid ${VID_TO_REPACK} | jq -r ". [0] | .filesLeftToArchive"` +nbDestinationVids=`admin_cta --json repack ls --vid ${VID_TO_REPACK} | jq -r ". [0] | .destinationInfos | length"` -echo "Amount of archived files = $amountArchivedFiles" +echo "Amount of archived files = $amountArchivedFiles (${nbDestinationVids}x$((amountArchivedFiles/nbDestinationVids))" echo "Amount of new recycled tape files = $amountRecyleTapeFiles" -if [[ $amountArchivedFiles -eq $amountRecyleTapeFiles ]] +echo "Amount of files left to retrieve = $filesLeftToRetrieve" +echo "Amount of files left to archive = $filesLeftToArchive" + +if [[ "$filesLeftToRetrieve" -ne "0" ]] || [[ "$filesLeftToArchive" -ne "0" ]] +then + echo "There were remaining files left to retrieve ($filesLeftToRetrieve) or archive ($filesLeftToArchive). Test FAILED" +fi +if [[ "$((amountArchivedFiles/nbDestinationVids))" -eq "$amountRecyleTapeFiles" ]] then echo "The amount of archived files is equal to the amount of new recycled tape files. Test OK" else diff --git a/continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh b/continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh index 912c0b417de9617d6d455d09f8a1b1347e92a5f0..04c68b4b8246a1b3dd3201c5372fe7478755fd92 100755 --- a/continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh +++ b/continuousintegration/orchestration/tests/repack_systemtest_wrapper.sh @@ -422,37 +422,42 @@ repackMoveAndAddCopies() { echo "STEP $1. Testing Repack \"Move and Add copies\" workflow" echo "*******************************************************" - tapepoolDestination1="ctasystest2" - tapepoolDestination2="ctasystest3" + defaultTapepool="ctasystest" + tapepoolDestination1_default="systest2_default" + tapepoolDestination2_default="systest3_default" + tapepoolDestination2_repack="systest3_repack" + + echo "Creating 2 destination tapepools : $tapepoolDestination1_default and $tapepoolDestination2_default" + kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tapepool add --name $tapepoolDestination1_default --vo vo --partialtapesnumber 2 --encrypted false --comment "$tapepoolDestination1_default tapepool" + kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tapepool add --name $tapepoolDestination2_default --vo vo --partialtapesnumber 2 --encrypted false --comment "$tapepoolDestination2_default tapepool" + echo "OK" - echo "Creating two destination tapepool : $tapepoolDestination1 and $tapepoolDestination2" - kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tapepool add --name $tapepoolDestination1 --vo vo --partialtapesnumber 2 --encrypted false --comment "$tapepoolDestination1 tapepool" - kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tapepool add --name $tapepoolDestination2 --vo vo --partialtapesnumber 2 --encrypted false --comment "$tapepoolDestination2 tapepool" + echo "Creating 1 destination tapepool for repack : $tapepoolDestination2_repack (will override $tapepoolDestination2_default)" + kubectl -n ${NAMESPACE} exec ctacli -- cta-admin tapepool add --name $tapepoolDestination2_repack --vo vo --partialtapesnumber 2 --encrypted false --comment "$tapepoolDestination2_repack tapepool" echo "OK" echo "Creating archive routes for adding two copies of the file" - kubectl -n ${NAMESPACE} exec ctacli -- cta-admin archiveroute add --storageclass ctaStorageClass --copynb 2 --tapepool $tapepoolDestination1 --comment "ArchiveRoute2" - kubectl -n ${NAMESPACE} exec ctacli -- cta-admin archiveroute add --storageclass ctaStorageClass --copynb 3 --tapepool $tapepoolDestination2 --comment "ArchiveRoute3" + kubectl -n ${NAMESPACE} exec ctacli -- cta-admin archiveroute add --storageclass ctaStorageClass --copynb 2 --tapepool $tapepoolDestination1_default --comment "ArchiveRoute2_default" + kubectl -n ${NAMESPACE} exec ctacli -- cta-admin archiveroute add --storageclass ctaStorageClass --copynb 3 --archiveroutetype DEFAULT --tapepool $tapepoolDestination2_default --comment "ArchiveRoute3_default" + kubectl -n ${NAMESPACE} exec ctacli -- cta-admin archiveroute add --storageclass ctaStorageClass --copynb 3 --archiveroutetype REPACK --tapepool $tapepoolDestination2_repack --comment "ArchiveRoute3_repack" echo "OK" echo "Will change the tapepool of the tapes" allVID=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tape ls --all | jq -r ". [] | .vid"` - read -a allVIDTable <<< $allVID + allVIDTable=($allVID) nbVid=${#allVIDTable[@]} allTapepool=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tapepool ls | jq -r ". [] .name"` - - read -a allTapepoolTable <<< $allTapepool + allTapepoolTable=($allTapepool) nbTapepool=${#allTapepoolTable[@]} nbTapePerTapepool=$(($nbVid / $nbTapepool)) allTapepool=`kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tapepool ls | jq -r ". [] .name"` - read -a allTapepoolTable <<< $allTapepool - + allTapepoolTable=($allTapepool) countChanging=0 tapepoolIndice=1 #We only change the vid of the remaining other tapes @@ -509,6 +514,28 @@ repackMoveAndAddCopies() { echo "ArchivedFiles ($archivedFiles) == totalFilesToArchive ($totalFilesToArchive), OK" fi + # Check that 2 copies were written to default tapepool (archive route 1 and 2) and 1 copy to repack tapepool (archive route 3) + TAPEPOOL_LIST=$(kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json repack ls --vid V00101 | jq ".[] | .destinationInfos[] | .vid" | xargs -I{} kubectl -n ${NAMESPACE} exec ctacli -- cta-admin --json tape ls --vid {} | jq -r '.[] .tapepool') + + if [[ $TAPEPOOL_LIST != *"$defaultTapepool"* ]]; then + echo "Did not find $defaultTapepool in repack archive destination pools. Archive route failed." + exit 1 + else + echo "Found $defaultTapepool in repack archive destination pools." + fi + if [[ $TAPEPOOL_LIST != *"$tapepoolDestination1_default"* ]]; then + echo "Did not find $tapepoolDestination1_default in repack archive destination pools. Archive route failed." + exit 1 + else + echo "Found $tapepoolDestination1_default in repack archive destination pools." + fi + if [[ $TAPEPOOL_LIST != *"$tapepoolDestination2_repack"* ]]; then + echo "Did not find $tapepoolDestination2_repack in repack archive destination pools. Archive route failed." + exit 1 + else + echo "Found $tapepoolDestination2_repack in repack archive destination pools." + fi + removeRepackRequest ${VID_TO_REPACK} echo "Setting the tape ${VID_TO_REPACK} back to ACTIVE" modifyTapeState ${VID_TO_REPACK} ACTIVE @@ -742,5 +769,6 @@ repackJustMove 4 repackTapeRepair 5 repackJustAddCopies 6 repackCancellation 7 -# repackMoveAndAddCopies 7 repackTapeRepairNoRecall 8 +# Keep this test for last - it adds new tapepools and archive routes +repackMoveAndAddCopies 9 diff --git a/frontend/common/AdminCmd.cpp b/frontend/common/AdminCmd.cpp index 8273bb2aa943b2a626eb174ba556b006b6791d3d..53cfdf21a0bfa956e6065b58eddac02d76fd1368 100644 --- a/frontend/common/AdminCmd.cpp +++ b/frontend/common/AdminCmd.cpp @@ -416,10 +416,14 @@ void AdminCmd::processArchiveRoute_Add(xrd::Response& response) { auto& scn = getRequired(OptionString::STORAGE_CLASS); auto& cn = getRequired(OptionUInt64::COPY_NUMBER); + auto art_opt = getOptional(OptionString::ARCHIVE_ROUTE_TYPE); auto& tapepool = getRequired(OptionString::TAPE_POOL); auto& comment = getRequired(OptionString::COMMENT); - m_catalogue.ArchiveRoute()->createArchiveRoute(m_cliIdentity, scn, static_cast<uint32_t>(cn), tapepool, comment); + // Adding a new archive route without type is allowed. If so, it should default to DEFAULT. + auto art = art_opt.has_value() ? common::dataStructures::strToArchiveRouteType(art_opt.value()) : cta::common::dataStructures::ArchiveRouteType::DEFAULT; + + m_catalogue.ArchiveRoute()->createArchiveRoute(m_cliIdentity, scn, static_cast<uint32_t>(cn), art, tapepool, comment); response.set_type(xrd::Response::RSP_SUCCESS); } @@ -427,14 +431,17 @@ void AdminCmd::processArchiveRoute_Add(xrd::Response& response) { void AdminCmd::processArchiveRoute_Ch(xrd::Response& response) { using namespace cta::admin; - auto& scn = getRequired(OptionString::STORAGE_CLASS); - auto& cn = getRequired(OptionUInt64::COPY_NUMBER); + auto& scn = getRequired(OptionString::STORAGE_CLASS); + auto& cn = getRequired(OptionUInt64::COPY_NUMBER); + auto& art_str = getRequired(OptionString::ARCHIVE_ROUTE_TYPE); + + auto art = common::dataStructures::strToArchiveRouteType(art_str); if(const auto comment = getOptional(OptionString::COMMENT); comment) { - m_catalogue.ArchiveRoute()->modifyArchiveRouteComment(m_cliIdentity, scn, static_cast<uint32_t>(cn), comment.value()); + m_catalogue.ArchiveRoute()->modifyArchiveRouteComment(m_cliIdentity, scn, static_cast<uint32_t>(cn), art, comment.value()); } if(const auto tapepool = getOptional(OptionString::TAPE_POOL); tapepool) { - m_catalogue.ArchiveRoute()->modifyArchiveRouteTapePoolName(m_cliIdentity, scn, static_cast<uint32_t>(cn), tapepool.value()); + m_catalogue.ArchiveRoute()->modifyArchiveRouteTapePoolName(m_cliIdentity, scn, static_cast<uint32_t>(cn), art, tapepool.value()); } response.set_type(xrd::Response::RSP_SUCCESS); @@ -443,10 +450,13 @@ void AdminCmd::processArchiveRoute_Ch(xrd::Response& response) { void AdminCmd::processArchiveRoute_Rm(xrd::Response& response) { using namespace cta::admin; - auto& scn = getRequired(OptionString::STORAGE_CLASS); - auto& cn = getRequired(OptionUInt64::COPY_NUMBER); + auto& scn = getRequired(OptionString::STORAGE_CLASS); + auto& cn = getRequired(OptionUInt64::COPY_NUMBER); + auto& art_str = getRequired(OptionString::ARCHIVE_ROUTE_TYPE); + + auto art = common::dataStructures::strToArchiveRouteType(art_str); - m_catalogue.ArchiveRoute()->deleteArchiveRoute(scn, static_cast<uint32_t>(cn)); + m_catalogue.ArchiveRoute()->deleteArchiveRoute(scn, static_cast<uint32_t>(cn), art); response.set_type(xrd::Response::RSP_SUCCESS); } diff --git a/scheduler/GenericSchedulerTest.cpp b/scheduler/GenericSchedulerTest.cpp index 972e411ae8caddad2c478f7eb0e2ae43a0e9d1f8..0a952b390e93eeef2e94f37101c7480e5e74a55e 100644 --- a/scheduler/GenericSchedulerTest.cpp +++ b/scheduler/GenericSchedulerTest.cpp @@ -267,8 +267,8 @@ public: tapePoolSupply, tapePoolComment); const uint32_t copyNb = 1; const std::string archiveRouteComment = "Archive-route comment"; - catalogue.ArchiveRoute()->createArchiveRoute(s_adminOnAdminHost, s_storageClassName, copyNb, s_tapePoolName, - archiveRouteComment); + catalogue.ArchiveRoute()->createArchiveRoute(s_adminOnAdminHost, s_storageClassName, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, + s_tapePoolName, archiveRouteComment); cta::catalogue::MediaType mediaType; mediaType.name = s_mediaType; @@ -938,10 +938,10 @@ TEST_P(SchedulerTest, archive_report_and_retrieve_new_dual_copy_file) { const std::string archiveRoute2Comment = "Archive-route for copy number 2"; const uint32_t archiveRoute1CopyNb = 1; const uint32_t archiveRoute2CopyNb = 2; - catalogue.ArchiveRoute()->createArchiveRoute(s_adminOnAdminHost, dualCopyStorageClassName, archiveRoute1CopyNb, tapePool1Name, - archiveRoute1Comment); - catalogue.ArchiveRoute()->createArchiveRoute(s_adminOnAdminHost, dualCopyStorageClassName, archiveRoute2CopyNb, tapePool2Name, - archiveRoute1Comment); + catalogue.ArchiveRoute()->createArchiveRoute(s_adminOnAdminHost, dualCopyStorageClassName, archiveRoute1CopyNb, + cta::common::dataStructures::ArchiveRouteType::DEFAULT, tapePool1Name, archiveRoute1Comment); + catalogue.ArchiveRoute()->createArchiveRoute(s_adminOnAdminHost, dualCopyStorageClassName, archiveRoute2CopyNb, + cta::common::dataStructures::ArchiveRouteType::DEFAULT, tapePool2Name, archiveRoute1Comment); cta::catalogue::MediaType mediaType; mediaType.name = s_mediaType; diff --git a/scheduler/Scheduler.cpp b/scheduler/Scheduler.cpp index 9d9da5a97d30477173d3f4654c277433d12d2780..e33c36de7342ae3c87dfffc22d7187686fc064e3 100644 --- a/scheduler/Scheduler.cpp +++ b/scheduler/Scheduler.cpp @@ -494,6 +494,11 @@ void Scheduler::expandRepackRequest(const std::unique_ptr<RepackRequest>& repack cta::common::dataStructures::ArchiveRoute::FullMap archiveRoutesMap; for(const auto& route: routes){ //insert the route into the map to allow a quick retrieval + if (route.type == common::dataStructures::ArchiveRouteType::DEFAULT + && archiveRoutesMap.count(route.storageClassName) && archiveRoutesMap[route.storageClassName].count(route.copyNb)) { + // A DEFAULT archive route type should not override a previously found one + continue; + } archiveRoutesMap[route.storageClassName][route.copyNb] = route; } uint64_t fSeq; diff --git a/scheduler/SchedulerTest.cpp b/scheduler/SchedulerTest.cpp index bd313994e0ad39820f2a209065040d3abb56d99a..32cc62a860599fc9b8d0904b79a58e323f804e68 100644 --- a/scheduler/SchedulerTest.cpp +++ b/scheduler/SchedulerTest.cpp @@ -281,12 +281,22 @@ public: const std::string tapePoolComment = "Tape-pool comment"; const bool tapePoolEncryption = false; const std::optional<std::string> tapePoolSupply("value for the supply pool mechanism"); - catalogue.TapePool()->createTapePool(s_adminOnAdminHost, s_tapePoolName, vo.name, nbPartialTapes, tapePoolEncryption, - tapePoolSupply, tapePoolComment); + const uint32_t copyNb = 1; - const std::string archiveRouteComment = "Archive-route comment"; - catalogue.ArchiveRoute()->createArchiveRoute(s_adminOnAdminHost, s_storageClassName, copyNb, s_tapePoolName, - archiveRouteComment); + + // Tape pool for default archive route + catalogue.TapePool()->createTapePool(s_adminOnAdminHost, s_tapePoolName_default, vo.name, nbPartialTapes, tapePoolEncryption, + tapePoolSupply, tapePoolComment); + const std::string archiveRouteComment_default = "Archive-route comment - default"; + catalogue.ArchiveRoute()->createArchiveRoute(s_adminOnAdminHost, s_storageClassName, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, + s_tapePoolName_default, archiveRouteComment_default); + + // Tape pool for repack archive route + catalogue.TapePool()->createTapePool(s_adminOnAdminHost, s_tapePoolName_repack, vo.name, nbPartialTapes, tapePoolEncryption, + tapePoolSupply, tapePoolComment); + const std::string archiveRouteComment_repack = "Archive-route comment - repack"; + catalogue.ArchiveRoute()->createArchiveRoute(s_adminOnAdminHost, s_storageClassName, copyNb, cta::common::dataStructures::ArchiveRouteType::REPACK, + s_tapePoolName_repack, archiveRouteComment_repack); cta::catalogue::MediaType mediaType; mediaType.name = s_mediaType; @@ -311,7 +321,7 @@ public: tape.mediaType = s_mediaType; tape.vendor = s_vendor; tape.logicalLibraryName = s_libraryName; - tape.tapePoolName = s_tapePoolName; + tape.tapePoolName = s_tapePoolName_default; tape.full = false; tape.state = common::dataStructures::Tape::ACTIVE; tape.comment = "Comment"; @@ -357,7 +367,8 @@ protected: const std::string s_diskInstance = "disk_instance"; const std::string s_storageClassName = "TestStorageClass"; const cta::common::dataStructures::SecurityIdentity s_adminOnAdminHost = { "admin1", "host1" }; - const std::string s_tapePoolName = "TapePool"; + const std::string s_tapePoolName_default = "TapePool_default"; + const std::string s_tapePoolName_repack = "TapePool_repack"; const std::string s_libraryName = "TestLogicalLibrary"; const std::string s_vid = "TESTVID"; const std::string s_mediaType = "TestMediaType"; @@ -424,7 +435,7 @@ TEST_P(SchedulerTest, archive_to_new_file) { auto poolItor = rqsts.cbegin(); ASSERT_FALSE(poolItor == rqsts.cend()); const std::string pool = poolItor->first; - ASSERT_TRUE(s_tapePoolName == pool); + ASSERT_TRUE(s_tapePoolName_default == pool); auto poolRqsts = poolItor->second; ASSERT_EQ(1, poolRqsts.size()); std::set<std::string> remoteFiles; @@ -598,8 +609,8 @@ TEST_P(SchedulerTest, archive_report_and_retrieve_new_file_no_report) { auto & osdb=getSchedulerDB(); auto mi=osdb.getMountInfo(lc); ASSERT_EQ(1, mi->existingOrNextMounts.size()); - ASSERT_EQ("TapePool", mi->existingOrNextMounts.front().tapePool); - ASSERT_EQ("TESTVID", mi->existingOrNextMounts.front().vid); + ASSERT_EQ(s_tapePoolName_default, mi->existingOrNextMounts.front().tapePool); + ASSERT_EQ(s_vid, mi->existingOrNextMounts.front().vid); std::unique_ptr<cta::ArchiveMount> archiveMount; archiveMount.reset(dynamic_cast<cta::ArchiveMount*>(mount.release())); ASSERT_NE(nullptr, archiveMount.get()); @@ -820,8 +831,8 @@ TEST_P(SchedulerTest, archive_report_and_retrieve_new_file_with_report) { auto & osdb=getSchedulerDB(); auto mi=osdb.getMountInfo(lc); ASSERT_EQ(1, mi->existingOrNextMounts.size()); - ASSERT_EQ("TapePool", mi->existingOrNextMounts.front().tapePool); - ASSERT_EQ("TESTVID", mi->existingOrNextMounts.front().vid); + ASSERT_EQ(s_tapePoolName_default, mi->existingOrNextMounts.front().tapePool); + ASSERT_EQ(s_vid, mi->existingOrNextMounts.front().vid); std::unique_ptr<cta::ArchiveMount> archiveMount; archiveMount.reset(dynamic_cast<cta::ArchiveMount*>(mount.release())); ASSERT_NE(nullptr, archiveMount.get()); @@ -1042,8 +1053,8 @@ TEST_P(SchedulerTest, archive_report_and_retrieve_new_file_with_specific_mount_p auto & osdb=getSchedulerDB(); auto mi=osdb.getMountInfo(lc); ASSERT_EQ(1, mi->existingOrNextMounts.size()); - ASSERT_EQ("TapePool", mi->existingOrNextMounts.front().tapePool); - ASSERT_EQ("TESTVID", mi->existingOrNextMounts.front().vid); + ASSERT_EQ(s_tapePoolName_default, mi->existingOrNextMounts.front().tapePool); + ASSERT_EQ(s_vid, mi->existingOrNextMounts.front().vid); std::unique_ptr<cta::ArchiveMount> archiveMount; archiveMount.reset(dynamic_cast<cta::ArchiveMount*>(mount.release())); ASSERT_NE(nullptr, archiveMount.get()); @@ -1263,10 +1274,10 @@ TEST_P(SchedulerTest, archive_report_and_retrieve_new_dual_copy_file) { const std::string archiveRoute2Comment = "Archive-route for copy number 2"; const uint32_t archiveRoute1CopyNb = 1; const uint32_t archiveRoute2CopyNb = 2; - catalogue.ArchiveRoute()->createArchiveRoute(s_adminOnAdminHost, dualCopyStorageClassName, archiveRoute1CopyNb, tapePool1Name, - archiveRoute1Comment); - catalogue.ArchiveRoute()->createArchiveRoute(s_adminOnAdminHost, dualCopyStorageClassName, archiveRoute2CopyNb, tapePool2Name, - archiveRoute1Comment); + catalogue.ArchiveRoute()->createArchiveRoute(s_adminOnAdminHost, dualCopyStorageClassName, archiveRoute1CopyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, + tapePool1Name, archiveRoute1Comment); + catalogue.ArchiveRoute()->createArchiveRoute(s_adminOnAdminHost, dualCopyStorageClassName, archiveRoute2CopyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, + tapePool2Name, archiveRoute1Comment); cta::catalogue::MediaType mediaType; mediaType.name = s_mediaType; @@ -1693,8 +1704,8 @@ TEST_P(SchedulerTest, archive_and_retrieve_failure) { auto & osdb=getSchedulerDB(); auto mi=osdb.getMountInfo(lc); ASSERT_EQ(1, mi->existingOrNextMounts.size()); - ASSERT_EQ("TapePool", mi->existingOrNextMounts.front().tapePool); - ASSERT_EQ("TESTVID", mi->existingOrNextMounts.front().vid); + ASSERT_EQ(s_tapePoolName_default, mi->existingOrNextMounts.front().tapePool); + ASSERT_EQ(s_vid, mi->existingOrNextMounts.front().vid); std::unique_ptr<cta::ArchiveMount> archiveMount; archiveMount.reset(dynamic_cast<cta::ArchiveMount*>(mount.release())); ASSERT_NE(nullptr, archiveMount.get()); @@ -1944,8 +1955,8 @@ TEST_P(SchedulerTest, archive_and_retrieve_report_failure) { auto & osdb=getSchedulerDB(); auto mi=osdb.getMountInfo(lc); ASSERT_EQ(1, mi->existingOrNextMounts.size()); - ASSERT_EQ("TapePool", mi->existingOrNextMounts.front().tapePool); - ASSERT_EQ("TESTVID", mi->existingOrNextMounts.front().vid); + ASSERT_EQ(s_tapePoolName_default, mi->existingOrNextMounts.front().tapePool); + ASSERT_EQ(s_vid, mi->existingOrNextMounts.front().vid); std::unique_ptr<cta::ArchiveMount> archiveMount; archiveMount.reset(dynamic_cast<cta::ArchiveMount*>(mount.release())); ASSERT_NE(nullptr, archiveMount.get()); @@ -2724,8 +2735,8 @@ TEST_P(SchedulerTest, expandRepackRequest) { JobQueueType::JobsToTransferForUser}) { ASSERT_EQ(0, re.dumpArchiveQueues(queueType).size()); } - // Now check we find all our requests in the archive queue. - cta::objectstore::ArchiveQueue aq(re.getArchiveQueueAddress(s_tapePoolName, JobQueueType::JobsToTransferForRepack), + // Now check we find all our requests in the archive queue for the repack tape pool + cta::objectstore::ArchiveQueue aq(re.getArchiveQueueAddress(s_tapePoolName_repack, JobQueueType::JobsToTransferForRepack), schedulerDB.getBackend()); aq.fetchNoLock(); std::set<uint64_t> archiveIdsSeen; @@ -3222,6 +3233,7 @@ TEST_P(SchedulerTest, expandRepackRequestArchiveSuccess) { { auto tape = getDefaultTape(); tape.vid = vidDestination; + tape.tapePoolName = s_tapePoolName_repack; catalogue.Tape()->createTape(s_adminOnAdminHost, tape); } @@ -3481,6 +3493,7 @@ TEST_P(SchedulerTest, expandRepackRequestArchiveFailed) { { auto tape = getDefaultTape(); tape.vid = vidDestinationRepack; + tape.tapePoolName = s_tapePoolName_repack; catalogue.Tape()->createTape(s_adminOnAdminHost, tape); } @@ -4510,7 +4523,7 @@ TEST_P(SchedulerTest, DISABLED_archiveReportMultipleAndQueueRetrievesWithActivit auto & osdb=getSchedulerDB(); auto mi=osdb.getMountInfo(lc); ASSERT_EQ(1, mi->existingOrNextMounts.size()); - ASSERT_EQ("TapePool", mi->existingOrNextMounts.front().tapePool); + ASSERT_EQ(s_tapePoolName_default, mi->existingOrNextMounts.front().tapePool); std::unique_ptr<cta::ArchiveMount> archiveMount; archiveMount.reset(dynamic_cast<cta::ArchiveMount*>(mount.release())); ASSERT_NE(nullptr, archiveMount.get()); @@ -4692,9 +4705,9 @@ TEST_P(SchedulerTest, expandRepackRequestAddCopiesOnly) { storageClass.comment = "Create storage class"; catalogue.StorageClass()->modifyStorageClassNbCopies(admin,storageClass.name,storageClass.nbCopies); - //Create the two archive routes for the new copies - catalogue.ArchiveRoute()->createArchiveRoute(admin,storageClass.name,2,tapepool2Name,"ArchiveRoute2"); - catalogue.ArchiveRoute()->createArchiveRoute(admin,storageClass.name,3,tapepool3Name,"ArchiveRoute3"); + //Create the two archive routes for the new copies, mark 3rd one as repack + catalogue.ArchiveRoute()->createArchiveRoute(admin,storageClass.name,2,cta::common::dataStructures::ArchiveRouteType::DEFAULT,tapepool2Name,"ArchiveRoute2"); + catalogue.ArchiveRoute()->createArchiveRoute(admin,storageClass.name,3,cta::common::dataStructures::ArchiveRouteType::REPACK,tapepool3Name,"ArchiveRoute3"); //Create two other destinationTape std::string vidDestination1 = "VIDDESTINATION1"; @@ -4943,7 +4956,7 @@ TEST_P(SchedulerTest, expandRepackRequestShouldFailIfArchiveRouteMissing) { catalogue.StorageClass()->modifyStorageClassNbCopies(admin,storageClass.name,storageClass.nbCopies); //Create the one archive route for the second copy - catalogue.ArchiveRoute()->createArchiveRoute(admin,storageClass.name,2,tapepool2Name,"ArchiveRoute3"); + catalogue.ArchiveRoute()->createArchiveRoute(admin,storageClass.name,2,cta::common::dataStructures::ArchiveRouteType::DEFAULT,tapepool2Name,"ArchiveRoute3"); //Create two other destinationTape std::string vidCopyNb2_source = "VIDCOPYNB2_SOURCE"; @@ -5035,7 +5048,7 @@ TEST_P(SchedulerTest, expandRepackRequestShouldFailIfArchiveRouteMissing) { } catalogue.Tape()->setTapeFull(admin,vidCopyNb2_source,true); //Delete the archive route of the second copy and repack the tape that contains these second copies - catalogue.ArchiveRoute()->deleteArchiveRoute(storageClass.name,2); + catalogue.ArchiveRoute()->deleteArchiveRoute(storageClass.name,2, cta::common::dataStructures::ArchiveRouteType::DEFAULT); { std::string vid = vidCopyNb2_source; cta::SchedulerDatabase::QueueRepackRequest qrr(vid,"file://"+tempDirectory.path(),common::dataStructures::RepackInfo::Type::MoveAndAddCopies, @@ -5128,12 +5141,12 @@ TEST_P(SchedulerTest, expandRepackRequestMoveAndAddCopies){ storageClass.comment = "Create storage class"; catalogue.StorageClass()->modifyStorageClassNbCopies(admin,storageClass.name,storageClass.nbCopies); - //Create the two archive routes for the new copies - catalogue.ArchiveRoute()->createArchiveRoute(admin,storageClass.name,2,tapepool2Name,"ArchiveRoute2"); - catalogue.ArchiveRoute()->createArchiveRoute(admin,storageClass.name,3,tapepool3Name,"ArchiveRoute3"); + //Create the two archive routes for the new copies, mark 3rd one as repack + catalogue.ArchiveRoute()->createArchiveRoute(admin,storageClass.name,2,cta::common::dataStructures::ArchiveRouteType::DEFAULT,tapepool2Name,"ArchiveRoute2"); + catalogue.ArchiveRoute()->createArchiveRoute(admin,storageClass.name,3,cta::common::dataStructures::ArchiveRouteType::REPACK,tapepool3Name,"ArchiveRoute3"); //Create two other destinationTape and one for the move workflow - std::string vidDestination1 = "VIDDESTINATION1"; + std::string vidDestination1 = "VIDDESTINATION2"; { auto tape = getDefaultTape(); tape.vid = vidDestination1; @@ -5141,7 +5154,7 @@ TEST_P(SchedulerTest, expandRepackRequestMoveAndAddCopies){ catalogue.Tape()->createTape(s_adminOnAdminHost, tape); } - std::string vidDestination2 = "VIDDESTINATION2"; + std::string vidDestination2 = "VIDDESTINATION3"; { auto tape = getDefaultTape(); tape.vid = vidDestination2; @@ -5154,6 +5167,7 @@ TEST_P(SchedulerTest, expandRepackRequestMoveAndAddCopies){ { auto tape = getDefaultTape(); tape.vid = vidMove; + tape.tapePoolName = s_tapePoolName_repack; catalogue.Tape()->createTape(s_adminOnAdminHost, tape); } @@ -5576,8 +5590,8 @@ TEST_P(SchedulerTest, cancelRepackRequest) { } } re.fetchNoLock(); - //Get all archive subrequests in the ArchiveQueue - cta::objectstore::ArchiveQueue aq(re.getArchiveQueueAddress(s_tapePoolName, JobQueueType::JobsToTransferForRepack),backend); + //Get all archive subrequests in the ArchiveQueue for repack + cta::objectstore::ArchiveQueue aq(re.getArchiveQueueAddress(s_tapePoolName_repack, JobQueueType::JobsToTransferForRepack), backend); aq.fetchNoLock(); for(auto & job: aq.dumpJobs()){ cta::objectstore::ArchiveRequest archiveReq(job.address,backend); @@ -5646,7 +5660,7 @@ TEST_P(SchedulerTest, getNextMountEmptyArchiveForRepackIfNbFilesQueuedIsLessThan aFile.fileSize = 667; aFile.storageClass = "sc"; ar->setArchiveFile(aFile); - ar->addJob(1, s_tapePoolName, agentReference.getAgentAddress(), 1, 1, 1); + ar->addJob(1, s_tapePoolName_default, agentReference.getAgentAddress(), 1, 1, 1); ar->setJobStatus(1, serializers::ArchiveJobStatus::AJS_ToTransferForRepack); cta::common::dataStructures::MountPolicy mp; mp.archiveMinRequestAge = 250000; @@ -5682,7 +5696,7 @@ TEST_P(SchedulerTest, getNextMountEmptyArchiveForRepackIfNbFilesQueuedIsLessThan aFile.fileSize = 667; aFile.storageClass = s_storageClassName; ar->setArchiveFile(aFile); - ar->addJob(1, s_tapePoolName, agentReference.getAgentAddress(), 1, 1, 1); + ar->addJob(1, s_tapePoolName_default, agentReference.getAgentAddress(), 1, 1, 1); ar->setJobStatus(1, serializers::ArchiveJobStatus::AJS_ToTransferForRepack); cta::common::dataStructures::MountPolicy mp; mp.archiveMinRequestAge = 250000; @@ -6451,6 +6465,7 @@ TEST_P(SchedulerTest, retrieveArchiveRepackQueueMaxDrivesVoInFlightChangeSchedul //Create a repack destination tape auto tape2 = getDefaultTape(); tape2.vid = "REPACK_DESTINATION_VID"; + tape2.tapePoolName = s_tapePoolName_repack; catalogue.Tape()->createTape(s_adminOnAdminHost, tape2); const std::string tapeDrive = "tape_drive"; @@ -6667,7 +6682,7 @@ TEST_P(SchedulerTest, retrieveArchiveAllTypesMaxDrivesVoInFlightChangeScheduleMo catalogue.StorageClass()->modifyStorageClassNbCopies(s_adminOnAdminHost,storageClass.name,storageClass.nbCopies); //Create the new archive routes for the second copy - catalogue.ArchiveRoute()->createArchiveRoute(s_adminOnAdminHost,storageClass.name,2,newTapepool,"ArchiveRoute2"); + catalogue.ArchiveRoute()->createArchiveRoute(s_adminOnAdminHost,storageClass.name,2,cta::common::dataStructures::ArchiveRouteType::DEFAULT,newTapepool,"ArchiveRoute2"); const std::string tapeDrive = "tape_drive"; const uint64_t nbArchiveFilesPerTape = 10; @@ -6734,7 +6749,7 @@ TEST_P(SchedulerTest, retrieveArchiveAllTypesMaxDrivesVoInFlightChangeScheduleMo aFile.fileSize = archiveFileSize; aFile.storageClass = "sc"; ar->setArchiveFile(aFile); - ar->addJob(1, s_tapePoolName, agentReference.getAgentAddress(), 1, 1, 1); + ar->addJob(1, s_tapePoolName_default, agentReference.getAgentAddress(), 1, 1, 1); ar->addJob(2, newTapepool, agentReference.getAgentAddress(), 1, 1, 1); ar->setJobStatus(1, serializers::ArchiveJobStatus::AJS_ToTransferForRepack); ar->setJobStatus(2, serializers::ArchiveJobStatus::AJS_ToTransferForUser); @@ -6931,7 +6946,7 @@ TEST_P(SchedulerTest, getQueuesAndMountSummariesTest) { cta::objectstore::ScopedExclusiveLock sel(re); re.fetch(); - archiveForUserQueueAddress = re.addOrGetArchiveQueueAndCommit(s_tapePoolName,agentReference,JobQueueType::JobsToTransferForUser); + archiveForUserQueueAddress = re.addOrGetArchiveQueueAndCommit(s_tapePoolName_default, agentReference, JobQueueType::JobsToTransferForUser); } cta::objectstore::ArchiveQueue::JobToAdd archiveJobToAdd; @@ -6953,7 +6968,7 @@ TEST_P(SchedulerTest, getQueuesAndMountSummariesTest) { cta::objectstore::ScopedExclusiveLock sel(re); re.fetch(); - archiveForRepackQueueAddress = re.addOrGetArchiveQueueAndCommit(s_tapePoolName,agentReference,JobQueueType::JobsToTransferForRepack); + archiveForRepackQueueAddress = re.addOrGetArchiveQueueAndCommit(s_tapePoolName_default, agentReference, JobQueueType::JobsToTransferForRepack); } cta::objectstore::ArchiveQueue::JobToAdd repackArchiveJob; @@ -6990,7 +7005,7 @@ TEST_P(SchedulerTest, getQueuesAndMountSummariesTest) ASSERT_EQ(cta::common::dataStructures::MountType::Retrieve,res->mountType); //Test the ArchiveForUser QueueAndMountSummary - std::string tapePool = s_tapePoolName; + std::string tapePool = s_tapePoolName_default; res = std::find_if(queuesAndMountSummaries.begin(), queuesAndMountSummaries.end(), [tapePool](const cta::common::dataStructures::QueueAndMountSummary & qams){ return qams.mountType == cta::common::dataStructures::MountType::ArchiveForUser && qams.tapePool == tapePool; }); @@ -7080,7 +7095,7 @@ TEST_P(SchedulerTest, getNextMountWithArchiveForUserAndArchiveForRepackShouldRet aFile.fileSize = fileSize; aFile.storageClass = "sc"; ar->setArchiveFile(aFile); - ar->addJob(1, s_tapePoolName, agentReference.getAgentAddress(), 1, 1, 1); + ar->addJob(1, s_tapePoolName_default, agentReference.getAgentAddress(), 1, 1, 1); ar->setJobStatus(1, serializers::ArchiveJobStatus::AJS_ToTransferForRepack); cta::common::dataStructures::MountPolicy mp; //We want the archiveMinRequestAge to be taken into account and trigger the mount @@ -7140,7 +7155,7 @@ TEST_P(SchedulerTest, getNextMountWithArchiveForUserAndArchiveForRepackShouldRet aFile.fileSize = fileSize; aFile.storageClass = "sc"; ar->setArchiveFile(aFile); - ar->addJob(1, s_tapePoolName, agentReference.getAgentAddress(), 1, 1, 1); + ar->addJob(1, s_tapePoolName_default, agentReference.getAgentAddress(), 1, 1, 1); ar->setJobStatus(1, serializers::ArchiveJobStatus::AJS_ToTransferForUser); cta::common::dataStructures::MountPolicy mp; //We want the archiveMinRequestAge to be taken into account and trigger the mount @@ -7217,7 +7232,7 @@ TEST_P(SchedulerTest, testCleaningUpKeepingTapePoolName) { const std::string driveName = "drive0"; auto tapeDrive = catalogue.DriveState()->getTapeDrive(driveName); // Insert tape pool name to the drive - tapeDrive.value().currentTapePool = s_tapePoolName; + tapeDrive.value().currentTapePool = s_tapePoolName_default; tapeDrive.value().driveStatus = common::dataStructures::DriveStatus::CleaningUp; catalogue.DriveState()->updateTapeDriveStatus(tapeDrive.value()); // And simulate the drive had a uncaught exception in CleaningUp state, and it didn't go to Down state @@ -7309,7 +7324,7 @@ TEST_P(SchedulerTest, testShutdownKeepingTapePoolName) { const std::string driveName = "drive0"; auto tapeDrive = catalogue.DriveState()->getTapeDrive(driveName); // Insert tape pool name to the drive - tapeDrive.value().currentTapePool = s_tapePoolName; + tapeDrive.value().currentTapePool = s_tapePoolName_default; tapeDrive.value().driveStatus = common::dataStructures::DriveStatus::Shutdown; catalogue.DriveState()->updateTapeDriveStatus(tapeDrive.value()); // And simulate the drive had a uncaught exception in Shutdown state, and it didn't go to Down state diff --git a/tapeserver/castor/tape/tapeserver/daemon/DataTransferSessionTest.cpp b/tapeserver/castor/tape/tapeserver/daemon/DataTransferSessionTest.cpp index d8d713cc37385718dba22704f8103745e296e5f3..07ae4bc3e88b63114813841c825cfe919a8dbac3 100644 --- a/tapeserver/castor/tape/tapeserver/daemon/DataTransferSessionTest.cpp +++ b/tapeserver/castor/tape/tapeserver/daemon/DataTransferSessionTest.cpp @@ -386,8 +386,8 @@ public: tapePoolEncryption, tapePoolSupply, tapePoolComment)); const uint32_t copyNb = 1; const std::string archiveRouteComment = "Archive-route comment"; - catalogue.ArchiveRoute()->createArchiveRoute(s_adminOnAdminHost, s_storageClassName, copyNb, s_tapePoolName, - archiveRouteComment); + catalogue.ArchiveRoute()->createArchiveRoute(s_adminOnAdminHost, s_storageClassName, copyNb, cta::common::dataStructures::ArchiveRouteType::DEFAULT, + s_tapePoolName, archiveRouteComment); cta::catalogue::MediaType mediaType; mediaType.name = s_mediaType; diff --git a/xroot_plugins/XrdCtaArchiveRouteLs.hpp b/xroot_plugins/XrdCtaArchiveRouteLs.hpp index 55d4fa392112f45be2680c26373b0841ff041406..a7c12baac055ad5fc7477a8db46171c4be131e8a 100644 --- a/xroot_plugins/XrdCtaArchiveRouteLs.hpp +++ b/xroot_plugins/XrdCtaArchiveRouteLs.hpp @@ -18,6 +18,7 @@ #pragma once #include <xroot_plugins/XrdCtaStream.hpp> +#include <common/dataStructures/ArchiveRouteTypeSerDeser.hpp> namespace cta::xrd { @@ -72,6 +73,7 @@ int ArchiveRouteLsStream::fillBuffer(XrdSsiPb::OStreamBuffer<Data> *streambuf) { ar_item->set_storage_class(ar.storageClassName); ar_item->set_copy_number(ar.copyNb); + ar_item->set_archive_route_type(cta::admin::ArchiveRouteTypeFormatToProtobuf(ar.type)); ar_item->set_tapepool(ar.tapePoolName); ar_item->mutable_creation_log()->set_username(ar.creationLog.username); ar_item->mutable_creation_log()->set_host(ar.creationLog.host); diff --git a/xrootd-ssi-protobuf-interface b/xrootd-ssi-protobuf-interface index 21095276b880d3d4d0618fc5b01851dc7dd419f0..7090f0278c55136071f280aea6ca86faff6d4904 160000 --- a/xrootd-ssi-protobuf-interface +++ b/xrootd-ssi-protobuf-interface @@ -1 +1 @@ -Subproject commit 21095276b880d3d4d0618fc5b01851dc7dd419f0 +Subproject commit 7090f0278c55136071f280aea6ca86faff6d4904