From 02f1abf80dd1f1459701a9cee66e894fc43846e7 Mon Sep 17 00:00:00 2001
From: Joao Afonso <joao.afonso@cern.ch>
Date: Fri, 28 Feb 2025 13:53:44 +0100
Subject: [PATCH 1/3] first draft
---
catalogue/TapeSearchCriteria.hpp | 5 ++++
catalogue/rdbms/RdbmsTapeCatalogue.cpp | 27 +++++++++++++++++++-
cmdline/CtaAdminCmdParse.hpp | 27 +++++++++++---------
frontend/grpc/ServerTapeLsRequestHandler.cpp | 23 +++++++++--------
xroot_plugins/XrdCtaTapeLs.hpp | 26 ++++++++++---------
5 files changed, 72 insertions(+), 36 deletions(-)
diff --git a/catalogue/TapeSearchCriteria.hpp b/catalogue/TapeSearchCriteria.hpp
index 1a0eaa0876..bef57db8da 100644
--- a/catalogue/TapeSearchCriteria.hpp
+++ b/catalogue/TapeSearchCriteria.hpp
@@ -95,6 +95,11 @@ struct TapeSearchCriteria {
*/
std::optional<std::vector<std::string>> diskFileIds;
+ /**
+ * Check tapes with incomplete tape file copies
+ */
+ std::optional<bool> checkIncompleteFileCopies;
+
/**
* The state of the tapes to look for
*/
diff --git a/catalogue/rdbms/RdbmsTapeCatalogue.cpp b/catalogue/rdbms/RdbmsTapeCatalogue.cpp
index 6461932ba4..ffb521aeee 100644
--- a/catalogue/rdbms/RdbmsTapeCatalogue.cpp
+++ b/catalogue/rdbms/RdbmsTapeCatalogue.cpp
@@ -1299,7 +1299,8 @@ std::list<common::dataStructures::Tape> RdbmsTapeCatalogue::getTapes(rdbms::Conn
searchCriteria.state ||
searchCriteria.fromCastor ||
searchCriteria.purchaseOrder ||
- searchCriteria.physicalLibraryName) {
+ searchCriteria.physicalLibraryName ||
+ searchCriteria.checkIncompleteFileCopies) {
sql += R"SQL( WHERE )SQL";
}
@@ -1425,6 +1426,30 @@ std::list<common::dataStructures::Tape> RdbmsTapeCatalogue::getTapes(rdbms::Conn
sql += R"SQL(
PHYSICAL_LIBRARY.PHYSICAL_LIBRARY_NAME = :PHYSICAL_LIBRARY_NAME
)SQL";
+ addedAWhereConstraint = true;
+ }
+ if (searchCriteria.checkIncompleteFileCopies) {
+ if (addedAWhereConstraint) {
+ sql += R"SQL( AND )SQL";
+ }
+ sql += R"SQL(
+ VID IN (
+ SELECT TF.VID FROM (
+ SELECT AF.ARCHIVE_FILE_ID, SC.NB_COPIES, COUNT(TF.ARCHIVE_FILE_ID) AS NB_TAPE_COPIES
+ FROM
+ ARCHIVE_FILE AF
+ INNER JOIN STORAGE_CLASS SC ON AF.STORAGE_CLASS_ID = SC.STORAGE_CLASS_ID
+ INNER JOIN TAPE_FILE TF ON AF.ARCHIVE_FILE_ID = TF.ARCHIVE_FILE_ID
+ WHERE
+ SC.NB_COPIES > 1
+ GROUP BY
+ AF.ARCHIVE_FILE_ID, SC.NB_COPIES
+ HAVING
+ SC.NB_COPIES <> COUNT(TF.ARCHIVE_FILE_ID)
+ ) MISSING_COPIES
+ INNER JOIN TAPE_FILE TF ON MISSING_COPIES.ARCHIVE_FILE_ID = TF.ARCHIVE_FILE_ID
+ )
+ )SQL";
}
sql += R"SQL( ORDER BY TAPE.VID )SQL";
diff --git a/cmdline/CtaAdminCmdParse.hpp b/cmdline/CtaAdminCmdParse.hpp
index e6029f801a..7f5c17562c 100644
--- a/cmdline/CtaAdminCmdParse.hpp
+++ b/cmdline/CtaAdminCmdParse.hpp
@@ -259,17 +259,18 @@ const std::map<std::string, OptionBoolean::Key> boolOptions = {
{"--fromcastor", OptionBoolean::FROM_CASTOR },
// hasOption options
- {"--disabledtape", OptionBoolean::DISABLED },
- {"--justarchive", OptionBoolean::JUSTARCHIVE },
- {"--justmove", OptionBoolean::JUSTMOVE },
- {"--justaddcopies", OptionBoolean::JUSTADDCOPIES },
- {"--justretrieve", OptionBoolean::JUSTRETRIEVE },
- {"--log", OptionBoolean::SHOW_LOG_ENTRIES},
- {"--lookupnamespace", OptionBoolean::LOOKUP_NAMESPACE},
- {"--summary", OptionBoolean::SUMMARY },
- {"--no-recall", OptionBoolean::NO_RECALL },
- {"--dirtybit", OptionBoolean::DIRTY_BIT },
- {"--isrepackvo", OptionBoolean::IS_REPACK_VO }
+ {"--disabledtape", OptionBoolean::DISABLED },
+ {"--justarchive", OptionBoolean::JUSTARCHIVE },
+ {"--justmove", OptionBoolean::JUSTMOVE },
+ {"--justaddcopies", OptionBoolean::JUSTADDCOPIES },
+ {"--justretrieve", OptionBoolean::JUSTRETRIEVE },
+ {"--log", OptionBoolean::SHOW_LOG_ENTRIES },
+ {"--lookupnamespace", OptionBoolean::LOOKUP_NAMESPACE },
+ {"--summary", OptionBoolean::SUMMARY },
+ {"--no-recall", OptionBoolean::NO_RECALL },
+ {"--dirtybit", OptionBoolean::DIRTY_BIT },
+ {"--isrepackvo", OptionBoolean::IS_REPACK_VO },
+ {"--incompletefilecopies", OptionBoolean::INCOMPLETE_FILE_COPIES },
};
/*!
@@ -516,6 +517,7 @@ const Option opt_archive_route_type {Option::OPT_STR,
std::string(R"( <")") +
cta::common::dataStructures::toString(ArchiveRouteType::DEFAULT) + R"(" or ")" +
cta::common::dataStructures::toString(ArchiveRouteType::REPACK) + R"(">)"};
+const Option opt_incompletefilecopes {Option::OPT_FLAG, "--incompletefilecopies", "--ifc", ""};
/*!
* Subset of commands that return streaming output
@@ -1034,7 +1036,8 @@ tape (ta)
opt_state.optional(),
opt_fromcastor.optional(),
opt_purchase_order.optional(),
- opt_physical_library.optional()} },
+ opt_physical_library.optional(),
+ opt_incompletefilecopes.optional()} },
/**md
tapefile (tf)
diff --git a/frontend/grpc/ServerTapeLsRequestHandler.cpp b/frontend/grpc/ServerTapeLsRequestHandler.cpp
index a2739d7b88..6c6367038f 100644
--- a/frontend/grpc/ServerTapeLsRequestHandler.cpp
+++ b/frontend/grpc/ServerTapeLsRequestHandler.cpp
@@ -83,17 +83,18 @@ bool cta::frontend::grpc::server::TapeLsRequestHandler::next(const bool bOk) {
// Get the search criteria from the optional options
- m_searchCriteria.full = requestMsg.getOptional(cta::admin::OptionBoolean::FULL, &bHasAny);
- m_searchCriteria.fromCastor = requestMsg.getOptional(cta::admin::OptionBoolean::FROM_CASTOR, &bHasAny);
- m_searchCriteria.capacityInBytes = requestMsg.getOptional(cta::admin::OptionUInt64::CAPACITY, &bHasAny);
- m_searchCriteria.logicalLibrary = requestMsg.getOptional(cta::admin::OptionString::LOGICAL_LIBRARY, &bHasAny);
- m_searchCriteria.tapePool = requestMsg.getOptional(cta::admin::OptionString::TAPE_POOL, &bHasAny);
- m_searchCriteria.vo = requestMsg.getOptional(cta::admin::OptionString::VO, &bHasAny);
- m_searchCriteria.vid = requestMsg.getOptional(cta::admin::OptionString::VID, &bHasAny);
- m_searchCriteria.mediaType = requestMsg.getOptional(cta::admin::OptionString::MEDIA_TYPE, &bHasAny);
- m_searchCriteria.vendor = requestMsg.getOptional(cta::admin::OptionString::VENDOR, &bHasAny);
- m_searchCriteria.purchaseOrder = requestMsg.getOptional(cta::admin::OptionString::MEDIA_PURCHASE_ORDER_NUMBER, &bHasAny);
- m_searchCriteria.diskFileIds = requestMsg.getOptional(cta::admin::OptionStrList::FILE_ID, &bHasAny);
+ m_searchCriteria.full = requestMsg.getOptional(cta::admin::OptionBoolean::FULL, &bHasAny);
+ m_searchCriteria.fromCastor = requestMsg.getOptional(cta::admin::OptionBoolean::FROM_CASTOR, &bHasAny);
+ m_searchCriteria.capacityInBytes = requestMsg.getOptional(cta::admin::OptionUInt64::CAPACITY, &bHasAny);
+ m_searchCriteria.logicalLibrary = requestMsg.getOptional(cta::admin::OptionString::LOGICAL_LIBRARY, &bHasAny);
+ m_searchCriteria.tapePool = requestMsg.getOptional(cta::admin::OptionString::TAPE_POOL, &bHasAny);
+ m_searchCriteria.vo = requestMsg.getOptional(cta::admin::OptionString::VO, &bHasAny);
+ m_searchCriteria.vid = requestMsg.getOptional(cta::admin::OptionString::VID, &bHasAny);
+ m_searchCriteria.mediaType = requestMsg.getOptional(cta::admin::OptionString::MEDIA_TYPE, &bHasAny);
+ m_searchCriteria.vendor = requestMsg.getOptional(cta::admin::OptionString::VENDOR, &bHasAny);
+ m_searchCriteria.purchaseOrder = requestMsg.getOptional(cta::admin::OptionString::MEDIA_PURCHASE_ORDER_NUMBER, &bHasAny);
+ m_searchCriteria.diskFileIds = requestMsg.getOptional(cta::admin::OptionStrList::FILE_ID, &bHasAny);
+ m_searchCriteria.checkIncompleteFileCopies = requestMsg.getOptional(cta::admin::OptionBoolean::INCOMPLETE_FILE_COPIES, &bHasAny);
auto stateOpt = requestMsg.getOptional(cta::admin::OptionString::STATE, &bHasAny);
if(stateOpt){
m_searchCriteria.state = common::dataStructures::Tape::stringToState(stateOpt.value());
diff --git a/xroot_plugins/XrdCtaTapeLs.hpp b/xroot_plugins/XrdCtaTapeLs.hpp
index 8210729141..bcb425b85a 100644
--- a/xroot_plugins/XrdCtaTapeLs.hpp
+++ b/xroot_plugins/XrdCtaTapeLs.hpp
@@ -68,19 +68,21 @@ TapeLsStream::TapeLsStream(const frontend::AdminCmdStream& requestMsg, cta::cata
// Get the search criteria from the optional options
- searchCriteria.full = requestMsg.getOptional(OptionBoolean::FULL, &has_any);
- searchCriteria.fromCastor = requestMsg.getOptional(OptionBoolean::FROM_CASTOR, &has_any);
- searchCriteria.capacityInBytes = requestMsg.getOptional(OptionUInt64::CAPACITY, &has_any);
- searchCriteria.logicalLibrary = requestMsg.getOptional(OptionString::LOGICAL_LIBRARY, &has_any);
- searchCriteria.tapePool = requestMsg.getOptional(OptionString::TAPE_POOL, &has_any);
- searchCriteria.vo = requestMsg.getOptional(OptionString::VO, &has_any);
- searchCriteria.vid = requestMsg.getOptional(OptionString::VID, &has_any);
- searchCriteria.mediaType = requestMsg.getOptional(OptionString::MEDIA_TYPE, &has_any);
- searchCriteria.vendor = requestMsg.getOptional(OptionString::VENDOR, &has_any);
- searchCriteria.purchaseOrder = requestMsg.getOptional(OptionString::MEDIA_PURCHASE_ORDER_NUMBER, &has_any);
- searchCriteria.physicalLibraryName = requestMsg.getOptional(OptionString::PHYSICAL_LIBRARY, &has_any);
- searchCriteria.diskFileIds = requestMsg.getOptional(OptionStrList::FILE_ID, &has_any);
+ searchCriteria.full = requestMsg.getOptional(OptionBoolean::FULL, &has_any);
+ searchCriteria.fromCastor = requestMsg.getOptional(OptionBoolean::FROM_CASTOR, &has_any);
+ searchCriteria.capacityInBytes = requestMsg.getOptional(OptionUInt64::CAPACITY, &has_any);
+ searchCriteria.logicalLibrary = requestMsg.getOptional(OptionString::LOGICAL_LIBRARY, &has_any);
+ searchCriteria.tapePool = requestMsg.getOptional(OptionString::TAPE_POOL, &has_any);
+ searchCriteria.vo = requestMsg.getOptional(OptionString::VO, &has_any);
+ searchCriteria.vid = requestMsg.getOptional(OptionString::VID, &has_any);
+ searchCriteria.mediaType = requestMsg.getOptional(OptionString::MEDIA_TYPE, &has_any);
+ searchCriteria.vendor = requestMsg.getOptional(OptionString::VENDOR, &has_any);
+ searchCriteria.purchaseOrder = requestMsg.getOptional(OptionString::MEDIA_PURCHASE_ORDER_NUMBER, &has_any);
+ searchCriteria.physicalLibraryName = requestMsg.getOptional(OptionString::PHYSICAL_LIBRARY, &has_any);
+ searchCriteria.diskFileIds = requestMsg.getOptional(OptionStrList::FILE_ID, &has_any);
+ searchCriteria.checkIncompleteFileCopies = requestMsg.getOptional(OptionBoolean::INCOMPLETE_FILE_COPIES, &has_any);
auto stateOpt = requestMsg.getOptional(OptionString::STATE, &has_any);
+
if(stateOpt){
searchCriteria.state = common::dataStructures::Tape::stringToState(stateOpt.value(), true);
}
--
GitLab
From 329b575674cf632ac631f5796c666846b5aa569e Mon Sep 17 00:00:00 2001
From: Joao Afonso <joao.afonso@cern.ch>
Date: Mon, 3 Mar 2025 13:57:08 +0100
Subject: [PATCH 2/3] Adding option to list tapes with incomplete tape file
copies
---
catalogue/rdbms/RdbmsTapeCatalogue.cpp | 2 +-
.../modules/ArchiveFileCatalogueTest.cpp | 168 ++++++++++++++++++
.../modules/ArchiveFileCatalogueTest.hpp | 1 +
xroot_plugins/XrdCtaTapeLs.hpp | 2 +-
xrootd-ssi-protobuf-interface | 2 +-
5 files changed, 172 insertions(+), 3 deletions(-)
diff --git a/catalogue/rdbms/RdbmsTapeCatalogue.cpp b/catalogue/rdbms/RdbmsTapeCatalogue.cpp
index ffb521aeee..fd580199d5 100644
--- a/catalogue/rdbms/RdbmsTapeCatalogue.cpp
+++ b/catalogue/rdbms/RdbmsTapeCatalogue.cpp
@@ -1428,7 +1428,7 @@ std::list<common::dataStructures::Tape> RdbmsTapeCatalogue::getTapes(rdbms::Conn
)SQL";
addedAWhereConstraint = true;
}
- if (searchCriteria.checkIncompleteFileCopies) {
+ if (searchCriteria.checkIncompleteFileCopies.value_or(false)) {
if (addedAWhereConstraint) {
sql += R"SQL( AND )SQL";
}
diff --git a/catalogue/tests/modules/ArchiveFileCatalogueTest.cpp b/catalogue/tests/modules/ArchiveFileCatalogueTest.cpp
index 5584cfe9b6..bea4b57d01 100644
--- a/catalogue/tests/modules/ArchiveFileCatalogueTest.cpp
+++ b/catalogue/tests/modules/ArchiveFileCatalogueTest.cpp
@@ -42,6 +42,7 @@ cta_catalogue_ArchiveFileTest::cta_catalogue_ArchiveFileTest()
: m_dummyLog("dummy", "dummy"),
m_tape1(CatalogueTestUtils::getTape1()),
m_tape2(CatalogueTestUtils::getTape2()),
+ m_tape3(CatalogueTestUtils::getTape3()),
m_mediaType(CatalogueTestUtils::getMediaType()),
m_admin(CatalogueTestUtils::getAdmin()),
m_diskInstance(CatalogueTestUtils::getDiskInstance()),
@@ -4987,4 +4988,171 @@ TEST_P(cta_catalogue_ArchiveFileTest, getArchiveFileQueueCriteria_ignore_repack_
ASSERT_EQ(tapePoolName_default_2, copyToPoolMap_it->second);
}
+TEST_P(cta_catalogue_ArchiveFileTest, getTapesWithMissingTapeFileCopies) {
+ const bool logicalLibraryIsDisabled= false;
+ const uint64_t nbPartialTapes = 2;
+ const bool isEncrypted = true;
+ const std::list<std::string> supply;
+ const std::string diskInstance = m_diskInstance.name;
+ std::optional<std::string> physicalLibraryName;
+
+ m_catalogue->MediaType()->createMediaType(m_admin, m_mediaType);
+ m_catalogue->LogicalLibrary()->createLogicalLibrary(m_admin, m_tape1.logicalLibraryName, logicalLibraryIsDisabled, physicalLibraryName, "Create logical library");
+ m_catalogue->DiskInstance()->createDiskInstance(m_admin, m_diskInstance.name, m_diskInstance.comment);
+ m_catalogue->VO()->createVirtualOrganization(m_admin, m_vo);
+ m_catalogue->TapePool()->createTapePool(m_admin, m_tape1.tapePoolName, m_vo.name, nbPartialTapes, isEncrypted, supply, "Create tape pool");
+ m_catalogue->Tape()->createTape(m_admin, m_tape1);
+ m_catalogue->Tape()->createTape(m_admin, m_tape2);
+ m_catalogue->Tape()->createTape(m_admin, m_tape3);
+ m_catalogue->StorageClass()->createStorageClass(m_admin, m_storageClassDualCopy);
+
+ // Storage class 'm_storageClassDualCopy' expects two tape copies for each file
+ // 1.1. Write 1 single file copy of 'archiveFileId_1' on 'm_tape1'
+ // 1.2. Write 1 single file copy of 'archiveFileId_2' on 'm_tape2' (different file)
+ // 2. Check that both tapes are identified as missing a second tape file copy
+
+ constexpr uint64_t archiveFileId_1 = 1234;
+ constexpr uint64_t archiveFileId_2 = 5678;
+
+ constexpr uint64_t archiveFileSize = 1;
+ const std::string tapeDrive = "tape_drive";
+
+ {
+ auto tapeFileWrittenPtr = std::make_unique<cta::catalogue::TapeFileWritten>();
+ auto & tapeFileWritten = *tapeFileWrittenPtr;
+
+ tapeFileWritten.archiveFileId = archiveFileId_1;
+ tapeFileWritten.diskInstance = diskInstance;
+ tapeFileWritten.diskFileId = "12340000";
+
+ tapeFileWritten.diskFileOwnerUid = PUBLIC_DISK_USER;
+ tapeFileWritten.diskFileGid = PUBLIC_DISK_GROUP;
+ tapeFileWritten.size = archiveFileSize;
+ tapeFileWritten.checksumBlob.insert(cta::checksum::ADLER32, "1234");
+ tapeFileWritten.storageClassName = m_storageClassDualCopy.name;
+ tapeFileWritten.vid = m_tape1.vid;
+ tapeFileWritten.fSeq = 1;
+ tapeFileWritten.blockId = 4321;
+ tapeFileWritten.copyNb = 1;
+ tapeFileWritten.tapeDrive = tapeDrive;
+
+ std::set<cta::catalogue::TapeItemWrittenPointer> fileWrittenSet;
+ fileWrittenSet.insert(tapeFileWrittenPtr.release());
+ m_catalogue->TapeFile()->filesWrittenToTape(fileWrittenSet);
+ }
+
+ {
+ auto tapeFileWrittenPtr = std::make_unique<cta::catalogue::TapeFileWritten>();
+ auto & tapeFileWritten = *tapeFileWrittenPtr;
+
+ tapeFileWritten.archiveFileId = archiveFileId_2;
+ tapeFileWritten.diskInstance = diskInstance;
+ tapeFileWritten.diskFileId = "56780000";
+
+ tapeFileWritten.diskFileOwnerUid = PUBLIC_DISK_USER;
+ tapeFileWritten.diskFileGid = PUBLIC_DISK_GROUP;
+ tapeFileWritten.size = archiveFileSize;
+ tapeFileWritten.checksumBlob.insert(cta::checksum::ADLER32, "8765");
+ tapeFileWritten.storageClassName = m_storageClassDualCopy.name;
+ tapeFileWritten.vid = m_tape2.vid;
+ tapeFileWritten.fSeq = 1;
+ tapeFileWritten.blockId = 5678;
+ tapeFileWritten.copyNb = 1;
+ tapeFileWritten.tapeDrive = tapeDrive;
+
+ std::set<cta::catalogue::TapeItemWrittenPointer> fileWrittenSet;
+ fileWrittenSet.insert(tapeFileWrittenPtr.release());
+ m_catalogue->TapeFile()->filesWrittenToTape(fileWrittenSet);
+ }
+
+ // Tape 1 and 2 should be identified as missing a 2nd copy
+ {
+ cta::catalogue::TapeSearchCriteria searchCriteria;
+ searchCriteria.checkIncompleteFileCopies = true;
+ const std::list<cta::common::dataStructures::Tape> tapes = m_catalogue->Tape()->getTapes(searchCriteria);
+
+ ASSERT_EQ(2, tapes.size());
+
+ const auto vidToTape = CatalogueTestUtils::tapeListToMap(tapes);
+ const cta::common::dataStructures::Tape & tape_1 = vidToTape.at(m_tape1.vid);
+ ASSERT_EQ(m_tape1.vid, tape_1.vid);
+ const cta::common::dataStructures::Tape & tape_2 = vidToTape.at(m_tape2.vid);
+ ASSERT_EQ(m_tape2.vid, tape_2.vid);
+ }
+
+ // 1.1. Write 1 new file copy of 'archiveFileId_1' on 'm_tape3'
+ // 2. Only 'm_tape3' should now be identified as missing a second tape file copy
+ {
+ auto tapeFileWrittenPtr = std::make_unique<cta::catalogue::TapeFileWritten>();
+ auto & tapeFileWritten = *tapeFileWrittenPtr;
+
+ tapeFileWritten.archiveFileId = archiveFileId_1;
+ tapeFileWritten.diskInstance = diskInstance;
+ tapeFileWritten.diskFileId = "12340000";
+
+ tapeFileWritten.diskFileOwnerUid = PUBLIC_DISK_USER;
+ tapeFileWritten.diskFileGid = PUBLIC_DISK_GROUP;
+ tapeFileWritten.size = archiveFileSize;
+ tapeFileWritten.checksumBlob.insert(cta::checksum::ADLER32, "1234");
+ tapeFileWritten.storageClassName = m_storageClassDualCopy.name;
+ tapeFileWritten.vid = m_tape3.vid;
+ tapeFileWritten.fSeq = 1;
+ tapeFileWritten.blockId = 4321;
+ tapeFileWritten.copyNb = 2;
+ tapeFileWritten.tapeDrive = tapeDrive;
+
+ std::set<cta::catalogue::TapeItemWrittenPointer> fileWrittenSet;
+ fileWrittenSet.insert(tapeFileWrittenPtr.release());
+ m_catalogue->TapeFile()->filesWrittenToTape(fileWrittenSet);
+ }
+
+ // Only tape 2 should now be identified as missing a 2nd copy
+ {
+ cta::catalogue::TapeSearchCriteria searchCriteria;
+ searchCriteria.checkIncompleteFileCopies = true;
+ const std::list<cta::common::dataStructures::Tape> tapes = m_catalogue->Tape()->getTapes(searchCriteria);
+
+ ASSERT_EQ(1, tapes.size());
+
+ const auto vidToTape = CatalogueTestUtils::tapeListToMap(tapes);
+ const cta::common::dataStructures::Tape & tape_2 = vidToTape.at(m_tape2.vid);
+ ASSERT_EQ(m_tape2.vid, tape_2.vid);
+ }
+
+ // 1.1. Write 1 new file copy of 'archiveFileId_2' on 'm_tape3'
+ // 2. No tapes should now be identified as missing a second tape file copy
+ {
+ auto tapeFileWrittenPtr = std::make_unique<cta::catalogue::TapeFileWritten>();
+ auto & tapeFileWritten = *tapeFileWrittenPtr;
+
+ tapeFileWritten.archiveFileId = archiveFileId_2;
+ tapeFileWritten.diskInstance = diskInstance;
+ tapeFileWritten.diskFileId = "56780000";
+
+ tapeFileWritten.diskFileOwnerUid = PUBLIC_DISK_USER;
+ tapeFileWritten.diskFileGid = PUBLIC_DISK_GROUP;
+ tapeFileWritten.size = archiveFileSize;
+ tapeFileWritten.checksumBlob.insert(cta::checksum::ADLER32, "8765");
+ tapeFileWritten.storageClassName = m_storageClassDualCopy.name;
+ tapeFileWritten.vid = m_tape3.vid;
+ tapeFileWritten.fSeq = 2;
+ tapeFileWritten.blockId = 5678;
+ tapeFileWritten.copyNb = 2;
+ tapeFileWritten.tapeDrive = tapeDrive;
+
+ std::set<cta::catalogue::TapeItemWrittenPointer> fileWrittenSet;
+ fileWrittenSet.insert(tapeFileWrittenPtr.release());
+ m_catalogue->TapeFile()->filesWrittenToTape(fileWrittenSet);
+ }
+
+ // No tapes should now be identified as missing a 2nd copy
+ {
+ cta::catalogue::TapeSearchCriteria searchCriteria;
+ searchCriteria.checkIncompleteFileCopies = true;
+ const std::list<cta::common::dataStructures::Tape> tapes = m_catalogue->Tape()->getTapes(searchCriteria);
+
+ ASSERT_TRUE(tapes.empty());
+ }
+}
+
} // namespace unitTests
diff --git a/catalogue/tests/modules/ArchiveFileCatalogueTest.hpp b/catalogue/tests/modules/ArchiveFileCatalogueTest.hpp
index da6516c647..b2e39c3265 100644
--- a/catalogue/tests/modules/ArchiveFileCatalogueTest.hpp
+++ b/catalogue/tests/modules/ArchiveFileCatalogueTest.hpp
@@ -46,6 +46,7 @@ protected:
const cta::catalogue::CreateTapeAttributes m_tape1;
const cta::catalogue::CreateTapeAttributes m_tape2;
+ const cta::catalogue::CreateTapeAttributes m_tape3;
const cta::catalogue::MediaType m_mediaType;
const cta::common::dataStructures::SecurityIdentity m_admin;
const cta::common::dataStructures::DiskInstance m_diskInstance;
diff --git a/xroot_plugins/XrdCtaTapeLs.hpp b/xroot_plugins/XrdCtaTapeLs.hpp
index bcb425b85a..278e75f88f 100644
--- a/xroot_plugins/XrdCtaTapeLs.hpp
+++ b/xroot_plugins/XrdCtaTapeLs.hpp
@@ -80,7 +80,7 @@ TapeLsStream::TapeLsStream(const frontend::AdminCmdStream& requestMsg, cta::cata
searchCriteria.purchaseOrder = requestMsg.getOptional(OptionString::MEDIA_PURCHASE_ORDER_NUMBER, &has_any);
searchCriteria.physicalLibraryName = requestMsg.getOptional(OptionString::PHYSICAL_LIBRARY, &has_any);
searchCriteria.diskFileIds = requestMsg.getOptional(OptionStrList::FILE_ID, &has_any);
- searchCriteria.checkIncompleteFileCopies = requestMsg.getOptional(OptionBoolean::INCOMPLETE_FILE_COPIES, &has_any);
+ searchCriteria.checkIncompleteFileCopies = requestMsg.getOptional(OptionBoolean::INCOMPLETE_FILE_COPIES, &has_any);
auto stateOpt = requestMsg.getOptional(OptionString::STATE, &has_any);
if(stateOpt){
diff --git a/xrootd-ssi-protobuf-interface b/xrootd-ssi-protobuf-interface
index 95ca6aca15..a0c94a22be 160000
--- a/xrootd-ssi-protobuf-interface
+++ b/xrootd-ssi-protobuf-interface
@@ -1 +1 @@
-Subproject commit 95ca6aca153a6e72a33a281258e3372d54fe6a74
+Subproject commit a0c94a22bece907d6038e3b1fc452621c64281af
--
GitLab
From 2164ab82c4c25621aaac1c17422618b1acea50c4 Mon Sep 17 00:00:00 2001
From: Joao Afonso <joao.afonso@cern.ch>
Date: Tue, 4 Mar 2025 11:06:30 +0100
Subject: [PATCH 3/3] Modifying --incompletefilecopes to --missingfilecopies
---
catalogue/TapeSearchCriteria.hpp | 4 +--
catalogue/rdbms/RdbmsTapeCatalogue.cpp | 4 +--
.../modules/ArchiveFileCatalogueTest.cpp | 6 ++--
cmdline/CtaAdminCmdParse.hpp | 28 +++++++++----------
frontend/grpc/ServerTapeLsRequestHandler.cpp | 24 ++++++++--------
xroot_plugins/XrdCtaTapeLs.hpp | 26 ++++++++---------
xrootd-ssi-protobuf-interface | 2 +-
7 files changed, 47 insertions(+), 47 deletions(-)
diff --git a/catalogue/TapeSearchCriteria.hpp b/catalogue/TapeSearchCriteria.hpp
index bef57db8da..05160b6b2e 100644
--- a/catalogue/TapeSearchCriteria.hpp
+++ b/catalogue/TapeSearchCriteria.hpp
@@ -96,9 +96,9 @@ struct TapeSearchCriteria {
std::optional<std::vector<std::string>> diskFileIds;
/**
- * Check tapes with incomplete tape file copies
+ * Check tapes with missing tape file copies
*/
- std::optional<bool> checkIncompleteFileCopies;
+ std::optional<bool> checkMissingFileCopies;
/**
* The state of the tapes to look for
diff --git a/catalogue/rdbms/RdbmsTapeCatalogue.cpp b/catalogue/rdbms/RdbmsTapeCatalogue.cpp
index fd580199d5..308a297ca2 100644
--- a/catalogue/rdbms/RdbmsTapeCatalogue.cpp
+++ b/catalogue/rdbms/RdbmsTapeCatalogue.cpp
@@ -1300,7 +1300,7 @@ std::list<common::dataStructures::Tape> RdbmsTapeCatalogue::getTapes(rdbms::Conn
searchCriteria.fromCastor ||
searchCriteria.purchaseOrder ||
searchCriteria.physicalLibraryName ||
- searchCriteria.checkIncompleteFileCopies) {
+ searchCriteria.checkMissingFileCopies) {
sql += R"SQL( WHERE )SQL";
}
@@ -1428,7 +1428,7 @@ std::list<common::dataStructures::Tape> RdbmsTapeCatalogue::getTapes(rdbms::Conn
)SQL";
addedAWhereConstraint = true;
}
- if (searchCriteria.checkIncompleteFileCopies.value_or(false)) {
+ if (searchCriteria.checkMissingFileCopies.value_or(false)) {
if (addedAWhereConstraint) {
sql += R"SQL( AND )SQL";
}
diff --git a/catalogue/tests/modules/ArchiveFileCatalogueTest.cpp b/catalogue/tests/modules/ArchiveFileCatalogueTest.cpp
index bea4b57d01..c443f2864f 100644
--- a/catalogue/tests/modules/ArchiveFileCatalogueTest.cpp
+++ b/catalogue/tests/modules/ArchiveFileCatalogueTest.cpp
@@ -5068,7 +5068,7 @@ TEST_P(cta_catalogue_ArchiveFileTest, getTapesWithMissingTapeFileCopies) {
// Tape 1 and 2 should be identified as missing a 2nd copy
{
cta::catalogue::TapeSearchCriteria searchCriteria;
- searchCriteria.checkIncompleteFileCopies = true;
+ searchCriteria.checkMissingFileCopies = true;
const std::list<cta::common::dataStructures::Tape> tapes = m_catalogue->Tape()->getTapes(searchCriteria);
ASSERT_EQ(2, tapes.size());
@@ -5109,7 +5109,7 @@ TEST_P(cta_catalogue_ArchiveFileTest, getTapesWithMissingTapeFileCopies) {
// Only tape 2 should now be identified as missing a 2nd copy
{
cta::catalogue::TapeSearchCriteria searchCriteria;
- searchCriteria.checkIncompleteFileCopies = true;
+ searchCriteria.checkMissingFileCopies = true;
const std::list<cta::common::dataStructures::Tape> tapes = m_catalogue->Tape()->getTapes(searchCriteria);
ASSERT_EQ(1, tapes.size());
@@ -5148,7 +5148,7 @@ TEST_P(cta_catalogue_ArchiveFileTest, getTapesWithMissingTapeFileCopies) {
// No tapes should now be identified as missing a 2nd copy
{
cta::catalogue::TapeSearchCriteria searchCriteria;
- searchCriteria.checkIncompleteFileCopies = true;
+ searchCriteria.checkMissingFileCopies = true;
const std::list<cta::common::dataStructures::Tape> tapes = m_catalogue->Tape()->getTapes(searchCriteria);
ASSERT_TRUE(tapes.empty());
diff --git a/cmdline/CtaAdminCmdParse.hpp b/cmdline/CtaAdminCmdParse.hpp
index 7f5c17562c..c73e72adf2 100644
--- a/cmdline/CtaAdminCmdParse.hpp
+++ b/cmdline/CtaAdminCmdParse.hpp
@@ -259,18 +259,18 @@ const std::map<std::string, OptionBoolean::Key> boolOptions = {
{"--fromcastor", OptionBoolean::FROM_CASTOR },
// hasOption options
- {"--disabledtape", OptionBoolean::DISABLED },
- {"--justarchive", OptionBoolean::JUSTARCHIVE },
- {"--justmove", OptionBoolean::JUSTMOVE },
- {"--justaddcopies", OptionBoolean::JUSTADDCOPIES },
- {"--justretrieve", OptionBoolean::JUSTRETRIEVE },
- {"--log", OptionBoolean::SHOW_LOG_ENTRIES },
- {"--lookupnamespace", OptionBoolean::LOOKUP_NAMESPACE },
- {"--summary", OptionBoolean::SUMMARY },
- {"--no-recall", OptionBoolean::NO_RECALL },
- {"--dirtybit", OptionBoolean::DIRTY_BIT },
- {"--isrepackvo", OptionBoolean::IS_REPACK_VO },
- {"--incompletefilecopies", OptionBoolean::INCOMPLETE_FILE_COPIES },
+ {"--disabledtape", OptionBoolean::DISABLED },
+ {"--justarchive", OptionBoolean::JUSTARCHIVE },
+ {"--justmove", OptionBoolean::JUSTMOVE },
+ {"--justaddcopies", OptionBoolean::JUSTADDCOPIES },
+ {"--justretrieve", OptionBoolean::JUSTRETRIEVE },
+ {"--log", OptionBoolean::SHOW_LOG_ENTRIES },
+ {"--lookupnamespace", OptionBoolean::LOOKUP_NAMESPACE },
+ {"--summary", OptionBoolean::SUMMARY },
+ {"--no-recall", OptionBoolean::NO_RECALL },
+ {"--dirtybit", OptionBoolean::DIRTY_BIT },
+ {"--isrepackvo", OptionBoolean::IS_REPACK_VO },
+ {"--missingfilecopies", OptionBoolean::MISSING_FILE_COPIES },
};
/*!
@@ -517,7 +517,7 @@ const Option opt_archive_route_type {Option::OPT_STR,
std::string(R"( <")") +
cta::common::dataStructures::toString(ArchiveRouteType::DEFAULT) + R"(" or ")" +
cta::common::dataStructures::toString(ArchiveRouteType::REPACK) + R"(">)"};
-const Option opt_incompletefilecopes {Option::OPT_FLAG, "--incompletefilecopies", "--ifc", ""};
+const Option opt_missingfilecopes {Option::OPT_FLAG, "--missingfilecopies", "--ifc", ""};
/*!
* Subset of commands that return streaming output
@@ -1037,7 +1037,7 @@ tape (ta)
opt_fromcastor.optional(),
opt_purchase_order.optional(),
opt_physical_library.optional(),
- opt_incompletefilecopes.optional()} },
+ opt_missingfilecopes.optional()} },
/**md
tapefile (tf)
diff --git a/frontend/grpc/ServerTapeLsRequestHandler.cpp b/frontend/grpc/ServerTapeLsRequestHandler.cpp
index 6c6367038f..04e069a1c3 100644
--- a/frontend/grpc/ServerTapeLsRequestHandler.cpp
+++ b/frontend/grpc/ServerTapeLsRequestHandler.cpp
@@ -83,18 +83,18 @@ bool cta::frontend::grpc::server::TapeLsRequestHandler::next(const bool bOk) {
// Get the search criteria from the optional options
- m_searchCriteria.full = requestMsg.getOptional(cta::admin::OptionBoolean::FULL, &bHasAny);
- m_searchCriteria.fromCastor = requestMsg.getOptional(cta::admin::OptionBoolean::FROM_CASTOR, &bHasAny);
- m_searchCriteria.capacityInBytes = requestMsg.getOptional(cta::admin::OptionUInt64::CAPACITY, &bHasAny);
- m_searchCriteria.logicalLibrary = requestMsg.getOptional(cta::admin::OptionString::LOGICAL_LIBRARY, &bHasAny);
- m_searchCriteria.tapePool = requestMsg.getOptional(cta::admin::OptionString::TAPE_POOL, &bHasAny);
- m_searchCriteria.vo = requestMsg.getOptional(cta::admin::OptionString::VO, &bHasAny);
- m_searchCriteria.vid = requestMsg.getOptional(cta::admin::OptionString::VID, &bHasAny);
- m_searchCriteria.mediaType = requestMsg.getOptional(cta::admin::OptionString::MEDIA_TYPE, &bHasAny);
- m_searchCriteria.vendor = requestMsg.getOptional(cta::admin::OptionString::VENDOR, &bHasAny);
- m_searchCriteria.purchaseOrder = requestMsg.getOptional(cta::admin::OptionString::MEDIA_PURCHASE_ORDER_NUMBER, &bHasAny);
- m_searchCriteria.diskFileIds = requestMsg.getOptional(cta::admin::OptionStrList::FILE_ID, &bHasAny);
- m_searchCriteria.checkIncompleteFileCopies = requestMsg.getOptional(cta::admin::OptionBoolean::INCOMPLETE_FILE_COPIES, &bHasAny);
+ m_searchCriteria.full = requestMsg.getOptional(cta::admin::OptionBoolean::FULL, &bHasAny);
+ m_searchCriteria.fromCastor = requestMsg.getOptional(cta::admin::OptionBoolean::FROM_CASTOR, &bHasAny);
+ m_searchCriteria.capacityInBytes = requestMsg.getOptional(cta::admin::OptionUInt64::CAPACITY, &bHasAny);
+ m_searchCriteria.logicalLibrary = requestMsg.getOptional(cta::admin::OptionString::LOGICAL_LIBRARY, &bHasAny);
+ m_searchCriteria.tapePool = requestMsg.getOptional(cta::admin::OptionString::TAPE_POOL, &bHasAny);
+ m_searchCriteria.vo = requestMsg.getOptional(cta::admin::OptionString::VO, &bHasAny);
+ m_searchCriteria.vid = requestMsg.getOptional(cta::admin::OptionString::VID, &bHasAny);
+ m_searchCriteria.mediaType = requestMsg.getOptional(cta::admin::OptionString::MEDIA_TYPE, &bHasAny);
+ m_searchCriteria.vendor = requestMsg.getOptional(cta::admin::OptionString::VENDOR, &bHasAny);
+ m_searchCriteria.purchaseOrder = requestMsg.getOptional(cta::admin::OptionString::MEDIA_PURCHASE_ORDER_NUMBER, &bHasAny);
+ m_searchCriteria.diskFileIds = requestMsg.getOptional(cta::admin::OptionStrList::FILE_ID, &bHasAny);
+ m_searchCriteria.checkMissingFileCopies = requestMsg.getOptional(cta::admin::OptionBoolean::MISSING_FILE_COPIES, &bHasAny);
auto stateOpt = requestMsg.getOptional(cta::admin::OptionString::STATE, &bHasAny);
if(stateOpt){
m_searchCriteria.state = common::dataStructures::Tape::stringToState(stateOpt.value());
diff --git a/xroot_plugins/XrdCtaTapeLs.hpp b/xroot_plugins/XrdCtaTapeLs.hpp
index 278e75f88f..07edc5b2d0 100644
--- a/xroot_plugins/XrdCtaTapeLs.hpp
+++ b/xroot_plugins/XrdCtaTapeLs.hpp
@@ -68,19 +68,19 @@ TapeLsStream::TapeLsStream(const frontend::AdminCmdStream& requestMsg, cta::cata
// Get the search criteria from the optional options
- searchCriteria.full = requestMsg.getOptional(OptionBoolean::FULL, &has_any);
- searchCriteria.fromCastor = requestMsg.getOptional(OptionBoolean::FROM_CASTOR, &has_any);
- searchCriteria.capacityInBytes = requestMsg.getOptional(OptionUInt64::CAPACITY, &has_any);
- searchCriteria.logicalLibrary = requestMsg.getOptional(OptionString::LOGICAL_LIBRARY, &has_any);
- searchCriteria.tapePool = requestMsg.getOptional(OptionString::TAPE_POOL, &has_any);
- searchCriteria.vo = requestMsg.getOptional(OptionString::VO, &has_any);
- searchCriteria.vid = requestMsg.getOptional(OptionString::VID, &has_any);
- searchCriteria.mediaType = requestMsg.getOptional(OptionString::MEDIA_TYPE, &has_any);
- searchCriteria.vendor = requestMsg.getOptional(OptionString::VENDOR, &has_any);
- searchCriteria.purchaseOrder = requestMsg.getOptional(OptionString::MEDIA_PURCHASE_ORDER_NUMBER, &has_any);
- searchCriteria.physicalLibraryName = requestMsg.getOptional(OptionString::PHYSICAL_LIBRARY, &has_any);
- searchCriteria.diskFileIds = requestMsg.getOptional(OptionStrList::FILE_ID, &has_any);
- searchCriteria.checkIncompleteFileCopies = requestMsg.getOptional(OptionBoolean::INCOMPLETE_FILE_COPIES, &has_any);
+ searchCriteria.full = requestMsg.getOptional(OptionBoolean::FULL, &has_any);
+ searchCriteria.fromCastor = requestMsg.getOptional(OptionBoolean::FROM_CASTOR, &has_any);
+ searchCriteria.capacityInBytes = requestMsg.getOptional(OptionUInt64::CAPACITY, &has_any);
+ searchCriteria.logicalLibrary = requestMsg.getOptional(OptionString::LOGICAL_LIBRARY, &has_any);
+ searchCriteria.tapePool = requestMsg.getOptional(OptionString::TAPE_POOL, &has_any);
+ searchCriteria.vo = requestMsg.getOptional(OptionString::VO, &has_any);
+ searchCriteria.vid = requestMsg.getOptional(OptionString::VID, &has_any);
+ searchCriteria.mediaType = requestMsg.getOptional(OptionString::MEDIA_TYPE, &has_any);
+ searchCriteria.vendor = requestMsg.getOptional(OptionString::VENDOR, &has_any);
+ searchCriteria.purchaseOrder = requestMsg.getOptional(OptionString::MEDIA_PURCHASE_ORDER_NUMBER, &has_any);
+ searchCriteria.physicalLibraryName = requestMsg.getOptional(OptionString::PHYSICAL_LIBRARY, &has_any);
+ searchCriteria.diskFileIds = requestMsg.getOptional(OptionStrList::FILE_ID, &has_any);
+ searchCriteria.checkMissingFileCopies = requestMsg.getOptional(OptionBoolean::MISSING_FILE_COPIES, &has_any);
auto stateOpt = requestMsg.getOptional(OptionString::STATE, &has_any);
if(stateOpt){
diff --git a/xrootd-ssi-protobuf-interface b/xrootd-ssi-protobuf-interface
index a0c94a22be..f59e44742c 160000
--- a/xrootd-ssi-protobuf-interface
+++ b/xrootd-ssi-protobuf-interface
@@ -1 +1 @@
-Subproject commit a0c94a22bece907d6038e3b1fc452621c64281af
+Subproject commit f59e44742c6436a421157e76e5ee062c88d813bf
--
GitLab