MGM/CONSOLE: add 'fs dropghosts' command to remove undrainable ghost entries in…

MGM/CONSOLE: add 'fs dropghosts' command to remove undrainable ghost entries in a filesystem view - additionally these entries are printed by 'dumpmd'
parent 8cd9b791
Pipeline #507527 passed with stages
in 47 minutes 50 seconds
......@@ -356,6 +356,32 @@ com_fs(char* arg1)
return (0);
}
if (subcommand == "dropghosts") {
XrdOucString arg = subtokenizer.GetToken();
if (!arg.c_str()) {
goto com_fs_usage;
}
XrdOucString in = "mgm.cmd=fs&mgm.subcmd=dropghosts";
int fsid = atoi(arg.c_str());
char r1fsid[128];
sprintf(r1fsid, "%d", fsid);
char r2fsid[128];
sprintf(r2fsid, "%04d", fsid);
if ((arg == r1fsid) || (arg == r2fsid)) {
// boot by fsid
in += "&mgm.fs.id=";
in += arg;
} else {
goto com_fs_usage;
}
global_retc = output_result(client_command(in, true));
return (0);
}
if (subcommand == "boot") {
XrdOucString arg = subtokenizer.GetToken();
XrdOucString option = subtokenizer.GetToken();
......@@ -832,7 +858,7 @@ com_fs(char* arg1)
com_fs_usage:
fprintf(stdout, "'[eos] fs ..' provides the filesystem interface of EOS.\n");
fprintf(stdout,
"Usage: fs add|boot|config|dropdeletion|dropfiles|dumpmd|mv|ls|rm|status [OPTIONS]\n");
"Usage: fs add|boot|config|dropdeletion|dropghosts|dropfiles|dumpmd|mv|ls|rm|status [OPTIONS]\n");
fprintf(stdout, "Options:\n");
fprintf(stdout,
"fs ls [-m|-l|-e|--io|--fsck|-d|--drain] [-s] [--brief|-b] [ [matchlist] ] :\n");
......@@ -956,6 +982,9 @@ com_fs_usage:
fprintf(stdout, "fs dropdeletion <fs-id> :\n");
fprintf(stdout,
" allows to drop all pending deletions on <fs-id> \n");
fprintf(stdout, "fs dropghosts <fs-id> :\n");
fprintf(stdout,
" allows to drop all filesystem view ghost entries (ids without meta data objects in the namespace) on <fs-id> \n");
fprintf(stdout, "fs dropfiles <fs-id> [-f] :\n");
fprintf(stdout,
" allows to drop all files on <fs-id> - force\n");
......
......@@ -62,7 +62,7 @@ int com_protofs(char* arg)
void com_fs_help()
{
std::ostringstream oss;
oss << "Usage: fs add|boot|config|dropdeletion|dropfiles|dumpmd|ls|mv|rm|status [OPTIONS]"
oss << "Usage: fs add|boot|config|dropdeletion|dropghosts|dropfiles|dumpmd|ls|mv|rm|status [OPTIONS]"
<< std::endl
<< " Options:" << std::endl
<< " fs add [-m|--manual <fsid>] <uuid> <node-queue>|<host>[:<port>] "
......@@ -91,7 +91,8 @@ void com_fs_help()
<< std::endl
<< " fs boot <fsid>|<uuid>|<node-queue>|* [--syncmgm]" << std::endl
<< " boot - filesystem identified by <fsid> or <uuid>" << std::endl
<< " - all filesystems on a node identified by <node-queue>" << std::endl
<< " - all filesystems on a node identified by <node-queue>" <<
std::endl
<< " - all filesystems registered" << std::endl
<< " --syncmgm : for MGM resynchronization during the booting"
<< std::endl
......@@ -164,6 +165,10 @@ void com_fs_help()
<< " fs dropdeletion <fsid> " << std::endl
<< " drop all pending deletions on the filesystem" << std::endl
<< std::endl
<< " fs dropghosts <fsid> " << std::endl
<< " drop all filesystem view ghost entries (ids without meta data objects in the namespace) on the filesystem"
<< std::endl
<< std::endl
<< " fs dropfiles <fsid> [-f]" << std::endl
<< " drop all files on the filesystem" << std::endl
<< " -f : unlink/remove files from the namespace (you have to remove"
......
......@@ -135,7 +135,7 @@ FsHelper::ParseCommand(const char* arg)
} else {
// Parse <fsid> or <uuid>
bool isUuid =
soption.find_first_not_of("0123456789") != std::string::npos;
soption.find_first_not_of("0123456789") != std::string::npos;
if (isUuid) {
boot->set_uuid(soption);
......@@ -143,7 +143,7 @@ FsHelper::ParseCommand(const char* arg)
try {
uint64_t fsid = std::stoull(soption);
boot->set_fsid(fsid);
} catch (const std::exception &e) {
} catch (const std::exception& e) {
std::cerr << "error: fsid needs to be numeric" << std::endl;
return false;
}
......@@ -237,7 +237,7 @@ FsHelper::ParseCommand(const char* arg)
} else {
// Parse <fsid> or <uuid>
bool isUuid =
soption.find_first_not_of("0123456789") != std::string::npos;
soption.find_first_not_of("0123456789") != std::string::npos;
if (isUuid) {
config->set_uuid(soption);
......@@ -245,7 +245,7 @@ FsHelper::ParseCommand(const char* arg)
try {
uint64_t fsid = std::stoull(soption);
config->set_fsid(fsid);
} catch (const std::exception &e) {
} catch (const std::exception& e) {
std::cerr << "error: fsid needs to be numeric" << std::endl;
return false;
}
......@@ -288,6 +288,23 @@ FsHelper::ParseCommand(const char* arg)
return false;
}
}
} else if (cmd == "dropghosts") {
using eos::console::FsProto_DropGhostsProto;
FsProto_DropGhostsProto* dropghosts = fs->mutable_dropghosts();
if (!(option = tokenizer.GetToken())) {
return false;
} else {
soption = option;
try {
uint64_t fsid = std::stoull(soption);
dropghosts->set_fsid(fsid);
} catch (const std::exception& e) {
std::cerr << "error: fsid needs to be numeric" << std::endl;
return false;
}
}
} else if (cmd == "dropfiles") {
using eos::console::FsProto_DropFilesProto;
FsProto_DropFilesProto* dropfiles = fs->mutable_dropfiles();
......@@ -515,8 +532,7 @@ FsHelper::ParseCommand(const char* arg)
oss << "/fst" << mountpoint;
rm->set_nodequeue(oss.str());
}
else {
} else {
// This needs to be an fsid
try {
uint64_t fsid = std::stoull(soption);
......
......@@ -155,6 +155,12 @@ ProcCommand::Fs()
eos::common::RWMutexReadLock rd_lock(FsView::gFsView.ViewMutex);
retc = proc_fs_dropdeletion(id, stdOut, stdErr, *pVid);
}
if (mSubCmd == "dropghosts") {
std::string id = pOpaque->Get("mgm.fs.id") ? pOpaque->Get("mgm.fs.id") : "";
eos::common::RWMutexReadLock rd_lock(FsView::gFsView.ViewMutex);
retc = proc_fs_dropghosts(id, stdOut, stdErr, *pVid);
}
}
if (mSubCmd == "boot") {
......
......@@ -60,6 +60,8 @@ eos::mgm::FsCmd::ProcessRequest()
reply.set_retc(Config(fs.config()));
} else if (subCmdCase == eos::console::FsProto::SubcmdCase::kDropdel) {
reply.set_retc(DropDeletion(fs.dropdel()));
} else if (subCmdCase == eos::console::FsProto::SubcmdCase::kDropghosts) {
reply.set_retc(DropGhosts(fs.dropghosts()));
} else if (subCmdCase == eos::console::FsProto::SubcmdCase::kDropfiles) {
reply.set_retc(DropFiles(fs.dropfiles()));
} else if (subCmdCase == eos::console::FsProto::SubcmdCase::kDumpmd) {
......@@ -152,10 +154,9 @@ FsCmd::Boot(const eos::console::FsProto::BootProto& bootProto)
for (const auto id : FsView::gFsView.mIdView) {
if ((id.second->GetConfigStatus() > eos::common::FileSystem::kOff)) {
eos::common::FileSystem::eBootConfig bootConfig = (forcemgmsync)
? eos::common::FileSystem::kBootResync // MGM resync
: eos::common::FileSystem::kBootForced; // local resync
? eos::common::FileSystem::kBootResync // MGM resync
: eos::common::FileSystem::kBootForced; // local resync
auto now = time(nullptr);
id.second->SetLongLong("bootcheck", bootConfig);
id.second->SetLongLong("bootsenttime", (unsigned long long) now);
outStream << " ";
......@@ -189,10 +190,9 @@ FsCmd::Boot(const eos::console::FsProto::BootProto& bootProto)
if (fs != nullptr) {
eos::common::FileSystem::eBootConfig bootConfig = (forcemgmsync)
? eos::common::FileSystem::kBootResync // MGM resync
: eos::common::FileSystem::kBootForced; // local resync
? eos::common::FileSystem::kBootResync // MGM resync
: eos::common::FileSystem::kBootForced; // local resync
auto now = time(nullptr);
fs->SetLongLong("bootcheck", bootConfig);
fs->SetLongLong("bootsenttime", ((now > 0) ? now : 0));
outStream << " ";
......@@ -204,10 +204,11 @@ FsCmd::Boot(const eos::console::FsProto::BootProto& bootProto)
}
} else {
// boot filesystem by fsid or uuid
FileSystem *fs = nullptr;
FileSystem* fs = nullptr;
if (fsid) {
eos::common::RWMutexReadLock lock(FsView::gFsView.ViewMutex);
if (FsView::gFsView.mIdView.count(fsid)) {
fs = FsView::gFsView.mIdView[fsid];
} else {
......@@ -217,6 +218,7 @@ FsCmd::Boot(const eos::console::FsProto::BootProto& bootProto)
}
} else if (fsuuid.length()) {
eos::common::RWMutexReadLock lock(FsView::gFsView.ViewMutex);
if (FsView::gFsView.GetMapping(fsuuid)
&& FsView::gFsView.mIdView.count(FsView::gFsView.GetMapping(fsuuid))) {
fs = FsView::gFsView.mIdView[FsView::gFsView.GetMapping(fsuuid)];
......@@ -229,12 +231,10 @@ FsCmd::Boot(const eos::console::FsProto::BootProto& bootProto)
if (fs != nullptr) {
eos::common::FileSystem::eBootConfig bootConfig = (forcemgmsync)
? eos::common::FileSystem::kBootResync // MGM resync
: eos::common::FileSystem::kBootForced; // local resync
? eos::common::FileSystem::kBootResync // MGM resync
: eos::common::FileSystem::kBootForced; // local resync
fs->SetLongLong("bootcheck", bootConfig);
fs->SetLongLong("bootsenttime", (unsigned long long) time(nullptr));
outStream << "success: boot message sent to ";
outStream << fs->GetString("host").c_str();
outStream << ":";
......@@ -248,7 +248,7 @@ FsCmd::Boot(const eos::console::FsProto::BootProto& bootProto)
} else {
retc = EPERM;
errStream << "error: you have to take role 'root' or connect via 'sss' "
"to execute this command";
"to execute this command";
}
mOut = outStream.str();
......@@ -308,6 +308,23 @@ FsCmd::DropDeletion(const eos::console::FsProto::DropDeletionProto&
}
//------------------------------------------------------------------------------
// DropGhosts subcommand
//------------------------------------------------------------------------------
int
FsCmd::DropGhosts(const eos::console::FsProto::DropGhostsProto&
dropghostsProto)
{
XrdOucString outLocal, errLocal;
eos::common::RWMutexReadLock rd_lock(FsView::gFsView.ViewMutex);
retc = proc_fs_dropghosts(std::to_string(dropghostsProto.fsid()), outLocal,
errLocal, mVid);
mOut = outLocal.c_str() != nullptr ? outLocal.c_str() : "";
mErr = errLocal.c_str() != nullptr ? errLocal.c_str() : "";
return retc;
}
//------------------------------------------------------------------------------
// Dumpmd subcommand
//------------------------------------------------------------------------------
......
......@@ -67,6 +67,8 @@ private:
int DropDeletion(const eos::console::FsProto::DropDeletionProto& dropdelProto);
int DropGhosts(const eos::console::FsProto::DropGhostsProto& dropghostsProto);
int Add(const eos::console::FsProto::AddProto& addProto);
int Boot(const eos::console::FsProto::BootProto& bootProto);
......@@ -84,12 +86,13 @@ private:
std::string DisplayModeToString(eos::console::FsProto::LsProto::DisplayMode
mode);
int SemaphoreProtectedProcDumpmd(std::string& fsid, XrdOucString& option, XrdOucString& dp,
int SemaphoreProtectedProcDumpmd(std::string& fsid, XrdOucString& option,
XrdOucString& dp,
XrdOucString& df, XrdOucString& ds, XrdOucString& out,
XrdOucString& err, size_t& entries);
template <class T, std::size_t N>
static constexpr std::size_t SizeOfArray(const T (&array)[N]) noexcept
static constexpr std::size_t SizeOfArray(const T(&array)[N]) noexcept
{
return N;
}
......
......@@ -151,7 +151,6 @@ proc_fs_dumpmd(std::string& sfsid, XrdOucString& option, XrdOucString& dp,
retc = EINVAL;
} else {
int fsid = atoi(sfsid.c_str());
std::shared_ptr<eos::IFileMD> fmd;
eos::Prefetcher::prefetchFilesystemFileListWithFileMDsAndParentsAndWait(
gOFS->eosView, gOFS->eosFsView, fsid);
......@@ -165,6 +164,8 @@ proc_fs_dumpmd(std::string& sfsid, XrdOucString& option, XrdOucString& dp,
for (auto it_fid = gOFS->eosFsView->getFileList(fsid);
(it_fid && it_fid->valid()); it_fid->next()) {
std::shared_ptr<eos::IFileMD> fmd;
try {
fmd = gOFS->eosFileService->getFileMD(it_fid->getElement());
......@@ -234,12 +235,21 @@ proc_fs_dumpmd(std::string& sfsid, XrdOucString& option, XrdOucString& dp,
"code: %d, message: %s", it_fid->getElement(),
e.getErrno(), e.getMessage().str().c_str());
}
if (!fmd) {
char sfid[1024];
snprintf(sfid, 1024, "# warning: ghost entry fid=%llu",
(unsigned long long) it_fid->getElement());
stdOut += sfid;
}
}
if (monitor) {
// Also add files which have yet to be unlinked
for (auto it_fid = gOFS->eosFsView->getUnlinkedFileList(fsid);
(it_fid && it_fid->valid()); it_fid->next()) {
std::shared_ptr<eos::IFileMD> fmd;
try {
fmd = gOFS->eosFileService->getFileMD(it_fid->getElement());
......@@ -1023,7 +1033,7 @@ proc_sort_groups_by_priority(FsView& fs_view, const std::string& space,
// without any file systems i.e highest priority
std::list<std::string> ret_grps(set_grps.begin(), set_grps.end());
for (auto && grp : grps) {
for (auto&& grp : grps) {
ret_grps.push_back(grp->mName);
}
......@@ -1066,7 +1076,7 @@ int proc_mv_grp_space(FsView& fs_view, const std::string& src,
if (!failed_fs.empty()) {
oss << "warning: the following file systems could not be moved ";
for (auto && elem : failed_fs) {
for (auto&& elem : failed_fs) {
oss << elem << " ";
}
......@@ -1129,7 +1139,7 @@ int proc_mv_space_space(FsView& fs_view, const std::string& src,
if (!failed_fs.empty()) {
oss << "warning: the following file systems could not be moved ";
for (auto && elem : failed_fs) {
for (auto&& elem : failed_fs) {
oss << elem << " ";
}
......@@ -1258,7 +1268,7 @@ proc_fs_dropdeletion(const std::string& id, XrdOucString& stdOut,
if (fsid) {
if ((vid_in.uid != 0)) {
stdErr = "error: filesystems can only be removed as 'root'\n";
stdErr = "error: filesystem deletions can oly be dropped as 'root'\n";
retc = EPERM;
} else {
eos::common::RWMutexWriteLock nslock(gOFS->eosViewRWMutex);
......@@ -1280,4 +1290,59 @@ proc_fs_dropdeletion(const std::string& id, XrdOucString& stdOut,
return retc;
}
//-------------------------------------------------------------------------------
// Clean ghost entries in a filesystem view
//-------------------------------------------------------------------------------
int
proc_fs_dropghosts(const std::string& id, XrdOucString& stdOut,
XrdOucString& stdErr,
eos::common::Mapping::VirtualIdentity& vid_in)
{
int retc = 0;
eos::common::FileSystem::fsid_t fsid = 0;
if (id.length()) {
fsid = atoi(id.c_str());
}
if (fsid) {
if ((vid_in.uid != 0)) {
stdErr = "error: filesystems view ghosts can only be removed as 'root'\n";
retc = EPERM;
} else {
eos::common::RWMutexWriteLock nslock(gOFS->eosViewRWMutex);
std::vector<uint64_t> todelete;
std::shared_ptr<eos::IFileMD> fmd;
for (auto it_fid = gOFS->eosFsView->getFileList(fsid);
(it_fid && it_fid->valid()); it_fid->next()) {
try {
fmd = gOFS->eosFileService->getFileMD(it_fid->getElement());
} catch (eos::MDException& e) {
if (e.getErrno() == ENOENT) {
stdOut += "# removing id: ";
stdOut += std::to_string(it_fid->getElement()).c_str();
stdOut += "\n";
todelete.push_back(it_fid->getElement());
}
}
}
for (auto it : todelete) {
gOFS->eosFsView->eraseEntry(fsid, it);
}
stdOut += "success: dropped ghost entries from fsid=";
stdOut += id.c_str();
}
} else {
stdErr = "error: there is no filesystem defined with fsid=";
stdErr += id.c_str();
stdErr += " ";
retc = EINVAL;
}
return retc;
}
EOSMGMNAMESPACE_END
......@@ -140,6 +140,21 @@ int proc_fs_dropdeletion(const std::string& id, XrdOucString& stdOut,
XrdOucString& stdErr,
eos::common::Mapping::VirtualIdentity& vid_in);
//------------------------------------------------------------------------------
//! Drop ghost entries from a filesystem view (file ids without meta data objects)
//!
//! @param id id of the filesystem
//! @param stdOut normal output string
//! @param stdErr error output string
//! @param vid_in virtual identify of the client
//!
//! @return 0 if successful, otherwise error code value
//------------------------------------------------------------------------------
int proc_fs_dropghosts(const std::string& id, XrdOucString& stdOut,
XrdOucString& stdErr,
eos::common::Mapping::VirtualIdentity& vid_in);
//------------------------------------------------------------------------------
//! Get type of entity. It can either be a filesystem, an eos group or an eos
//! space.
......
......@@ -47,6 +47,10 @@ message FsProto {
uint64 Fsid = 1;
}
message DropGhostsProto {
uint64 Fsid = 1;
}
message DropFilesProto {
uint64 Fsid = 1;
bool Force = 2;
......@@ -116,5 +120,6 @@ message FsProto {
LsProto ls = 10;
RmProto rm = 11;
StatusProto status = 12;
DropGhostsProto dropghosts = 13;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment