FUSEX: remove stacked inodes in a separate thread - remove all in-memory…

FUSEX: remove stacked inodes in a separate thread - remove all in-memory structures when no cap was held anymore for 3 minutes - cache entries in the kernel cache for the lifetime of the CAP not with static 180s since this can lead to inconsistencies
parent 4602b34f
Pipeline #506974 passed with stages
in 59 minutes and 17 seconds
......@@ -472,6 +472,27 @@ cap::capx::valid(bool debug)
}
}
/* -------------------------------------------------------------------------- */
double
/* -------------------------------------------------------------------------- */
cap::capx::lifetime()
/* -------------------------------------------------------------------------- */
{
struct timespec ts;
ts.tv_sec = vtime();
ts.tv_nsec = vtime_ns();
double lifetime = 1.0 * (eos::common::Timing::GetCoarseAgeInNs(&ts,
0)) / 1000000000.0;
eos_static_debug("inode=%08lx client-id=%s lifetime=%.02f",
id(), clientid().c_str(), lifetime);
if (lifetime < 0) {
lifetime = 0.000000001;
}
return lifetime;
}
/* -------------------------------------------------------------------------- */
void
/* -------------------------------------------------------------------------- */
......@@ -482,11 +503,22 @@ cap::capflush(ThreadAssistant& assistant)
{
cmap capdelmap;
cinodes capdelinodes;
static time_t capemptytime = 0;
capmap.Lock();
time_t now = time(NULL);
if (!capmap.size()) {
eos_static_debug("forgetting all md from mdmap");
mds->forget_all();
if (!capemptytime) {
capemptytime = now;
}
if ((now - capemptytime) > CAP_EXTENSION_TIME) {
eos_static_crit("forgetting all md from mdmap");
mds->forget_all();
capemptytime = now;
}
} else {
capemptytime = 0;
}
for (auto it = capmap.begin(); it != capmap.end(); ++it) {
......
......@@ -111,6 +111,8 @@ public:
bool valid(bool debug = true);
double lifetime();
void use()
{
lastusage = time(NULL);
......
......@@ -1217,6 +1217,7 @@ EosFuse::run(int argc, char* argv[], void* userdata)
tStatCirculate.reset(&EosFuse::StatCirculate, this);
tMetaCacheFlush.reset(&metad::mdcflush, &mds);
tMetaSizeFlush.reset(&metad::mdsizeflush, &mds);
tMetaStackFree.reset(&metad::mdstackfree, &mds);
tMetaCommunicate.reset(&metad::mdcommunicate, &mds);
tCapFlush.reset(&cap::capflush, &caps);
eos_static_warning("********************************************************************************");
......@@ -1331,6 +1332,7 @@ EosFuse::run(int argc, char* argv[], void* userdata)
tStatCirculate.join();
tMetaCacheFlush.join();
tMetaSizeFlush.join();
tMetaStackFree.join();
tMetaCommunicate.join();
tCapFlush.join();
Mounter().terminate();
......@@ -1560,7 +1562,7 @@ EosFuse::getattr(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi)
if (pcap->errc()) {
rc = pcap->errc();
} else {
md->convert(e);
md->convert(e, pcap->lifetime());
eos_static_info("%s", md->dump(e).c_str());
}
}
......@@ -1922,7 +1924,7 @@ EosFuse::setattr(fuse_req_t req, fuse_ino_t ino, struct stat* attr, int op,
} else {
struct fuse_entry_param e;
memset(&e, 0, sizeof(e));
md->convert(e);
md->convert(e, pcap->lifetime());
eos_static_info("%s", md->dump(e).c_str());
Instance().mds.update(req, md, pcap->authid());
md->Locker().UnLock();
......@@ -1954,11 +1956,11 @@ EosFuse::lookup(fuse_req_t req, fuse_ino_t parent, const char* name)
if (md->id() && !md->deleted()) {
XrdSysMutexHelper mLock(md->Locker());
md->set_pid(parent);
md->convert(e);
eos_static_info("%s", md->dump(e).c_str());
md->lookup_inc();
cap::shared_cap pcap = Instance().caps.acquire(req, parent,
R_OK);
md->convert(e, pcap->lifetime());
} else {
// negative cache entry
e.ino = 0;
......@@ -2515,7 +2517,7 @@ EROFS pathname refers to a file on a read-only filesystem.
if (!rc) {
memset(&e, 0, sizeof(e));
md->convert(e);
md->convert(e, pcap->lifetime());
md->lookup_inc();
eos_static_info("%s", md->dump(e).c_str());
} else {
......@@ -3086,7 +3088,7 @@ EosFuse::open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi)
capLock.UnLock();
struct fuse_entry_param e;
memset(&e, 0, sizeof(e));
md->convert(e);
md->convert(e, pcap->lifetime());
mLock.UnLock();
data::data_fh* io = data::data_fh::Instance(Instance().datas.get(req, md->id(),
md), md, (mode == W_OK));
......@@ -3380,7 +3382,7 @@ The O_NONBLOCK flag was specified, and an incompatible lease was held on the fil
if (!rc) {
Instance().caps.book_inode(pcap);
md->convert(e);
md->convert(e, pcap->lifetime());
md->lookup_inc();
if (fi) {
......@@ -4569,7 +4571,7 @@ EosFuse::symlink(fuse_req_t req, const char* link, fuse_ino_t parent,
}
memset(&e, 0, sizeof(e));
md->convert(e);
md->convert(e, pcap->lifetime());
}
}
......@@ -4682,7 +4684,7 @@ EosFuse::link(fuse_req_t req, fuse_ino_t ino, fuse_ino_t parent,
if (!rc) {
XrdSysMutexHelper tmLock(tmd->Locker());
memset(&e, 0, sizeof(e));
tmd->convert(e);
tmd->convert(e, pcap->lifetime());
if (EOS_LOGS_DEBUG) {
eos_static_debug("hlnk tmd %s %s", tmd->name().c_str(), tmd->dump(e).c_str());
......
......@@ -361,6 +361,7 @@ private:
AssistedThread tStatCirculate;
AssistedThread tMetaCacheFlush;
AssistedThread tMetaSizeFlush;
AssistedThread tMetaStackFree;
AssistedThread tMetaCommunicate;
AssistedThread tCapFlush;
......
......@@ -305,12 +305,8 @@ metad::forget_all()
for (auto it = mdmap.begin(); it != mdmap.end();) {
if (it->first != 1) {
if (!S_ISDIR(it->second->mode()) || it->second->deleted()) {
it = mdmap.erase(it);
stat.inodes_dec();
} else {
it++;
}
it = mdmap.erase(it);
stat.inodes_dec();
} else {
it++;
}
......@@ -319,7 +315,7 @@ metad::forget_all()
/* -------------------------------------------------------------------------- */
void
metad::mdx::convert(struct fuse_entry_param& e)
metad::mdx::convert(struct fuse_entry_param& e, double lifetime)
{
const char* k_mdino = "sys.eos.mdino";
const char* k_fifo = "sys.eos.fifo";
......@@ -372,8 +368,8 @@ metad::mdx::convert(struct fuse_entry_param& e)
e.attr.CTIMESPEC.tv_nsec = ctime_ns();
if (EosFuse::Instance().Config().options.md_kernelcache) {
e.attr_timeout = 180.0;
e.entry_timeout = 180.0;
e.attr_timeout = lifetime;
e.entry_timeout = lifetime;;
} else {
e.attr_timeout = 0;
e.entry_timeout = 0;
......@@ -2202,6 +2198,50 @@ metad::mdsizeflush(ThreadAssistant& assistant)
return;
}
/* -------------------------------------------------------------------------- */
void
metad::mdstackfree(ThreadAssistant& assistant)
{
size_t cnt = 0;
while (!assistant.terminationRequested()) {
cnt++;
std::this_thread::sleep_for(std::chrono::milliseconds(500));
// do this ~every 128 seconds
if (!cnt % 256) {
XrdSysMutexHelper mLock(mdmap);
for (auto it = mdmap.begin(); it != mdmap.end();) {
// if the parent is gone, we can remove the child
if ((!mdmap.count(it->second->pid())) &&
(!S_ISDIR(it->second->mode()) || it->second->deleted())) {
eos_static_warning("removing orphaned inode from mdmap ino=%016x path=%s",
it->first, it->second->fullpath().c_str());
it = mdmap.erase(it);
stat.inodes_dec();
} else {
if (it->second->deleted()) {
if (!has_flush(it->first)) {
eos_static_warning("removing deleted inode from mdmap ino=%016x path=%s",
it->first, it->second->fullpath().c_str());
it = mdmap.erase(it);
stat.inodes_dec();
} else {
it++;
}
} else {
it++;
}
}
}
}
}
return;
}
/* -------------------------------------------------------------------------- */
bool
metad::determineLockOrder(shared_md md1, shared_md md2)
......
......@@ -97,7 +97,7 @@ public:
return mLock;
}
void convert(fuse_entry_param& e);
void convert(fuse_entry_param& e, double lifetime = 180.0);
std::string dump();
static std::string dump(struct fuse_entry_param& e);
......@@ -509,6 +509,9 @@ public:
void mdsizeflush(ThreadAssistant&
assistant); // thread updating filesize during long lasting writes
void mdstackfree(ThreadAssistant&
assistant); // thread removing stacked inodes
int connect(std::string zmqtarget, std::string zmqidentity = "",
std::string zmqname = "", std::string zmqclienthost = "",
std::string zmqclientuuid = "");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment