GitLab unavailability on July 18, 22, 23 due to hypervisor security updates: http://cern.ch/go/BP7D

...
 
......@@ -121,9 +121,10 @@ before_script:
- if [[ "$CI_JOB_NAME" == system_test_qdb* ]]; then sudo ./eos-docker/scripts/start_services.sh -q -i gitlab-registry.cern.ch/dss/${EOS_IMG_NAME}${CI_COMMIT_TAG-$CI_PIPELINE_ID}; else sudo ./eos-docker/scripts/start_services.sh -i gitlab-registry.cern.ch/dss/${EOS_IMG_NAME}${CI_COMMIT_TAG-$CI_PIPELINE_ID}; fi
- docker exec -i eos-client1-test git clone https://gitlab.cern.ch/dss/eosclient-tests.git
- docker exec -i eos-mgm-test /bin/bash -c 'eos vid enable krb5'
- docker exec -i eos-mgm-test eos-instance-test-ci
- docker exec -di eos-client1-test /bin/bash -c 'mkdir /eos1/; mount -t fuse eosxd /eos1/'
- docker exec -di eos-client1-test /bin/bash -c 'mkdir /eos2/; mount -t fuse eosxd /eos2/'
- docker exec -i eos-client1-test env EOS_MGM_URL=root://eos-mgm-test.eoscluster.cern.ch bash -c 'cd /eosclient-tests; git checkout rtb_clone; ./clone_test.sh'
- docker exec -i eos-mgm-test eos-instance-test-ci
- docker exec -i -u eos-user eos-client1-test /bin/bash -c 'mkdir /eos1/dockertest/fusex_tests/; cd /eos1/dockertest/fusex_tests/; fusex-benchmark'
# @todo(esindril): run "all" tests in schedule mode once these are properly supported
# - if [ "$CI_PIPELINE_SOURCE" == "schedule" ]; then
......
......@@ -1028,6 +1028,7 @@ Contains an instance and fuse test script and some test executables and test arc
%{_sbindir}/eos-mq-tests
%{_sbindir}/eos-instance-test
%{_sbindir}/eos-instance-test-ci
%{_sbindir}/eos-backup-test
%{_sbindir}/eos-rain-test
%{_sbindir}/eos-drain-test
%{_sbindir}/eos-http-upload-test
......
......@@ -454,6 +454,51 @@ XrdFstOfsFile::open(const char* path, XrdSfsFileOpenMode open_mode,
}
}
char *sCloneFST = mCapOpaque->Get("mgm.cloneFST");
if (sCloneFST) {
XrdOucString mcFstPath;
eos::common::FileId::FidPrefix2FullPath(sCloneFST, mLocalPrefix.c_str(), mcFstPath);
struct stat clone_stat;
int clonerc = ::stat(mcFstPath.c_str(), &clone_stat) ? errno : 0;
eos_info("fstpath=%s clonepath=%s clonerc=%d len=%d", mFstPath.c_str(), mcFstPath.c_str(), clonerc, clonerc ? -1 : clone_stat.st_size);
/* clone handling:
* if read-write and clone does not exist, create it
* if read-only switch to clone if it exists (note: if several clones were allowed, we'd might have to search!)
*/
FmdHelper* gMd;
if (isRW && clonerc != 0) { /* for RW, only if clone not yet created */
if (open_mode & SFS_O_TRUNC) {
/* rename data file to clone, it will be re-created */
int rc = ::rename(mFstPath.c_str(), mcFstPath.c_str()) ? errno : 0;
eos_info("copy-on-write: rename %s %s rc=%d",mFstPath.c_str(), mcFstPath.c_str(), rc);
} else {
/* copy data file to clone before modyfying */
char sbuff[1024];
snprintf(sbuff, sizeof(sbuff), "cp --preserve=xattr,ownership,mode --reflink=auto %s %s", mFstPath.c_str(), mcFstPath.c_str());
int rc = system(sbuff);
eos_info("copy-on-write: %s rc=%d", sbuff, rc);
}
/* Populate local DB (future reads need it) */
unsigned long long clFid = eos::common::FileId::Hex2Fid(sCloneFST);
gMd = gFmdDbMapHandler.LocalGetFmd(clFid, mFsId, vid.uid, vid.gid, mLid, isRW);
gMd->mProtoFmd.set_checksum(fMd->mProtoFmd.checksum());
gMd->mProtoFmd.set_diskchecksum(fMd->mProtoFmd.diskchecksum());
gMd->mProtoFmd.set_mgmchecksum(fMd->mProtoFmd.mgmchecksum());
if (!gFmdDbMapHandler.Commit(gMd)) {
eos_err("copy-on-write unable to commit meta data to local database");
(void) gOFS.Emsg(epname, this->error, EIO,
"copy-on-write - unable to commit meta data", mNsPath.c_str());
}
} else {
gMd = fMd;
}
eos_debug("fid %lld cs %s diskcs %s mgmcs %s", gMd->mProtoFmd.fid(), gMd->mProtoFmd.checksum().c_str(), gMd->mProtoFmd.diskchecksum().c_str(), gMd->mProtoFmd.mgmchecksum().c_str());
}
XrdOucString oss_opaque = "";
oss_opaque += "&mgm.lid=";
oss_opaque += std::to_string(mLid).c_str();
......
......@@ -35,6 +35,7 @@
#include "mgm/Quota.hh"
#include "mgm/Recycle.hh"
#include "mgm/XrdMgmOfs.hh"
#include "mgm/XrdMgmOfsFile.hh"
#include "mgm/ZMQ.hh"
#include "mgm/Stat.hh"
......@@ -2357,11 +2358,16 @@ Server::OpDeleteFile(const std::string& id,
}
}
if (doDelete) {
uint64_t cloneId;
if (doDelete && ( ((cloneId = fmd->getCloneId()) == 0) || !(fmd->getCloneFST().empty()) )) {
pcmd->removeFile(fmd->getName());
fmd->setContainerId(0);
fmd->unlinkAllLocations();
gOFS->WriteRmRecord(fmd);
} else if (doDelete) { /* delete, but clone first */
XrdOucErrInfo error;
XrdMgmOfsFile::create_cow(true, cloneId, pcmd, fmd, vid, error);
gOFS->WriteRmRecord(fmd);
}
gOFS->eosFileService->updateStore(fmd.get());
......
This diff is collapsed.
......@@ -233,9 +233,6 @@ XrdMgmOfs::_rem(const char* path,
"remove existing file - you are write-once user");
}
eos_debug("vid.uid %d vid.gid %d CanotDelete %d CUid %d", vid.uid, vid.gid,
acl.CanNotDelete(), fmd->getCUid());
// if there is a !d policy we cannot delete files which we don't own
if (((vid.uid) && (vid.uid != 3) && (vid.gid != 4) && (acl.CanNotDelete())) &&
((fmd->getCUid() != vid.uid))) {
......@@ -290,6 +287,11 @@ XrdMgmOfs::_rem(const char* path,
if (!doRecycle) {
try {
uint64_t cloneId;
if (!simulate && fmd->getCloneFST().empty() && (cloneId = fmd->getCloneId()) != 0) {
eos_info("Creating cow clone (delete) for %s fxid:%lx cloneId %lld", path, fmd->getId(), cloneId);
errno = XrdMgmOfsFile::create_cow(true, cloneId, container, fmd, vid, error);
}
if (!simulate) {
eos_info("unlinking from view %s", path);
Workflow workflow;
......
......@@ -2178,9 +2178,11 @@ XrdMgmOfs::SetupProcFiles()
procpathreconnect += "/reconnect";
XrdOucString procpathmaster = MgmProcPath;
procpathmaster += "/master";
XrdOucString clonePath(MgmProcPath + "/clone");
XrdOucErrInfo error;
eos::common::VirtualIdentity vid = eos::common::VirtualIdentity::Root();
std::shared_ptr<eos::IFileMD> fmd;
std::shared_ptr<eos::IContainerMD> cmd;
try {
fmd.reset();
......@@ -2234,6 +2236,10 @@ XrdMgmOfs::SetupProcFiles()
eosView->updateFileStore(fmd.get());
}
try {
cmd = eosView->createContainer(clonePath.c_str());
} catch (eos::MDException& e) {};
try {
fmd.reset();
fmd = eosView->getFile(procpathmaster.c_str());
......
......@@ -66,6 +66,109 @@
/* MGM File Interface */
/******************************************************************************/
/* copied for "eos_static_..." */
static int
emsg(XrdOucErrInfo& error, int ec, const char *txt, const char *txt2) {
// Get the reason for the error
if (ec < 0) ec = -ec;
char *etext = strerror(ec);
char sbuff[1024];
char ebuff[64];
if (etext == NULL) {
etext = ebuff;
snprintf(ebuff, sizeof(ebuff), "error code %d", ec);
}
snprintf(sbuff, sizeof(sbuff), "create_cow: unable to %s %s: %s", txt, txt2, etext);
eos_static_err(sbuff);
error.setErrInfo(ec, sbuff);
return SFS_ERROR;
}
/*
* Auxiliary routine: creates the copy-on-write clone an intermediate directories
*/
int
XrdMgmOfsFile::create_cow(bool isDelete, uint64_t cloneId,
std::shared_ptr<eos::IContainerMD> dmd, std::shared_ptr<eos::IFileMD> fmd,
eos::common::VirtualIdentity& vid, XrdOucErrInfo& error)
{
char sbuff[1024];
snprintf(sbuff, sizeof(sbuff), "%s/clone/%ld", gOFS->MgmProcPath.c_str(), cloneId);
std::shared_ptr<eos::IContainerMD> cloneMd, dirMd;
try {
cloneMd = gOFS->eosView->getContainer(sbuff);
} catch (eos::MDException& e) {
eos_static_debug("caught exception %d %s path %s\n", e.getErrno(), e.getMessage().str().c_str(), sbuff);
return emsg(error, ENOENT /*EEXIST*/, "open file ()", sbuff);
}
if (!dmd) return emsg(error, ENOENT, "determine parent", fmd->getName().c_str());
/* set up directory for clone */
int tlen = strlen(sbuff);
snprintf(sbuff+tlen, sizeof(sbuff)-tlen, "/%lx", dmd->getId());
try {
dirMd = gOFS->eosView->getContainer(sbuff);
} catch (eos::MDException& e) {
dirMd = gOFS->eosView->createContainer(sbuff, true);
dirMd->setMode(dmd->getMode());
eos::IFileMD::XAttrMap xattrs = dmd->getAttributes();
for (const auto& a : xattrs) {
if (a.first == "sys.acl" || a.first == "user.acl" || a.first == "sys.eval.useracl") {
dirMd->setAttribute(a.first, a.second);
}
}
}
/* create the clone */
if (isDelete) { /* effectively a "mv" */
dmd->removeFile(fmd->getName());
snprintf(sbuff, sizeof(sbuff), "%lx", fmd->getId());
fmd->setName(sbuff);
fmd->setCloneId(0); /* don't ever cow this again! */
dirMd->addFile(fmd.get());
gOFS->eosFileService->updateStore(fmd.get());
} else { /* prepare a "cp --reflink" (to be performed on the FSTs) */
std::shared_ptr<eos::IFileMD> gmd;
eos::IFileMD::ctime_t ttime;
tlen = strlen(sbuff);
snprintf(sbuff+tlen, sizeof(sbuff)-tlen, "/%lx", fmd->getId());
gmd = gOFS->eosView->createFile(sbuff, vid.uid, vid.gid);
gmd->setAttribute("sys.clone.targetFid", sbuff+tlen+1);
fmd->getCTime(ttime);
gmd->setCTime(ttime);
fmd->getMTime(ttime);
gmd->setMTime(ttime);
gmd->setCUid(fmd->getCUid());
gmd->setCGid(fmd->getCGid());
gmd->setFlags(fmd->getFlags());
gmd->setLayoutId(fmd->getLayoutId());
gmd->setSize(fmd->getSize());
gmd->setChecksum(fmd->getChecksum());
gmd->setContainerId(dirMd->getId());
for (unsigned int i = 0; i < fmd->getNumLocation(); i++)
gmd->addLocation(fmd->getLocation(i));
gOFS->eosFileService->updateStore(gmd.get());
fmd->setCloneFST(eos::common::FileId::Fid2Hex(gmd->getId()));
gOFS->eosFileService->updateStore(fmd.get());
}
gOFS->eosDirectoryService->updateStore(dirMd.get());
gOFS->FuseXCastContainer(dirMd->getIdentifier());
gOFS->FuseXCastContainer(dirMd->getParentIdentifier()); /* cloneMd */
gOFS->FuseXCastRefresh(dirMd->getIdentifier(), dirMd->getParentIdentifier());
gOFS->FuseXCastRefresh(cloneMd->getIdentifier(), cloneMd->getParentIdentifier());
return 0;
}
/*----------------------------------------------------------------------------*/
int
XrdMgmOfsFile::open(const char* inpath,
......@@ -998,6 +1101,20 @@ XrdMgmOfsFile::open(const char* inpath,
} else {
capability += "&mgm.access=create";
}
uint64_t cloneId;
if (fmd && (cloneId = fmd->getCloneId()) != 0) {
char sbuff[1024];
std::string cloneFST = fmd->getCloneFST();
if (cloneFST == "") { /* This triggers the copy-on-write */
if (int rc = create_cow(false, cloneId, dmd, fmd, vid, error)) return rc;
}
eos_debug("file %s cloneid %ld cloneFST %s trunc %d", path, fmd->getCloneId(), fmd->getCloneFST().c_str(), open_mode & SFS_O_TRUNC);
snprintf(sbuff, sizeof(sbuff), "&mgm.cloneid=%ld&mgm.cloneFST=%s", cloneId, fmd->getCloneFST().c_str());
capability += sbuff;
eos_debug("capability write: %s", capability.c_str());
}
} else {
capability += "&mgm.access=read";
}
......@@ -1082,7 +1199,7 @@ XrdMgmOfsFile::open(const char* inpath,
}
}
if ((!isInjection) && (isCreation || ((open_mode == SFS_O_TRUNC)))) {
if ((!isInjection) && (isCreation || (open_mode == SFS_O_TRUNC))) {
eos_info("blocksize=%llu lid=%x",
eos::common::LayoutId::GetBlocksize(new_lid), new_lid);
layoutId = new_lid;
......@@ -1193,7 +1310,18 @@ XrdMgmOfsFile::open(const char* inpath,
capability += "&mgm.manager=";
capability += gOFS->ManagerId.c_str();
capability += "&mgm.fid=";
const std::string hex_fid = eos::common::FileId::Fid2Hex(fileId);
std::string hex_fid;
if (!isRW) {
const char* val;
if ((val = openOpaque->Get("eos.clonefst")) && (strlen(val) < 32)) {
hex_fid = fmd->getCloneFST();
eos_debug("open read eos.clonefst %s hex_fid %s", val, hex_fid.c_str());
if (hex_fid != val) return Emsg(epname, error, EINVAL, "open - invalid clonefst argument", path);
}
}
if (hex_fid.empty()) {
hex_fid = eos::common::FileId::Fid2Hex(fileId);
}
capability += hex_fid.c_str();
XrdOucString sizestring;
capability += "&mgm.cid=";
......@@ -2356,9 +2484,9 @@ XrdMgmOfsFile::open(const char* inpath,
}
}
if (openOpaque->Get("eos.checksum")) {
if (openOpaque->Get("eos.checksum") || openOpaque->Get("eos.cloneid")) {
redirectionhost += "&mgm.checksum=";
redirectionhost += openOpaque->Get("eos.checksum");
redirectionhost += openOpaque->Get("eos.cloneidxxx") ? "ignore" : openOpaque->Get("eos.checksum");
}
if (openOpaque->Get("eos.mtime")) {
......
......@@ -86,6 +86,13 @@ public:
//----------------------------------------------------------------------------
virtual ~XrdMgmOfsFile();
//----------------------------------------------------------------------------
// utility function: create copy-on-write clone
//----------------------------------------------------------------------------
static int create_cow(bool isDelete, uint64_t cloneId,
std::shared_ptr<eos::IContainerMD> dmd, std::shared_ptr<eos::IFileMD> fmd,
eos::common::VirtualIdentity& vid, XrdOucErrInfo& error);
//----------------------------------------------------------------------------
// open a file
//----------------------------------------------------------------------------
......
......@@ -297,6 +297,26 @@ public:
//----------------------------------------------------------------------------
virtual void setCGid(gid_t gid) = 0;
//----------------------------------------------------------------------------
//! Get cloneId
//----------------------------------------------------------------------------
virtual time_t getCloneId() const = 0;
//----------------------------------------------------------------------------
//! Set cloneId
//----------------------------------------------------------------------------
virtual void setCloneId(time_t id) = 0;
//----------------------------------------------------------------------------
//! Get cloneFST
//----------------------------------------------------------------------------
virtual const std::string getCloneFST() const = 0;
//----------------------------------------------------------------------------
//! Set set cloneFST
//----------------------------------------------------------------------------
virtual void setCloneFST(const std::string& data) = 0;
//----------------------------------------------------------------------------
//! Get mode
//----------------------------------------------------------------------------
......
......@@ -108,6 +108,41 @@ public:
//----------------------------------------------------------------------------
virtual void setMTimeNow() = 0;
//----------------------------------------------------------------------------
//! get sync time
//----------------------------------------------------------------------------
virtual void getSyncTime(ctime_t& stime) const = 0;
//----------------------------------------------------------------------------
//! Set sync time
//----------------------------------------------------------------------------
virtual void setSyncTime(ctime_t stime) = 0;
//----------------------------------------------------------------------------
//! Set sync time
//----------------------------------------------------------------------------
virtual void setSyncTimeNow() = 0;
//----------------------------------------------------------------------------
//! Get clone id
//----------------------------------------------------------------------------
virtual uint64_t getCloneId() const = 0;
//----------------------------------------------------------------------------
//! Set clone id
//----------------------------------------------------------------------------
virtual void setCloneId(uint64_t id) = 0;
//----------------------------------------------------------------------------
//! Get cloneFST
//----------------------------------------------------------------------------
virtual const std::string getCloneFST() const = 0;
//----------------------------------------------------------------------------
//! Set cloneFST
//----------------------------------------------------------------------------
virtual void setCloneFST(const std::string& data) = 0;
//----------------------------------------------------------------------------
//! Get size
//----------------------------------------------------------------------------
......
......@@ -453,6 +453,35 @@ public:
pMode = mode;
}
//----------------------------------------------------------------------------
//! Get cloneId (dummy)
//----------------------------------------------------------------------------
time_t getCloneId() const override
{
return 0;
}
//----------------------------------------------------------------------------
//! Set cloneId (dummy)
//----------------------------------------------------------------------------
void setCloneId(time_t id) override {
}
//----------------------------------------------------------------------------
//! Get cloneFST (dummy)
//----------------------------------------------------------------------------
const std::string getCloneFST() const override
{
return std::string("");
}
//----------------------------------------------------------------------------
//! Set cloneFST (dummy)
//----------------------------------------------------------------------------
void setCloneFST(const std::string& data) override
{
}
//----------------------------------------------------------------------------
//! Add extended attribute
//----------------------------------------------------------------------------
......
......@@ -151,6 +151,58 @@ public:
#endif
}
//----------------------------------------------------------------------------
//! Get sync time
//----------------------------------------------------------------------------
void getSyncTime(ctime_t& mtime) const override
{
getMTime(mtime);
}
//----------------------------------------------------------------------------
//! Set sync time
//----------------------------------------------------------------------------
void setSyncTime(ctime_t mtime) override
{
}
//----------------------------------------------------------------------------
//! Set sync time
//----------------------------------------------------------------------------
void setSyncTimeNow() override
{
}
//----------------------------------------------------------------------------
//! Get cloneId (dummy)
//----------------------------------------------------------------------------
uint64_t getCloneId() const override
{
return 0;
}
//----------------------------------------------------------------------------
//! Set cloneId (dummy)
//----------------------------------------------------------------------------
void setCloneId(uint64_t id) override
{
}
//----------------------------------------------------------------------------
//! Get cloneFST (dummy)
//----------------------------------------------------------------------------
const std::string getCloneFST() const override
{
return std::string("");
}
//----------------------------------------------------------------------------
//! Set cloneFST (dummy)
//----------------------------------------------------------------------------
void setCloneFST(const std::string& data) override
{
}
//----------------------------------------------------------------------------
//! Get size
//----------------------------------------------------------------------------
......
......@@ -579,6 +579,11 @@ QuarkContainerMD::setTMTime(tmtime_t tmtime)
std::unique_lock<std::shared_timed_mutex> lock(mMutex);
tmtime_t tmt;
getTMTimeNoLock(tmt);
tmtime_t now;
clock_gettime(CLOCK_REALTIME, &now);
if (tmtime.tv_sec == 0 || tmtime.tv_sec > now.tv_sec) tmtime = now;
if (((tmt.tv_sec == 0) && (tmt.tv_nsec == 0)) ||
(tmtime.tv_sec > tmt.tv_sec) ||
......@@ -598,6 +603,7 @@ void
QuarkContainerMD::setTMTimeNow()
{
tmtime_t tmtime = {0};
#if 0
#ifdef __APPLE__
struct timeval tv;
gettimeofday(&tv, 0);
......@@ -605,6 +611,7 @@ QuarkContainerMD::setTMTimeNow()
tmtime.tv_nsec = tv.tv_usec * 1000;
#else
clock_gettime(CLOCK_REALTIME, &tmtime);
#endif
#endif
setTMTime(tmtime);
}
......@@ -616,6 +623,8 @@ void
QuarkContainerMD::getTMTimeNoLock(tmtime_t& tmtime)
{
(void) memcpy(&tmtime, mCont.stime().data(), sizeof(tmtime));
if (tmtime.tv_sec == 0)
(void) memcpy(&tmtime, mCont.mtime().data(), sizeof(tmtime));
}
//------------------------------------------------------------------------------
......
......@@ -333,6 +333,45 @@ public:
mCont.set_gid(gid);
}
//----------------------------------------------------------------------------
//! Get cloneId
//----------------------------------------------------------------------------
inline time_t
getCloneId() const
{
std::shared_lock<std::shared_timed_mutex> lock(mMutex);
return mCont.cloneid();
}
//----------------------------------------------------------------------------
//! Set cloneId
//----------------------------------------------------------------------------
inline void
setCloneId(time_t id)
{
std::unique_lock<std::shared_timed_mutex> lock(mMutex);
mCont.set_cloneid(id);
}
//----------------------------------------------------------------------------
//! Get cloneFST
//----------------------------------------------------------------------------
inline const std::string
getCloneFST() const
{
std::shared_lock<std::shared_timed_mutex> lock(mMutex);
return mCont.clonefst();
}
//----------------------------------------------------------------------------
//! Set cloneFST
//----------------------------------------------------------------------------
void setCloneFST(const std::string& data) {
std::unique_lock<std::shared_timed_mutex> lock(mMutex);
mCont.set_clonefst(data);
}
//----------------------------------------------------------------------------
//! Get mode
//----------------------------------------------------------------------------
......
......@@ -434,6 +434,56 @@ QuarkFileMD::setMTimeNow()
clock_gettime(CLOCK_REALTIME, &tnow);
#endif
setMTime(tnow);
struct timespec default_ts = {0, 0};
setSyncTime(default_ts);
}
/* SyncTime: whenever the file is changed, SyncTime becomes MTime. It is only
* when mtime is set explicitely that the two diverge. Hencex. the logic
* here is that if SyncTime is 0, use MTime. And reset SyncTime to
* zero when MTime is set to now (thus logically setting them both).
*/
//------------------------------------------------------------------------------
// Get sync time, no lock
//------------------------------------------------------------------------------
void
QuarkFileMD::getSyncTimeNoLock(ctime_t& stime) const
{
(void) memcpy(&stime, mFile.stime().data(), sizeof(stime));
if (stime.tv_sec == 0) /* fall back to mtime if default */
(void) memcpy(&stime, mFile.mtime().data(), sizeof(stime));
}
//------------------------------------------------------------------------------
// Get sync time
//------------------------------------------------------------------------------
void
QuarkFileMD::getSyncTime(ctime_t& stime) const
{
std::shared_lock<std::shared_timed_mutex> lock(mMutex);
getSyncTimeNoLock(stime);
}
//------------------------------------------------------------------------------
// Set sync time
//------------------------------------------------------------------------------
void
QuarkFileMD::setSyncTime(ctime_t stime)
{
std::unique_lock<std::shared_timed_mutex> lock(mMutex);
mFile.set_stime(&stime, sizeof(stime));
}
//------------------------------------------------------------------------------
// Set sync time to now
//------------------------------------------------------------------------------
void
QuarkFileMD::setSyncTimeNow()
{
struct timespec tnow;
clock_gettime(CLOCK_REALTIME, &tnow);
setSyncTime(tnow);
}
//------------------------------------------------------------------------------
......
......@@ -108,6 +108,21 @@ public:
//----------------------------------------------------------------------------
void setMTimeNow() override;
//----------------------------------------------------------------------------
//! get sync time
//----------------------------------------------------------------------------
void getSyncTime(ctime_t& stime) const override;
//----------------------------------------------------------------------------
//! set sync time
//----------------------------------------------------------------------------
void setSyncTime(ctime_t stime) override;
//----------------------------------------------------------------------------
//! set sync time to now
//----------------------------------------------------------------------------
void setSyncTimeNow() override;
//----------------------------------------------------------------------------
//! Get file id
//----------------------------------------------------------------------------
......@@ -142,6 +157,42 @@ public:
//----------------------------------------------------------------------------
void setSize(uint64_t size) override;
//----------------------------------------------------------------------------
//! Get cloneId
//----------------------------------------------------------------------------
inline uint64_t
getCloneId() const
{
std::shared_lock<std::shared_timed_mutex> lock(mMutex);
return mFile.cloneid();
}
//----------------------------------------------------------------------------
//! Set cloneId
//----------------------------------------------------------------------------
void setCloneId(uint64_t id) {
std::shared_lock<std::shared_timed_mutex> lock(mMutex);
mFile.set_cloneid(id);
}
//----------------------------------------------------------------------------
//! Get cloneFST
//----------------------------------------------------------------------------
const std::string
getCloneFST() const
{
std::shared_lock<std::shared_timed_mutex> lock(mMutex);
return mFile.clonefst();
}
//----------------------------------------------------------------------------
//! Set cloneFST
//----------------------------------------------------------------------------
void setCloneFST(const std::string& data) {
std::shared_lock<std::shared_timed_mutex> lock(mMutex);
mFile.set_clonefst(data);
}
//----------------------------------------------------------------------------
//! Get parent id
//----------------------------------------------------------------------------
......@@ -632,6 +683,11 @@ private:
//----------------------------------------------------------------------------
void getMTimeNoLock(ctime_t& mtime) const;
//----------------------------------------------------------------------------
//! Get modification time, no locks
//----------------------------------------------------------------------------
void getSyncTimeNoLock(ctime_t& stime) const;
//----------------------------------------------------------------------------
//! Get creation time, no locks
//----------------------------------------------------------------------------
......
......@@ -17,4 +17,7 @@ message ContainerMdProto {
bytes mtime = 10; // modification time
bytes stime = 11; // sync time
map<string, bytes> xattrs = 12;
uint64 cloneid = 256; // transient
bytes clonefst = 257; // transient
}
......@@ -20,4 +20,8 @@ message FileMdProto {
repeated uint32 locations = 13;
repeated uint32 unlink_locations = 14;
map<string, bytes> xattrs = 15;
bytes stime = 16; // server sync time
uint64 cloneid = 256; // transient
bytes clonefst = 257; // transient
}
......@@ -138,7 +138,7 @@ install(
install(
PROGRAMS xrdstress eos-instance-test eos-instance-test-ci fuse/eos-fuse-test
eos-rain-test eoscp-rain-test eos-io-test eos-oc-test eos-drain-test
eos-http-upload-test eos-mq-tests
eos-http-upload-test eos-mq-tests eos-backup-test
DESTINATION ${CMAKE_INSTALL_FULL_SBINDIR}
PERMISSIONS OWNER_READ OWNER_EXECUTE
GROUP_READ GROUP_EXECUTE
......
#!/usr/bin/env python
# sample_backup backup -B /BackupPrefix [-P <parentId>] /eos/pathName
# sample_backup restore -B /BackupPrefix -F /inputCatalog1[,/inputCatalog2[,...]] /outputDirectory
# for backup the data are stored in /BackupPrefix.cloneId and /BackupPrefix.cloneId.catalog
# the /eos pathnames are as seen by the MGM, not necessarily fuse-mounted paths in the local file system
# environment:
# EOS_MGM_URL=root://eos-mgm-test.eoscluster.cern.ch must point to the MGM serving /eos/pathName
import getopt, os, sys, tempfile
import pdb
backupDirPrefix = "/tmp/backupDir"
eos_instance = os.popen("eos version").readlines()[0].split('=')[1].rstrip()
eos_mgm_url = os.environ['EOS_MGM_URL']
def do_clone(args):
global backupDirPrefix
parentId = 10
opts, args = getopt.getopt(args, "B:P:")
for opt in opts:
if opt[0] == '-B': backupDirPrefix = opt[1]
elif opt[0] == '-P':
try:
parentId = int(opt[1])
except ValueError:
print "invalid parent Id"
sys.exit(22)
catalogFile = tempfile.mkstemp(prefix=backupDirPrefix)
try:
f = os.fdopen(catalogFile[0], "w")
except IOError as e:
print "cannot open %s: %s" % (catalogFile[1], e.message)
sys.exit(5)
cmd = "eos find -f -x sys.clone=+%d %s" % (parentId, args[0])
findOut = os.popen(cmd).readlines()
(rootPath, ttt) = findOut[0].split(' ') # the rootPath includes the trailing '/'
cloneInfo = ttt.split(':')
cloneId = cloneInfo[1]
for l in findOut: f.write(l)
f.close()
newCatFile = "%s.%s.catalog" % (backupDirPrefix, cloneId)
os.rename(catalogFile[1], newCatFile)
print "cloneId %s catalog %s" % (cloneId, newCatFile)
return cloneId
def do_backup(args):
global backupDirPrefix
opts, args = getopt.getopt(args, "F:B:P:")
catalogFile = None
for opt in opts:
if opt[0] == '-B': backupDirPrefix = opt[1]
elif opt[0] == '-F': catalogFile = opt[1]
elif opt[0] == '-P':
try:
parentId = int(opt[1])
except ValueError:
print "invalid parent Id"
sys.exit(22)
if catalogFile is None:
cloneId = do_clone(args)
catalogFile = "%s.%s.catalog" % (backupDirPrefix, cloneId)
try:
f = open(catalogFile, "r")
except IOError as e:
print "cannot open %s: %s" % (catalogFile, e.message)
sys.exit(5)
findOut = f.readlines()
(rootPath, ttt) = findOut[0].split(' ') # the rootPath includes the trailing '/'
cloneInfo = ttt.split(':')
cloneId = cloneInfo[1]
backupDir = "%s.%s/" % (backupDirPrefix, cloneId)
cloneDir = "/eos/%s/proc/clone/" % (eos_instance)
print "cloneId %s backup media %s" % (cloneId, backupDir)
for l in findOut:
path, cloneInfo = l.split()
rpath = path[len(rootPath):] # skip the "root" part
if path.endswith('/'): # this is a directory
os.makedirs(backupDir + rpath)
else: # this is a file
stime, clonePath = cloneInfo.split(':')[0:2]
fCloneId = clonePath.split('/')[0]
if fCloneId != cloneId: continue # file has not been modified in incremental dump
cpath = cloneDir + clonePath
# try clone first
cmd1 = "xrdcp %s/%s %s/%s 2>&1" % (eos_mgm_url, cpath, backupDir, rpath)
p1 = os.popen(cmd1)
x1 = p1.readlines()
rc = p1.close()
if rc > 256: rc = rc >> 8 # recover xrdcp return code
if rc > 0:
# clone live file
cmd2 = "xrdcp %s/%s %s/%s 2>&1" % (eos_mgm_url, path, backupDir, rpath)
p2 = os.popen(cmd2)
x2 = p2.readlines()
rc = p2.close()
if rc > 256: rc = rc >> 8 # recover xrdcp return code
# retry clone if created in the meantime
p1 = os.popen(cmd1)
x1 = p1.readlines()
rc = p1.close()
if rc > 256: rc = rc >> 8 # recover xrdcp return code
f.close()
cmd = "eos find -f -x sys.clone=-%s %s > /dev/null" % (cloneId, args[0])
os.system(cmd)
def do_restore(args):
#pdb.set_trace()
global backupDirPrefix
opts, args = getopt.getopt(args, "B:F:P:")
trg = args[0] + "/"
for opt in opts:
if opt[0] == '-F': catalogFiles = opt[1].split(',')
elif opt[0] == '-B': backupDirPrefix = opt[1]
cat = dict()
backupDirPrefix = '.'.join(catalogFiles[0].split('.')[:-2])
for fn in catalogFiles:
lastCatFile = fn == catalogFiles[-1]
ff = open(fn, "r").readlines()
(rootPath, ttt) = ff[0].split(' ') # the rootPath includes the trailing '/'
cloneInfo = ttt.split(':')
cloneId = cloneInfo[1]
for l in ff:
path, ttt = l.split(' ')
rpath = path[len(rootPath):] # skip the "root" part
stime, clonePath = ttt.split(":")[0:2]
if rpath in cat:
if stime < cat[rpath]['stime'] or clonePath.startswith('0'): # file exists but not part of this backup
if lastCatFile: cat[rpath]['keep'] = True
continue # restore latest version
else: cat[rpath] = dict()
cat[rpath]['stime'] = stime
cat[rpath]['cloneId'] = cloneId
if lastCatFile: cat[rpath]['keep'] = True
# restore files
for p in sorted(cat.keys()): # sorted so that dirs come before their files
if not 'keep' in cat[p]: continue
if p.endswith('/') or p == '':
os.makedirs(trg + p)
else:
cmd = "cp %s.%s/%s %s/%s" % (backupDirPrefix, cat[p]['cloneId'], p, trg, p)
os.system(cmd)
# main
try:
#pdb.set_trace()
if sys.argv[1] == 'clone':
do_clone(sys.argv[2:])
elif sys.argv[1] == 'backup':
do_backup(sys.argv[2:])
elif sys.argv[1] == 'restore':
do_restore(sys.argv[2:])
else:
raise NameError
sys.exit(0)
except (IndexError, NameError) as e:
print "incorrect argument", "" if type(e) == type(IndexError()) else sys.argv[1]
sys.exit(1)