GitLab unavailability on July 18, 22, 23 due to hypervisor security updates: http://cern.ch/go/BP7D

Commit a4c3d944 authored by Elvin Sindrilaru's avatar Elvin Sindrilaru

FST: Refactor the ChecksumPlugins interface to always return a unique_ptr to

the checksum object.

* Add helper methods to get a checksum object also from a string representation
of the checksum type.

* Always use the GetHexChecksum for retriving a consistent hex representation
of the binary checksum. Note the adler is interpreted as little-endian while
the rest are just a hex representation of the memory layout.

* When resetting the disk info in the local db set the file and block checksum
errors to false i.e. 0.
parent 09e44602
Pipeline #905049 canceled with stages
in 14 seconds
......@@ -797,6 +797,7 @@ FmdDbMapHandler::UpdateWithScanInfo(eos::common::FileSystem::fsid_t fsid,
if ((orphaned == false) &&
((fmd->mProtoFmd.layouterror() & eos::common::LayoutId::kUnregistered)
== false)) {
eos_info("msg=\"trigger auto repair\" fid=%08llx", fid);
CallAutoRepair(manager.c_str(), fid);
}
}
......@@ -825,8 +826,8 @@ FmdDbMapHandler::ResetDiskInformation(eos::common::FileSystem::fsid_t fsid)
f.set_disksize(Fmd::UNDEF);
f.set_diskchecksum("");
f.set_checktime(0);
f.set_filecxerror(-1);
f.set_blockcxerror(-1);
f.set_filecxerror(0);
f.set_blockcxerror(0);
val = *v;
f.SerializeToString(&val.value);
mDbMap[fsid]->set(*k, val);
......@@ -917,12 +918,11 @@ FmdDbMapHandler::ResyncDisk(const char* path,
struct stat buf;
if ((!io->fileStat(&buf)) && S_ISREG(buf.st_mode)) {
disk_size = buf.st_size;
std::string sxs_type, scheck_stamp, filexs_err, blockxs_err;
char xs_val[SHA_DIGEST_LENGTH];
size_t xs_len = 0;
disk_size = buf.st_size;
size_t xs_len = SHA_DIGEST_LENGTH;
memset(xs_val, 0, sizeof(xs_val));
xs_len = SHA_DIGEST_LENGTH;
io->attrGet("user.eos.checksumtype", sxs_type);
io->attrGet("user.eos.filecxerror", filexs_err);
io->attrGet("user.eos.blockcxerror", blockxs_err);
......@@ -937,7 +937,13 @@ FmdDbMapHandler::ResyncDisk(const char* path,
std::string disk_xs;
if (io->attrGet("user.eos.checksum", xs_val, xs_len) == 0) {
disk_xs = eos::common::StringConversion::BinData2HexString(xs_val, xs_len);
std::unique_ptr<CheckSum> xs_obj {ChecksumPlugins::GetXsObj(sxs_type)};
if (xs_obj) {
if (xs_obj->SetBinChecksum(xs_val, xs_len)) {
disk_xs = xs_obj->GetHexChecksum();
}
}
}
// Update the DB
......@@ -945,7 +951,7 @@ FmdDbMapHandler::ResyncDisk(const char* path,
(filexs_err == "1"), (blockxs_err == "1"),
flaglayouterror)) {
eos_err("msg=\"failed to update DB\" dbpath=%s fsid=%lu fid=%08llx",
eos::common::DbMap::getDbType().c_str(), (unsigned long) fsid, fid);
eos::common::DbMap::getDbType().c_str(), fsid, fid);
return false;
}
} else {
......@@ -1643,7 +1649,7 @@ FmdDbMapHandler::FileHasXsError(const std::string& lpath,
auto fid = eos::common::FileId::PathToFid(lpath.c_str());
auto fmd = LocalGetFmd(fid, fsid, true);
if (fmd && (fmd->mProtoFmd.filecxerror() != -1)) {
if (fmd && fmd->mProtoFmd.filecxerror()) {
has_xs_err = true;
}
......
......@@ -294,8 +294,8 @@ Fsck::CheckFile(const char* filepath)
errors["detached"]++;
} else {
if (checksumLen) {
std::unique_ptr<CheckSum> checksum(ChecksumPlugins::GetChecksumObject(
mMd[fid].lid(), false));
std::unique_ptr<CheckSum> checksum =
ChecksumPlugins::GetChecksumObject(mMd[fid].lid(), false);
checksum->SetBinChecksum(checksumVal, checksumLen);
std::string hex_checksum = checksum->GetHexChecksum();
mMd[fid].set_diskchecksum(hex_checksum);
......
......@@ -185,6 +185,8 @@ ScanDir::Run(ThreadAssistant& assistant) noexcept
}
}
// @todo(esindril): wait for all FSes to be booted before starting
if (mBgThread) {
// Get a random smearing and avoid that all start at the same time! 0-4 hours
size_t sleeper = (1.0 * mRerunIntervalSec * random() / RAND_MAX);
......@@ -328,8 +330,7 @@ ScanDir::CheckFile(const std::string& fpath)
bool filexs_err = false;
unsigned long long scan_size {0ull};
if (!ScanFileLoadAware(io, scan_size, lfn, filexs_err, blockxs_err)) {
eos_err("msg=\"failed scan file\" path=%s", fpath.c_str());
if (!ScanFileLoadAware(io, scan_size, filexs_err, blockxs_err)) {
return;
}
......@@ -422,7 +423,7 @@ ScanDir::GetBlockXS(const std::string& file_path)
auto layoutid = LayoutId::GetId(LayoutId::kPlain, LayoutId::kNone, 0,
bxs_size_type, bxs_type);
std::unique_ptr<eos::fst::CheckSum> checksum =
eos::fst::ChecksumPlugins::GetChecksumObjectPtr(layoutid, true);
eos::fst::ChecksumPlugins::GetChecksumObject(layoutid, true);
if (checksum) {
if (checksum->OpenMap(filexs_path.c_str(), info.st_size, bxs_size, false)) {
......@@ -492,19 +493,19 @@ ScanDir::DoRescan(const std::string& timestamp_sec) const
bool
ScanDir::ScanFileLoadAware(const std::unique_ptr<eos::fst::FileIo>& io,
unsigned long long& scan_size,
const std::string& lfn, bool& filexs_err,
bool& blockxs_err)
bool& filexs_err, bool& blockxs_err)
{
scan_size = 0ull;
filexs_err = blockxs_err = false;
int scan_rate = mRateBandwidth;
std::string file_path = io->GetPath();
struct stat info;
if (io->fileStat(&info)) {
eos_err("msg=\"failed stat\" path=%s\"", file_path.c_str());
return false;
}
std::string file_path = io->GetPath();
// Get checksum type and value
std::string xs_type;
char xs_val[SHA_DIGEST_LENGTH];
......@@ -512,19 +513,16 @@ ScanDir::ScanFileLoadAware(const std::unique_ptr<eos::fst::FileIo>& io,
size_t xs_len = SHA_DIGEST_LENGTH;
io->attrGet("user.eos.checksumtype", xs_type);
io->attrGet("user.eos.checksum", xs_val, xs_len);
auto lid = LayoutId::GetId(LayoutId::kPlain,
LayoutId::GetChecksumFromString(xs_type));
std::unique_ptr<eos::fst::CheckSum> normalXS
{eos::fst::ChecksumPlugins::GetChecksumObjectPtr(lid)};
auto comp_file_xs = eos::fst::ChecksumPlugins::GetXsObj(xs_type);
std::unique_ptr<eos::fst::CheckSum> blockXS {GetBlockXS(file_path)};
// If no checksum then there is nothing to check
if (!normalXS && !blockXS) {
if (!comp_file_xs && !blockXS) {
return false;
}
if (normalXS) {
normalXS->Reset();
if (comp_file_xs) {
comp_file_xs->Reset();
}
size_t nread = 0;
......@@ -540,6 +538,8 @@ ScanDir::ScanFileLoadAware(const std::unique_ptr<eos::fst::FileIo>& io,
blockXS->CloseMap();
}
eos_err("msg=\"failed read\" offset=%llu path=%s", offset,
file_path.c_str());
return false;
}
......@@ -550,8 +550,8 @@ ScanDir::ScanFileLoadAware(const std::unique_ptr<eos::fst::FileIo>& io,
}
}
if (normalXS) {
normalXS->Add(mBuffer, nread, offset);
if (comp_file_xs) {
comp_file_xs->Add(mBuffer, nread, offset);
}
offset += nread;
......@@ -561,27 +561,32 @@ ScanDir::ScanFileLoadAware(const std::unique_ptr<eos::fst::FileIo>& io,
scan_size = (unsigned long long) offset;
if (normalXS) {
normalXS->Finalize();
if (comp_file_xs) {
comp_file_xs->Finalize();
}
// Check file checksum only for replica layouts
if (normalXS && (!normalXS->Compare(xs_val))) {
LogMsg(LOG_ERR, "msg=\file checksum error\" expected_xs=%s computed_xs=%s "
"scan_size=%llu", xs_val, normalXS->GetHexChecksum(), scan_size);
++mNumCorruptedFiles;
filexs_err = true;
if (comp_file_xs) {
if (!comp_file_xs->Compare(xs_val)) {
auto exp_file_xs = eos::fst::ChecksumPlugins::GetXsObj(xs_type);
exp_file_xs->SetBinChecksum(xs_val, xs_len);
LogMsg(LOG_ERR, "msg=\"file checksum error\" expected_file_xs=%s "
"computed_file_xs=%s scan_size=%llu",
exp_file_xs->GetHexChecksum(), comp_file_xs->GetHexChecksum(),
scan_size);
++mNumCorruptedFiles;
filexs_err = true;
}
}
// Check block checksum
if (blockxs_err) {
LogMsg(LOG_ERR, "msg=\"corrupted block checksum\" path=%s "
"blockxs_path=%s.xsmap lfn=%s", file_path.c_str(), file_path.c_str(),
lfn.c_str());
LogMsg(LOG_ERR, "msg=\"corrupted block checksum\" path=%s ",
"blockxs_path=%s.xsmap", file_path.c_str(), file_path.c_str());
if (mBgThread) {
syslog(LOG_ERR, "corrupted block checksum path=%s blockxs_path=%s.xsmap "
"lfn=%s\n", file_path.c_str(), file_path.c_str(), lfn.c_str());
syslog(LOG_ERR, "corrupted block checksum path=%s blockxs_path=%s.xsmap\n",
file_path.c_str(), file_path.c_str());
}
}
......
......@@ -112,7 +112,6 @@ public:
//!
//! @param io io object attached to the file
//! @param scan_size final scan size
//! @param lfn logical file name (NS path)
//! @param filexs_err set to true if file has a checksum error
//! @param blockxs_err set to true if file has a block checksum errror
//!
......@@ -121,8 +120,7 @@ public:
//----------------------------------------------------------------------------
bool ScanFileLoadAware(const std::unique_ptr<eos::fst::FileIo>& io,
unsigned long long& scan_size,
const std::string& lfn, bool& filexs_err,
bool& blockxs_err);
bool& filexs_err, bool& blockxs_err);
//----------------------------------------------------------------------------
//! Get clock reference for testing purposes
......
......@@ -245,7 +245,6 @@ XrdFstOfs::xrdfstofs_coverage(int sig)
__gcov_flush();
return;
#endif
eos_static_notice("msg=\"compiled without coverage support\"");
}
......@@ -1712,7 +1711,7 @@ XrdFstOfs::GetKeytabChecksum(const std::string& kt_path) const
if (nread > 0) {
std::unique_ptr<CheckSum> KeyCKS =
ChecksumPlugins::GetChecksumObjectPtr(eos::common::LayoutId::kAdler);
ChecksumPlugins::GetChecksumObject(eos::common::LayoutId::kAdler);
if (KeyCKS) {
KeyCKS->Add(buffer, nread, 0);
......
......@@ -1182,8 +1182,6 @@ XrdFstOfsFile::close()
fMd->mProtoFmd.set_filecxerror(0);
fMd->mProtoFmd.set_blockcxerror(0);
fMd->mProtoFmd.set_locations(""); // reset locations
fMd->mProtoFmd.set_filecxerror(0);
fMd->mProtoFmd.set_blockcxerror(0);
fMd->mProtoFmd.set_mtime(statinfo.st_mtime);
#ifdef __APPLE__
fMd->mProtoFmd.set_mtime_ns(0);
......@@ -1273,13 +1271,13 @@ XrdFstOfsFile::close()
// The entry server commits size and checksum
capOpaqueFile += "&mgm.commit.size=1&mgm.commit.checksum=1";
} else {
if (mCheckSum ) {
// if we computed a checksum, we verify it
capOpaqueFile += "&mgm.replication=1&mgm.verify.checksum=1";
} else {
// if we didn't compute a checksum, we disable checksum verification
capOpaqueFile += "&mgm.replication=1&mgm.verify.checksum=0";
}
if (mCheckSum) {
// if we computed a checksum, we verify it
capOpaqueFile += "&mgm.replication=1&mgm.verify.checksum=1";
} else {
// if we didn't compute a checksum, we disable checksum verification
capOpaqueFile += "&mgm.replication=1&mgm.verify.checksum=0";
}
}
}
......@@ -2887,7 +2885,7 @@ XrdFstOfsFile::ProcessMixedOpaque()
// Call the checksum factory function with the selected layout
if (opaqueCheckSum != "ignore") {
mCheckSum = eos::fst::ChecksumPlugins::GetChecksumObjectPtr(mLid);
mCheckSum = eos::fst::ChecksumPlugins::GetChecksumObject(mLid);
eos_debug("checksum requested %d %u", mCheckSum.get(), mLid);
}
......
......@@ -83,7 +83,7 @@ XrdFstOssFile::Open(const char* path, int flags, mode_t mode, XrdOucEnv& env)
const char* val = 0;
unsigned long lid = 0;
off_t booking_size = 0;
eos_info("path=%s", path);
eos_debug("path=%s", path);
mPath = path;
if (fd >= 0) {
......@@ -115,7 +115,10 @@ XrdFstOssFile::Open(const char* path, int flags, mode_t mode, XrdOucEnv& env)
mBlockXs = pair_value.second;
if (!mBlockXs) {
mBlockXs = ChecksumPlugins::GetChecksumObject(lid, true);
auto xs_ptr = ChecksumPlugins::GetChecksumObject(lid, true);
mBlockXs = xs_ptr.get();
// Management of the xs object lifetime is handled by the OSS class
xs_ptr.release();
if (mBlockXs) {
XrdOucString xs_path = mBlockXs->MakeBlockXSPath(mPath.c_str());
......
......@@ -21,12 +21,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.*
************************************************************************/
#ifndef __EOSFST_CHECKSUMPLUGIN_HH__
#define __EOSFST_CHECKSUMPLUGIN_HH__
/*----------------------------------------------------------------------------*/
#include "common/LayoutId.hh"
#pragma once
#include "fst/Namespace.hh"
#include "common/LayoutId.hh"
#include "fst/checksum/CheckSum.hh"
#include "fst/checksum/Adler.hh"
#include "fst/checksum/CRC32.hh"
......@@ -34,79 +31,70 @@
#include "fst/checksum/MD5.hh"
#include "fst/checksum/SHA1.hh"
/*----------------------------------------------------------------------------*/
EOSFSTNAMESPACE_BEGIN
//------------------------------------------------------------------------------
//! Class ChecksumPluging
//------------------------------------------------------------------------------
class ChecksumPlugins
{
public:
ChecksumPlugins () { };
~ChecksumPlugins () { };
//----------------------------------------------------------------------------
//! Get checksum object depending on the given type
//!
//! @param xs_type checksum type given usigned long
//!
//! @return checksum object
//----------------------------------------------------------------------------
static CheckSum*
GetChecksumObject (unsigned int layoutid, bool blockchecksum = false)
GetXsObj(unsigned long xs_type)
{
if (blockchecksum)
{
if (eos::common::LayoutId::GetBlockChecksum(layoutid) == eos::common::LayoutId::kAdler)
{
return (CheckSum*)new Adler;
}
if (eos::common::LayoutId::GetBlockChecksum(layoutid) == eos::common::LayoutId::kCRC32)
{
return (CheckSum*)new CRC32;
}
if (eos::common::LayoutId::GetBlockChecksum(layoutid) == eos::common::LayoutId::kCRC32C)
{
return (CheckSum*)new CRC32C;
}
if (eos::common::LayoutId::GetBlockChecksum(layoutid) == eos::common::LayoutId::kMD5)
{
return (CheckSum*)new MD5;
}
if (eos::common::LayoutId::GetBlockChecksum(layoutid) == eos::common::LayoutId::kSHA1)
{
return (CheckSum*)new SHA1;
}
}
else
{
if (eos::common::LayoutId::GetChecksum(layoutid) == eos::common::LayoutId::kAdler)
{
return (CheckSum*)new Adler;
}
if (eos::common::LayoutId::GetChecksum(layoutid) == eos::common::LayoutId::kCRC32)
{
return (CheckSum*)new CRC32;
}
if (eos::common::LayoutId::GetChecksum(layoutid) == eos::common::LayoutId::kCRC32C)
{
return (CheckSum*)new CRC32C;
}
if (eos::common::LayoutId::GetChecksum(layoutid) == eos::common::LayoutId::kMD5)
{
return (CheckSum*)new MD5;
}
if (eos::common::LayoutId::GetChecksum(layoutid) == eos::common::LayoutId::kSHA1)
{
return (CheckSum*)new SHA1;
}
if (xs_type == eos::common::LayoutId::kAdler) {
return static_cast<CheckSum*>(new Adler());
} else if (xs_type == eos::common::LayoutId::kCRC32) {
return static_cast<CheckSum*>(new CRC32());
} else if (xs_type == eos::common::LayoutId::kCRC32C) {
return static_cast<CheckSum*>(new CRC32C());
} else if (xs_type == eos::common::LayoutId::kMD5) {
return static_cast<CheckSum*>(new MD5());
} else if (xs_type == eos::common::LayoutId::kSHA1) {
return static_cast<CheckSum*>(new SHA1());
}
return 0;
return nullptr;
}
//----------------------------------------------------------------------------
//! Get checksum object depending on the given type
//!
//! @param xs_type checksum type given as string
//!
//! @return checksum object
//----------------------------------------------------------------------------
static std::unique_ptr<CheckSum>
GetChecksumObjectPtr(unsigned int layoutid, bool blockchecksum = false)
GetXsObj(const std::string& xs_type)
{
return std::unique_ptr<CheckSum>(GetChecksumObject(layoutid, blockchecksum));
return std::unique_ptr<CheckSum>
(GetXsObj(eos::common::LayoutId::GetChecksumFromString(xs_type)));
}
//----------------------------------------------------------------------------
//! Get checksum object given the layoutid
//!
//! @param layoutid layout id endcoding see eos::common::LayoutId
//! @param blockchecksum if true then return the checksum object for the
//! block checksum part encoded in the layout id
//!
//! @return checksum object
//----------------------------------------------------------------------------
static std::unique_ptr<CheckSum>
GetChecksumObject(unsigned long layoutid, bool blockchecksum = false)
{
unsigned int xs_type = (blockchecksum ?
eos::common::LayoutId::GetBlockChecksum(layoutid) :
eos::common::LayoutId::GetChecksum(layoutid));
return std::unique_ptr<CheckSum>(GetXsObj(xs_type));
}
};
EOSFSTNAMESPACE_END
#endif
......@@ -158,7 +158,7 @@ Storage::Verify()
fMd->mProtoFmd.set_lid(verifyfile->lId);
fMd->mProtoFmd.set_cid(verifyfile->cId);
std::unique_ptr<CheckSum> checksummer =
ChecksumPlugins::GetChecksumObjectPtr(fMd->mProtoFmd.lid());
ChecksumPlugins::GetChecksumObject(fMd->mProtoFmd.lid());
unsigned long long scansize = 0;
float scantime = 0; // is ms
eos::fst::CheckSum::ReadCallBack::callback_data_t cbd;
......
......@@ -26,32 +26,31 @@
#include "fst/checksum/ChecksumPlugins.hh"
int
main (int argc, char* argv[])
main(int argc, char* argv[])
{
if (argc != 2)
{
if (argc != 2) {
fprintf(stderr, "error: you have to provide a path name\n");
exit(-1);
}
std::unique_ptr<eos::fst::CheckSum> normalXS =
eos::fst::ChecksumPlugins::GetChecksumObjectPtr(eos::common::LayoutId::kAdler);
if (normalXS)
{
eos::fst::ChecksumPlugins::GetChecksumObject(eos::common::LayoutId::kAdler);
if (normalXS) {
XrdOucString path = (argv[1]) ? argv[1] : "";
unsigned long long scansize;
float scantime;
if (!normalXS->ScanFile(path.c_str(), scansize, scantime))
{
fprintf(stderr,"error: unable to scan file path=%s\n", argv[1]);
if (!normalXS->ScanFile(path.c_str(), scansize, scantime)) {
fprintf(stderr, "error: unable to scan file path=%s\n", argv[1]);
exit(-1);
}
else
{
fprintf(stdout, "path=%s size=%llu time=%.02f adler32=%s\n", argv[1], scansize, scantime, normalXS->GetHexChecksum());
} else {
fprintf(stdout, "path=%s size=%llu time=%.02f adler32=%s\n", argv[1], scansize,
scantime, normalXS->GetHexChecksum());
exit(0);
}
}
fprintf(stderr, "error: failed to get checksum object\n");
exit(-1);
}
......@@ -80,7 +80,7 @@ main(int argc, const char* argv[])
eos::common::LayoutId::kNone, 0,
blockSizeSymbol, checksumType);
std::unique_ptr<eos::fst::CheckSum> checksum =
eos::fst::ChecksumPlugins::GetChecksumObjectPtr(layoutid, true);
eos::fst::ChecksumPlugins::GetChecksumObject(layoutid, true);
if (!checksum) {
fprintf(stderr, "error: failed to get checksum object for file %s",
......
......@@ -81,7 +81,7 @@ main(int argc, const char* argv[])
eos::common::LayoutId::kNone, 0,
blockSizeSymbol, checksumType);
std::unique_ptr<eos::fst::CheckSum> checksum =
eos::fst::ChecksumPlugins::GetChecksumObjectPtr(layoutid, true);
eos::fst::ChecksumPlugins::GetChecksumObject(layoutid, true);
if (!checksum) {
fprintf(stderr, "error: failed to get checksum object for file %s",
......
......@@ -617,7 +617,7 @@ main(int argc, char* argv[])
layoutId = LayoutId::GetId(layout, LayoutId::kCRC32C);
}
xsObj = eos::fst::ChecksumPlugins::GetChecksumObjectPtr(layoutId);
xsObj = eos::fst::ChecksumPlugins::GetChecksumObject(layoutId);
if (xsObj) {
xsObj->Reset();
......
......@@ -110,7 +110,7 @@ int main(int argc, char* argv[])
for (size_t i = 0; i < checksumnames.size(); i++) {
eos_static_info("benchmarking checksum algorithm %s", checksumnames[i].c_str());
std::unique_ptr<eos::fst::CheckSum> checksum =
eos::fst::ChecksumPlugins::GetChecksumObjectPtr(checksumids[i]);
eos::fst::ChecksumPlugins::GetChecksumObject(checksumids[i]);
if (!checksum) {
eos_static_err("failed to get checksum algorithm %s", checksumnames[i].c_str());
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment