GitLab unavailability on July 18, 22, 23 due to hypervisor security updates: http://cern.ch/go/BP7D

...
 
Commits (58)
......@@ -23,7 +23,7 @@
include_directories(
${CMAKE_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR}
${CMAKE_BINARY_DIR}
${Z_INCLUDE_DIRS}
${PROTOBUF_INCLUDE_DIRS}
${XROOTD_INCLUDE_DIRS}
......@@ -64,6 +64,7 @@ set_target_properties(EosCrc32c-Static PROPERTIES
# eosCommon-Objects
#-------------------------------------------------------------------------------
add_library(eosCommon-Objects OBJECT
Fmd.cc
SymKeys.cc
Mapping.cc
RWMutex.cc
......@@ -90,6 +91,12 @@ add_library(eosCommon-Objects OBJECT
set_target_properties(eosCommon-Objects PROPERTIES
POSITION_INDEPENDENT_CODE TRUE)
#-------------------------------------------------------------------------------
# Add dependency to protobuf objects so we guarantee that the protocol
# buffer files are generated when we try to build XrdEosFst
#-------------------------------------------------------------------------------
add_dependencies(eosCommon-Objects EosFstProto-Objects)
target_compile_definitions(eosCommon-Objects PUBLIC
-DDAEMONUID=${DAEMONUID} -DDAEMONGID=${DAEMONGID})
......
......@@ -23,20 +23,28 @@
#pragma once
#include "common/Namespace.hh"
EOSCOMMONNAMESPACE_BEGIN
static constexpr auto TAPE_FS_ID = 65535u;
static constexpr auto RETRIEVE_REQID_ATTR_NAME = "sys.retrieve.req_id"; //!< List of Prepare request IDs for this file
static constexpr auto RETRIEVE_REQTIME_ATTR_NAME = "sys.retrieve.req_time"; //!< Last time the Prepare request was actioned
static constexpr auto RETRIEVE_ERROR_ATTR_NAME = "sys.retrieve.error"; //!< Prepare request failure reason
static constexpr auto ARCHIVE_ERROR_ATTR_NAME = "sys.archive.error"; //!< Archive request failure reason
//! List of Prepare request IDs for this file
static constexpr auto RETRIEVE_REQID_ATTR_NAME = "sys.retrieve.req_id";
//! Last time the Prepare request was actioned
static constexpr auto RETRIEVE_REQTIME_ATTR_NAME = "sys.retrieve.req_time";
//!< Prepare request failure reason
static constexpr auto RETRIEVE_ERROR_ATTR_NAME = "sys.retrieve.error";
//! Archive request failure reason
static constexpr auto ARCHIVE_ERROR_ATTR_NAME = "sys.archive.error";
static constexpr auto RETRIEVE_WRITTEN_WORKFLOW_NAME = "retrieve_written";
static constexpr auto RETRIEVE_FAILED_WORKFLOW_NAME = "sync::retrieve_failed";
static constexpr auto ARCHIVE_FAILED_WORKFLOW_NAME = "sync::archive_failed";
static constexpr auto WF_CUSTOM_ATTRIBUTES_TO_FST_EQUALS = "=";
static constexpr auto WF_CUSTOM_ATTRIBUTES_TO_FST_SEPARATOR = ";;;";
//! Max rate in MB/s at which the scanner should run
static constexpr auto SCAN_RATE_NAME = "scanrate";
//! Time interval after which a scanned filed is rescanned
static constexpr auto SCAN_INTERVAL_NAME = "scaninterval";
//! Time interval after which the scanner will rerun
static constexpr auto SCAN_RERUNINTERVAL_NAME = "scanreruninterval";
EOSCOMMONNAMESPACE_END
......@@ -26,6 +26,7 @@
#include "XrdOuc/XrdOucString.hh"
#include <chrono>
#include <cmath>
#include <cstring>
EOSCOMMONNAMESPACE_BEGIN
......@@ -54,7 +55,7 @@ public:
//----------------------------------------------------------------------------
static unsigned long long Hex2Fid(const char* hexstring)
{
if (hexstring) {
if (hexstring && strlen(hexstring)) {
return strtoll(hexstring, 0, 16);
} else {
return 0;
......
......@@ -27,6 +27,7 @@
#include "common/TransferQueue.hh"
#include "common/StringUtils.hh"
#include "common/ParseUtils.hh"
#include "common/Constants.hh"
EOSCOMMONNAMESPACE_BEGIN;
......@@ -1049,8 +1050,9 @@ FileSystem::SnapShotFileSystem(FileSystem::fs_snapshot_t& fs, bool dolock)
fs.mDiskWopen = (long) hash->GetLongLong("stat.wopen");
fs.mWeightRead = 1.0;
fs.mWeightWrite = 1.0;
fs.mScanRate = (time_t) hash->GetLongLong("scanrate");
fs.mScanInterval = (time_t) hash->GetLongLong("scaninterval");
fs.mScanRate = hash->GetLongLong(eos::common::SCAN_RATE_NAME);
fs.mScanInterval = hash->GetLongLong(eos::common::SCAN_INTERVAL_NAME);
fs.mScanRerunInterval = hash->GetLongLong(eos::common::SCAN_RERUNINTERVAL_NAME);
fs.mGracePeriod = (time_t) hash->GetLongLong("graceperiod");
fs.mDrainPeriod = (time_t) hash->GetLongLong("drainperiod");
fs.mBalThresh = hash->GetDouble("stat.balance.threshold");
......@@ -1108,6 +1110,8 @@ FileSystem::SnapShotFileSystem(FileSystem::fs_snapshot_t& fs, bool dolock)
fs.mDiskRopen = 0;
fs.mDiskWopen = 0;
fs.mScanRate = 0;
fs.mScanInterval = 0;
fs.mScanRerunInterval = 0;
fs.mBalThresh = 0.0;
return false;
}
......
......@@ -557,7 +557,8 @@ public:
long mDiskRopen;
long mDiskWopen;
long mScanRate; ///< Maximum scan rate in MB/s
time_t mScanInterval;
long mScanInterval; ///< Time after which a scanned file is rescanned
long mScanRerunInterval; ///< Time after which the scanner runs again
time_t mGracePeriod;
time_t mDrainPeriod;
......
......@@ -16,19 +16,77 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.*
************************************************************************/
#include "fst/Fmd.hh"
#include "common/Fmd.hh"
#include "common/StringConversion.hh"
#include "common/LayoutId.hh"
EOSFSTNAMESPACE_BEGIN
EOSCOMMONNAMESPACE_BEGIN
//------------------------------------------------------------------------------
// Convert an FST env representation to an Fmd struct
//------------------------------------------------------------------------------
bool EnvToFstFmd(XrdOucEnv& env, FmdHelper& fmd)
{
// Check that all tags are present
std::set<std::string> tags {"id", "cid", "fsid", "ctime", "ctime_ns", "mtime",
"mtime_ns", "atime", "atime_ns", "size", "disksize", "mgmsize", "lid",
"uid", "gid", "filecxerror", "blockcxerror", "layouterror", "locations"};
for (const auto& tag : tags) {
if (env.Get(tag.c_str()) == nullptr) {
int envlen = 0;
eos_static_crit("msg=\"missing fields in fmd encoding\" field=%s "
"encoding=\"%s\"", tag.c_str(), env.Env(envlen));
return false;
}
}
fmd.mProtoFmd.set_fid(strtoull(env.Get("id"), 0, 10));
fmd.mProtoFmd.set_cid(strtoull(env.Get("cid"), 0, 10));
fmd.mProtoFmd.set_fsid(strtoull(env.Get("fsid"), 0, 10));
fmd.mProtoFmd.set_ctime(strtoul(env.Get("ctime"), 0, 10));
fmd.mProtoFmd.set_ctime_ns(strtoul(env.Get("ctime_ns"), 0, 10));
fmd.mProtoFmd.set_mtime(strtoul(env.Get("mtime"), 0, 10));
fmd.mProtoFmd.set_mtime_ns(strtoul(env.Get("mtime_ns"), 0, 10));
fmd.mProtoFmd.set_size(strtoull(env.Get("size"), 0, 10));
fmd.mProtoFmd.set_disksize(strtoull(env.Get("disksize"), 0, 10));
fmd.mProtoFmd.set_lid(strtoul(env.Get("lid"), 0, 10));
fmd.mProtoFmd.set_uid((uid_t) strtoul(env.Get("uid"), 0, 10));
fmd.mProtoFmd.set_gid((gid_t) strtoul(env.Get("gid"), 0, 10));
fmd.mProtoFmd.set_checksum(env.Get("checksum"));
if (fmd.mProtoFmd.checksum() == "none") {
fmd.mProtoFmd.set_checksum("");
}
fmd.mProtoFmd.set_diskchecksum(env.Get("diskchecksum"));
if (fmd.mProtoFmd.diskchecksum() == "none") {
fmd.mProtoFmd.set_diskchecksum("");
}
fmd.mProtoFmd.set_mgmchecksum(env.Get("mgmchecksum"));
if (fmd.mProtoFmd.mgmchecksum() == "none") {
fmd.mProtoFmd.set_mgmchecksum("");
}
fmd.mProtoFmd.set_locations(env.Get("locations"));
if (fmd.mProtoFmd.locations() == "none") {
fmd.mProtoFmd.set_locations("");
}
return true;
}
//------------------------------------------------------------------------------
// Compute layout error
//------------------------------------------------------------------------------
int
FmdHelper::LayoutError(const Fmd& fmd, eos::common::FileSystem::fsid_t fsid)
FmdHelper::LayoutError(eos::common::FileSystem::fsid_t fsid)
{
uint32_t lid = fmd.lid();
uint32_t lid = mProtoFmd.lid();
if (lid == 0) {
// An orphan has no lid at the MGM e.g. lid=0
......@@ -36,7 +94,7 @@ FmdHelper::LayoutError(const Fmd& fmd, eos::common::FileSystem::fsid_t fsid)
}
size_t valid_replicas = 0;
auto location_set = GetLocations(fmd, valid_replicas);
auto location_set = GetLocations(valid_replicas);
size_t nstripes = eos::common::LayoutId::GetStripeNumber(lid) + 1;
int lerror = 0;
......@@ -55,41 +113,42 @@ FmdHelper::LayoutError(const Fmd& fmd, eos::common::FileSystem::fsid_t fsid)
// Reset file meta data object
//---------------------------------------------------------------------------
void
FmdHelper::Reset(Fmd& fmd)
FmdHelper::Reset()
{
fmd.set_fid(0);
fmd.set_cid(0);
fmd.set_ctime(0);
fmd.set_ctime_ns(0);
fmd.set_mtime(0);
fmd.set_mtime_ns(0);
fmd.set_atime(0);
fmd.set_atime_ns(0);
fmd.set_checktime(0);
fmd.set_size(0xfffffff1ULL);
fmd.set_disksize(0xfffffff1ULL);
fmd.set_mgmsize(0xfffffff1ULL);
fmd.set_checksum("");
fmd.set_diskchecksum("");
fmd.set_mgmchecksum("");
fmd.set_lid(0);
fmd.set_uid(0);
fmd.set_gid(0);
fmd.set_filecxerror(0);
fmd.set_blockcxerror(0);
fmd.set_layouterror(0);
fmd.set_locations("");
mProtoFmd.set_fid(0);
mProtoFmd.set_cid(0);
mProtoFmd.set_ctime(0);
mProtoFmd.set_ctime_ns(0);
mProtoFmd.set_mtime(0);
mProtoFmd.set_mtime_ns(0);
mProtoFmd.set_atime(0);
mProtoFmd.set_atime_ns(0);
mProtoFmd.set_checktime(0);
mProtoFmd.set_size(UNDEF);
mProtoFmd.set_disksize(UNDEF);
mProtoFmd.set_mgmsize(UNDEF);
mProtoFmd.set_checksum("");
mProtoFmd.set_diskchecksum("");
mProtoFmd.set_mgmchecksum("");
mProtoFmd.set_lid(0);
mProtoFmd.set_uid(0);
mProtoFmd.set_gid(0);
mProtoFmd.set_filecxerror(0);
mProtoFmd.set_blockcxerror(0);
mProtoFmd.set_layouterror(0);
mProtoFmd.set_locations("");
}
//---------------------------------------------------------------------------
// Get the set of all file system id locations of the current file
//---------------------------------------------------------------------------
std::set<eos::common::FileSystem::fsid_t>
FmdHelper::GetLocations(const Fmd& fmd, size_t& valid_replicas)
FmdHelper::GetLocations(size_t& valid_replicas)
{
valid_replicas = 0;
std::vector<std::string> location_vector;
eos::common::StringConversion::Tokenize(fmd.locations(), location_vector, ",");
eos::common::StringConversion::Tokenize(mProtoFmd.locations(), location_vector,
",");
std::set<eos::common::FileSystem::fsid_t> location_set;
for (size_t i = 0; i < location_vector.size(); i++) {
......@@ -113,55 +172,55 @@ FmdHelper::GetLocations(const Fmd& fmd, size_t& valid_replicas)
std::unique_ptr<XrdOucEnv>
FmdHelper::FmdToEnv()
{
std::ostringstream serializedStream;
serializedStream << "id=" << mProtoFmd.fid()
<< "&cid=" << mProtoFmd.cid()
<< "&ctime=" << mProtoFmd.ctime()
<< "&ctime_ns=" << mProtoFmd.ctime_ns()
<< "&mtime=" << mProtoFmd.mtime()
<< "&mtime_ns=" << mProtoFmd.mtime_ns()
<< "&size=" << mProtoFmd.size()
<< "&checksum=" << mProtoFmd.checksum()
<< "&diskchecksum=" << mProtoFmd.diskchecksum()
<< "&lid=" << mProtoFmd.lid()
<< "&uid=" << mProtoFmd.uid()
<< "&gid=" << mProtoFmd.gid() << '&';
return std::unique_ptr<XrdOucEnv>(new XrdOucEnv(
serializedStream.str().c_str()));
}
std::ostringstream oss;
oss << "id=" << mProtoFmd.fid()
<< "&cid=" << mProtoFmd.cid()
<< "&fsid=" << mProtoFmd.fsid()
<< "&ctime=" << mProtoFmd.ctime()
<< "&ctime_ns=" << mProtoFmd.ctime_ns()
<< "&mtime=" << mProtoFmd.mtime()
<< "&mtime_ns=" << mProtoFmd.mtime_ns()
<< "&atime=" << mProtoFmd.atime()
<< "&atime_ns=" << mProtoFmd.atime_ns()
<< "&size=" << mProtoFmd.size()
<< "&disksize=" << mProtoFmd.disksize()
<< "&mgmsize=" << mProtoFmd.mgmsize()
<< "&lid=0x" << std::hex << mProtoFmd.lid() << std::dec
<< "&uid=" << mProtoFmd.uid()
<< "&gid=" << mProtoFmd.gid()
<< "&filecxerror=0x" << std::hex << mProtoFmd.filecxerror()
<< "&blockcxerror=0x" << mProtoFmd.blockcxerror()
<< "&layouterror=0x" << mProtoFmd.layouterror();
//-------------------------------------------------------------------------------
// Convert fmd object to env representation
//-------------------------------------------------------------------------------
std::unique_ptr<XrdOucEnv>
FmdHelper::FullFmdToEnv()
{
std::ostringstream serializedStream;
serializedStream << "id=" << mProtoFmd.fid()
<< "&cid=" << mProtoFmd.cid()
<< "&fsid=" << mProtoFmd.fsid()
<< "&ctime=" << mProtoFmd.ctime()
<< "&ctime_ns=" << mProtoFmd.ctime_ns()
<< "&mtime=" << mProtoFmd.mtime()
<< "&mtime_ns=" << mProtoFmd.mtime_ns()
<< "&atime=" << mProtoFmd.atime()
<< "&atime_ns=" << mProtoFmd.atime_ns()
<< "&size=" << mProtoFmd.size()
<< "&disksize=" << mProtoFmd.disksize()
<< "&mgmsize=" << mProtoFmd.mgmsize()
<< "&checksum=" << mProtoFmd.checksum()
<< "&diskchecksum=" << mProtoFmd.diskchecksum()
<< "&mgmchecksum=" << mProtoFmd.mgmchecksum()
<< "&lid=0x" << std::hex << mProtoFmd.lid() << std::dec
<< "&uid=" << mProtoFmd.uid()
<< "&gid=" << mProtoFmd.gid()
<< "&filecxerror=0x" << std::hex << mProtoFmd.filecxerror()
<< "&blockcxerror=0x" << mProtoFmd.blockcxerror()
<< "&layouterror=0x" << mProtoFmd.layouterror()
<< "&locations=" << std::dec << mProtoFmd.locations()
<< '&';
return std::unique_ptr<XrdOucEnv>(new XrdOucEnv(
serializedStream.str().c_str()));
// Take care at string fields since XrdOucEnv does not deal well with empty
// values
if (mProtoFmd.checksum().empty()) {
oss << "&checksum=none";
} else {
oss << "&checksum=" << mProtoFmd.checksum();
}
if (mProtoFmd.diskchecksum().empty()) {
oss << "&diskchecksum=none";
} else {
oss << "&diskchecksum=" << mProtoFmd.diskchecksum();
}
if (mProtoFmd.mgmchecksum().empty()) {
oss << "&mgmchecksum=none";
} else {
oss << "&mgmchecksum=" << mProtoFmd.mgmchecksum();
}
if (mProtoFmd.locations().empty()) {
oss << "&locations=none";
} else {
oss << "&locations=" << std::dec << mProtoFmd.locations();
}
oss << '&';
return std::unique_ptr<XrdOucEnv>
(new XrdOucEnv(oss.str().c_str()));
}
EOSFSTNAMESPACE_END
EOSCOMMONNAMESPACE_END
/************************************************************************
* EOS - the CERN Disk Storage System *
* Copyright (C) 2017 CERN/Switzerland *
......@@ -17,72 +18,66 @@
************************************************************************/
#pragma once
#include "fst/Namespace.hh"
#include "common/Namespace.hh"
#include "proto/FmdBase.pb.h"
#include "common/FileSystem.hh"
#include "common/Logging.hh"
#include "common/FileId.hh"
EOSFSTNAMESPACE_BEGIN
//------------------------------------------------------------------------------
//! Structure holding file metadata
//------------------------------------------------------------------------------
struct Fmd : public FmdBase {
public:
virtual ~Fmd() {}
};
EOSCOMMONNAMESPACE_BEGIN
//------------------------------------------------------------------------------
//! Class implementing file meta data
//! Class modelling the file metadata stored on the FST. It wrapps an FmdBase
//! class generated from the ProtoBuffer specification and adds some extra
//! functionality for conversion to/from string/env representation.
//------------------------------------------------------------------------------
class FmdHelper : public eos::common::LogId
{
public:
static constexpr uint64_t UNDEF = 0xfffffffffff1ULL;
//---------------------------------------------------------------------------
//! Constructor
//---------------------------------------------------------------------------
FmdHelper(eos::common::FileId::fileid_t fid = 0, int fsid = 0)
{
Reset();
mProtoFmd.set_fid(fid);
mProtoFmd.set_fsid(fsid);
}
//---------------------------------------------------------------------------
//! Destructor
//---------------------------------------------------------------------------
virtual ~FmdHelper() = default;
//---------------------------------------------------------------------------
//! Compute layout error
//!
//! @param fmd protobuf file meta data
//! @param fsid file system id to check against
//!
//! @return 0 if there are no errors, otherwise encoded type of layout error
//! stored in the int.
//---------------------------------------------------------------------------
static int LayoutError(const Fmd& fmd, eos::common::FileSystem::fsid_t fsid);
int LayoutError(eos::common::FileSystem::fsid_t fsid);
//---------------------------------------------------------------------------
//! Reset file meta data object
//!
//! @param fmd protobuf file meta data
//---------------------------------------------------------------------------
static void Reset(Fmd& fmd);
void Reset();
//---------------------------------------------------------------------------
//! Get set of locations for the given fmd
//!
//! @param fmd file metadata object
//! @param valid_replicas number of valid replicas <= size of the returned
//! set i.e. replicas which are not unlinked
//!
//! @return set of file system ids representing the locations
//---------------------------------------------------------------------------
static std::set<eos::common::FileSystem::fsid_t>
GetLocations(const Fmd& fmd, size_t& valid_replicas);
//---------------------------------------------------------------------------
//! Constructor
//---------------------------------------------------------------------------
FmdHelper(eos::common::FileId::fileid_t fid = 0, int fsid = 0): LogId()
{
Reset(mProtoFmd);
mProtoFmd.set_fid(fid);
mProtoFmd.set_fsid(fsid);
}
//---------------------------------------------------------------------------
//! Destructor
//---------------------------------------------------------------------------
virtual ~FmdHelper() = default;
std::set<eos::common::FileSystem::fsid_t>
GetLocations(size_t& valid_replicas);
//---------------------------------------------------------------------------
//! Convert fmd object to env representation
......@@ -91,23 +86,26 @@ public:
//---------------------------------------------------------------------------
std::unique_ptr<XrdOucEnv> FmdToEnv();
//---------------------------------------------------------------------------
//! Convert fmd object to env representation
//!
//! @return XrdOucEnv holding information about current object
//---------------------------------------------------------------------------
std::unique_ptr<XrdOucEnv> FullFmdToEnv();
//---------------------------------------------------------------------------
//! File meta data object replication function (copy constructor)
//---------------------------------------------------------------------------
void
Replicate(Fmd& fmd)
Replicate(FmdHelper& fmd)
{
mProtoFmd = fmd;
mProtoFmd = fmd.mProtoFmd;
}
Fmd mProtoFmd; ///< Protobuf file metadata info
eos::fst::FmdBase mProtoFmd; ///< Protobuf file metadata info
};
EOSFSTNAMESPACE_END
//------------------------------------------------------------------------------
//! Convert an FST env representation to an Fmd struct
//!
//! @param env env representation
//! @param fmd reference to Fmd struct
//!
//! @return true if successful, otherwise false
//------------------------------------------------------------------------------
bool EnvToFstFmd(XrdOucEnv& env, FmdHelper& fmd);
EOSCOMMONNAMESPACE_END
......@@ -151,19 +151,23 @@ public:
//--------------------------------------------------------------------------
//! Get a reed solomon layout by number of redundancystripess
//--------------------------------------------------------------------------
static int
GetReedSLayoutByParity(int redundancystripes)
static int
GetReedSLayoutByParity(int redundancystripes)
{
switch (redundancystripes) {
case 1:
return kRaid5;
case 2:
return kRaid6;
case 3:
return kArchive;
case 4:
return kQrain;
default:
default:
return 0;
}
}
......@@ -191,7 +195,7 @@ public:
// already set explicitly
if (redundancystripes == 0) {
if (layout == kRaid5) {
redundancystripes = 1;
redundancystripes = 1;
} else if (layout == kRaidDP) {
redundancystripes = 2;
} else if (layout == kRaid6) {
......@@ -199,8 +203,8 @@ public:
} else if (layout == kArchive) {
redundancystripes = 3;
} else if (layout == kQrain) {
redundancystripes = 4;
}
redundancystripes = 4;
}
}
id |= ((redundancystripes & 0x7) << 28);
......@@ -213,24 +217,31 @@ public:
static unsigned long
BlockSize(int blocksize)
{
switch (blocksize) {
case k4k:
return (4 * 1024);
case k64k:
return (64 * 1024);
case k128k:
return (128 * 1024);
case k512k:
return (512 * 1024);
case k1M:
return (1024 * 1024);
case k4M:
return (4 * 1024 * 1024);
case k16M:
return (16 * 1024 * 1024);
case k64M:
return (64 * 1024 * 1024);
default:
return 0;
}
......@@ -245,20 +256,28 @@ public:
switch (blocksize) {
case (4*1024):
return k4k;
case (64*1024):
return k64k;
case (128*1024):
return k128k;
case (512*1024):
return k512k;
case (1024 * 1024):
return k1M;
case (4 * 1024 * 1024):
return k4M;
case (16 * 1024 * 1024):
return k16M;
case (64 * 1024 * 1024):
return k64M;
default:
return 0;
}
......@@ -274,7 +293,7 @@ public:
}
//--------------------------------------------------------------------------
//! Get Length of Layout checksum in bytes
//! Get length of Layout checksum in bytes
//--------------------------------------------------------------------------
static unsigned long
GetChecksumLen(unsigned long layout)
......@@ -302,6 +321,27 @@ public:
return 0;
}
//--------------------------------------------------------------------------
//! Get length of Layout checksum in bytes
//--------------------------------------------------------------------------
static unsigned long
GetChecksumLen(const std::string& xs_type)
{
if (xs_type == "adler") {
return 4;
} else if (xs_type == "crc32") {
return 4;
} else if (xs_type == "crc32c") {
return 4;
} else if (xs_type == "md5") {
return 16;
} else if (xs_type == "sha") {
return 20;
} else {
return 0;
}
}
static std::string
GetEmptyFileChecksum(unsigned long layout)
{
......@@ -358,13 +398,12 @@ public:
//! Test for RAIN layout e.g. raid6,archive,qrain
//--------------------------------------------------------------------------
static bool
IsRainLayout(unsigned long layout)
IsRain(unsigned long layout)
{
// everything but plain and replica
return (GetLayoutType(layout)>kReplica);
return (GetLayoutType(layout) > kReplica);
}
//--------------------------------------------------------------------------
//! Set file layout type in the layout encoding
//!
......@@ -565,7 +604,7 @@ public:
static unsigned long
GetOnlineStripeNumber(unsigned long layout)
{
return (GetStripeNumber(layout)+1);
return (GetStripeNumber(layout) + 1);
}
//--------------------------------------------------------------------------
......@@ -772,13 +811,13 @@ public:
static std::string
GetStripeNumberString(unsigned long layout)
{
int n = GetStripeNumber(layout) + 1;
if (n<256)
if (n < 256) {
return std::to_string(n);
else
} else {
return "none";
}
}
//--------------------------------------------------------------------------
......@@ -936,7 +975,7 @@ public:
}
if (typ == "qrain") {
return kQrain;
return kQrain;
}
}
......
......@@ -102,7 +102,7 @@ EOSCOMMONNAMESPACE_BEGIN
//------------------------------------------------------------------------------
#define eos_log(__EOSCOMMON_LOG_PRIORITY__ , ...) \
eos::common::Logging::GetInstance().log(__FUNCTION__,__FILE__, __LINE__, this->logId, \
vid, this->cident, LOG_MASK(__EOSCOMMON_LOG_PRIORITY__), __VA_ARGS__)
vid, this->cident, __EOSCOMMON_LOG_PRIORITY__, __VA_ARGS__)
#define eos_debug(...) \
eos::common::Logging::GetInstance().log(__FUNCTION__,__FILE__, __LINE__, this->logId, \
vid, this->cident, (LOG_DEBUG), __VA_ARGS__)
......
......@@ -195,12 +195,76 @@ StringConversion::char_to_hex(const char input)
return output;
}
//------------------------------------------------------------------------------
// Convert binary string given as a char* and length to hex string representation
//------------------------------------------------------------------------------
std::string
StringConversion::BinData2HexString(const char* buf, const size_t buf_len,
const size_t nominal_len,
const char separator)
{
std::string out;
if (buf_len == 0) {
return out;
}
char hb[4];
for (size_t i = 0; i < nominal_len; ++i) {
unsigned char target = 0x00;
if (i < buf_len) {
target = buf[i];
}
if ((separator != 0x00) && (i != (nominal_len - 1))) {
sprintf(hb, "%02x%c", target, separator);
} else {
sprintf(hb, "%02x", target);
}
out += hb;
}
return out;
}
//------------------------------------------------------------------------------
// Convert checksum hex representation to binary string
//------------------------------------------------------------------------------
std::unique_ptr<char>
StringConversion::Hex2BinDataChar(const std::string& shex, size_t& out_size)
{
out_size = 0;
std::unique_ptr<char> buf {new char[SHA_DIGEST_LENGTH]};
if ((buf == nullptr) || shex.empty()) {
return nullptr;
}
memset(buf.get(), 0, SHA_DIGEST_LENGTH);
char hex[3];
for (size_t i = 0;
((i < shex.length() - 1) && (i / 2 < SHA_DIGEST_LENGTH)); i += 2) {
hex[0] = shex.at(i);
hex[1] = shex.at(i + 1);
hex[2] = '\0';
buf.get()[i / 2] = std::stol(hex, 0, 16);
++out_size;
}
return buf;
}
//----------------------------------------------------------------------------
//! Get size from the given string, return true if parsing was successful,
//! false otherwise
//----------------------------------------------------------------------------
bool
StringConversion::GetSizeFromString(const std::string& sizestring, uint64_t &out)
StringConversion::GetSizeFromString(const std::string& sizestring,
uint64_t& out)
{
out = GetSizeFromString(sizestring.c_str());
return (errno == 0);
......@@ -638,11 +702,12 @@ StringConversion::LoadFileIntoString(const char* filename, std::string& out)
return out.c_str();
}
// ---------------------------------------------------------------------------
// Save a string into a text file <name>
// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------
// Save a string into a text file <name>
// ---------------------------------------------------------------------------
bool
StringConversion::SaveStringIntoFile(const char* filename, const std::string& in)
StringConversion::SaveStringIntoFile(const char* filename,
const std::string& in)
{
std::ofstream save(filename);
save.write(in.c_str(), in.size());
......
......@@ -31,18 +31,21 @@
#include "common/Namespace.hh"
#include "XrdOuc/XrdOucString.hh"
#include "XrdOuc/XrdOucEnv.hh"
#include "fmt/format.h"
#include <string>
#include <vector>
#include <algorithm>
#include <map>
#include <set>
#include <memory>
#include <stdio.h>
#include <limits.h>
#include <errno.h>
#include <string.h>
#include <fstream>
#include <sstream>
#include <openssl/sha.h>
typedef void CURL;
......@@ -106,6 +109,36 @@ public:
static std::string string_to_hex(const std::string& input);
static std::string char_to_hex(const char input);
//----------------------------------------------------------------------------
//! Convert binary string given as a char* and length to hex string
//! representation
//!
//! @param buf buffer holding the checksum in binary format
//! @param buf_len size of the checksum buffer
//! @param nominal_len expected size of the checksum in bytes
//! @param separator possible separator for the display of the converted
//! string
//! @param return string holding the hex representation of the checksum or
//! empty strin if nothing is converted
//----------------------------------------------------------------------------
static std::string
BinData2HexString(const char* buf, const size_t buf_len,
const size_t nominal_len = SHA_DIGEST_LENGTH,
const char separator = 0x00);
//----------------------------------------------------------------------------
//! Convert checksum hex representation to binary string
//!
//! @param shex string hex representation of checksum
//! @param out_size buffer size of the binary data
//!
//! @return array of chars holding the binary data representaion of the
//! checksum or nullptr if there were any errors
//----------------------------------------------------------------------------
static std::unique_ptr<char>
Hex2BinDataChar(const std::string& shex, size_t& out_size);
// ---------------------------------------------------------------------------
/**
* Convert a long long value into time s,m,h,d scale
......@@ -180,7 +213,7 @@ public:
//! Get size from the given string, return true if parsing was successful,
//! false otherwise
//----------------------------------------------------------------------------
static bool GetSizeFromString(const std::string& sizestring, uint64_t &out);
static bool GetSizeFromString(const std::string& sizestring, uint64_t& out);
// ---------------------------------------------------------------------------
/**
......@@ -409,7 +442,7 @@ public:
// ---------------------------------------------------------------------------
/**
* Save a string into a text file <name>
* Save a string into a text file <name>
*
* @param filename where to save the contents
* @param in string with the contents
......
......@@ -295,6 +295,14 @@ public:
return mPoolSize;
}
//----------------------------------------------------------------------------
//! Get size of the queue of jobs
//----------------------------------------------------------------------------
size_t GetQueueSize() const
{
return mTasks.size();
}
// Disable copy/move constructors and assignment operators
ThreadPool(const ThreadPool&) = delete;
ThreadPool(ThreadPool&&) = delete;
......
......@@ -30,7 +30,7 @@
#include "common/DbMap.hh"
#include "google/protobuf/text_format.h"
#include "google/protobuf/io/zero_copy_stream_impl.h"
#include "test.pb.h"
#include "common/test.pb.h"
#include <iostream>
#include <fstream>
#include <string>
......
......@@ -48,6 +48,7 @@ add_library(
commands/helpers/FsHelper.cc commands/helpers/FsHelper.hh
commands/helpers/AclHelper.cc commands/helpers/AclHelper.hh
commands/helpers/RecycleHelper.cc commands/helpers/RecycleHelper.hh
commands/helpers/FsckHelper.cc commands/helpers/FsckHelper.hh
commands/HealthCommand.cc commands/HealthCommand.hh
commands/com_access.cc
commands/com_accounting.cc
......@@ -66,7 +67,6 @@ add_library(
commands/com_file.cc
commands/com_find.cc
commands/com_fs.cc
commands/com_fsck.cc
commands/com_fuse.cc
commands/com_fusex.cc
commands/com_geosched.cc
......@@ -122,6 +122,7 @@ add_library(
#@todo (esindril) drop com_recycle when we drop beryl_aquamarine
commands/com_recycle.cc
commands/com_proto_recycle.cc
commands/com_proto_fsck.cc
commands/com_stagerrm.cc)
add_dependencies(EosConsoleCommands-Objects EosCliProto-Objects)
......@@ -139,7 +140,6 @@ set_target_properties(
add_executable(
eos
ConsoleMainExecutable.cc
${CMAKE_SOURCE_DIR}/fst/Fmd.cc
# @TODO (esindril) Add the table formatter in common
${CMAKE_SOURCE_DIR}/mgm/TableFormatter/TableFormatterBase.cc
$<TARGET_OBJECTS:EosCliProto-Objects>
......
......@@ -73,7 +73,7 @@ extern int com_find(char*);
extern int com_find_new(char*);
//extern int com_fs(char*);
extern int com_protofs(char*);
extern int com_fsck(char*);
extern int com_proto_fsck(char*);
extern int com_fuse(char*);
extern int com_fusex(char*);
extern int com_geosched(char*);
......@@ -148,7 +148,7 @@ COMMAND commands[] = {
{ (char*) "find", com_find, (char*) "Find files/directories"},
{ (char*) "newfind", com_find_new, (char*) "Find files/directories (new implementation)"},
{ (char*) "fs", com_protofs, (char*) "File System configuration"},
{ (char*) "fsck", com_fsck, (char*) "File System Consistency Checking"},
{ (char*) "fsck", com_proto_fsck, (char*) "File System Consistency Checking"},
{ (char*) "fuse", com_fuse, (char*) "Fuse Mounting"},
{ (char*) "fusex", com_fusex, (char*) "Fuse(x) Administration"},
{ (char*) "geosched", com_geosched, (char*) "Geoscheduler Interface"},
......
......@@ -100,7 +100,10 @@ public:
//----------------------------------------------------------------------------
std::string GetError();
inline bool NeedsConfirmation()
//----------------------------------------------------------------------------
//! Check if commands needs confirmation from the client
//----------------------------------------------------------------------------
inline bool NeedsConfirmation() const
{
return mNeedsConfirmation;
}
......
This diff is collapsed.
// ----------------------------------------------------------------------
// FiBle: com_fsck.cc
// Author: Andreas-Joachim Peters - CERN
// ----------------------------------------------------------------------
/************************************************************************
* EOS - the CERN Disk Storage System *
* Copyright (C) 2011 CERN/Switzerland *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>.*
************************************************************************/
/*----------------------------------------------------------------------------*/
#include "console/ConsoleMain.hh"
#include "common/StringTokenizer.hh"
#include "common/StringConversion.hh"
#include <map>
#include <iostream>
/*----------------------------------------------------------------------------*/
extern int com_protofs(char* arg);
/* Namespace Interface */
int
com_fsck(char* arg1)
{
eos::common::StringTokenizer subtokenizer(arg1);
subtokenizer.GetLine();
XrdOucString cmd = subtokenizer.GetToken();
XrdOucString option;
XrdOucString options = "";
XrdOucString path = "";
XrdOucString in = "";
XrdOucString selection = "";
if (wants_help(arg1)) {
goto com_fsck_usage;
}
if ((cmd != "stat") && (cmd != "enable") && (cmd != "disable") &&
(cmd != "report") && (cmd != "repair") && (cmd != "search")) {
goto com_fsck_usage;
}
in = "mgm.cmd=fsck&";
if (cmd == "enable") {
XrdOucString interval = subtokenizer.GetToken();
if (interval.length() && ((atoi(interval.c_str())) <= 0)) {
goto com_fsck_usage;
}
in += "mgm.subcmd=enable";
if (interval.length()) {
in += "&mgm.fsck.interval=";
in += interval.c_str();
}
}
if (cmd == "disable") {
in += "mgm.subcmd=disable";
}
if (cmd == "stat") {
in += "mgm.subcmd=stat";
}
if (cmd == "report") {
in += "mgm.subcmd=report";
do {
option = subtokenizer.GetToken();
if (option.length()) {
if (option == "--error") {
selection = subtokenizer.GetToken();
if (!selection.length()) {
goto com_fsck_usage;
}
continue;
}
while (option.replace("-", "")) {
}
options += option;
}
} while (option.length());
}
if (cmd == "search") {
size_t nrep=0;
path = subtokenizer.GetToken();
XrdOucString srep = subtokenizer.GetToken();
if (srep.length()) {
nrep = std::stoi(srep.c_str());
}
filesystems fs;
fs.Load();
fs.Connect();
files f;
f.Find(path.c_str());
fprintf(stdout, "# found %lu files\n", f.Size());
f.Lookup(fs);
f.Report(nrep);
return 0;
}
if (cmd == "repair") {
in += "mgm.subcmd=repair";
option = subtokenizer.GetToken();
if ((!option.length()) ||
((option != "--checksum") &&
(option != "--checksum-commit") &&
(option != "--resync") &&
(option != "--unlink-unregistered") &&
(option != "--unlink-orphans") &&
(option != "--adjust-replicas") &&
(option != "--adjust-replicas-nodrop") &&
(option != "--drop-missing-replicas") &&
(option != "--unlink-zero-replicas") &&
(option != "--replace-damaged-replicas") &&
(option != "--all"))) {
goto com_fsck_usage;
}
option.replace("--", "");
in += "&mgm.option=";
in += option;
}
if (options.length()) {
in += "&mgm.option=";
in += options;
}
if (selection.length()) {
in += "&mgm.fsck.selection=";
in += selection;
}
global_retc = output_result(client_command(in, true));
return (0);
com_fsck_usage:
fprintf(stdout,
"usage: fsck stat : print status of consistency check\n");
fprintf(stdout,
" fsck enable [<interval>] : enable fsck\n");
fprintf(stdout,
" <interval> : check interval in minutes - default 30 minutes");
fprintf(stdout,
" fsck disable : disable fsck\n");
fprintf(stdout,
" fsck report [-h] [-a] [-i] [-l] [--json] [--error <tag> ] : report consistency check results");
fprintf(stdout,
" -a : break down statistics per filesystem\n");
fprintf(stdout,
" -i : print concerned file ids\n");
fprintf(stdout,
" -l : print concerned logical names\n");
fprintf(stdout,
" --json : select JSON output format\n");
fprintf(stdout,
" --error : select to report only error tag <tag>\n");
fprintf(stdout,
" -h : print help explaining the individual tags!\n");
fprintf(stdout, " fsck repair --checksum\n");
fprintf(stdout,
" : issues a 'verify' operation on all files with checksum errors\n");
fprintf(stdout, " fsck repair --checksum-commit\n");
fprintf(stdout,
" : issues a 'verify' operation on all files with checksum errors and forces a commit of size and checksum to the MGM\n");
fprintf(stdout, " fsck repair --resync\n");
fprintf(stdout,
" : issues a 'resync' operation on all files with any error. This will resync the MGM meta data to the storage node and will clean-up 'ghost' entries in the FST meta data cache.\n");
fprintf(stdout, " fsck repair --unlink-unregistered\n");
fprintf(stdout,
" : unlink replicas which are not connected/registered to their logical name\n");
fprintf(stdout, " fsck repair --unlink-orphans\n");
fprintf(stdout,
" : unlink replicas which don't belong to any logical name\n");
fprintf(stdout, " fsck repair --adjust-replicas[-nodrop]\n");
fprintf(stdout,
" : try to fix all replica inconsistencies - if --adjust-replicas-nodrop is used replicas are only added but never removed!\n");
fprintf(stdout, " fsck repair --drop-missing-replicas\n");
fprintf(stdout,
" : just drop replicas from the namespace if they cannot be found on disk\n");
fprintf(stdout, " fsck repair --unlink-zero-replicas\n");
fprintf(stdout,
" : drop all files which have no replica's attached and are older than 48 hours!\n");
fprintf(stdout, " fsck repair --replace-damaged-replicas\n");
fprintf(stdout,
" : drop the damaged replica of the file and recover with a healthy one if possible!\n");
fprintf(stdout,
" fsck repair --all : do all the repair actions besides <checksum-commit>\n");
fprintf(stdout, " fsck search <searchpath> [<nrep expected>] : do a forward FSCK scan from namespace to FSTs - will report missing files and show files, which are lost - be careful with the tree given to searchpath because everything has to stay in memory - don't use /eos/ !\n");
global_retc = EINVAL;
return (0);
}
//------------------------------------------------------------------------------
//! @file com_fsck.cc
//! @autor Elvin Sindrilaru - CERN
//------------------------------------------------------------------------------
/************************************************************************
* EOS - the CERN Disk Storage System *
* Copyright (C) 2019 CERN/Switzerland *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>.*
************************************************************************/
#include "console/ConsoleMain.hh"
#include "console/commands/helpers/FsckHelper.hh"
void com_fsck_help();
//------------------------------------------------------------------------------
// Fsck command entry point
//------------------------------------------------------------------------------
int com_proto_fsck(char* arg)
{
if (wants_help(arg)) {
com_fsck_help();
global_retc = EINVAL;
return EINVAL;
}
FsckHelper fsck;
if (!fsck.ParseCommand(arg)) {
com_fsck_help();
global_retc = EINVAL;
return EINVAL;
}
global_retc = fsck.Execute();
return global_retc;
}
//------------------------------------------------------------------------------
// Print help message
//------------------------------------------------------------------------------
void com_fsck_help()
{
std::ostringstream oss;
oss << "Usage: fsck [stat|enable|disable|report|repair|search]" << std::endl
<< " control and display file system check information" << std::endl
<< std::endl
<< " fsck enable [<interval>]" << std::endl
<< " enable fsck with interval in minutes (default 30 minutes)" << std::endl
<< std::endl
<< " fsck disable" << std::endl
<< " disable fsck" << std::endl
<< std::endl
<< " fsck stat" << std::endl
<< " print summary of consistency checks" << std::endl
<< std::endl
<< " fsck config <key> <value>" << std::endl
<< " configure the fsck with the following possible options:"
<< std::endl
<< " show-dark-files : yes/no [default no]" << std::endl
<< " show-offline : yes/no [default no]" << std::endl
<< std::endl
<< " fsck report [-a] [-h] [-i] [-l] [-j|--json] [--error <tag1> <tag2> ...]"
<< std::endl
<< " report consistency check results, with the following options"
<< std::endl
<< " -a : break down statistics per file system" << std::endl
<< " -h : display explanation for individual tags" << std::endl
<< " -i : display file identifiers" << std::endl
<< " -l : display logical file name" << std::endl
<< " -j|--json : display in JSON output format" << std::endl
<< " --error : dispaly information about the following error tags"
<< std::endl
<< std::endl
<< " fsck repair [--checksum|--checksum-commit|--resync|--unlink-unregistered|"
<< std::endl
<< " --unlink-orphans|--adjust-replicas[-nodrop]|--drop-missing-replicas|"
<< std::endl
<< " --unlink-zero-replicas|--replace-damaged-replicas|--all]"
<< std::endl
<< " trigger the repair procedure for the various types of errors. Options:"
<< std::endl
<< " checksum : issue 'verify' operation on all files with checksum errors"
<< std::endl
<< " checksum-commit : issue 'verify' operation on all files with checkusm errors"
<< std::endl
<< " and force a commit of size and checksum to the MGM"
<< std::endl
<< " resync : issue a 'resync' operation on all files with any errors."
<< std::endl
<< " This will resync the MGM metadata to the storage node and will clean-up"
<< std::endl
<< " 'ghost' entries from the FST metadata cache."
<< std::endl
<< " unlink-unregistered : unlink replicas which are not connected to their"
<< std::endl
<< " logical name"
<< std::endl
<< " unlink-orphans : unlink replicas which don't belong to any logical name"
<< std::endl
<< " adjust-replicas : try to fix all replica inconsistencies. If 'nodrop' is used"
<< std::endl
<< " replicas are only added and never removed"