Commit 5a4fd0b6 authored by Georgios Bitzes's avatar Georgios Bitzes

Merge branch 'locality-hashes-for-ns'

parents c29b90e5 feada2a2
Pipeline #387127 failed with stages
in 29 minutes 15 seconds
......@@ -26,6 +26,7 @@
#include "namespace/Namespace.hh"
#include "namespace/utils/Buffer.hh"
#include "namespace/utils/LocalityHint.hh"
#include "namespace/MDException.hh"
#include "namespace/interface/Identifiers.hh"
#include "common/Murmur3.hh"
......@@ -372,6 +373,13 @@ public:
mIsDeleted = true;
}
//----------------------------------------------------------------------------
//! Get locality hint for this container.
//----------------------------------------------------------------------------
virtual std::string getLocalityHint() const {
return LocalityHint::build(ContainerIdentifier(getParentId()), getName());
}
private:
friend class FileMapIterator;
friend class ContainerMapIterator;
......
......@@ -26,6 +26,7 @@
#include "namespace/Namespace.hh"
#include "namespace/utils/Buffer.hh"
#include "namespace/utils/LocalityHint.hh"
#include "namespace/interface/IContainerMD.hh"
#include "namespace/interface/Identifiers.hh"
#include <stdint.h>
......@@ -388,6 +389,13 @@ public:
}
//----------------------------------------------------------------------------
//! Get locality hint for this file.
//----------------------------------------------------------------------------
virtual std::string getLocalityHint() const {
return LocalityHint::build(ContainerIdentifier(getContainerId()), getName());
}
//----------------------------------------------------------------------------
//! Get env representation of the file object
//!
//! @param env string where representation is stored
......
......@@ -32,10 +32,10 @@ EOSNSNAMESPACE_BEGIN
//! Variables associated with the HierarchcalView
namespace constants
{
//! Suffix for container metadata in Redis
static const std::string sContKeySuffix{":c_bucket"};
//! Sufix for file metadata in Redis
static const std::string sFileKeySuffix{":f_bucket"};
//! Key for container metadata locality hash.
static const std::string sContainerKey {"eos-container-md"};
//! Key for file metadata locality hash.
static const std::string sFileKey {"eos-file-md"};
//! Suffix for set of subcontainers in a container
static const std::string sMapDirsSuffix{":map_conts"};
//! Suffix for set of files in a container
......@@ -49,6 +49,13 @@ static const std::string sLastUsedCid{"last_used_cid"};
//! Set of orphans files, which are not anymore attached to a container
//! they were unlinked but not removed yet
static const std::string sOrphanFiles{"orphan_files"};
//! Suffix for container metadata in Redis
static const std::string sContKeySuffix{":c_bucket"};
//! Sufix for file metadata in Redis
static const std::string sFileKeySuffix{":f_bucket"};
}
//! Variable associated with the QuotaView
......
......@@ -357,6 +357,7 @@ ContainerMDSvc::ComputeNumberOfContainers()
(void) ah.Wait();
auto resp = ah.GetResponses();
mNumConts.store(std::accumulate(resp.begin(), resp.end(), 0ull));
mNumConts += pQcl->execute(RequestBuilder::getNumberOfContainers()).get()->integer;
}
//------------------------------------------------------------------------------
......
......@@ -316,6 +316,8 @@ FileMDSvc::ComputeNumberOfFiles()
(void) ah.Wait();
std::list<long long int> resp = ah.GetResponses();
mNumFiles.store(std::accumulate(resp.begin(), resp.end(), 0ull));
mNumFiles += pQcl->execute(RequestBuilder::getNumberOfFiles()).get()->integer;
}
//------------------------------------------------------------------------------
......
......@@ -40,18 +40,18 @@ RequestBuilder::writeContainerProto(IContainerMD *obj)
eos::Buffer ebuff;
obj->serialize(ebuff);
std::string buffer(ebuff.getDataPtr(), ebuff.getSize());
return writeContainerProto(ContainerIdentifier(obj->getId()), buffer);
return writeContainerProto(ContainerIdentifier(obj->getId()), obj->getLocalityHint(), buffer);
}
//------------------------------------------------------------------------------
//! Write container protobuf metadata - low level API.
//------------------------------------------------------------------------------
RedisRequest
RequestBuilder::writeContainerProto(ContainerIdentifier id, const std::string &blob)
RequestBuilder::writeContainerProto(ContainerIdentifier id, const std::string &hint, const std::string &blob)
{
std::string sid = stringify(id.getUnderlyingUInt64());
return { "HSET", RequestBuilder::getContainerBucketKey(id.getUnderlyingUInt64()), sid, blob };
// TODO(gbitzes): Remove compatibility hack eventually.
return { "LHSET-AND-DEL-FALLBACK", constants::sContainerKey, sid, hint, blob, RequestBuilder::getContainerBucketKey(id) };
}
//------------------------------------------------------------------------------
......@@ -64,17 +64,18 @@ RequestBuilder::writeFileProto(IFileMD *obj)
obj->serialize(ebuff);
std::string buffer(ebuff.getDataPtr(), ebuff.getSize());
return writeFileProto(FileIdentifier(obj->getId()), buffer);
return writeFileProto(FileIdentifier(obj->getId()), obj->getLocalityHint(), buffer);
}
//------------------------------------------------------------------------------
//! Write file protobuf metadata - low level API.
//------------------------------------------------------------------------------
RedisRequest
RequestBuilder::writeFileProto(FileIdentifier id, const std::string &blob)
RequestBuilder::writeFileProto(FileIdentifier id, const std::string &hint, const std::string &blob)
{
std::string sid = stringify(id.getUnderlyingUInt64());
return { "HSET", RequestBuilder::getFileBucketKey(id.getUnderlyingUInt64()), sid, blob };
// TODO(gbitzes): Remove compatibility hack eventually.
return { "LHSET-AND-DEL-FALLBACK", constants::sFileKey, sid, hint, blob, RequestBuilder::getFileBucketKey(id) };
}
//------------------------------------------------------------------------------
......@@ -83,7 +84,9 @@ RequestBuilder::writeFileProto(FileIdentifier id, const std::string &blob)
RedisRequest
RequestBuilder::readContainerProto(ContainerIdentifier id)
{
return { "HGET", RequestBuilder::getContainerBucketKey(id.getUnderlyingUInt64()), SSTR(id.getUnderlyingUInt64()) };
// TODO(gbitzes): Pass locality hint when available.
// TODO(gbitzes): Remove compatibility hack eventually.
return { "LHGET-WITH-FALLBACK", constants::sContainerKey, SSTR(id.getUnderlyingUInt64()), RequestBuilder::getContainerBucketKey(id) };
}
//------------------------------------------------------------------------------
......@@ -92,7 +95,9 @@ RequestBuilder::readContainerProto(ContainerIdentifier id)
RedisRequest
RequestBuilder::readFileProto(FileIdentifier id)
{
return { "HGET", RequestBuilder::getFileBucketKey(id.getUnderlyingUInt64()), SSTR(id.getUnderlyingUInt64()) };
// TODO(gbitzes): Pass locality hint when available.
// TODO(gbitzes): Remove compatibility hack eventually.
return { "LHGET-WITH-FALLBACK", constants::sFileKey, SSTR(id.getUnderlyingUInt64()), RequestBuilder::getFileBucketKey(id) };
}
//------------------------------------------------------------------------------
......@@ -101,7 +106,8 @@ RequestBuilder::readFileProto(FileIdentifier id)
RedisRequest
RequestBuilder::deleteContainerProto(ContainerIdentifier id)
{
return { "HDEL", RequestBuilder::getContainerBucketKey(id.getUnderlyingUInt64()), SSTR(id.getUnderlyingUInt64()) };
// TODO(gbitzes): Remove compatibility hack eventually.
return { "LHDEL-WITH-FALLBACK", constants::sContainerKey, SSTR(id.getUnderlyingUInt64()), RequestBuilder::getContainerBucketKey(id) };
}
//------------------------------------------------------------------------------
......@@ -110,32 +116,51 @@ RequestBuilder::deleteContainerProto(ContainerIdentifier id)
RedisRequest
RequestBuilder::deleteFileProto(FileIdentifier id)
{
return { "HDEL", RequestBuilder::getFileBucketKey(id.getUnderlyingUInt64()), SSTR(id.getUnderlyingUInt64()) };
// TODO(gbitzes): Remove compatibility hack eventually.
return { "LHDEL-WITH-FALLBACK", constants::sFileKey, SSTR(id.getUnderlyingUInt64()), RequestBuilder::getFileBucketKey(id) };
}
//------------------------------------------------------------------------------
//! Calculate number of containers.
//------------------------------------------------------------------------------
RedisRequest RequestBuilder::getNumberOfContainers()
{
return { "LHLEN", constants::sContainerKey };
}
//------------------------------------------------------------------------------
//! Calculate number of files.
//------------------------------------------------------------------------------
RedisRequest RequestBuilder::getNumberOfFiles()
{
return { "LHLEN", constants::sFileKey };
}
//------------------------------------------------------------------------------
// Get container bucket
//------------------------------------------------------------------------------
std::string
RequestBuilder::getContainerBucketKey(IContainerMD::id_t id)
RequestBuilder::getContainerBucketKey(ContainerIdentifier identifier)
{
uint64_t id = identifier.getUnderlyingUInt64();
id = id & (sNumContBuckets - 1);
std::string bucket_key = stringify(id);
std::string bucket_key = stringify(identifier.getUnderlyingUInt64());
bucket_key += constants::sContKeySuffix;
return bucket_key;
}
//------------------------------------------------------------------------------
// Get file bucket
//! Calculate number of files.
//------------------------------------------------------------------------------
std::string
RequestBuilder::getFileBucketKey(IContainerMD::id_t id)
RequestBuilder::getFileBucketKey(FileIdentifier identifier)
{
uint64_t id = identifier.getUnderlyingUInt64();
id = id & (sNumFileBuckets - 1);
std::string bucket_key = stringify(id);
std::string bucket_key = stringify(identifier.getUnderlyingUInt64());
bucket_key += constants::sFileKeySuffix;
return bucket_key;
}
EOSNSNAMESPACE_END
......@@ -47,7 +47,7 @@ public:
//----------------------------------------------------------------------------
//! Write container protobuf metadata - low level API.
//----------------------------------------------------------------------------
static RedisRequest writeContainerProto(ContainerIdentifier id, const std::string &blob);
static RedisRequest writeContainerProto(ContainerIdentifier id, const std::string &hint, const std::string &blob);
//----------------------------------------------------------------------------
//! Write file protobuf metadata.
......@@ -57,7 +57,7 @@ public:
//----------------------------------------------------------------------------
//! Write file protobuf metadata - low level API.
//----------------------------------------------------------------------------
static RedisRequest writeFileProto(FileIdentifier id, const std::string &blob);
static RedisRequest writeFileProto(FileIdentifier id, const std::string &hint, const std::string &blob);
//----------------------------------------------------------------------------
//! Read container protobuf metadata.
......@@ -80,6 +80,16 @@ public:
static RedisRequest deleteFileProto(FileIdentifier id);
//----------------------------------------------------------------------------
//! Calculate number of containers.
//----------------------------------------------------------------------------
static RedisRequest getNumberOfContainers();
//----------------------------------------------------------------------------
//! Calculate number of files.
//----------------------------------------------------------------------------
static RedisRequest getNumberOfFiles();
//----------------------------------------------------------------------------
//! Get container bucket which is computed as the id of the container modulo
//! the number of container buckets.
//!
......@@ -87,7 +97,7 @@ public:
//!
//! @return container bucket key
//----------------------------------------------------------------------------
static std::string getContainerBucketKey(IContainerMD::id_t id);
static std::string getContainerBucketKey(ContainerIdentifier id);
//----------------------------------------------------------------------------
//! Get file bucket which is computed as the id of the container modulo the
......@@ -96,8 +106,9 @@ public:
//! @param id file id
//!
//! @return file bucket key
//! Calculate number of files.
//----------------------------------------------------------------------------
static std::string getFileBucketKey(IContainerMD::id_t id);
static std::string getFileBucketKey(FileIdentifier id);
//----------------------------------------------------------------------------
//! Override number of container buckets
......@@ -115,6 +126,7 @@ public:
static std::uint64_t sNumContBuckets; ///< Number of buckets power of 2
static std::uint64_t sNumFileBuckets; ///< Number of buckets power of 2
};
......
......@@ -302,39 +302,39 @@ TEST_F(FileSystemViewF, RandomFilePicking)
//------------------------------------------------------------------------------
// Test file iterator on top of QHash object
//------------------------------------------------------------------------------
TEST_F(FileSystemViewF, FileIterator)
{
std::srand(std::time(0));
std::unordered_set<eos::IFileMD::id_t> input_set;
for (std::uint64_t i = 0ull; i < 50000; ++i) {
double frac = std::rand() / (double)RAND_MAX;
(void)input_set.insert((uint64_t)(UINT64_MAX * frac));
}
// Push the set to QuarkDB
qclient::AsyncHandler ah;
const std::string key = "set_iter_test";
qclient::QSet set(qcl(), key);
for (auto elem : input_set) {
set.sadd_async(elem, &ah);
}
ASSERT_TRUE(ah.Wait());
std::unordered_set<eos::IFileMD::id_t> result_set;
auto iter = std::shared_ptr<eos::ICollectionIterator<eos::IFileMD::id_t>>
(new eos::QdbFileIterator(qcl(), key));
for (; (iter && iter->valid()); iter->next()) {
result_set.insert(iter->getElement());
}
ASSERT_EQ(input_set.size(), result_set.size());
for (auto elem : input_set) {
ASSERT_TRUE(result_set.find(elem) != result_set.end());
}
qcl().del(key);
}
// TEST_F(FileSystemViewF, FileIterator)
// {
// std::srand(std::time(0));
// std::unordered_set<eos::IFileMD::id_t> input_set;
//
// for (std::uint64_t i = 0ull; i < 50000; ++i) {
// double frac = std::rand() / (double)RAND_MAX;
// (void)input_set.insert((uint64_t)(UINT64_MAX * frac));
// }
//
// // Push the set to QuarkDB
// qclient::AsyncHandler ah;
// const std::string key = "set_iter_test";
// qclient::QSet set(qcl(), key);
//
// for (auto elem : input_set) {
// set.sadd_async(elem, &ah);
// }
//
// ASSERT_TRUE(ah.Wait());
// std::unordered_set<eos::IFileMD::id_t> result_set;
// auto iter = std::shared_ptr<eos::ICollectionIterator<eos::IFileMD::id_t>>
// (new eos::QdbFileIterator(qcl(), key));
//
// for (; (iter && iter->valid()); iter->next()) {
// result_set.insert(iter->getElement());
// }
//
// ASSERT_EQ(input_set.size(), result_set.size());
//
// for (auto elem : input_set) {
// ASSERT_TRUE(result_set.find(elem) != result_set.end());
// }
//
// qcl().del(key);
// }
......@@ -60,10 +60,6 @@ NsTestsFixture::NsTestsFixture() {
NsTestsFixture::~NsTestsFixture() {
shut_down_everything();
// Restore default values
// FileMDSvc::OverrideNumberOfBuckets();
// ContainerMDSvc::OverrideNumberOfBuckets();
}
qclient::Members NsTestsFixture::getMembers() {
......
......@@ -277,7 +277,7 @@ TEST_F(FileMDFetching, CorruptionTest) {
shut_down_everything();
qcl().exec("HSET", RequestBuilder::getFileBucketKey(1), "1", "chicken_chicken_chicken_chicken").get();
qcl().exec(RequestBuilder::writeFileProto(FileIdentifier(1), "hint", "chicken_chicken_chicken_chicken")).get();
try {
MetadataFetcher::getFileFromId(qcl(), FileIdentifier(1)).get();
......@@ -289,8 +289,8 @@ TEST_F(FileMDFetching, CorruptionTest) {
shut_down_everything();
qcl().exec("DEL", RequestBuilder::getFileBucketKey(1)).get();
qcl().exec("SADD", RequestBuilder::getFileBucketKey(1), "zzzz").get();
qcl().exec("DEL", constants::sFileKey).get();
qcl().exec("SADD", constants::sFileKey, "zzz").get();
try {
MetadataFetcher::getFileFromId(qcl(), FileIdentifier(1)).get();
......
......@@ -502,7 +502,7 @@ ConvertContainerMDSvc::commitToBackend()
std::string buffer;
conv_cont->serializeToStr(buffer);
ah.Register(&qclient, RequestBuilder::writeContainerProto(container->getIdentifier(), buffer));
ah.Register(&qclient, RequestBuilder::writeContainerProto(container->getIdentifier(), container->getLocalityHint(), buffer));
// Commit subcontainers and files only if not empty otherwise the hmset
// command will fail
......@@ -719,7 +719,7 @@ const
// @todo (esindril): Could use compression when storing the entries
file->updateInternal();
file->serializeToStr(buffer);
ah.Register(&qclient, RequestBuilder::writeFileProto(file->getIdentifier(), buffer));
ah.Register(&qclient, RequestBuilder::writeFileProto(file->getIdentifier(), file->getLocalityHint(), buffer));
} catch (std::runtime_error& qdb_err) {
MDException e(ENOENT);
e.getMessage() << "File #" << file->getId() << " failed to contact backend";
......
/************************************************************************
* EOS - the CERN Disk Storage System *
* Copyright (C) 2018 CERN/Switzerland *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>.*
************************************************************************/
//------------------------------------------------------------------------------
//! @author Georgios Bitzes <georgios.bitzes@cern.ch>
//! @brief Helper class to generate locality hints
//------------------------------------------------------------------------------
#ifndef EOS_NS_LOCALITY_HINT_HH
#define EOS_NS_LOCALITY_HINT_HH
#include <iostream>
#include "namespace/interface/Identifiers.hh"
#include "namespace/Namespace.hh"
EOSNSNAMESPACE_BEGIN
class LocalityHint {
public:
static std::string build(ContainerIdentifier parent, const std::string &name) {
std::ostringstream ss;
ss << unsignedIntToBinaryString(parent.getUnderlyingUInt64());
ss << ":" << name;
return ss.str();
}
private:
static inline void unsignedIntToBinaryString(uint64_t num, char* buff) {
uint64_t be = htobe64(num);
memcpy(buff, &be, sizeof(be));
}
static std::string unsignedIntToBinaryString(uint64_t num) {
char buff[sizeof(num)];
unsignedIntToBinaryString(num, buff);
return std::string(buff, sizeof(num));
}
};
EOSNSNAMESPACE_END
#endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment