...
 
Commits (279)
......@@ -32,6 +32,7 @@ Namespace/tests/text_runner
valgrind.supp
eos-log-repair
/build/
/build_fuzz/
__pycache__
kineticio-dist.tgz
# clion specific configs
......@@ -43,3 +44,4 @@ kineticio-dist.tgz
.project
.vscode
nbproject
my_clang_cache.cmake
......@@ -119,22 +119,24 @@ before_script:
- ./eos-docker/scripts/remove_unused_images.sh
- docker pull gitlab-registry.cern.ch/dss/${EOS_IMG_NAME}${CI_COMMIT_TAG-$CI_PIPELINE_ID}
- if [[ "$CI_JOB_NAME" == system_test_qdb* ]]; then sudo ./eos-docker/scripts/start_services.sh -q -i gitlab-registry.cern.ch/dss/${EOS_IMG_NAME}${CI_COMMIT_TAG-$CI_PIPELINE_ID}; else sudo ./eos-docker/scripts/start_services.sh -i gitlab-registry.cern.ch/dss/${EOS_IMG_NAME}${CI_COMMIT_TAG-$CI_PIPELINE_ID}; fi
- docker exec -i eos-client1-test git clone https://gitlab.cern.ch/dss/eosclient-tests.git
- docker exec -i eos-mgm-test /bin/bash -c 'eos vid enable krb5'
- docker exec -i eos-mgm-test eos-instance-test-ci
- docker exec -di eos-client1-test /bin/bash -c 'mkdir /eos1/; mount -t fuse eosxd -ofsname=mount-1 /eos1/'
- docker exec -di eos-client1-test /bin/bash -c 'mkdir /eos2/; mount -t fuse eosxd -ofsname=mount-2 /eos2/'
- docker exec -i -u eos-user eos-client1-test /bin/bash -c 'mkdir /eos1/dockertest/fusex_tests/; cd /eos1/dockertest/fusex_tests/; fusex-benchmark'
- docker exec -i eos-cli1 git clone https://gitlab.cern.ch/dss/eosclient-tests.git
- if [[ "$CI_JOB_NAME" == system_test_qdb* ]]; then docker cp eos-cli1:/usr/sbin/eos-fsck-test .; ./eos-fsck-test --type docker; rm -rf eos-fsck-test; fi
- docker exec -i eos-mgm1 eos-instance-test-ci
- docker exec -di eos-cli1 /bin/bash -c 'mkdir /eos1/; mount -t fuse eosxd -ofsname=mount-1 /eos1/'
- docker exec -di eos-cli1 /bin/bash -c 'mkdir /eos2/; mount -t fuse eosxd -ofsname=mount-2 /eos2/'
- docker exec -i -u eos-user eos-cli1 /bin/bash -c 'mkdir /eos1/dockertest/fusex_tests/; cd /eos1/dockertest/fusex_tests/; fusex-benchmark'
# @todo(esindril): run "all" tests in schedule mode once these are properly supported
# - if [ "$CI_PIPELINE_SOURCE" == "schedule" ]; then
# docker exec -i eos-mgm-test eos vid add gateway eos-client1-test.eoscluster.cern.ch unix;
# docker exec -i eos-client1-test env EOS_FUSE_NO_ROOT_SQUASH=1 python /eosclient-tests/run.py --workdir="/eos1/dockertest /eos2/dockertest" ci;
# docker exec -i eos-mgm1 eos vid add gateway eos-cli1.eoscluster.cern.ch unix;
# docker exec -i eos-cli1 env EOS_FUSE_NO_ROOT_SQUASH=1 python /eosclient-tests/run.py --workdir="/eos1/dockertest /eos2/dockertest" ci;
# fi
# until then just run the "ci" tests
- docker exec -i -u eos-user eos-client1-test python /eosclient-tests/run.py --workdir="/eos1/dockertest /eos2/dockertest" ci;
# - docker exec -i eos-client1-test env EOS_MGM_URL=root://eos-mgm-test.eoscluster.cern.ch eos fuse mount /eos_fuse
# - docker exec -i eos-client1-test env EOS_MGM_URL=root://eos-mgm-test.eoscluster.cern.ch eos fuse mount /eos_fuse2
# - docker exec -i eos-client1-test python /eosclient-tests/run.py --workdir="/eos_fuse/dockertest /eos_fuse2/dockertest" ci
- docker exec -i eos-cli1 /bin/bash -c 'cd eosclient-tests && for n in prepare/*.sh; do /bin/bash $n prepare; done'
- docker exec -i -u eos-user eos-cli1 python /eosclient-tests/run.py --workdir="/eos1/dockertest /eos2/dockertest" ci;
- docker exec -i eos-cli1 /bin/bash -c 'cd eosclient-tests && for n in prepare/*.sh; do /bin/bash $n cleanup; done'
# - docker exec -i eos-cli1 env EOS_MGM_URL=root://eos-mgm1.eoscluster.cern.ch eos fuse mount /eos_fuse
# - docker exec -i eos-cli1 env EOS_MGM_URL=root://eos-mgm1.eoscluster.cern.ch eos fuse mount /eos_fuse2
# - docker exec -i eos-cli1 python /eosclient-tests/run.py --workdir="/eos_fuse/dockertest /eos_fuse2/dockertest" ci
<<: *system_test_after_script_template
artifacts:
when: on_failure
......@@ -151,19 +153,17 @@ before_script:
- docker pull gitlab-registry.cern.ch/dss/${UBUNTU_EOS_IMG_NAME}${CI_COMMIT_TAG-$CI_PIPELINE_ID}
# Run the following tests only on the NS in QuarkDB
- sudo ./eos-docker/scripts/start_services.sh -q -i gitlab-registry.cern.ch/dss/eos:${CI_COMMIT_TAG-$CI_PIPELINE_ID} -u gitlab-registry.cern.ch/dss/${UBUNTU_EOS_IMG_NAME}${CI_COMMIT_TAG-$CI_PIPELINE_ID};
- docker exec -i eos-client1-test git clone https://gitlab.cern.ch/dss/eosclient-tests.git
- docker exec -i eos-mgm-test /bin/bash -c 'eos vid enable krb5'
- docker exec -i eos-mgm-test /bin/bash -c 'eos chmod 2777 /eos/dockertest/'
- docker exec -di eos-client1-test /bin/bash -c 'mkdir /eos1/; mount -t fuse eosxd -ofsname=mount-1 /eos1/'
- docker exec -di eos-client1-test /bin/bash -c 'mkdir /eos2/; mount -t fuse eosxd -ofsname=mount-2 /eos2/'
- docker exec -i -u eos-user eos-client1-test /bin/bash -c 'mkdir /eos1/dockertest/fusex_tests/; cd /eos1/dockertest/fusex_tests/; fusex-benchmark'
- docker exec -i eos-cli1 git clone https://gitlab.cern.ch/dss/eosclient-tests.git
- docker exec -di eos-cli1 /bin/bash -c 'mkdir /eos1/; mount -t fuse eosxd -ofsname=mount-1 /eos1/'
- docker exec -di eos-cli1 /bin/bash -c 'mkdir /eos2/; mount -t fuse eosxd -ofsname=mount-2 /eos2/'
- docker exec -i -u eos-user eos-cli1 /bin/bash -c 'mkdir /eos1/dockertest/fusex_tests/; cd /eos1/dockertest/fusex_tests/; fusex-benchmark'
# @todo(esindril): run "all" tests in schedule mode once these are properly supported
# - if [ "$CI_PIPELINE_SOURCE" == "schedule" ]; then
# docker exec -i eos-mgm-test eos vid add gateway eos-client1-test.eoscluster.cern.ch unix;
# docker exec -i eos-client1-test env EOS_FUSE_NO_ROOT_SQUASH=1 python /eosclient-tests/run.py --workdir="/eos1/dockertest /eos2/dockertest" ci;
# docker exec -i eos-mgm1 eos vid add gateway eos-cli1.eoscluster.cern.ch unix;
# docker exec -i eos-cli1 env EOS_FUSE_NO_ROOT_SQUASH=1 python /eosclient-tests/run.py --workdir="/eos1/dockertest /eos2/dockertest" ci;
# fi
# until then just run the "ci" tests
- docker exec -i -u eos-user eos-client1-test python /eosclient-tests/run.py --workdir="/eos1/dockertest /eos2/dockertest" ci;
- docker exec -i -u eos-user eos-cli1 python /eosclient-tests/run.py --workdir="/eos1/dockertest /eos2/dockertest" ci;
<<: *system_test_after_script_template
artifacts:
when: on_failure
......@@ -221,7 +221,7 @@ before_script:
else
IMAGE_TAG="${BASETAG}${CI_PIPELINE_ID}";
fi;
- git clone https://gitlab.cern.ch/faluchet/eos-on-k8s.git
- git clone https://gitlab.cern.ch/eos/eos-on-k8s.git
.k8s_after_script_template: &k8s_after_script_template
......@@ -242,6 +242,10 @@ before_script:
elif [[ "$CI_JOB_NAME" == k8s_system_qdb* ]]; then ./eos-on-k8s/create-all.sh -i ${IMAGE_TAG} -n ${NAMESPACE} -q;
else echo "CI_JOB_NAME:$CI_JOB_NAME not found"; exit 1;
fi
- if [[ "$CI_JOB_NAME" == k8s_system_qdb* ]]; then
kubectl cp $NAMESPACE/$(kubectl get pods -n $NAMESPACE --no-headers -o custom-columns=":metadata.name" -l app=eos-cli1):/usr/sbin/eos-fsck-test ./eos-fsck-test;
chmod +x eos-fsck-test; ./eos-fsck-test --type k8s $NAMESPACE ; rm -rf eos-fsck-test;
fi
- if [[ "$CI_JOB_NAME" == k8s_ubuntu_system ]]; then ./gitlab-ci/k8s_system_test_script.sh --only-client ${NAMESPACE};
else ./gitlab-ci/k8s_system_test_script.sh ${NAMESPACE};
fi
......@@ -792,7 +796,6 @@ k8s_stress:
<<: *k8s_before_script_template
script:
- ./eos-on-k8s/create-all.sh -i ${IMAGE_TAG} -n ${NAMESPACE}
- kubectl exec -n $NAMESPACE -i $(kubectl get pods -n $NAMESPACE --no-headers -o custom-columns=":metadata.name" -l app=eos-mgm1) -- eos chmod 2777 /eos/dockertest
- kubectl exec -n $NAMESPACE -i $(kubectl get pods -n $NAMESPACE --no-headers -o custom-columns=":metadata.name" -l app=eos-mgm1) -- hammer-runner.py --strict-exit-code 1 --gitlab --url eos-mgm1.eos-mgm1.$NAMESPACE.svc.cluster.local//eos/dockertest/hammer/ --protocols xroot --threads 1 2 10 100 --operations write stat read delete --runs 3 --nfiles 10000
<<: *k8s_after_script_template
tags:
......@@ -939,8 +942,7 @@ stress_test:
- ./eos-docker/scripts/remove_unused_images.sh
- docker pull gitlab-registry.cern.ch/dss/eos:${CI_COMMIT_TAG-$CI_PIPELINE_ID}
- sudo ./eos-docker/scripts/start_services.sh -i gitlab-registry.cern.ch/dss/eos:${CI_COMMIT_TAG-$CI_PIPELINE_ID}
- docker exec -i eos-mgm-test eos chmod 2777 /eos/dockertest
- docker exec -i eos-mgm-test hammer-runner.py --strict-exit-code 1 --gitlab --url eos-mgm-test.eoscluster.cern.ch//eos/dockertest/hammer/ --protocols xroot --threads 1 2 10 100 --operations write stat read delete --runs 3 --nfiles 10000
- docker exec -i eos-mgm1 hammer-runner.py --strict-exit-code 1 --gitlab --url eos-mgm1.eoscluster.cern.ch//eos/dockertest/hammer/ --protocols xroot --threads 1 2 10 100 --operations write stat read delete --runs 3 --nfiles 10000
- sudo ./eos-docker/scripts/shutdown_services.sh
tags:
- shell-with-docker
......
......@@ -89,6 +89,13 @@ add_executable(
$<TARGET_OBJECTS:EosGrpcClient-Objects>
)
add_executable(
eos-grpc-ns-stat
grpc/NsStat.cc
$<TARGET_OBJECTS:EosGrpcProto-Objects>
$<TARGET_OBJECTS:EosGrpcClient-Objects>
)
#-------------------------------------------------------------------------------
# Add dependency which guarantees that the protocol buffer files are generated
# when we build the "eos" executable.
......@@ -159,6 +166,16 @@ target_link_libraries(
${CMAKE_THREAD_LIBS_INIT}
eosCommon)
target_link_libraries(
eos-grpc-ns-stat PUBLIC
${RT_LIBRARIES}
${NCURSES_LIBRARY}
${PROTOBUF_LIBRARY}
${GRPC_LIBRARY}
${GRPC_GRPC++_LIBRARY}
${CMAKE_THREAD_LIBS_INIT}
eosCommon)
set_target_properties(
eos PROPERTIES
INSTALL_RPATH "${EOS_RPATH}"
......@@ -168,7 +185,8 @@ set_target_properties(
INSTALL_RPATH_USE_LINK_PATH TRUE)
install(
TARGETS eos-grpc-ping eos-grpc-md eos-grpc-insert eos-grpc-manila eos-grpc-ns eos-grpc-find
TARGETS eos-grpc-ping eos-grpc-md eos-grpc-insert eos-grpc-manila
eos-grpc-ns eos-grpc-find eos-grpc-ns-stat
LIBRARY DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}
RUNTIME DESTINATION ${CMAKE_INSTALL_FULL_BINDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR})
......@@ -53,6 +53,8 @@ using eos::rpc::MDResponse;
using eos::rpc::FindRequest;
using eos::rpc::NSRequest;
using eos::rpc::NSResponse;
using eos::rpc::NsStatRequest;
using eos::rpc::NsStatResponse;
using eos::rpc::FileInsertRequest;
using eos::rpc::ContainerInsertRequest;
using eos::rpc::InsertReply;
......@@ -577,6 +579,31 @@ GrpcClient::Create(std::string endpoint,
return p;
}
int
GrpcClient::NsStat(const eos::rpc::NsStatRequest& request,
eos::rpc::NsStatResponse& reply)
{
ClientContext context;
CompletionQueue cq;
Status status;
std::unique_ptr<ClientAsyncResponseReader<NsStatResponse>> rpc(
stub_->AsyncNsStat(&context, request, &cq));
rpc->Finish(&reply, &status, (void*) 1);
void* got_tag;
bool ok = false;
GPR_ASSERT(cq.Next(&got_tag, &ok));
GPR_ASSERT(got_tag == (void*) 1);
GPR_ASSERT(ok);
// Act upon the status of the actual RPC
if (status.ok()) {
return reply.code();
} else {
return -1;
}
}
int
GrpcClient::Exec(const eos::rpc::NSRequest& request,
eos::rpc::NSResponse& reply)
......
......@@ -70,6 +70,9 @@ public:
int ManilaRequest(const eos::rpc::ManilaRequest& request,
eos::rpc::ManilaResponse& reply);
int NsStat(const eos::rpc::NsStatRequest& request,
eos::rpc::NsStatResponse& reply);
int Exec(const eos::rpc::NSRequest& request,
eos::rpc::NSResponse& reply);
......
/************************************************************************
* EOS - the CERN Disk Storage System *
* Copyright (C) 2018 CERN/Switzerland *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>.*
************************************************************************/
#include <sstream>
#include <iostream>
#include <iomanip>
#include <getopt.h>
#include "client/grpc/GrpcClient.hh"
int usage(const char* name)
{
std::ostringstream oss;
oss << "usage: " << name
<< " [--key <ssl-key-file> --cert <ssl-cert-file> --ca <ca-cert-file>]"
<< " [--token <auth-token>]"
<< std::endl << std::setw(strlen(name) + 8) << ""
<< "[--endpoint <host:port>] [-d|--debug] [-h|--help]"
<< std::endl;
std::cerr << oss.str();
return -1;
}
int main(int argc, char* argv[])
{
using eos::client::GrpcClient;
std::string endpoint{"localhost:50051"};
std::string keyfile;
std::string certfile;
std::string cafile;
std::string token;
bool debug = false;
while (true) {
static struct option long_options[] {
{"key", required_argument, 0, 'k'},
{"cert", required_argument, 0, 'c'},
{"ca", required_argument, 0, 'a'},
{"endpoint", required_argument, 0, 'e'},
{"token", required_argument, 0, 't'},
{"debug", no_argument, 0, 'd'},
{"help", no_argument, 0, 'h'},
{0, 0, 0, 0}
};
int option_index = 0;
int c = getopt_long(argc, argv, "k:c:a:e:t:dh", long_options, &option_index);
// Detect end of the options
if (c == -1) {
break;
}
switch (c) {
case 'k':
keyfile = optarg;
break;
case 'c':
certfile = optarg;
break;
case 'a':
cafile = optarg;
break;
case 'e':
endpoint = optarg;
break;
case 't':
token = optarg;
break;
case 'd':
debug = true;
break;
case 'h':
return usage(argv[0]);
default:
return usage(argv[0]);
}
}
// Make sure all elements are present if certificate authentication is used
if (keyfile.length() || certfile.length() || cafile.length()) {
if (!keyfile.length() || !certfile.length() || !cafile.length()) {
return usage(argv[0]);
}
}
std::unique_ptr<GrpcClient> eosgrpc =
GrpcClient::Create(endpoint, token, keyfile, certfile, cafile);
if (!eosgrpc) {
std::cerr << "Failed to create grpc client object!" << std::endl;
return -1;
}
auto start_time = std::chrono::steady_clock::now();
google::protobuf::util::JsonPrintOptions options;
options.always_print_primitive_fields = true;
options.add_whitespace = true;
std::string jsonstring;
eos::rpc::NsStatRequest request;
eos::rpc::NsStatResponse reply;
request.set_authkey(token);
if (debug) {
google::protobuf::util::MessageToJsonString(request, &jsonstring, options);
std::cout << "request: " << std::endl << jsonstring << std::endl;
}
if (eosgrpc->NsStat(request, reply)) {
std::cerr << "GRPC request field" << std::endl;
debug = true;
}
if (debug) {
std::cout << "reply: " << std::endl;
}
jsonstring = "";
google::protobuf::util::MessageToJsonString(reply, &jsonstring, options);
std::cout << jsonstring << std::endl;
auto elapsed_time = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now() - start_time);
if (debug) {
std::cout << "request took " << elapsed_time.count() << " microseconds"
<< std::endl;
}
return reply.code();
}
......@@ -23,10 +23,11 @@
include_directories(
${CMAKE_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR}
${CMAKE_BINARY_DIR}
${Z_INCLUDE_DIRS}
${PROTOBUF_INCLUDE_DIRS}
${XROOTD_INCLUDE_DIRS}
${JSONCPP_INCLUDE_DIR}
${NCURSES_INCLUDE_DIRS}
${ZMQ_INCLUDE_DIRS}
${SPARSEHASH_INCLUDE_DIRS}
......@@ -64,6 +65,7 @@ set_target_properties(EosCrc32c-Static PROPERTIES
# eosCommon-Objects
#-------------------------------------------------------------------------------
add_library(eosCommon-Objects OBJECT
Fmd.cc
SymKeys.cc
InstanceName.cc
Mapping.cc
......@@ -79,19 +81,28 @@ add_library(eosCommon-Objects OBJECT
Report.cc
StringTokenizer.cc
CommentLog.cc
RateLimit.cc
IntervalStopwatch.cc IntervalStopwatch.hh
VirtualIdentity.cc VirtualIdentity.hh
XrdConnPool.cc XrdConnPool.hh
XrdErrorMap.cc XrdErrorMap.hh
OAuth.cc OAuth.hh
JeMallocHandler.cc
plugin_manager/Plugin.hh
plugin_manager/PluginManager.cc
plugin_manager/DynamicLibrary.cc
${CMAKE_SOURCE_DIR}/mgm/TableFormatter/TableCell.cc)
table_formatter/TableCell.cc
table_formatter/TableFormatterBase.cc)
set_target_properties(eosCommon-Objects PROPERTIES
POSITION_INDEPENDENT_CODE TRUE)
#-------------------------------------------------------------------------------
# Add dependency to protobuf objects so we guarantee that the protocol
# buffer files are generated when we try to build XrdEosFst
#-------------------------------------------------------------------------------
add_dependencies(eosCommon-Objects EosFstProto-Objects)
target_compile_definitions(eosCommon-Objects PUBLIC
-DDAEMONUID=${DAEMONUID} -DDAEMONGID=${DAEMONGID})
......@@ -105,6 +116,7 @@ set(EOSCOMMON_LIBS
${UUID_LIBRARIES}
${ATTR_LIBRARIES}
${NCURSES_LIBRARY}
${JSONCPP_LIBRARIES}
${PROTOBUF_LIBRARY}
${XROOTD_CL_LIBRARY}
${XROOTD_UTILS_LIBRARY}
......@@ -182,6 +194,7 @@ if (Linux)
${UUID_LIBRARIES}
${ATTR_LIBRARIES}
${NCURSES_LIBRARY}
${JSONCPP_LIBRARIES}
${LEVELDB_LIBRARIES}
${GLIBC_RT_LIBRARY}
${XROOTD_CL_LIBRARY}
......
......@@ -23,20 +23,33 @@
#pragma once
#include "common/Namespace.hh"
EOSCOMMONNAMESPACE_BEGIN
static constexpr auto TAPE_FS_ID = 65535u;
static constexpr auto RETRIEVE_REQID_ATTR_NAME = "sys.retrieve.req_id"; //!< List of Prepare request IDs for this file
static constexpr auto RETRIEVE_REQTIME_ATTR_NAME = "sys.retrieve.req_time"; //!< Last time the Prepare request was actioned
static constexpr auto RETRIEVE_ERROR_ATTR_NAME = "sys.retrieve.error"; //!< Prepare request failure reason
static constexpr auto ARCHIVE_ERROR_ATTR_NAME = "sys.archive.error"; //!< Archive request failure reason
//! List of Prepare request IDs for this file
static constexpr auto RETRIEVE_REQID_ATTR_NAME = "sys.retrieve.req_id";
//! Last time the Prepare request was actioned
static constexpr auto RETRIEVE_REQTIME_ATTR_NAME = "sys.retrieve.req_time";
//!< Prepare request failure reason
static constexpr auto RETRIEVE_ERROR_ATTR_NAME = "sys.retrieve.error";
//! Archive request failure reason
static constexpr auto ARCHIVE_ERROR_ATTR_NAME = "sys.archive.error";
static constexpr auto RETRIEVE_WRITTEN_WORKFLOW_NAME = "retrieve_written";
static constexpr auto RETRIEVE_FAILED_WORKFLOW_NAME = "sync::retrieve_failed";
static constexpr auto ARCHIVE_FAILED_WORKFLOW_NAME = "sync::archive_failed";
static constexpr auto WF_CUSTOM_ATTRIBUTES_TO_FST_EQUALS = "=";
static constexpr auto WF_CUSTOM_ATTRIBUTES_TO_FST_SEPARATOR = ";;;";
//! Max rate in MB/s at which the scanner should run
static constexpr auto SCAN_IO_RATE_NAME = "scanrate";
//! Time interval after which a scanned filed is rescanned
static constexpr auto SCAN_ENTRY_INTERVAL_NAME = "scaninterval";
//! Time interval after which the scanner will rerun
static constexpr auto SCAN_DISK_INTERVAL_NAME = "scan_disk_interval";
//! Maximum ns scan rate when it comes to stat requests done against the
//! local disks on the FSTs
static constexpr auto SCAN_NS_RATE_NAME = "scan_ns_rate";
//! Time interval after which the ns scanner will rerun
static constexpr auto SCAN_NS_INTERVAL_NAME = "scan_ns_interval";
EOSCOMMONNAMESPACE_END
......@@ -26,6 +26,7 @@
#include "XrdOuc/XrdOucString.hh"
#include <chrono>
#include <cmath>
#include <cstring>
EOSCOMMONNAMESPACE_BEGIN
......@@ -54,7 +55,7 @@ public:
//----------------------------------------------------------------------------
static unsigned long long Hex2Fid(const char* hexstring)
{
if (hexstring) {
if (hexstring && strlen(hexstring)) {
return strtoll(hexstring, 0, 16);
} else {
return 0;
......@@ -127,30 +128,26 @@ public:
//----------------------------------------------------------------------------
//! Compute a path from a fid and localprefix
//----------------------------------------------------------------------------
static void FidPrefix2FullPath(const char* hexstring, const char* localprefix,
XrdOucString& fullpath, unsigned int subindex = 0)
static std::string FidPrefix2FullPath(const char* hexstring,
const char* localprefix)
{
if ((!hexstring) || (!localprefix)) {
fullpath = "";
return;
std::string fullpath;
if (!hexstring || !localprefix) {
return fullpath;
}
unsigned long long fid = Hex2Fid(hexstring);
char sfullpath[16384];
XrdOucString slocalprefix = localprefix;
std::string slocalprefix = localprefix;
if (!slocalprefix.endswith("/")) {
if (*slocalprefix.rbegin() != '/') {
slocalprefix += "/";
}
if (subindex) {
sprintf(sfullpath, "%s%08llx/%s.%u", slocalprefix.c_str(), fid / 10000,
hexstring, subindex);
} else {
sprintf(sfullpath, "%s%08llx/%s", slocalprefix.c_str(), fid / 10000, hexstring);
}
char sfullpath[16384];
sprintf(sfullpath, "%s%08llx/%s", slocalprefix.c_str(), fid / 10000, hexstring);
fullpath = sfullpath;
return fullpath;
}
//----------------------------------------------------------------------------
......
This diff is collapsed.
......@@ -138,11 +138,6 @@ public:
//----------------------------------------------------------------------------
void setId(fsid_t fsid);
//----------------------------------------------------------------------------
//! Set the draining status - durable.
//----------------------------------------------------------------------------
void setDrainStatus(DrainStatus status);
//----------------------------------------------------------------------------
//! Set the draining status - local.
//----------------------------------------------------------------------------
......@@ -405,9 +400,6 @@ protected:
//! to avoid race conditions in deletion.
XrdMqSharedObjectManager* mSom;
//! Handle to the drain queue
TransferQueue* mDrainQueue;
//! Handle to the balance queue
TransferQueue* mBalanceQueue;
......@@ -485,8 +477,11 @@ public:
long mDiskNameLen;
long mDiskRopen;
long mDiskWopen;
long mScanRate; ///< Maximum scan rate in MB/s
time_t mScanInterval;
long mScanIoRate; ///< Maximum scan rate in MB/s
long mScanEntryInterval; ///< Time after which a scanned file is rescanned
long mScanDiskInterval; ///< Time after which the disk scanner runs again
long mScanNsInterval; ///< Time after which the disk scanner runs again
long mScanNsRate; ///< Max ns scan rate in entries/s
time_t mGracePeriod;
time_t mDrainPeriod;
......@@ -513,7 +508,7 @@ public:
//! Fields which are not present in coreParams (ie mNetInRateMiB) remain
//! unchanged.
//--------------------------------------------------------------------------
void fillFromCoreParams(const FileSystemCoreParams &coreParams);
void fillFromCoreParams(const FileSystemCoreParams& coreParams);
};
struct host_snapshot_t {
......@@ -694,9 +689,9 @@ public:
//! Set the draining status
//----------------------------------------------------------------------------
bool
SetDrainStatus(DrainStatus status, bool broadcast = true)
SetDrainStatus(DrainStatus status)
{
return SetString("stat.drain", GetDrainStatusAsString(status), broadcast);
return SetString("stat.drain", GetDrainStatusAsString(status), false);
}
//----------------------------------------------------------------------------
......@@ -735,15 +730,6 @@ public:
return ParseDouble(GetString(key));
}
//----------------------------------------------------------------------------
//! Return handle to the drain queue.
//----------------------------------------------------------------------------
TransferQueue*
GetDrainQueue()
{
return mDrainQueue;
}
//----------------------------------------------------------------------------
//! Return handle to the balance queue.
//----------------------------------------------------------------------------
......
This diff is collapsed.
/************************************************************************
* EOS - the CERN Disk Storage System *
* Copyright (C) 2017 CERN/Switzerland *
......@@ -17,72 +18,62 @@
************************************************************************/
#pragma once
#include "fst/Namespace.hh"
#include "common/Namespace.hh"
#include "proto/FmdBase.pb.h"
#include "common/FileSystem.hh"
#include "common/Logging.hh"
#include "common/FileId.hh"
EOSFSTNAMESPACE_BEGIN
//------------------------------------------------------------------------------
//! Structure holding file metadata
//------------------------------------------------------------------------------
struct Fmd : public FmdBase {
public:
virtual ~Fmd() {}
};
EOSCOMMONNAMESPACE_BEGIN
//------------------------------------------------------------------------------
//! Class implementing file meta data
//! Class modelling the file metadata stored on the FST. It wrapps an FmdBase
//! class generated from the ProtoBuffer specification and adds some extra
//! functionality for conversion to/from string/env representation.
//------------------------------------------------------------------------------
class FmdHelper : public eos::common::LogId
{
public:
static constexpr uint64_t UNDEF = 0xfffffffffff1ULL;
//---------------------------------------------------------------------------
//! Constructor
//---------------------------------------------------------------------------
FmdHelper(eos::common::FileId::fileid_t fid = 0, int fsid = 0)
{
Reset();
mProtoFmd.set_fid(fid);
mProtoFmd.set_fsid(fsid);
}
//---------------------------------------------------------------------------
//! Destructor
//---------------------------------------------------------------------------
virtual ~FmdHelper() = default;
//---------------------------------------------------------------------------
//! Compute layout error
//!
//! @param fmd protobuf file meta data
//! @param fsid file system id to check against
//!
//! @return 0 if there are no errors, otherwise encoded type of layout error
//! stored in the int.
//---------------------------------------------------------------------------
static int LayoutError(const Fmd& fmd, eos::common::FileSystem::fsid_t fsid);
int LayoutError(eos::common::FileSystem::fsid_t fsid);
//---------------------------------------------------------------------------
//! Reset file meta data object
//!
//! @param fmd protobuf file meta data
//---------------------------------------------------------------------------
static void Reset(Fmd& fmd);
void Reset();
//---------------------------------------------------------------------------
//! Get set of locations for the given fmd
//!
//! @param fmd file metadata object
//! @param valid_replicas number of valid replicas <= size of the returned
//! set i.e. replicas which are not unlinked
//! Get set of valid locations (not unlinked) for the given fmd
//!
//! @return set of file system ids representing the locations
//---------------------------------------------------------------------------
static std::set<eos::common::FileSystem::fsid_t>
GetLocations(const Fmd& fmd, size_t& valid_replicas);
//---------------------------------------------------------------------------
//! Constructor
//---------------------------------------------------------------------------
FmdHelper(eos::common::FileId::fileid_t fid = 0, int fsid = 0): LogId()
{
Reset(mProtoFmd);
mProtoFmd.set_fid(fid);
mProtoFmd.set_fsid(fsid);
}
//---------------------------------------------------------------------------
//! Destructor
//---------------------------------------------------------------------------
virtual ~FmdHelper() = default;
std::set<eos::common::FileSystem::fsid_t> GetLocations() const;
//---------------------------------------------------------------------------
//! Convert fmd object to env representation
......@@ -91,23 +82,26 @@ public:
//---------------------------------------------------------------------------
std::unique_ptr<XrdOucEnv> FmdToEnv();
//---------------------------------------------------------------------------
//! Convert fmd object to env representation
//!
//! @return XrdOucEnv holding information about current object
//---------------------------------------------------------------------------
std::unique_ptr<XrdOucEnv> FullFmdToEnv();
//---------------------------------------------------------------------------
//! File meta data object replication function (copy constructor)
//---------------------------------------------------------------------------
void
Replicate(Fmd& fmd)
Replicate(FmdHelper& fmd)
{
mProtoFmd = fmd;
mProtoFmd = fmd.mProtoFmd;
}
Fmd mProtoFmd; ///< Protobuf file metadata info
eos::fst::FmdBase mProtoFmd; ///< Protobuf file metadata info
};
EOSFSTNAMESPACE_END
//------------------------------------------------------------------------------
//! Convert an FST env representation to an Fmd struct
//!
//! @param env env representation
//! @param fmd reference to Fmd struct
//!
//! @return true if successful, otherwise false
//------------------------------------------------------------------------------
bool EnvToFstFmd(XrdOucEnv& env, FmdHelper& fmd);
EOSCOMMONNAMESPACE_END
......@@ -34,8 +34,9 @@ GlobalConfig::gConfig; //! Singleton for global configuration access
// Get the global MGM configuration queue
//------------------------------------------------------------------------------
std::string
GlobalConfig::GetGlobalMgmConfigQueue() const {
return SSTR("/config/" << InstanceName::get() << "/mgm");
GlobalConfig::GetGlobalMgmConfigQueue() const
{
return SSTR("/config/" << InstanceName::get() << "/mgm/");
}
EOSCOMMONNAMESPACE_END
......@@ -151,19 +151,23 @@ public:
//--------------------------------------------------------------------------
//! Get a reed solomon layout by number of redundancystripess
//--------------------------------------------------------------------------
static int
GetReedSLayoutByParity(int redundancystripes)
static int
GetReedSLayoutByParity(int redundancystripes)
{
switch (redundancystripes) {
case 1:
return kRaid5;
case 2:
return kRaid6;
case 3:
return kArchive;
case 4:
return kQrain;
default:
default:
return 0;
}
}
......@@ -191,7 +195,7 @@ public:
// already set explicitly
if (redundancystripes == 0) {
if (layout == kRaid5) {
redundancystripes = 1;
redundancystripes = 1;
} else if (layout == kRaidDP) {
redundancystripes = 2;
} else if (layout == kRaid6) {
......@@ -199,8 +203,8 @@ public:
} else if (layout == kArchive) {
redundancystripes = 3;
} else if (layout == kQrain) {
redundancystripes = 4;
}
redundancystripes = 4;
}
}
id |= ((redundancystripes & 0x7) << 28);
......@@ -213,24 +217,31 @@ public:
static unsigned long
BlockSize(int blocksize)
{
switch (blocksize) {
case k4k:
return (4 * 1024);
case k64k:
return (64 * 1024);
case k128k:
return (128 * 1024);
case k512k:
return (512 * 1024);
case k1M:
return (1024 * 1024);
case k4M:
return (4 * 1024 * 1024);
case k16M:
return (16 * 1024 * 1024);
case k64M:
return (64 * 1024 * 1024);
default:
return 0;
}
......@@ -245,20 +256,28 @@ public:
switch (blocksize) {
case (4*1024):
return k4k;
case (64*1024):
return k64k;
case (128*1024):
return k128k;
case (512*1024):
return k512k;
case (1024 * 1024):
return k1M;
case (4 * 1024 * 1024):
return k4M;
case (16 * 1024 * 1024):
return k16M;
case (64 * 1024 * 1024):
return k64M;
default:
return 0;
}
......@@ -274,7 +293,7 @@ public:
}
//--------------------------------------------------------------------------
//! Get Length of Layout checksum in bytes
//! Get length of Layout checksum in bytes
//--------------------------------------------------------------------------
static unsigned long
GetChecksumLen(unsigned long layout)
......@@ -302,6 +321,27 @@ public:
return 0;
}
//--------------------------------------------------------------------------
//! Get length of Layout checksum in bytes
//--------------------------------------------------------------------------
static unsigned long
GetChecksumLen(const std::string& xs_type)
{
if (xs_type == "adler") {
return 4;
} else if (xs_type == "crc32") {
return 4;
} else if (xs_type == "crc32c") {
return 4;
} else if (xs_type == "md5") {
return 16;
} else if (xs_type == "sha") {
return 20;
} else {
return 0;
}
}
static std::string
GetEmptyFileChecksum(unsigned long layout)
{
......@@ -358,13 +398,12 @@ public:
//! Test for RAIN layout e.g. raid6,archive,qrain
//--------------------------------------------------------------------------
static bool
IsRainLayout(unsigned long layout)
IsRain(unsigned long layout)
{
// everything but plain and replica
return (GetLayoutType(layout)>kReplica);
return (GetLayoutType(layout) > kReplica);
}
//--------------------------------------------------------------------------
//! Set file layout type in the layout encoding
//!
......@@ -565,7 +604,7 @@ public:
static unsigned long
GetOnlineStripeNumber(unsigned long layout)
{
return (GetStripeNumber(layout)+1);
return (GetStripeNumber(layout) + 1);
}
//--------------------------------------------------------------------------
......@@ -772,13 +811,13 @@ public:
static std::string
GetStripeNumberString(unsigned long layout)
{
int n = GetStripeNumber(layout) + 1;
if (n<256)
if (n < 256) {
return std::to_string(n);
else
} else {
return "none";
}
}
//--------------------------------------------------------------------------
......@@ -936,7 +975,7 @@ public:
}
if (typ == "qrain") {
return kQrain;
return kQrain;
}
}
......
......@@ -211,7 +211,8 @@ std::string FileSystemLocator::getTransientChannel() const
//------------------------------------------------------------------------------
// Empty constructor
//------------------------------------------------------------------------------
SharedHashLocator::SharedHashLocator() {
SharedHashLocator::SharedHashLocator()
{
mInitialized = false;
}
......@@ -220,54 +221,61 @@ SharedHashLocator::SharedHashLocator() {
//
// Once we drop the MQ entirely, the instance name can be removed.
//------------------------------------------------------------------------------
SharedHashLocator::SharedHashLocator(const std::string &instanceName, Type type,
const std::string &name)
: mInitialized(true), mInstanceName(instanceName), mType(type), mName(name) {
switch(type) {
case Type::kSpace: {
mMqSharedHashPath = SSTR("/config/" << instanceName << "/space/" << name);
mBroadcastQueue = "/eos/*/mgm";
break;
}
case Type::kGroup: {
mMqSharedHashPath = SSTR("/config/" << instanceName << "/group/" << name);
mBroadcastQueue = "/eos/*/mgm";
break;
}
case Type::kNode: {
std::string hostPort = eos::common::StringConversion::GetHostPortFromQueue(name.c_str()).c_str();
mMqSharedHashPath = SSTR("/config/" << instanceName << "/node/" << hostPort);
mBroadcastQueue = SSTR("/eos/" << hostPort << "/fst");
break;
}
case Type::kGlobalConfigHash: {
mMqSharedHashPath = SSTR("/config/" << instanceName << "/mgm");
mBroadcastQueue = "/eos/*/mgm";
break;
}
default: {
eos_assert("should never reach here");
}
SharedHashLocator::SharedHashLocator(const std::string& instanceName, Type type,
const std::string& name)
: mInitialized(true), mInstanceName(instanceName), mType(type), mName(name)
{
switch (type) {
case Type::kSpace: {
mMqSharedHashPath = SSTR("/config/" << instanceName << "/space/" << name);
mBroadcastQueue = "/eos/*/mgm";
break;
}
case Type::kGroup: {
mMqSharedHashPath = SSTR("/config/" << instanceName << "/group/" << name);
mBroadcastQueue = "/eos/*/mgm";
break;
}
case Type::kNode: {
std::string hostPort = eos::common::StringConversion::GetHostPortFromQueue(
name.c_str()).c_str();
mMqSharedHashPath = SSTR("/config/" << instanceName << "/node/" << hostPort);
mBroadcastQueue = SSTR("/eos/" << hostPort << "/fst");
break;
}
case Type::kGlobalConfigHash: {
mMqSharedHashPath = SSTR("/config/" << instanceName << "/mgm/");
mBroadcastQueue = "/eos/*/mgm";
break;
}
default: {
eos_assert("should never reach here");
}
}
}
//------------------------------------------------------------------------------
//! Constructor: Same as above, but auto-discover instance name.
//------------------------------------------------------------------------------
SharedHashLocator::SharedHashLocator(Type type, const std::string &name)
: SharedHashLocator(InstanceName::get(), type, name) {}
SharedHashLocator::SharedHashLocator(Type type, const std::string& name)
: SharedHashLocator(InstanceName::get(), type, name) {}
//------------------------------------------------------------------------------
// Constructor: Special case for FileSystems, as they work a bit differently
// than the rest.
//------------------------------------------------------------------------------
SharedHashLocator::SharedHashLocator(const FileSystemLocator &fsLocator, bool bc2mgm) {
SharedHashLocator::SharedHashLocator(const FileSystemLocator& fsLocator,
bool bc2mgm)
{
mInitialized = true;
mMqSharedHashPath = fsLocator.getQueuePath();
mBroadcastQueue = fsLocator.getFSTQueue();
if(bc2mgm) {
if (bc2mgm) {
mBroadcastQueue = "/eos/*/mgm";
}
}
......@@ -275,58 +283,67 @@ SharedHashLocator::SharedHashLocator(const FileSystemLocator &fsLocator, bool bc
//------------------------------------------------------------------------------
//! Convenience "Constructors": Make locator for space, group, node
//------------------------------------------------------------------------------
SharedHashLocator SharedHashLocator::makeForSpace(const std::string &name) {
SharedHashLocator SharedHashLocator::makeForSpace(const std::string& name)
{
return SharedHashLocator(Type::kSpace, name);
}
SharedHashLocator SharedHashLocator::makeForGroup(const std::string &name) {
SharedHashLocator SharedHashLocator::makeForGroup(const std::string& name)
{
return SharedHashLocator(Type::kGroup, name);
}
SharedHashLocator SharedHashLocator::makeForNode(const std::string &name) {
SharedHashLocator SharedHashLocator::makeForNode(const std::string& name)
{
return SharedHashLocator(Type::kNode, name);
}
SharedHashLocator SharedHashLocator::makeForGlobalHash() {
SharedHashLocator SharedHashLocator::makeForGlobalHash()
{
return SharedHashLocator(Type::kGlobalConfigHash, "");
}
//------------------------------------------------------------------------------
// Get "config queue" for shared hash
//------------------------------------------------------------------------------
std::string SharedHashLocator::getConfigQueue() const {
std::string SharedHashLocator::getConfigQueue() const
{
return mMqSharedHashPath;
}
//------------------------------------------------------------------------------
// Get "broadcast queue" for shared hash
//------------------------------------------------------------------------------
std::string SharedHashLocator::getBroadcastQueue() const {
std::string SharedHashLocator::getBroadcastQueue() const
{
return mBroadcastQueue;
}
//----------------------------------------------------------------------------
// Check if this object is actually pointing to something
//----------------------------------------------------------------------------
bool SharedHashLocator::empty() const {
bool SharedHashLocator::empty() const
{
return !mInitialized;
}
//------------------------------------------------------------------------------
// Produce SharedHashLocator by parsing config queue
//------------------------------------------------------------------------------
bool SharedHashLocator::fromConfigQueue(const std::string &configQueue, SharedHashLocator &out) {
bool SharedHashLocator::fromConfigQueue(const std::string& configQueue,
SharedHashLocator& out)
{
std::vector<std::string> parts =
common::StringTokenizer::split<std::vector<std::string>>(configQueue, '/');
std::reverse(parts.begin(), parts.end());
if(parts.empty() || parts.back() != "config") {
if (parts.empty() || parts.back() != "config") {
return false;
}
parts.pop_back();
if(parts.empty()) {
if (parts.empty()) {
return false;
}
......@@ -334,41 +351,36 @@ bool SharedHashLocator::fromConfigQueue(const std::string &configQueue, SharedHa
std::string instanceName = parts.back();
parts.pop_back();
if(parts.empty()) {
if (parts.empty()) {
return false;
}
else if(parts.back() == "node") {
} else if (parts.back() == "node") {
type = Type::kNode;
}
else if(parts.back() == "space") {
} else if (parts.back() == "space") {
type = Type::kSpace;
}
else if(parts.back() == "group") {
} else if (parts.back() == "group") {
type = Type::kGroup;
}
else if(parts.back() == "mgm" && parts.size() == 1u) {
} else if (parts.back() == "mgm" && parts.size() == 1u) {
type = Type::kGlobalConfigHash;
out = SharedHashLocator(instanceName, type, "");
return true;
}
else {
} else {
return false;
}
parts.pop_back();
if(parts.empty()) {
if (parts.empty()) {
return false;
}
std::string name = parts.back();
parts.pop_back();
if(!parts.empty()) {
if (!parts.empty()) {
return false;
}
if(instanceName.empty() || name.empty()) {
if (instanceName.empty() || name.empty()) {
return false;
}
......
......@@ -102,7 +102,7 @@ EOSCOMMONNAMESPACE_BEGIN
//------------------------------------------------------------------------------
#define eos_log(__EOSCOMMON_LOG_PRIORITY__ , ...) \
eos::common::Logging::GetInstance().log(__FUNCTION__,__FILE__, __LINE__, this->logId, \
vid, this->cident, LOG_MASK(__EOSCOMMON_LOG_PRIORITY__), __VA_ARGS__)
vid, this->cident, __EOSCOMMON_LOG_PRIORITY__, __VA_ARGS__)
#define eos_debug(...) \
eos::common::Logging::GetInstance().log(__FUNCTION__,__FILE__, __LINE__, this->logId, \
vid, this->cident, (LOG_DEBUG), __VA_ARGS__)
......@@ -278,7 +278,7 @@ public:
void
SetLogId(const char* newlogid, const char* td)
{
if (newlogid != logId) {
if (newlogid && (newlogid != logId)) {
snprintf(logId, sizeof(logId) - 1, "%s", newlogid);
}
......
......@@ -66,6 +66,10 @@ std::map<std::string, uid_t> Mapping::gPhysicalUserIdCache;
std::map<std::string, gid_t> Mapping::gPhysicalGroupIdCache;
Mapping::ip_cache Mapping::gIpCache(300);
OAuth Mapping::gOAuth;
/*----------------------------------------------------------------------------*/
/**
* Initialize Google maps
......@@ -85,6 +89,9 @@ Mapping::Init()
!strcmp("1", getenv("EOS_FUSE_NO_ROOT_SQUASH"))) {
gRootSquash = false;
}
gOAuth.Init();