...
 
Commits (125)
......@@ -36,6 +36,8 @@ __pycache__
kineticio-dist.tgz
# clion specific configs
/cmake-build-debug/
/cmake-build-relwithdebinfo/
/cmake-build-release/
/.idea/
# eclipse specific configs
/.settings/
......
This diff is collapsed.
......@@ -27,7 +27,7 @@
#include "XrdOfs/XrdOfs.hh"
#include "Namespace.hh"
#include "common/ConcurrentQueue.hh"
#include "mgm/ZMQ.hh"
#include <zmq.hpp>
#include <string>
//! Forward declaration
......
......@@ -492,6 +492,7 @@ FileSystem::SnapShotFileSystem(FileSystem::fs_snapshot_t& fs, bool dolock)
fs.mHostPort = hash->Get("hostport");
fs.mProxyGroup = hash->Get("proxygroup");
fs.mS3Credentials = hash->Get("s3credentials");
fs.mLogicalPath = hash->Get("logicalpath");
fs.mFileStickyProxyDepth = -1;
if (hash->Get("filestickyproxydepth").size()) {
......@@ -595,6 +596,7 @@ FileSystem::SnapShotFileSystem(FileSystem::fs_snapshot_t& fs, bool dolock)
fs.mHostPort = "";
fs.mProxyGroup = "";
fs.mS3Credentials = "";
fs.mLogicalPath = "";
fs.mFileStickyProxyDepth = -1;
fs.mPort = "";
fs.mErrMsg = "";
......
......@@ -115,6 +115,7 @@ public:
std::string mHostPort;
std::string mProxyGroup;
std::string mS3Credentials;
std::string mLogicalPath;
int8_t mFileStickyProxyDepth;
std::string mPort;
std::string mGeoTag;
......
......@@ -31,7 +31,7 @@
/*----------------------------------------------------------------------------*/
//! Macros defining the comman namespace
//! Macros defining the common namespace
/*----------------------------------------------------------------------------*/
#ifndef __EOSCOMMON_NAMESPACE_HH__
......
......@@ -577,7 +577,7 @@ private:
#endif
//------------------------------------------------------------------------------
//! Class RWMutexReadLock
//! Class RWMutexWriteLock
//------------------------------------------------------------------------------
class RWMutexWriteLock
{
......@@ -656,7 +656,7 @@ private:
};
//------------------------------------------------------------------------------
//! RW Mutex prefereing the reader
//! RW Mutex preferring the reader
//------------------------------------------------------------------------------
class RWMutexR : public RWMutex
{
......@@ -666,7 +666,7 @@ public:
};
//------------------------------------------------------------------------------
//! RW Mutex prefereing the writerr
//! RW Mutex preferring the writer
//------------------------------------------------------------------------------
class RWMutexW : public RWMutex
{
......
......@@ -876,6 +876,27 @@ StringConversion::CreateUrl(const char* protocol, const char* hostport,
return 0;
}
//------------------------------------------------------------------------------
// Builds the physical path of a file on a filesystem,
// given that filesystem's local prefix and the file path suffix.
//------------------------------------------------------------------------------
XrdOucString
StringConversion::BuildPhysicalPath(const char *localprefix,
const char *pathsuffix)
{
XrdOucString physicalPath = localprefix;
int lprefixLen = physicalPath.length();
if (!physicalPath.endswith("/")) {
physicalPath += "/";
}
physicalPath += pathsuffix;
physicalPath.replace("//", "/", lprefixLen - 1);
return physicalPath;
}
//------------------------------------------------------------------------------
// Check if string is hex number
//------------------------------------------------------------------------------
......
......@@ -486,6 +486,19 @@ public:
CreateUrl(const char* protocol, const char* hostport, const char* path,
XrdOucString& url);
// ---------------------------------------------------------------------------
/**
* Builds the physical path of a file on a filesystem,
* given that filesystem's local prefix and the file path suffix.
*
* @param localprefix the filesystem local prefix
* @param pathsuffix the file path suffix
* @return the file physical path
*/
// ---------------------------------------------------------------------------
static XrdOucString
BuildPhysicalPath(const char* localprefix, const char* pathsuffix);
// ---------------------------------------------------------------------------
/**
* Check if a string is a hexadecimal number
......
......@@ -80,7 +80,7 @@ public:
}
//----------------------------------------------------------------------------
//! Get time elapsed between the two tags in miliseconds
//! Get time elapsed between the two tags in milliseconds
//----------------------------------------------------------------------------
float
GetTagTimelapse(const std::string& tagBegin, const std::string& tagEnd)
......@@ -257,11 +257,87 @@ public:
}
}
//----------------------------------------------------------------------------
//! Wrapper Function to hide difference between Apple and Linux
//----------------------------------------------------------------------------
static void
GetTimeSpec(struct timespec& ts, bool coarse = false)
{
#ifdef __APPLE__
struct timeval tv;
gettimeofday(&tv, 0);
ts.tv_sec = tv.tv_sec;
ts.tv_nsec = tv.tv_usec * 1000;
#else
if (coarse) {
#ifdef CLOCK_REALTIME_COARSE
_clock_gettime(CLOCK_REALTIME_COARSE, &ts);
#else
_clock_gettime(CLOCK_REALTIME, &ts);
#endif
} else {
_clock_gettime(CLOCK_REALTIME, &ts);
}
#endif
}
//----------------------------------------------------------------------------
//! Convert a timespec string representation to timespec.
//! (Timespec string: tv_sec.tv_nsec)
//!
//! Returns 0 for successful conversion, -1 otherwise
//!
//! Note: the function resets the value of errno
//----------------------------------------------------------------------------
static int
TimespecString_to_Timespec(std::string tsString, struct timespec &ts)
{
size_t pos = tsString.find(".");
const char *nptr = tsString.c_str();
char *endptr = NULL;
errno = 0;
if (pos == std::string::npos) {
ts.tv_sec = strtoull(nptr, &endptr, 10);
ts.tv_nsec = 0;
} else {
nptr = tsString.substr(0, pos).c_str();
ts.tv_sec = strtoull(nptr, &endptr, 10);
ts.tv_nsec = strtoull(tsString.substr(pos + 1, 9).c_str(), 0, 10);
}
// Failed conversion or no digits found
if ((errno != 0) || (nptr == endptr)) {
return -1;
}
return 0;
}
//----------------------------------------------------------------------------
//! Convert a timespec string representation to nanoseconds.
//! (Timespec string: tv_sec.tv_nsec)
//!
//! Returns time in nanoseconds if successful, -1 otherwise
//!
//! Note: the function resets the value of errno
//----------------------------------------------------------------------------
static long long
TimespecString_to_Ns(std::string tsString)
{
struct timespec ts;
int rc = TimespecString_to_Timespec(tsString, ts);
return (rc == 0) ? (ts.tv_sec * 1000000000 + ts.tv_nsec) : -1;
}
//----------------------------------------------------------------------------
//! Time Conversion Function for timestamp time strings
//----------------------------------------------------------------------------
static std::string
UnixTimstamp_to_Day(time_t when)
UnixTimestamp_to_Day(time_t when)
{
struct tm* now = localtime(&when);
std::string year;
......@@ -289,37 +365,11 @@ public:
return ts;
}
//----------------------------------------------------------------------------
//! Wrapper Function to hide difference between Apple and Linux
//----------------------------------------------------------------------------
static void
GetTimeSpec(struct timespec& ts, bool coarse = false)
{
#ifdef __APPLE__
struct timeval tv;
gettimeofday(&tv, 0);
ts.tv_sec = tv.tv_sec;
ts.tv_nsec = tv.tv_usec * 1000;
#else
if (coarse) {
#ifdef CLOCK_REALTIME_COARSE
_clock_gettime(CLOCK_REALTIME_COARSE, &ts);
#else
_clock_gettime(CLOCK_REALTIME, &ts);
#endif
} else {
_clock_gettime(CLOCK_REALTIME, &ts);
}
#endif
}
//----------------------------------------------------------------------------
//! Time Conversion Function for ISO8601 time strings
//----------------------------------------------------------------------------
static std::string
UnixTimstamp_to_ISO8601(time_t now)
UnixTimestamp_to_ISO8601(time_t now)
{
struct tm* utctime;
char str[21];
......
......@@ -164,6 +164,10 @@ void com_fs_help()
<< " s3credentials=<accesskey>:<secretkey>" << std::endl
<< " the access and secret key pair used to authenticate" << std::endl
<< " with the S3 storage endpoint" << std::endl
<< " logicalpath=<true>|<false>" << std::endl
<< " keep the file names on the storage endpoint" << std::endl
<< " as they appear in the namespace " << std::endl
<< " True and 1 are synonyms." << std::endl
<< std::endl
<< " fs dropdeletion <fsid> " << std::endl
<< " drop all pending deletions on the filesystem" << std::endl
......@@ -186,6 +190,19 @@ void com_fs_help()
<< " -m : print full metadata record in env format" << std::endl
<< " -s : silent mode (will keep an internal reference)" << std::endl
<< std::endl
<< " fs import start <src_fsid> <ext_path> <lcl_path>" << std::endl
<< " scan an external endpoint and register all files" << std::endl
<< " into the namespace as belonging to the given filesystem" << std::endl
<< " at the specified local path"
<< std::endl
<< " src_fsid : source filesystem identifier" <<std::endl
<< " ext_path : complete path to an external location" << std::endl
<< " lcl_path : path to a location within the selected" << std::endl
<< " filesystem" << std::endl
<< " fs import query <import_id>" << std::endl
<< " perform status query about ongoing import procedure" << std::endl
<< " import_id : id to identify import operation" << std::endl
<< std::endl
<< " fs ls [-m|-l|-e|--io|--fsck|-d|--drain] [-s] [-b|--brief] [[matchlist]]"
<< std::endl
<< " list filesystems using the default output format" << std::endl
......
......@@ -452,6 +452,71 @@ FsHelper::ParseCommand(const char* arg)
}
}
}
} else if (cmd == "import") {
using eos::console::FsProto_ImportProto;
FsProto_ImportProto* import = fs->mutable_import();
if (!(option = tokenizer.GetToken())) {
return false;
} else {
soption = option;
if (soption == "start") {
import->set_command(FsProto_ImportProto::START);
if (!(option = tokenizer.GetToken())) {
std::cerr << "error: missing <fsid>" << std::endl;
return false;
}
soption = option;
// Parse fsid
try {
uint64_t fsid = std::stoull(soption);
import->set_fsid(fsid);
} catch (const std::exception& e) {
std::cerr << "error: fsid needs to be numeric" << std::endl;
return false;
}
if (!(option = tokenizer.GetToken())) {
std::cerr << "error: missing <external_path>" << std::endl;
return false;
}
soption = option;
import->set_externalpath(soption);
if (!(option = tokenizer.GetToken())) {
std::cerr << "error: missing <local_path>" << std::endl;
return false;
}
soption = option;
// Local path must be an absolute path
if (soption.find("/") != 0) {
std::cerr << "error: <local_path> must be an absolute path"
<< std::endl;
return false;
}
import->set_localpath(soption);
} else if (soption == "query") {
import->set_command(FsProto_ImportProto::QUERY);
if (!(option = tokenizer.GetToken())) {
std::cerr << "error: missing <import_id>" << std::endl;
return false;
}
soption = option;
import->set_importid(soption);
} else {
std::cerr << "error: unknown import command <"
<< option << ">" << std::endl;
return false;
}
}
} else if (cmd == "rm") {
using eos::console::FsProto_RmProto;
FsProto_RmProto* rm = fs->mutable_rm();
......
......@@ -3,6 +3,7 @@ usr/sbin/eos-fuse-test
usr/sbin/eos-fusex-certify
usr/sbin/eos-instance-test
usr/sbin/eos-instance-test-ci
usr/sbin/eos-extstorage-test
usr/sbin/eos-io-test
usr/sbin/eos-io-tool
usr/sbin/eos-oc-test
......
......@@ -28,7 +28,9 @@ This chapter discusses several components of EOS and how they are configured.
configuration/geotags
configuration/groupbalancer
configuration/http
configuration/import
configuration/kinetic
configuration/logicalpath
configuration/lru
configuration/master
configuration/master_quarkdb
......
.. highlight:: rst
.. index::
triple: Import; HTTP; S3
Importing files
===============
EOS can use multiple protocols when communicating with a storage device.
Most commonly, the data is stored on local disks. However, data can also be
stored on an external storage system, such as an S3, WebDAV or XRootD endpoint.
More information can be found `here <logicalpath.html>`__.
Importing files works with any underlying storage device
as file importation is a filesystem operation.
This means the endpoint must be mapped and accessible
through an EOS filesystem.
The importation procedure does a scan of the remote path,
recursively traversing all directories from this point on.
All discovered files will be imported into the namespace,
together with their size and time of creation/modification.
Upon registering the files into the namespace,
they can be managed like any other EOS file.
.. note::
File importation is only a metadata operation.
The files themselves will not be moved.
Configuration
-------------
Preconditions
+++++++++++++
To set-up the importation procedure, a filesystem must be
associated with that particular endpoint.
Registering a filesystem which uses an external endpoint for storage
is done the same way. For example, to import from ``s3://s3.cern.ch/bucket``,
we would do the following:
.. code-block:: bash
eos fs add [-m 1] <uuid> s3-fst.cern.ch:1095 s3://s3.cern.ch/bucket/ s3 rw
Once registered, make sure the filesystem is online:
.. code-block:: bash
┌────────────────────────┬────┬──────┬──────────────────────────────────────────┬────────────────┬────────────────┬────────────┬──────────────┬────────────┬────────┬────────────────┐
│host │port│ id│ path│ schedgroup│ geotag│ boot│ configstatus│ drainstatus│ active│ health│
└────────────────────────┴────┴──────┴──────────────────────────────────────────┴────────────────┴────────────────┴────────────┴──────────────┴────────────┴────────┴────────────────┘
s3-fst.cern.ch 1095 1 s3://s3.cern.ch/bucket/ s3.0 xdc booted rw nodrain online N/A
Filesystem set-up
+++++++++++++++++
Although not mandatory, configuring the filesystem to use `logicalpath <logicalpath.html>`__
will be needed in most cases.
.. code-block:: bash
eos fs config <fsid> logicalpath=1
Depending on the remote endpoint, additional configuration might be needed.
S3
##
FSTs with S3 filesystems registered will need an access/secret key pair.
These can be done on a filesystem basis or FST wide.
To set keys only for a particular, use the ``fs config`` option:
.. code-block:: bash
eos fs config <fsid> s3credentials=<accesskey>:<secretkey>
To set keys FST wide, export the following environment variables:
.. code-block:: bash
export EOS_FST_S3_ACCESS_KEY=<accesskey>
export EOS_FST_S3_SECRET_KEY=<secretkey>
WebDAV with x509
################
To access WebDAV storage endpoints, you will need a way to authenticate.
Support is provided for x509 certificates.
The setting applies FST wide and is retrieved
from the following environment variable:
.. code-block:: bash
export EOS_FST_HTTPS_X509_CERTIFICATE_PAT=/path/to/x509/certificate
Import procedure
----------------
The import procedure is triggered via the `fs import start` admin command,
which is documented `here <../clicommands/fs.html>`__.
It offers the option to start an import procedure
or to check the status of an ongoing import.
When starting a new import operation, the command is sent to the MGM
which does the following checks:
- external path begins with the filesystem local prefix
- destination path is a directory
- destination path is in same scheduling group as the filesystem
A new id, together with an import status object
are generated for this import operation.
A signal is sent to the responsible FST,
which will place the import request in a queue.
Import requests are processed on a dedicated thread.
This will do a traversal of the endpoint path.
For each file discovered, a stat is performed to retrieve
needed information. This info is encoded into a message
which is sent back to the MGM.
The MGM receives this message, updates the namespace
and the import status object.
.. highlight:: rst
.. index::
pair: Logicalpath; Import
Logicalpath
===========
The way files are stored on EOS filesystems is transparent to the users.
They identify a file in the namespace either via a path
or a file id (fid / fxid). How this translates into the placement
of the file is the responsibility of the system.
So far, the physical location of files was computed from the file id.
With the introduction of the logical path setting,
filesystems may store files under a 'path-like' location.
Example:
.. code-block:: bash
/fst/00000000/0000a12d vs /fst/eos/instance/path/filename
To activate the logical path setting,
the following command must be executed:
.. code-block:: bash
eos fs config <fsid> logicalpath=1
The setting can be switched on or off at any time, with immediate effects.
Translation mechanism
---------------------
File created on a filesystem with the logicalpath setting on will have
an additional extended attribute ``sys.eos.lpath``
which keeps a mapping of the file's location.
The extended attribute stores string information of a file's
physical location on a given file system.
Operations regarding the file's physical path make use of a namespace
utility class which takes into account both the possibility of a logicalpath,
as well as constructing the path from the file's id.
For example, the file ``/eos/s3/testfile`` with 3 replicas,
with 2 of those using logicalpath, will look like this:
.. code-block:: bash
> eos fileinfo /eos/s3/testfile --fullpath
...
┌───┬──────┬────────────────────────┬────────────────┬────────────────────────┬──────────┬──────────────┬────────────┬────────┬────────────────────────┬───────────────────────────────────────┐
│no.│ fs-id│ host│ schedgroup│ path│ boot│ configstatus│ drainstatus│ active│ geotag│ physical location│
└───┴──────┴────────────────────────┴────────────────┴────────────────────────┴──────────┴──────────────┴────────────┴────────┴────────────────────────┴───────────────────────────────────────┘
0 1 xdc-fst1.cern.ch default.0 /fst1 booted rw nodrain online xdc /fst1/00000013/0002f914
1 2 xdc-fst1.cern.ch lpath.0 /fst1_lpath booted rw nodrain online xdc /fst1_lpath/eos/s3/testfile
2 3 xdc-fst2.cern.ch s3.0 s3://s3.cern.ch/bucket/ booted rw nodrain online xdc s3://s3.cern.ch/bucket/eos/s3/testfile
> eos attr ls /eos/s3/testfile
sys.eos.lpath="2|/eos/s3/testfile&3|/eos/s3/testfile"
The two replicas, stored on filesystems using logicalpath,
can be seen in the file's extended attributes.
......@@ -100,6 +100,7 @@ BuildRequires: jemalloc, jemalloc-devel
BuildRequires: glibc-headers
BuildRequires: binutils-devel
BuildRequires: cppunit-devel
BuildRequires: davix-devel
BuildRequires: help2man
%if %{?_with_server:1}%{!?_with_server:0}
......@@ -135,8 +136,10 @@ BuildRequires: centos-release-scl
%if 0%{distribution} == 6 || 0%{distribution} == 7
BuildRequires: devtoolset-6
BuildRequires: devtoolset-6-binutils-devel
BuildRequires: devtoolset-6-libasan-devel
%else
BuildRequires: binutils-devel
BuildRequires: libasan
%endif
%if %{?_with_asan:1}%{!?_with_asan:0}
......@@ -736,7 +739,7 @@ esac
Summary: The EOS test package
Group: Applications/File
Requires: cadaver bc davix
Requires: cadaver bc davix dmidecode
%description -n eos-test
Contains an instance and fuse test script and some test executables and test archives.
......@@ -749,6 +752,7 @@ Contains an instance and fuse test script and some test executables and test arc
%{_sbindir}/eos-mq-tests
%{_sbindir}/eos-instance-test
%{_sbindir}/eos-instance-test-ci
%{_sbindir}/eos-extstorage-test
%{_sbindir}/eos-rain-test
%{_sbindir}/eos-drain-test
%{_sbindir}/eos-http-upload-test
......
......@@ -299,10 +299,10 @@ set(XRDEOSFST_SRCS
XrdFstOfsFile.cc XrdFstOfsFile.hh
# Storage interface
storage/Balancer.cc
storage/Cleaner.cc storage/Communicator.cc
storage/Drainer.cc storage/ErrorReport.cc
storage/FileSystem.cc storage/MgmSyncer.cc
storage/Balancer.cc storage/Cleaner.cc
storage/Communicator.cc storage/Drainer.cc
storage/ErrorReport.cc storage/FileSystem.cc
storage/ImportScan.cc storage/MgmSyncer.cc
storage/Publish.cc storage/Remover.cc
storage/Report.cc storage/Scrub.cc
storage/Storage.cc storage/Supervisor.cc
......
......@@ -38,6 +38,7 @@ class Config
public:
bool autoBoot; // -> indicates if the node tries to boot automatically or waits for a boot message from a master
XrdOucString FstMetaLogDir; // Directory containing the meta data log files
XrdOucString FstAuthDir; // Directory needed for file transfers among FSTs
XrdOucString FstOfsBrokerUrl; // Url of the message broker
XrdOucString
FstDefaultReceiverQueue; // Queue where we are sending to by default
......
......@@ -42,16 +42,25 @@ EOSFSTNAMESPACE_BEGIN
/*----------------------------------------------------------------------------*/
class Deletion
{
private:
class FileDeletion
{
public:
unsigned long long fId;
XrdOucString logicalPath;
XrdOucString cTime;
};
public:
std::vector<unsigned long long> fIdVector;
std::vector<FileDeletion> fileVector;
unsigned long fsId;
XrdOucString localPrefix;
XrdOucString managerId;
XrdOucString opaque;
Deletion (std::vector<unsigned long long> &idvector, unsigned long fsid, const char* localprefix, const char* managerid, const char* inopaque)
Deletion (std::vector<FileDeletion> &filevector, unsigned long fsid, const char* localprefix, const char* managerid, const char* inopaque)
{
fIdVector = idvector;
fileVector = filevector;
fsId = fsid;
localPrefix = localprefix;
managerId = managerid;
......@@ -66,13 +75,15 @@ public:
XrdOucString hexfids = "";
XrdOucString hexfid = "";
XrdOucString access = "";
XrdOucString lpath = "";
XrdOucString ctime = "";
const char* sfsid = 0;
const char* smanager = 0;
std::vector <unsigned long long> idvector;
unsigned long long fileid = 0;
unsigned long fsid = 0;
std::vector <FileDeletion> filevector;
FileDeletion *fDeletion;
localprefix = capOpaque->Get("mgm.localprefix");
hexfids = capOpaque->Get("mgm.fids");
sfsid = capOpaque->Get("mgm.fsid");
......@@ -99,8 +110,25 @@ public:
hexfid = subtokenizer.GetToken();
if (hexfid.length())
{
fileid = eos::common::FileId::Hex2Fid(hexfid.c_str());
idvector.push_back(fileid);
fDeletion = new FileDeletion();
// extract logical path and creation time -- format hexfid[:lpath:ctime]
int pos1 = hexfid.find(":");
if (pos1 != STR_NPOS) {
int pos2 = hexfid.find(":", pos1 + 1);
lpath = ctime = hexfid;
lpath.keep(pos1 + 1, pos2 - pos1 - 1);
ctime.keep(pos2 + 1);
hexfid.erase(pos1);
fDeletion->logicalPath = lpath;
fDeletion->cTime = ctime;
}
// convert hexfid to fid
fDeletion->fId = eos::common::FileId::Hex2Fid(hexfid.c_str());
filevector.push_back(*fDeletion);
}
else
{
......@@ -109,7 +137,7 @@ public:
}
fsid = atoi(sfsid);
return new Deletion(idvector, fsid, localprefix, smanager, capOpaque->Env(envlen));
return new Deletion(filevector, fsid, localprefix, smanager, capOpaque->Env(envlen));
};
~Deletion () { };
......
//------------------------------------------------------------------------------
// File: MonitorVarPartitionTest.hh
// Author: Jozsef Makai <jmakai@cern.ch>
//------------------------------------------------------------------------------
// ----------------------------------------------------------------------
// File: ImportScan.hh
// Author: Mihai Patrascoiu - CERN
// ----------------------------------------------------------------------
/************************************************************************
* EOS - the CERN Disk Storage System *
* Copyright (C) 2017 CERN/Switzerland *
* Copyright (C) 2018 CERN/Switzerland *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
......@@ -21,15 +21,86 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.*
************************************************************************/
#ifndef EOS_MONITORVARPARTITIONTEST_HH
#define EOS_MONITORVARPARTITIONTEST_HH
#ifndef __EOSFST_IMPORTSCAN_HH__
#define __EOSFST_IMPORTSCAN_HH__
#include "Namespace.hh"
/*----------------------------------------------------------------------------*/
#include "fst/Namespace.hh"
#include "common/FileId.hh"
/*----------------------------------------------------------------------------*/
#include "XrdOuc/XrdOucString.hh"
#include "XrdOuc/XrdOucEnv.hh"
#include "XrdOuc/XrdOucTokenizer.hh"
EOSFSTTEST_NAMESPACE_BEGIN
/*----------------------------------------------------------------------------*/
void VarPartitionMonitoringTest();
class XrdOucEnv;
EOSFSTTEST_NAMESPACE_END
EOSFSTNAMESPACE_BEGIN
#endif //EOS_MONITORVARPARTITIONTEST_HH
/*----------------------------------------------------------------------------*/
class ImportScan
{
public:
unsigned long fsId;
XrdOucString id;
XrdOucString managerId;
XrdOucString extPath;
XrdOucString lclPath;
XrdOucString opaque;
ImportScan(const char* importid, unsigned long fsid, const char* managerid,
const char* extpath, const char* lclpath, const char* inopaque)
{
id = importid;
fsId = fsid;
managerId = managerid;
extPath = extpath;
lclPath = lclpath;
opaque = inopaque;
}
static ImportScan*
Create(XrdOucEnv* capOpaque)
{
// decode the opaque tags
const char* id = 0;
const char* sfsid = 0;
const char* smanager = 0;
const char* extPath = 0;
const char* lclPath = 0;
unsigned long fsid = 0;
id = capOpaque->Get("mgm.id");
sfsid = capOpaque->Get("mgm.fsid");
smanager = capOpaque->Get("mgm.manager");
extPath = capOpaque->Get("mgm.extpath");
lclPath = capOpaque->Get("mgm.lclpath");
if (!id || !sfsid || !smanager || !extPath || !lclPath) {
return 0;
}
int envlen = 0;
fsid = atoi(sfsid);
return new ImportScan(id, fsid, smanager, extPath,
lclPath, capOpaque->Env(envlen));
};
~ImportScan() { };
//----------------------------------------------------------------------------
//! Display information about current import scan job
//----------------------------------------------------------------------------
void
Show(const char* show = "")
{
eos_static_info("ImportScan[id=%s] fs=%u external_path=%s local_path=%s %s",
id.c_str(), fsId, extPath.c_str(), lclPath.c_str(), show);
}
};
EOSFSTNAMESPACE_END
#endif
......@@ -27,6 +27,7 @@
#include "fst/Messaging.hh"
#include "fst/Deletion.hh"
#include "fst/Verify.hh"
#include "fst/ImportScan.hh"
#include "fst/XrdFstOfs.hh"
#include "fst/FmdDbMap.hh"
#include "common/ShellCmd.hh"
......@@ -53,7 +54,7 @@ Messaging::Listen(ThreadAssistant& assistant) noexcept
}
//------------------------------------------------------------------------------
// Process incomming messages
// Process incoming messages
//------------------------------------------------------------------------------
void
Messaging::Process(XrdMqMessage* newmessage)
......@@ -167,6 +168,20 @@ Messaging::Process(XrdMqMessage* newmessage)
}
}
if (cmd == "importscan") {
eos_info("importscan");
XrdOucEnv* capOpaque = &action;
int envlen = 0;
eos_debug("opaque is %s", capOpaque->Env(envlen));
ImportScan* new_importScan = ImportScan::Create(capOpaque);
if (new_importScan) {
gOFS.Storage->PushImportScan(new_importScan);
} else {
eos_err("Cannot create an importScan entry - illegal opaque information");
}
}
if (cmd == "resync") {
eos::common::FileSystem::fsid_t fsid = (action.Get("mgm.fsid") ? strtoul(
action.Get("mgm.fsid"), 0, 10) : 0);
......
......@@ -793,8 +793,16 @@ ScanDir::ScanFileLoadAware(const std::unique_ptr<eos::fst::FileIo>& io,
eos::fst::CheckSum* normalXS, *blockXS;
scansize = 0;
scantime = 0;
filePath = io->GetPath();
fileXSPath = filePath + ".xsmap";
fileXSPath = filePath = io->GetPath();
// TODO: Refactor FileIo to provide path, opaque and fullpath
// File path might have opaque info
size_t insertPos = filePath.rfind("?");
if (insertPos == std::string::npos) {
insertPos = filePath.length();
}
fileXSPath.insert(insertPos, ".xsmap");
normalXS = eos::fst::ChecksumPlugins::GetChecksumObject(layoutid);
gettimeofday(&opentime, &tz);
struct stat current_stat;
......
......@@ -52,6 +52,7 @@ public:
XrdOucString opaque;
XrdOucString container;
XrdOucString path;
XrdOucString lPath;
bool computeChecksum;
bool commitChecksum;
......@@ -63,8 +64,8 @@ public:
Verify(unsigned long long fid, unsigned long fsid, const char* localprefix,
const char* managerid, const char* inopaque, const char* incontainer,
unsigned long incid, unsigned long inlid, const char* inpath,
bool inComputeChecksum, bool inCommitChecksum, bool inCommitSize,
bool inCommitFmd, unsigned int inVerifyRate)
const char* inlpath, bool inComputeChecksum, bool inCommitChecksum,
bool inCommitSize, bool inCommitFmd, unsigned int inVerifyRate)
{
fId = fid;
fsId = fsid;
......@@ -74,6 +75,7 @@ public:
container = incontainer;
cId = incid;
path = inpath;
lPath = inlpath;
lId = inlid;
computeChecksum = inComputeChecksum;
commitChecksum = inCommitChecksum;
......@@ -94,6 +96,7 @@ public:
const char* scid = 0;
const char* layout = 0;
const char* path = 0;
const char* lpath = 0;
bool computeChecksum = false;
bool commitChecksum = false;
bool commitSize = false;
......@@ -119,6 +122,7 @@ public:
scid = capOpaque->Get("mgm.cid");
path = capOpaque->Get("mgm.path");
layout = capOpaque->Get("mgm.lid");
lpath = capOpaque->Get("mgm.lpath");
if (capOpaque->Get("mgm.verify.compute.checksum")) {
computeChecksum = atoi(capOpaque->Get("mgm.verify.compute.checksum"));
......@@ -156,8 +160,8 @@ public:
fid = eos::common::FileId::Hex2Fid(hexfid.c_str());
fsid = atoi(sfsid);
return new Verify(fid, fsid, localprefix, smanager, capOpaque->Env(envlen),
container, cid, lid, path, computeChecksum, commitChecksum, commitSize,
commitFmd, verifyRate);
container, cid, lid, path, lpath, computeChecksum,
commitChecksum, commitSize, commitFmd, verifyRate);
};
~Verify() { };
......@@ -168,11 +172,12 @@ public:
void
Show(const char* show = "")
{
eos_static_info("Verify fid=%llu on fs=%u path=%s compute_checksum=%d "
"commit_checksum=%d commit_size=%d commit_fmd=%d "
"verify_rate=%d %s", fId, fsId, path.c_str(),
computeChecksum, commitChecksum, commitSize, commitFmd,
verifyRate, show);
const char* lpath = lPath.length() ? lPath.c_str() : "\"\"";
eos_static_info("Verify fid=%llu on fs=%u path=%s lpath=%s "
"compute_checksum=%d commit_checksum=%d commit_size=%d "
"commit_fmd=%d verify_rate=%d %s", fId, fsId, path.c_str(),
lpath, computeChecksum, commitChecksum, commitSize,
commitFmd, verifyRate, show);
}
};
......
......@@ -40,6 +40,7 @@
#include "common/eos_cta_pb/EosCtaAlertHandler.hh"
#include "common/Constants.hh"
#include "common/StringConversion.hh"
#include "common/Timing.hh"
#include "XrdNet/XrdNetOpts.hh"
#include "XrdOfs/XrdOfs.hh"
#include "XrdOfs/XrdOfsTrace.hh"
......@@ -396,6 +397,7 @@ XrdFstOfs::Configure(XrdSysError& Eroute, XrdOucEnv* envP)
}
eos::fst::Config::gConfig.FstMetaLogDir = "/var/tmp/eos/md/";
eos::fst::Config::gConfig.FstAuthDir = "/var/eos/auth/";
setenv("XrdClientEUSER", "daemon", 1);
// Set short timeout resolution, connection window, connection retry and
// stream error window
......@@ -468,6 +470,21 @@ XrdFstOfs::Configure(XrdSysError& Eroute, XrdOucEnv* envP)
}
}
if (!strcmp("authdir", var)) {
if (!(val = Config.GetWord())) {
Eroute.Emsg("Config", "argument 2 for authdir missing");
NoGo = 1;
} else {
if (strlen(val)) {
eos::fst::Config::gConfig.FstAuthDir = val;
if (val[strlen(val) - 1] != '/') {
eos::fst::Config::gConfig.FstAuthDir += '/';
}
}
}
}
if (!strcmp("protowfendpoint", var)) {
if ((val = Config.GetWord())) {
eos::fst::Config::gConfig.ProtoWFEndpoint = val;
......@@ -564,7 +581,7 @@ XrdFstOfs::Configure(XrdSysError& Eroute, XrdOucEnv* envP)
if (pos2 != STR_NPOS) {
eos::fst::Config::gConfig.FstQueue.erase(0, pos2 + 1);
} else {
Eroute.Emsg("Config", "cannot determin my queue name: ",
Eroute.Emsg("Config", "cannot determine my queue name: ",
eos::fst::Config::gConfig.FstQueue.c_str());
return 1;
}
......@@ -602,7 +619,7 @@ XrdFstOfs::Configure(XrdSysError& Eroute, XrdOucEnv* envP)
}
Eroute.Say("=====> eoscp-log : ", eoscpTransferLog.c_str());
// Compute checkusm of the keytab file
// Compute checksum of the keytab file
std::string kt_cks = GetKeytabChecksum("/etc/eos.keytab");
eos::fst::Config::gConfig.KeyTabAdler = kt_cks.c_str();
// Create the messaging object(recv thread)
......@@ -623,7 +640,7 @@ XrdFstOfs::Configure(XrdSysError& Eroute, XrdOucEnv* envP)
ObjectManager.mEnableQueue = true;
ObjectManager.SetAutoReplyQueue("/eos/*/mgm");
ObjectManager.SetDebug(false);
// create the specific listener class
// Create the specific listener class
Messaging = new eos::fst::Messaging(
eos::fst::Config::gConfig.FstOfsBrokerUrl.c_str(),
eos::fst::Config::gConfig.FstDefaultReceiverQueue.c_str(),
......@@ -643,6 +660,31 @@ XrdFstOfs::Configure(XrdSysError& Eroute, XrdOucEnv* envP)
return NoGo;
}
// Setup auth dir
{
XrdOucString scmd = "mkdir -p ";
scmd += eos::fst::Config::gConfig.FstAuthDir;
scmd += " ; chown -R daemon ";
scmd += eos::fst::Config::gConfig.FstAuthDir;
scmd += " ; chmod 700 ";
scmd += eos::fst::Config::gConfig.FstAuthDir;
int src = system(scmd.c_str());
if (src) {
eos_err("%s returned %d", scmd.c_str(), src);
}
if (access(eos::fst::Config::gConfig.FstAuthDir.c_str(),
R_OK | W_OK | X_OK)) {
Eroute.Emsg("Config", "cannot access the auth directory for r/w: ",
eos::fst::Config::gConfig.FstAuthDir.c_str());
return 1;
}
Eroute.Say("=====> fstofs.authdir : ",
eos::fst::Config::gConfig.FstAuthDir.c_str());
}
// Attach Storage to the meta log dir
Storage = eos::fst::Storage::Create(
eos::fst::Config::gConfig.FstMetaLogDir.c_str());
......@@ -662,7 +704,7 @@ XrdFstOfs::Configure(XrdSysError& Eroute, XrdOucEnv* envP)
eos_crit("error starting the shared object change notifier");
}
eos_notice("sending broadcast's ...");
eos_notice("sending broadcasts ...");
// Create a wildcard broadcast
XrdMqSharedHash* hash = 0;
XrdMqSharedQueue* queue = 0;
......@@ -927,6 +969,21 @@ again:
rc = EREMCHG;
}
if (msg.find("[ENOTDIR]") != STR_NPOS) {
rc = ENOTDIR;
}
if (msg.find("[EEXIST]") != STR_NPOS) {
rc = EEXIST;
}
// Avoid duplication of "Unable to" prefix in error message
// (gOFS.Emsg will add its own "Unable to" prefix to the message)
int pos = msg.rfind("Unable to ");
if (pos != STR_NPOS) {
msg.erase(pos, 10);
}
if (rc != SFS_ERROR) {
return gOFS.Emsg(epname, *error, rc, msg.c_str(), path);
} else {
......@@ -1222,9 +1279,12 @@ XrdFstOfs::_rem(const char* path, XrdOucErrInfo& error,
EPNAME("rem");
XrdOucString fstPath = "";
const char* localprefix = 0;
const char* lPath = 0;
const char* hexfid = 0;
const char* sfsid = 0;
eos_debug("");
// Retrieve logical path, if one is used
lPath = capOpaque->Get("mgm.lpath");
if ((!fstpath) && (!fsid) && (!fid)) {
// Standard deletion brings all information via the opaque info
......@@ -1243,7 +1303,13 @@ XrdFstOfs::_rem(const char* path, XrdOucErrInfo& error,
"open - no file system id in capability", path);
}
eos::common::FileId::FidPrefix2FullPath(hexfid, localprefix, fstPath);
if (lPath) {
fstPath = eos::common::StringConversion::BuildPhysicalPath(localprefix,
lPath);
} else {
eos::common::FileId::FidPrefix2FullPath(hexfid, localprefix, fstPath);
}
fid = eos::common::FileId::Hex2Fid(hexfid);
fsid = atoi(sfsid);
} else {
......@@ -1252,6 +1318,54 @@ XrdFstOfs::_rem(const char* path, XrdOucErrInfo& error,
}
eos_info("fstpath=%s", fstPath.c_str());
// Gather extra opaque info for IO object construction
std::string sFstPath = fstPath.c_str();
std::string s3credentials =
gOFS.Storage->GetFileSystemById(fsid)->GetString("s3credentials");
if (!s3credentials.empty()) {
sFstPath += "?s3credentials=" + s3credentials;
}
// Construct IO object
std::unique_ptr<FileIo> io(eos::fst::FileIoPlugin::GetIoObject(
sFstPath.c_str()));
if (!io) {
return gOFS.Emsg(epname, error, EINVAL, "open - no IO plug-in available",
sFstPath.c_str());
}
// Prevent a scheduled delete from erasing a newer file with the same lpath
if (lPath) {
std::string stime = capOpaque->Get("mgm.ctime") ?
capOpaque->Get("mgm.ctime") : "";
// Retrieve creation time from MGM
long long delctime = eos::common::Timing::TimespecString_to_Ns(stime);
if (delctime == -1) {
eos_err("could not parse creation time sent from MGM fstpath=%s "
"mgm.ctime=%s", fstPath.c_str(), stime.c_str());
return gOFS.Emsg(epname, error, errno, "delete file -- ctime parse error",
fstPath.c_str());
}
// Retrieve creation time from file extended attributes
io->attrGet("user.eos.ctime", stime);
long long ioctime = eos::common::Timing::TimespecString_to_Ns(stime);
// File is newer and should not be deleted
if (ioctime > delctime) {
eos_info("fstpath=%s -- won't delete newer version", fstPath.c_str());
return SFS_OK;
} else if (ioctime == -1) {
eos_notice("could not retrieve creation time for file %s fstpath=%s "
"fsid=%lu fid=%llu", path, fstPath.c_str(), fsid, fid);
}
}
int rc = 0;
errno = 0; // If file not found this will be ENOENT
struct stat sbd;
......@@ -1264,29 +1378,8 @@ XrdFstOfs::_rem(const char* path, XrdOucErrInfo& error,
// get the size before deletion
XrdOfs::stat(fstPath.c_str(), &sbd, error, client, 0);
rc = XrdOfs::rem(fstPath.c_str(), error, client, 0);
if (rc) {
eos_info("rc=%i, errno=%i", rc, errno);
}
} else {
// Check for additional opaque info to create remote IO object
std::string sFstPath = fstPath.c_str();
std::string s3credentials =
gOFS.Storage->GetFileSystemById(fsid)->GetString("s3credentials");
if (!s3credentials.empty()) {
sFstPath += "?s3credentials=" + s3credentials;
}
std::unique_ptr<FileIo> io(eos::fst::FileIoPlugin::GetIoObject(
sFstPath.c_str()));
if (!io) {
return gOFS.Emsg(epname, error, EINVAL, "open - no IO plug-in avaialble",
sFstPath.c_str());
}
// get the size before deletion
// Get the size before deletion
io->fileStat(&sbd);
rc = io->fileRemove();
}
......@@ -1296,6 +1389,8 @@ XrdFstOfs::_rem(const char* path, XrdOucErrInfo& error,
(void) gOFS.Storage->CloseTransaction(fsid, fid);
if (rc) {
eos_info("rc=%i, errno=%i", rc, errno);
if (errno == ENOENT) {
// Ignore error if a file to be deleted doesn't exist
if (ignoreifnotexist) {
......
......@@ -26,6 +26,7 @@
#include "common/Path.hh"
#include "common/http/OwnCloud.hh"
#include "common/StringTokenizer.hh"
#include "common/StringConversion.hh"
#include "common/SecEntity.hh"
#include "common/xrootd-ssi-protobuf-interface/eos_cta/include/CtaFrontendApi.hpp"
#include "fst/FmdDbMap.hh"
......@@ -55,13 +56,12 @@ XrdFstOfsFile::XrdFstOfsFile(const char* user, int MonID) :
writeDelete(false), mRainSize(0), mNsPath(""), mLocalPrefix(""),
mRedirectManager(""), mSecString(""), mTpcKey(""), mEtag(""), mFileId(0),
mFsId(0), mLid(0), mCid(0), mForcedMtime(1), mForcedMtime_ms(0), mFusex(false),
mFusexIsUnlinked(false),
closed(false), opened(false), mHasWrite(false), hasWriteError(false),
hasReadError(false), isRW(false), mIsTpcDst(false), mIsDevNull(false),
isCreation(false), isReplication(false), mIsInjection(false),
mRainReconstruct(false), deleteOnClose(false), repairOnClose(false),
commitReconstruction(false), mEventOnClose(false), mEventWorkflow(""),
mSyncEventOnClose(false),
mFusexIsUnlinked(false), closed(false), opened(false), mHasWrite(false),
hasWriteError(false), hasReadError(false), isRW(false), mUselPath(false),
mIsTpcDst(false), mIsDevNull(false), isCreation(false), forceCreation(false),
isReplication(false), mIsInjection(false), mRainReconstruct(false),
deleteOnClose(false), repairOnClose(false), commitReconstruction(false),
mEventOnClose(false), mEventWorkflow(""), mSyncEventOnClose(false),
mIsOCchunk(false), writeErrorFlag(false), mTpcFlag(kTpcNone),
fMd(nullptr), mCheckSum(nullptr), layOut(nullptr), maxOffsetWritten(0),
openSize(0), closeSize(0),
......@@ -330,40 +330,50 @@ XrdFstOfsFile::open(const char* path, XrdSfsFileOpenMode open_mode,
}
layOut->SetLogId(logId, client, tident);
// Check file existence
errno = 0;
retc = layOut->GetFileIo()->fileExists();
bool file_exists = (retc == 0);
if ((retc = layOut->GetFileIo()->fileExists())) {
// We have to distinguish if an Exists call fails or return ENOENT, otherwise
// we might trigger an automatic clean-up of a file !!!
if (errno != ENOENT) {
delete fMd;
return gOFS.Emsg(epname, error, EIO, "open - unable to check for existance"
" of file ", mCapOpaque->Env(envlen));
}
// Distinguish if the fileExists() call failed or returned ENOENT.
// Otherwise, we might trigger an automatic clean-up of a file !!!
if (!file_exists && (errno != ENOENT)) {
delete fMd;
return gOFS.Emsg(epname, error, EIO, "open - unable to check file existence",
mCapOpaque->Env(envlen));
}
if (isRW || (mCapOpaque->Get("mgm.zerosize"))) {
// File does not exist, keep the create flag for writers and readers with 0-size at MGM
// Enter creation setup if file doesn't exist
// or the MGM set the creation opaque flag
if (!file_exists || forceCreation) {
if (forceCreation || isRW || (mCapOpaque->Get("mgm.zerosize"))) {
// File does not exist, keep the create flag for writers
// and readers with 0-size at MGM
isCreation = true;
openSize = 0;
// Used to indicate if a file was written in the meanwhile by someone else
updateStat.st_mtime = 0;
open_mode |= SFS_O_CREAT;
create_mode |= SFS_O_MKPTH;
eos_debug("adding creation flag because of %d %d", retc, errno);
if (!file_exists) {
open_mode |= SFS_O_CREAT;
create_mode |= SFS_O_MKPTH;
eos_debug("adding creation flag -- file doesn't exist retc=%d errno=%d",
retc, errno);
}
} else {
// The open will fail but the client will get a recoverable error,
// therefore it will try to read again from the other replicas.
eos_warning("open for read, local file does not exists");
return gOFS.Emsg(epname, error, ENOENT, "open, file does not exist ",
return gOFS.Emsg(epname, error, ENOENT, "open - file does not exist ",
mCapOpaque->Env(envlen));
}
} else {
eos_debug("removing creation flag because of %d %d", retc, errno);
}
// Remove the creat flag
if (open_mode & SFS_O_CREAT) {
open_mode -= SFS_O_CREAT;
}
if (file_exists && (open_mode & SFS_O_CREAT)) {
// Remove the create flag
eos_debug("removing creation flag because file exists errno=%d", errno);
open_mode -= SFS_O_CREAT;
}
// Capability access distinction
......@@ -590,11 +600,12 @@ XrdFstOfsFile::open(const char* path, XrdSfsFileOpenMode open_mode,
std::string filecxerror = "0";
if (!rc) {
// Set the eos lfn as extended attribute
// Set extended attributes
std::unique_ptr<FileIo> io(FileIoPlugin::GetIoObject(
layOut->GetLocalReplicaPath(), this));
if (isRW) {
// Set the eos lfn attribute
if (mNsPath.beginswith("/replicate:") || mNsPath.beginswith("/fusex-open")) {
if (mCapOpaque->Get("mgm.path")) {
XrdOucString unsealedpath = mCapOpaque->Get("mgm.path");
......@@ -612,6 +623,13 @@ XrdFstOfsFile::open(const char* path, XrdSfsFileOpenMode open_mode,
eos_err("unable to set extended attribute <eos.lfn> errno=%d", errno);
}
}
// Set the eos creation time attribute
if (mUselPath && isCreation && mCapOpaque->Get("mgm.ctime")) {
if (io->attrSet("user.eos.ctime", mCapOpaque->Get("mgm.ctime"))) {
eos_err("unable to set extended attribute <eos.ctime> errno=%d", errno);
}
}
}
// Try to get error if the file has a scan error
......@@ -1178,13 +1196,13 @@ XrdFstOfsFile::close()
// ---- add error simulation for checksum errors on read
if ((!isRW) && gOFS.Simulate_XS_read_error) {
checksumerror = true;
eos_warning("simlating checksum errors on read");
eos_warning("simulating checksum errors on read");
}
// ---- add error simulation for checksum errors on write
if (isRW && gOFS.Simulate_XS_write_error) {
checksumerror = true;
eos_warning("simlating checksum errors on write");
eos_warning("simulating checksum errors on write");
}
if (isRW && (checksumerror || targetsizeerror || minimumsizeerror)) {
......@@ -1253,7 +1271,7 @@ XrdFstOfsFile::close()
// Set the container id
fMd->mProtoFmd.set_cid(mCid);
// For replicat's set the original uid/gid/lid values
// For replicas set the original uid/gid/lid values
if (mCapOpaque->Get("mgm.source.lid")) {
fMd->mProtoFmd.set_lid(strtoul(mCapOpaque->Get("mgm.source.lid"), 0, 10));
}
......@@ -1269,7 +1287,7 @@ XrdFstOfsFile::close()
// Commit local
try {
if (!gFmdDbMapHandler.Commit(fMd)) {
eos_err("unabel to commit meta data to local database");
eos_err("unable to commit meta data to local database");
(void) gOFS.Emsg(epname, this->error, EIO, "close - unable to "
"commit meta data", mNsPath.c_str());
}
......@@ -1301,6 +1319,19 @@ XrdFstOfsFile::close()
(mForcedMtime != 1) ? mForcedMtime_ms : (unsigned long long)
fMd->mProtoFmd.mtime_ns());
if (mUselPath) {
// Extract logical path from full fstpath
XrdOucString lPath = mFstPath;
lPath.keep(mLocalPrefix.length());
// Make sure it starts with '/'
if (!lPath.beginswith("/")) {
lPath.insert("/", 0);
}
capOpaqueFile += "&mgm.lpath=";
capOpaqueFile += lPath;
}
if (mFusex) {
capOpaqueFile += "&mgm.fusex=1";
}
......@@ -2917,13 +2948,14 @@ XrdFstOfsFile::ProcessCapOpaque(bool& is_repair_read,
//----------------------------------------------------------------------------
// Process mixed opaque information - decisions that need to be taken based
// on both the ecrypted and un-encrypted opaque info
// on both the encrypted and un-encrypted opaque info
//----------------------------------------------------------------------------
int
XrdFstOfsFile::ProcessMixedOpaque()
{