Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
eos
QuarkDB
Commits
35edc863
Commit
35edc863
authored
Feb 21, 2020
by
Georgios Bitzes
Browse files
Implement quarkdb-validate-checkpoint tool
parent
891d4246
Pipeline
#1431834
passed with stages
in 63 minutes and 3 seconds
Changes
6
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
CHANGELOG.md
View file @
35edc863
...
...
@@ -4,6 +4,9 @@
### New features
-
Implement command
``quarkdb-verify-checksum``
for manually running a full checksum scan.
-
Addition of
``quarkdb-validate-checkpoint``
tool for ensuring that a given
checkpoint is valid -- useful to run in backup scripts before streaming a given
checkpoint for long-term storage.
### Improvements
-
Security hardening of the redis parser for unauthenticated clients.
...
...
packaging/quarkdb.spec.in
View file @
35edc863
...
...
@@ -90,7 +90,8 @@ rm -rf $RPM_BUILD_ROOT
%{_bindir}/quarkdb-sudo-tests
%{_bindir}/quarkdb-bench
%{_bindir}/quarkdb-create
%{_bindir}/quarkdb-ldb
%{_bindir}/quarkdb-recovery
%{_bindir}/quarkdb-server
%{_bindir}/quarkdb-sst-inspect
%{_bindir}/quarkdb-
ldb
%{_bindir}/quarkdb-
validate-checkpoint
src/ShardDirectory.cc
View file @
35edc863
...
...
@@ -105,11 +105,15 @@ StateMachine* ShardDirectory::getStateMachine() {
return
smptr
;
}
bool
ShardDirectory
::
hasRaftJournal
(
std
::
string
&
err
)
const
{
return
directoryExists
(
raftJournalPath
(),
err
);
}
RaftJournal
*
ShardDirectory
::
getRaftJournal
()
{
if
(
journalptr
)
return
journalptr
;
std
::
string
suberr
;
if
(
!
directoryExists
(
r
aftJournal
Path
(),
suberr
))
{
if
(
!
hasR
aftJournal
(
suberr
))
{
qdb_throw
(
"Cannot open raft journal: "
<<
suberr
);
}
...
...
@@ -117,15 +121,15 @@ RaftJournal* ShardDirectory::getRaftJournal() {
return
journalptr
;
}
std
::
string
ShardDirectory
::
currentPath
()
{
std
::
string
ShardDirectory
::
currentPath
()
const
{
return
pathJoin
(
path
,
"current"
);
}
std
::
string
ShardDirectory
::
stateMachinePath
()
{
std
::
string
ShardDirectory
::
stateMachinePath
()
const
{
return
pathJoin
(
currentPath
(),
"state-machine"
);
}
std
::
string
ShardDirectory
::
raftJournalPath
()
{
std
::
string
ShardDirectory
::
raftJournalPath
()
const
{
return
pathJoin
(
currentPath
(),
"raft-journal"
);
}
...
...
src/ShardDirectory.hh
View file @
35edc863
...
...
@@ -61,6 +61,7 @@ public:
StateMachine
*
getStateMachineForBulkload
();
StateMachine
*
getStateMachine
();
RaftJournal
*
getRaftJournal
();
bool
hasRaftJournal
(
std
::
string
&
err
)
const
;
// Reset the contents of both the state machine and the raft journal.
// Physical paths remain the same.
...
...
@@ -109,9 +110,9 @@ private:
ResilveringHistory
resilveringHistory
;
std
::
string
stateMachinePath
();
std
::
string
raftJournalPath
();
std
::
string
currentPath
();
std
::
string
stateMachinePath
()
const
;
std
::
string
raftJournalPath
()
const
;
std
::
string
currentPath
()
const
;
std
::
string
resilveringHistoryPath
();
std
::
string
getSupplanted
(
const
ResilveringEventID
&
id
);
...
...
tools/CMakeLists.txt
View file @
35edc863
...
...
@@ -16,6 +16,9 @@ target_link_libraries(quarkdb-server jemalloc XrdQuarkDB ${CMAKE_THREAD_LIBS_INI
add_executable
(
quarkdb-sst-inspect quarkdb-sst-inspect.cc
)
target_link_libraries
(
quarkdb-sst-inspect XrdQuarkDB
${
CMAKE_THREAD_LIBS_INIT
}
)
add_executable
(
quarkdb-validate-checkpoint quarkdb-validate-checkpoint.cc
)
target_link_libraries
(
quarkdb-validate-checkpoint XrdQuarkDB
${
CMAKE_THREAD_LIBS_INIT
}
)
include_directories
(
${
XROOTD_INCLUDE_DIRS
}
${
CMAKE_SOURCE_DIR
}
/src
# need access to the private headers
...
...
@@ -27,6 +30,6 @@ include_directories(
)
install
(
TARGETS quarkdb-create quarkdb-ldb quarkdb-recovery quarkdb-server quarkdb-sst-inspect
TARGETS quarkdb-create quarkdb-ldb quarkdb-recovery quarkdb-server quarkdb-sst-inspect
quarkdb-validate-checkpoint
RUNTIME DESTINATION
${
CMAKE_INSTALL_FULL_BINDIR
}
)
tools/quarkdb-validate-checkpoint.cc
0 → 100644
View file @
35edc863
// ----------------------------------------------------------------------
// File: quarkdb-validate-checkpoint.cc
// Author: Georgios Bitzes - CERN
// ----------------------------------------------------------------------
/************************************************************************
* quarkdb - a redis-like highly available key-value store *
* Copyright (C) 2020 CERN/Switzerland *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>.*
************************************************************************/
#include
"../deps/CLI11.hpp"
#include
"utils/FileUtils.hh"
#include
"ShardDirectory.hh"
#include
"StateMachine.hh"
#include
"raft/RaftJournal.hh"
#define SSTR(message) static_cast<std::ostringstream&>(std::ostringstream().flush() << message).str()
struct
PathValidator
:
public
CLI
::
Validator
{
PathValidator
()
:
Validator
(
"PATH"
)
{
func_
=
[](
const
std
::
string
&
path
)
{
std
::
string
err
;
if
(
!
quarkdb
::
directoryExists
(
path
,
err
))
{
return
SSTR
(
"'"
<<
path
<<
"' does not exist."
);
}
return
std
::
string
();
};
}
};
int
main
(
int
argc
,
char
**
argv
)
{
//----------------------------------------------------------------------------
// Setup variables
//----------------------------------------------------------------------------
CLI
::
App
app
(
"Tool to validate QuarkDB checkpoints (backups)"
);
PathValidator
pathValidator
;
std
::
string
optPath
;
bool
acceptStandalone
=
false
;
bool
eos
=
false
;
//----------------------------------------------------------------------------
// Setup options
//----------------------------------------------------------------------------
app
.
add_option
(
"--path"
,
optPath
,
"The path to the QuarkDB checkpoint"
)
->
required
()
->
check
(
pathValidator
);
app
.
add_flag
(
"--accept-standalone"
,
acceptStandalone
,
"No need to ensure that the raft journal is present -- use this flag for standalone instances"
);
app
.
add_flag
(
"--eos"
,
eos
,
"This QuarkDB instance contains the EOS namespace; additionally check eos-files-md and eos-containers-md"
);
//----------------------------------------------------------------------------
// Parse
//----------------------------------------------------------------------------
try
{
app
.
parse
(
argc
,
argv
);
}
catch
(
const
CLI
::
ParseError
&
e
)
{
return
app
.
exit
(
e
);
}
//----------------------------------------------------------------------------
// Can we set-up a ShardDirectory?
//----------------------------------------------------------------------------
qdb_info
(
"Attempting to open ShardDirectory..."
);
quarkdb
::
ShardDirectory
shardDirectory
(
optPath
);
qdb_info
(
"--- OK!"
);
//----------------------------------------------------------------------------
// Can we open the StateMachine?
//----------------------------------------------------------------------------
qdb_info
(
"Attempting to open StateMachine..."
);
quarkdb
::
StateMachine
*
stateMachine
=
shardDirectory
.
getStateMachine
();
qdb_info
(
"--- OK! LAST-APPLIED: "
<<
stateMachine
->
getLastApplied
());
//----------------------------------------------------------------------------
// Check eos namespace parameters?
//----------------------------------------------------------------------------
if
(
eos
)
{
size_t
len
=
0
;
rocksdb
::
Status
st
=
stateMachine
->
lhlen
(
"eos-files-md"
,
len
);
if
(
!
st
.
ok
())
{
qdb_error
(
"Status not ok when retrieving eos-files-md: "
<<
st
.
ToString
());
return
1
;
}
if
(
len
==
0
)
{
qdb_error
(
"eos-files-md length is zero!"
);
return
1
;
}
qdb_info
(
"eos-files-md length: "
<<
len
);
st
=
stateMachine
->
lhlen
(
"eos-containers-md"
,
len
);
if
(
!
st
.
ok
())
{
qdb_error
(
"Status not ok when retrieving eos-containers-md: "
<<
st
.
ToString
());
return
1
;
}
if
(
len
==
0
)
{
qdb_error
(
"eos-containers-md length is zero!"
);
return
1
;
}
qdb_info
(
"eos-containers-md length: "
<<
len
);
}
//----------------------------------------------------------------------------
// Does the raft journal directory even exist?
//----------------------------------------------------------------------------
std
::
string
err
;
if
(
!
shardDirectory
.
hasRaftJournal
(
err
))
{
if
(
acceptStandalone
)
{
qdb_info
(
"raft-journal directory not found, likely a standalone instance"
);
return
0
;
}
qdb_error
(
"raft-journal not found!"
);
return
1
;
}
//----------------------------------------------------------------------------
// Yes, let's open it
//----------------------------------------------------------------------------
qdb_info
(
"Attempting to open RaftJournal..."
);
quarkdb
::
RaftJournal
*
raftJournal
=
shardDirectory
.
getRaftJournal
();
qdb_info
(
"--- OK! LOG-SIZE: "
<<
raftJournal
->
getLogSize
()
<<
", COMMIT-INDEX: "
<<
raftJournal
->
getCommitIndex
()
<<
", LOG-START: "
<<
raftJournal
->
getLogStart
());
//----------------------------------------------------------------------------
// Ensure LAST-APPLIED makes sense
//----------------------------------------------------------------------------
if
(
stateMachine
->
getLastApplied
()
>
raftJournal
->
getCommitIndex
())
{
qdb_error
(
"LAST-APPLIED does not make sense given the current COMMIT-INDEX!"
);
return
1
;
}
if
(
stateMachine
->
getLastApplied
()
<
raftJournal
->
getLogStart
())
{
qdb_error
(
"LAST-APPLIED does not make sense given the current LOG-START!"
);
return
1
;
}
return
0
;
}
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment