Commit 0d713b7c authored by Georgios Bitzes's avatar Georgios Bitzes
Browse files

Speed up KEYS for common case of prefix matching by implementing in terms of SCAN

parent 869f0abd
Pipeline #993034 passed with stages
in 38 minutes and 55 seconds
......@@ -9,6 +9,9 @@ causing all writes to stall). From now on, similar kind of corruption should onl
take out a single node, and not spread to the entire cluster.
- Add command ``RAFT-JOURNAL-SCAN`` to make searching through the contents of the
raft journal easier.
- ``KEYS`` is now implemented in terms of ``SCAN``, making prefix scanning of the
keyspace just as efficient as with ``SCAN``. (Note: The use of ``KEYS`` is still
generally discouraged due to potentially huge response size, don't use in production)
## 0.3.8 (2019-05-27)
- Prevent elections from hanging on the TCP timeout when one of the member hosts
......
......@@ -1700,24 +1700,24 @@ rocksdb::Status StateMachine::exists(StagingArea &stagingArea, const ReqIterator
rocksdb::Status StateMachine::keys(StagingArea &stagingArea, std::string_view pattern, std::vector<std::string> &result) {
result.clear();
bool allkeys = (pattern.length() == 1 && pattern[0] == '*');
IteratorPtr iter(stagingArea.getIterator());
std::string oldCursor;
std::string newCursor;
std::string searchPrefix(1, char(InternalKeyType::kDescriptor));
for(iter->Seek(searchPrefix); iter->Valid(); iter->Next()) {
std::string rkey = iter->key().ToString();
if(rkey.size() == 0 || rkey[0] != char(InternalKeyType::kDescriptor)) break;
while(true) {
rocksdb::Status st = scan(stagingArea, oldCursor, pattern, std::numeric_limits<size_t>::max()-1, newCursor, result);
if(!st.ok()) return st;
if(allkeys || stringmatchlen(pattern.data(), pattern.length(), rkey.data()+1, rkey.length()-1, 0)) {
result.push_back(rkey.substr(1));
if(newCursor.empty()) {
break;
}
oldCursor = newCursor;
}
return rocksdb::Status::OK();
}
rocksdb::Status StateMachine::scan(StagingArea &stagingArea, std::string_view cursor, std::string_view pattern, size_t count, std::string &newcursor, std::vector<std::string> &results) {
results.clear();
// Any hits *must* start with patternPrefix. This will allow us in many
// circumstances to eliminate checking large parts of the keyspace, without
......
......@@ -654,30 +654,37 @@ TEST_F(State_Machine, scan) {
ASSERT_EQ(keys, make_vec("key1", "key2"));
ASSERT_EQ(newcursor, "key3");
keys.clear();
ASSERT_OK(stateMachine()->scan(newcursor, "key*", 2, newcursor, keys));
ASSERT_EQ(keys, make_vec("key3", "key4"));
ASSERT_EQ(newcursor, "key5");
keys.clear();
ASSERT_OK(stateMachine()->scan(newcursor, "key*", 2, newcursor, keys));
ASSERT_EQ(keys, make_vec("key5", "key6"));
ASSERT_EQ(newcursor, "");
keys.clear();
ASSERT_OK(stateMachine()->scan("", "*key1", 2, newcursor, keys));
ASSERT_EQ(keys, make_vec("key1"));
ASSERT_EQ(newcursor, "key3");
keys.clear();
ASSERT_OK(stateMachine()->scan(newcursor, "*key1", 2, newcursor, keys));
ASSERT_TRUE(keys.empty());
ASSERT_EQ(newcursor, "key5");
keys.clear();
ASSERT_OK(stateMachine()->scan(newcursor, "*key1", 2, newcursor, keys));
ASSERT_TRUE(keys.empty());
ASSERT_EQ(newcursor, "otherkey1");
keys.clear();
ASSERT_OK(stateMachine()->scan(newcursor, "*key1", 2, newcursor, keys));
ASSERT_EQ(keys, make_vec("otherkey1"));
ASSERT_EQ(newcursor, "otherkey3");
keys.clear();
ASSERT_OK(stateMachine()->scan(newcursor, "*key1", 2, newcursor, keys));
ASSERT_TRUE(keys.empty());
ASSERT_EQ(newcursor, "");
......@@ -687,20 +694,24 @@ TEST_F(State_Machine, scan) {
ASSERT_OK(stateMachine()->set("abc", "8"));
ASSERT_OK(stateMachine()->set("abcd", "8"));
keys.clear();
ASSERT_OK(stateMachine()->scan("", "ab?", 3, newcursor, keys));
ASSERT_EQ(keys, make_vec("aba", "abb", "abc"));
ASSERT_EQ(newcursor, "abcd");
keys.clear();
ASSERT_OK(stateMachine()->scan(newcursor, "ab?", 3, newcursor, keys));
ASSERT_TRUE(keys.empty());
ASSERT_EQ(newcursor, "");
// Using a non-sense cursor
keys.clear();
ASSERT_OK(stateMachine()->scan("zz", "ab?", 100, newcursor, keys));
ASSERT_TRUE(keys.empty());
ASSERT_EQ(newcursor, "");
// Match only a single key
keys.clear();
ASSERT_OK(stateMachine()->scan("", "abc", 100, newcursor, keys));
ASSERT_EQ(keys, make_vec("abc"));
ASSERT_EQ(newcursor, "");
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment