Implement first version of quarkdb-health-local, only StateMachine free space for now

src/Commands.cc

@@ -150,6 +150,8 @@ struct cmdMapInit {
     redis_cmd_map["quarkdb_compression_stats"] = {RedisCommand::QUARKDB_COMPRESSION_STATS, CommandType::QUARKDB};
     redis_cmd_map["quarkdb_version"] = {RedisCommand::QUARKDB_VERSION, CommandType::QUARKDB};
     redis_cmd_map["quarkdb_checkpoint"] = {RedisCommand::QUARKDB_CHECKPOINT, CommandType::QUARKDB};
+    redis_cmd_map["quarkdb_health_local"] = {RedisCommand::QUARKDB_HEALTH_LOCAL, CommandType::QUARKDB};
+
     // Compatibility: Keep raft_checkpoint, make identical to quarkdb_checkpoint.
     // Maybe remove in a few versions.
     redis_cmd_map["raft_checkpoint"] = {RedisCommand::QUARKDB_CHECKPOINT, CommandType::QUARKDB};

src/Commands.hh

@@ -157,6 +157,7 @@ enum class RedisCommand {
   QUARKDB_COMPRESSION_STATS,
   QUARKDB_VERSION,
   QUARKDB_CHECKPOINT,
+  QUARKDB_HEALTH_LOCAL,
 
   RECOVERY_GET,
   RECOVERY_SET,

src/Shard.cc

@@ -29,6 +29,7 @@
 #include "raft/RaftDispatcher.hh"
 #include "redis/LeaseFilter.hh"
 #include "utils/ScopedAdder.hh"
+#include "utils/VectorUtils.hh"
 
 using namespace quarkdb;
 
@@ -220,6 +221,22 @@ LinkStatus Shard::dispatch(Connection *conn, RedisRequest &req) {
 
       return conn->status(ss.str());
     }
+    case RedisCommand::QUARKDB_HEALTH_LOCAL: {
+      if(req.size() != 1) return conn->errArgs(req[0]);
+
+      InFlightRegistration registration(inFlightTracker);
+      if(!registration.ok()) {
+        return conn->err("unavailable");
+      }
+
+      std::vector<HealthIndicator> healthIndicators;
+      VectorUtils::appendToVector(healthIndicators, stateMachine->getHealthIndicators());
+
+      std::vector<std::string> outVector;
+      VectorUtils::appendToVector(outVector, healthIndicatorsAsStrings(healthIndicators));
+
+      return conn->statusVector(outVector);
+    }
     case RedisCommand::COMMAND_STATS: {
       if(req.size() != 1) return conn->errArgs(req[0]);
       InFlightRegistration registration(inFlightTracker);

src/StateMachine.cc

@@ -42,6 +42,7 @@
 #include <rocksdb/filter_policy.h>
 #include <rocksdb/table.h>
 #include <thread>
+#include <sys/vfs.h>
 
 #define RETURN_ON_ERROR(st) { rocksdb::Status st2 = st; if(!st2.ok()) return st2; }
 #define THROW_ON_ERROR(st) { rocksdb::Status st2 = st; if(!st2.ok()) qdb_throw(st2.ToString()); }
@@ -1822,6 +1823,42 @@ rocksdb::Status StateMachine::noop(LogIndex index) {
   return stagingArea.commit(index);
 }
 
+//------------------------------------------------------------------------------
+// Return health information about the state machine
+//------------------------------------------------------------------------------
+std::vector<HealthIndicator> StateMachine::getHealthIndicators() {
+  std::string description = "Free space in state-machine filesystem";
+
+  struct statfs out;
+  if(statfs(filename.c_str(), &out) != 0) {
+    return { HealthIndicator(HealthStatus::kRed, description, SSTR("Could not statfs '" << filename << "'")) };
+  }
+
+  HealthStatus status = HealthStatus::kGreen;
+  constexpr int64_t bytesInGiB = 1024 * 1024 * 1024;
+
+  int64_t freeBytes = out.f_bavail * out.f_bsize;
+  double percentFree = 100 - (100.0 * (double) (out.f_blocks - out.f_bfree) / (double) (out.f_blocks - out.f_bfree + out.f_bavail));
+
+  // Red if less than 1 GiB, yellow if less than 5 GiB, green otherwise
+  if(freeBytes <= bytesInGiB) {
+    status = chooseWorstHealth(status, HealthStatus::kRed);
+  }
+  else if(freeBytes <= 5 * bytesInGiB) {
+    status = chooseWorstHealth(status, HealthStatus::kYellow);
+  }
+
+  // Red if less than 3% of total capacity, yellow if less than 10%, green otherwise
+  if(percentFree <= 3) {
+    status = chooseWorstHealth(status, HealthStatus::kRed);
+  }
+  else if(percentFree <= 10) {
+    status = chooseWorstHealth(status, HealthStatus::kYellow);
+  }
+
+  return { HealthIndicator(status, description, SSTR(freeBytes << " bytes (" << percentFree << "%)")) };
+}
+
 rocksdb::Status StateMachine::manualCompaction() {
   qdb_event("Triggering manual compaction.. auto-compaction will be disabled while the manual one is running.");
   // Disabling auto-compactions is a hack to prevent write-stalling. Pending compaction

src/StateMachine.hh

@@ -34,6 +34,7 @@
 #include "storage/ConsistencyScanner.hh"
 #include "storage/KeyConstants.hh"
 #include "storage/LeaseInfo.hh"
+#include "health/HealthIndicator.hh"
 #include <rocksdb/db.h>
 #include <rocksdb/utilities/write_batch_with_index.h>
 #include <rocksdb/utilities/debug.h>
@@ -252,6 +253,11 @@ public:
     return bulkLoad;
   }
 
+  //----------------------------------------------------------------------------
+  // Return health information about the state machine
+  //----------------------------------------------------------------------------
+  std::vector<HealthIndicator> getHealthIndicators();
+
   rocksdb::Status manualCompaction();
   void finalizeBulkload();
   IteratorPtr getRawIterator();

src/health/HealthIndicator.hh

@@ -24,9 +24,10 @@
 #ifndef QUARKDB_HEALTH_INDICATOR_HH
 #define QUARKDB_HEALTH_INDICATOR_HH
 
+#include "../utils/Macros.hh"
 #include <string>
 #include <string_view>
-#include "../utils/Macros.hh"
+#include <vector>
 
 namespace quarkdb {
 
@@ -100,6 +101,16 @@ private:
   const std::string message;
 };
 
+inline std::vector<std::string> healthIndicatorsAsStrings(const std::vector<HealthIndicator> &source) {
+  std::vector<std::string> out;
+
+  for(size_t i = 0; i < source.size(); i++) {
+    out.emplace_back(source[i].toString());
+  }
+
+  return out;
+}
+
 }
 
 #endif

src/utils/VectorUtils.hh

@@ -40,6 +40,13 @@ bool checkEquality(const T& one, const T& two) {
   return true;
 }
 
+template<typename T>
+void appendToVector(std::vector<T> &target, const std::vector<T> &source) {
+  for(size_t i = 0; i < source.size(); i++) {
+    target.emplace_back(source[i]);
+  }
+}
+
 } }