Properly handle connection shutdown, prevent deadlock with xrootd

CHANGELOG.md

@@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file.
 - A newly elected leader now stalls writers in addition to readers, until its leadership marker entry in the raft journal has been committed and applied.
 
 ### Fixed
+- Sockets and threads from closed connections were not being cleaned due to misunderstanding how XRootD handles connection shutdown. Each connection would hog a socket and a deadlocked thread on the server forever. (oops)
 - A particularly rare race condition was able to trigger an assertion in the Raft subsystem, causing the current cluster leader to crash.
 
 ## 0.2.7 (2018-06-22)

src/Link.cc

@@ -83,6 +83,10 @@ std::string Link::describe() const {
   return SSTR(host << " [" << uuid << "]");
 }
 
+void Link::preventXrdLinkClose() {
+  xrdLinkCloseDisabled = true;
+}
+
 LinkStatus Link::rawRecv(char *buff, int blen, int timeout) {
   if(link) return link->Recv(buff, blen, timeout);
   if(fd >= 0) return fdRecv(buff, blen, timeout);
@@ -100,7 +104,10 @@ LinkStatus Link::Recv(char *buff, int blen, int timeout) {
 
 LinkStatus Link::Close(int defer) {
   if(tlsconfig.active) tlsfilter.close(defer);
-  if(link) return link->Close(defer);
+  if(link) {
+    if(xrdLinkCloseDisabled) return 1;
+    return link->Close(defer);
+  }
   if(fd >= 0) return fdClose(defer);
   return streamClose(defer);
 }

src/Link.hh

@@ -67,6 +67,12 @@ public:
 
   // Set global connection logging config
   static void setConnectionLogging(bool val);
+
+  // Prevent closing an underlying XrdLink, if any. Use this if
+  // the xrootd machinery calls XrdProtocol::Reset, which takes care
+  // of closing the link on its own.
+  void preventXrdLinkClose();
+
 private:
   qclient::TlsConfig tlsconfig;
   qclient::TlsFilter tlsfilter;
@@ -75,6 +81,7 @@ private:
   XrdLink *link = nullptr;
   bool dead = false;
   int fd = -1;
+  bool xrdLinkCloseDisabled = false;
 
   std::string uuid;
   std::string host;

src/XrdQuarkDB.cc

@@ -118,6 +118,7 @@ void XrdQuarkDB::Reset() {
     conn = nullptr;
   }
   if(link) {
+    link->preventXrdLinkClose();
     delete link;
     link = nullptr;
   }