Fixed corrupted batch-produced single-event data files. Details:...

Fixed corrupted batch-produced single-event data files. Details: https://its.cern.ch/jira/browse/ATLASVPONE-316 (VP1UtilsBase-00-00-06)

2016-05-17 Riccardo.Maria.Bianchi@cern.ch
    * Added removing of 'PoolFileCatalog.xml' after the copy of files,
      because it causes errors while accessing copied files later in the same Athena
      Details: https://its.cern.ch/jira/browse/ATLASVPONE-316
    * new tag: VP1UtilsBase-00-00-05

2016-02-17 Attila Krasznahorkay <Attila.Krasznahorkay@cern.ch>
	* Added a hand-written CMakeLists.txt file to the package.
	* Removed the executable flag from the source files.
	* Tagging as VP1UtilsBase-00-00-05

2016-02-11 Riccardo.Maria.Bianchi@cern.ch
    * Added extra arguments to the VP1FileUtilities constructor, with default values:
      - outputDir = "",
      - bool forceMakeOutputDir = false,
      - bool removeInputFile = true

2014-11-06  Vakho Tsulaia  <tsulaia@cern.ch>

	* Remove the source file after copying
...
(Long ChangeLog diff - truncated)

graphics/VP1/VP1UtilsBase/CMakeLists.txt

+################################################################################
+# Package: VP1UtilsBase
+################################################################################
+# Author: Riccardo Maria BIANCHI <rbianchi@cern.ch>
+################################################################################
+
+# Declare the package name:
+atlas_subdir( VP1UtilsBase )
+
+# External dependencies:
+find_package( Qt4 COMPONENTS QtCore )
+
+# Component(s) in the package:
+atlas_add_library( VP1UtilsBase src/*.cxx
+   PUBLIC_HEADERS VP1UtilsBase
+   PRIVATE_LINK_LIBRARIES Qt4::QtCore )

graphics/VP1/VP1UtilsBase/VP1UtilsBase/VP1FileUtilities.h

@@ -10,6 +10,9 @@
 //                                                         //
 //  Initial version: October 2007                          //
 //                                                         //
+//  Updates:                                               //
+//      Riccardo Maria BIANCHI <rbianchi@cern.ch>          //
+//                                                         //
 /////////////////////////////////////////////////////////////
 
 #ifndef VP1FILEUTILITIES_H
@@ -32,7 +35,7 @@ public:
   //    * Writing: maximum number of files to keep in 'inputDirectory'
   //    * Reading: maximum number of files to keep in 'copyDirectory' (see below)
   VP1FileUtilities(const std::string& inputDirectory,
-		   unsigned int fileLimit);
+		   unsigned int fileLimit, const std::string& outputDir = "", bool forceMakeOutputDir = false, bool removeInputFile = true);
   virtual ~VP1FileUtilities();
 
   //
@@ -62,7 +65,10 @@ private:
   void cleanUp();
 
   std::string m_inputDirectory;
-  unsigned int m_fileLimit;
+  int m_fileLimit;
+  std::string m_outputDirectory;
+  bool m_forceMakeOutputDir;
+  bool m_removeInputFile;
 };
 
 #endif

graphics/VP1/VP1UtilsBase/src/VP1FileUtilities.cxx

@@ -25,11 +25,16 @@
 #include <iomanip>
 #include <iostream>
 #include <stdexcept>
+#include <thread>         // std::this_thread::sleep_for // C++11
+#include <chrono>         // std::chrono::seconds // C++11
 
 VP1FileUtilities::VP1FileUtilities(const std::string& inputDirectory,
-				   unsigned int fileLimit):
+				   unsigned int fileLimit, const std::string& outputDir, bool forceMakeOutputDir, bool removeInputFile):
   m_inputDirectory(inputDirectory),
-  m_fileLimit(fileLimit)
+  m_fileLimit(fileLimit),
+  m_outputDirectory(outputDir),
+  m_forceMakeOutputDir(forceMakeOutputDir),
+  m_removeInputFile(removeInputFile)
 {
   // Check if the input directory exists and is writable
   QFileInfo inpDir(m_inputDirectory.c_str());
@@ -37,6 +42,21 @@ VP1FileUtilities::VP1FileUtilities(const std::string& inputDirectory,
     std::string errMessage = std::string("VP1FileUtilities: ERROR!! The directory ") + inputDirectory + std::string(" either does not exist or is not writable");
     throw std::runtime_error(errMessage.c_str());
   }
+
+  // Check if the output directory exists and is writable
+  if (m_outputDirectory != "") {
+	  QFileInfo inpDir(m_outputDirectory.c_str());
+	  if(!inpDir.exists()||!inpDir.isDir()||!inpDir.isReadable()||!inpDir.isWritable()) {
+		  std::string errMessage = std::string("VP1FileUtilities: ERROR!! The directory ") + m_outputDirectory + std::string(" does not exist.");
+		  if (m_forceMakeOutputDir) {
+			  errMessage += "\nforceMakeOutputDir=True --> Creating the output folder now...";
+			  QDir().mkdir(m_outputDirectory.c_str());
+		  } else {
+			  throw std::runtime_error(errMessage.c_str());
+		  }
+	  }
+  }
+
 }
 
 VP1FileUtilities::~VP1FileUtilities()
@@ -56,30 +76,85 @@ void VP1FileUtilities::produceNewFile(const std::string& sourceFile,
     throw std::runtime_error("Source file does not exist!");
 
   QString inpDirName(m_inputDirectory.c_str());
+  QString outDirName(m_outputDirectory.c_str());
 
   // Construct new filename
   QString newFileName = inpDirName;
   std::ostringstream newFileStr;
 
   // if input directory name is empty don't add / to the file name
-  // also take into account a possibility od trailing slash in directory name
+  // also take into account a possibility of trailing slash in directory name
   if(m_inputDirectory!="" && m_inputDirectory.rfind("/")!=m_inputDirectory.size()-1)
     newFileStr << "/";
 
   QString latestEventFileName = inpDirName + QString(newFileStr.str().c_str()) + QString("latest_vp1event");
 
   if (textLabel != "" ) {
-    newFileStr << "vp1_" << runNumber << "_" << eventNumber << "_" << timeStamp << "_" << textLabel << ".pool.root";
+    newFileStr << "vp1_r" << runNumber << "_ev" << eventNumber << "_u" << timeStamp << "_t" << textLabel << ".pool.root";
   } else {
-    newFileStr << "vp1_" << runNumber << "_" << eventNumber << "_" << timeStamp << ".pool.root";
+    newFileStr << "vp1_r" << runNumber << "_ev" << eventNumber << "_u" << timeStamp << ".pool.root";
   }
 
   newFileName += QString(newFileStr.str().c_str());
 
+
+  // adding the output folder, if not empty
+  if (outDirName != "")
+	  newFileName = outDirName + QDir::separator() + newFileName;
+
   // do copy
+  std::cout << "copying '" <<  (srcFile.fileName()).toStdString() << "' to: '" << newFileName.toStdString() << "'..." << std::endl;
+
+
+    		  // get size of the file to be copied
+  qint64 inputSize = srcFile.size();
+    		  std::cout << "Size of the input file to be copied: " << inputSize << std::endl;
+
+
   if(!srcFile.copy(newFileName))
     throw std::runtime_error("Unable to produce new vp1 event file");
 
+
+
+      // the operation to time (for elapsed time)
+      char ch; std::cout << '?' && std::cin >> ch ;
+
+
+
+      // remove the input file (if not disabled by user)
+      if (m_removeInputFile) {
+    	  bool copyDone = false;
+    	  std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now() ;
+    	  typedef std::chrono::duration<int,std::milli> millisecs_t ;
+    	  while (!copyDone) {
+
+    		  // get handle on new file
+    		  QFileInfo checkFile(newFileName);
+
+
+    		  // check if file exists (and it is a file, and not a directory)
+    		  if (checkFile.exists() && checkFile.isFile() && (checkFile.size() == inputSize) )
+    		  {
+    			  std::cout << "Size of the copied file: " << checkFile.size() << std::endl;
+    			  copyDone = true;
+    			  if(!srcFile.remove())
+    				  std::cerr << "VP1FileUtilities WARNING! Unable to delete " << sourceFile << std::endl;
+    		  }
+    		  else
+    		  {
+    			  std::cout << "I could not find the output file, so probably the copy action is not finished yet. I'll wait for a short while and I will retry..." << std::endl;
+    			  std::this_thread::sleep_for(std::chrono::milliseconds(500)); //make the program waiting for 0.5 seconds
+    			  std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now() ;
+    			  millisecs_t duration( std::chrono::duration_cast<millisecs_t>(end-start) ) ;
+    			  if (duration.count() > 2000.0 )
+    			  {
+    				  std::cout << "WARNING!!! " << duration.count() << " milliseconds passed and still I cannot find the output file. Probably there was a problem. Giving up with the removal of the source file...\n" ;
+    				  copyDone = true;
+    			  }
+    		  }
+    	  }
+      }
+
   // create/update the latest event file
   QFile latestEvent(latestEventFileName);
   if(latestEvent.exists() && !latestEvent.remove())
@@ -90,25 +165,34 @@ void VP1FileUtilities::produceNewFile(const std::string& sourceFile,
   latestEvent.write(newFileName.toStdString().c_str());
   latestEvent.close();
 
-  // do cleanup if needed
-  cleanUp();
+  // do cleanup if requested. '-1' means 'KEEP ALL FILES'
+  if (m_fileLimit != -1) {
+	  std::cout << "VP1FileUtilities - cleaning up..." << std::endl;
+	  cleanUp();
+  }
 }
 
+// if the user set a "m_fileLimit" then clean the outputDirectory,
+// to only keep this predefined number of output files.
+// Please notice: default behaviours is: keep ALL output events.
 void VP1FileUtilities::cleanUp()
 {
+  //std::cout << "VP1FileUtilities::cleanUp()" << std::endl;
+
   QDir _dir;
-  _dir.setPath(QString(m_inputDirectory.c_str()));
+  _dir.setPath(QString(m_outputDirectory.c_str()));
 
   QStringList nameFilters;
   nameFilters << "vp1_*.pool.root";
 
-
   _dir.setFilter(QDir::Files);
   _dir.setNameFilters(nameFilters);
   _dir.setSorting(QDir::Time|QDir::Reversed);
   QFileInfoList list = _dir.entryInfoList();
 
-  if(unsigned(list.size()) > m_fileLimit)
+  //std::cout << "m_fileLimit: " << m_fileLimit << " - list.size(): " << list.size() << " - list: " << list << std::endl;
+
+  if(int(list.size()) > m_fileLimit)
   {
     QFileInfo fileInfo = list.at(0);
 
@@ -116,6 +200,17 @@ void VP1FileUtilities::cleanUp()
       throw std::runtime_error("VP1FileUtilities::cleanup() - WARNING: Unable to do the clean up!");
   }
 
+  QString poolCatalog("PoolFileCatalog.xml");
+  std::string poolCatalogStr = poolCatalog.toStdString();
+  std::cout << "looking for " << poolCatalogStr << " in " <<  QDir::currentPath().toStdString() << std::endl;
+  if ( QDir::current().entryList().contains( poolCatalog ) ) {
+	  std::cout << "VP1FileUtilities::cleanUp() - removing the file '" << poolCatalogStr << "' because it causes problems for subsequent Athena commands opening the copied file." << std::endl;
+	  QDir cwd = QDir::currentPath();
+	  if ( ! cwd.remove( poolCatalog ) )
+		std::cerr << "VP1FileUtilities WARNING! Unable to delete " << poolCatalogStr << std::endl;
+  }
+
+
   return;
 }