From 51e9fd8b02cdac164d0dd46d3b3fd953bf809233 Mon Sep 17 00:00:00 2001
From: Riccardo Maria Bianchi <riccardo.maria.bianchi@cern.ch>
Date: Tue, 17 May 2016 15:42:17 +0200
Subject: [PATCH] Fixed corrupted batch-produced single-event data files.
Details: https://its.cern.ch/jira/browse/ATLASVPONE-316
(VP1UtilsBase-00-00-06)
2016-05-17 Riccardo.Maria.Bianchi@cern.ch
* Added removing of 'PoolFileCatalog.xml' after the copy of files,
because it causes errors while accessing copied files later in the same Athena
Details: https://its.cern.ch/jira/browse/ATLASVPONE-316
* new tag: VP1UtilsBase-00-00-05
2016-02-17 Attila Krasznahorkay <Attila.Krasznahorkay@cern.ch>
* Added a hand-written CMakeLists.txt file to the package.
* Removed the executable flag from the source files.
* Tagging as VP1UtilsBase-00-00-05
2016-02-11 Riccardo.Maria.Bianchi@cern.ch
* Added extra arguments to the VP1FileUtilities constructor, with default values:
- outputDir = "",
- bool forceMakeOutputDir = false,
- bool removeInputFile = true
2014-11-06 Vakho Tsulaia <tsulaia@cern.ch>
* Remove the source file after copying
...
(Long ChangeLog diff - truncated)
Former-commit-id: d62c984f9ccbed43ed217f11b46dca4aab892f9b
---
graphics/VP1/VP1UtilsBase/CMakeLists.txt | 16 +++
.../VP1UtilsBase/VP1FileUtilities.h | 10 +-
.../VP1/VP1UtilsBase/src/VP1FileUtilities.cxx | 115 ++++++++++++++++--
3 files changed, 129 insertions(+), 12 deletions(-)
create mode 100644 graphics/VP1/VP1UtilsBase/CMakeLists.txt
mode change 100755 => 100644 graphics/VP1/VP1UtilsBase/VP1UtilsBase/VP1FileUtilities.h
mode change 100755 => 100644 graphics/VP1/VP1UtilsBase/src/VP1FileUtilities.cxx
diff --git a/graphics/VP1/VP1UtilsBase/CMakeLists.txt b/graphics/VP1/VP1UtilsBase/CMakeLists.txt
new file mode 100644
index 000000000000..8c05f5d3cce6
--- /dev/null
+++ b/graphics/VP1/VP1UtilsBase/CMakeLists.txt
@@ -0,0 +1,16 @@
+################################################################################
+# Package: VP1UtilsBase
+################################################################################
+# Author: Riccardo Maria BIANCHI <rbianchi@cern.ch>
+################################################################################
+
+# Declare the package name:
+atlas_subdir( VP1UtilsBase )
+
+# External dependencies:
+find_package( Qt4 COMPONENTS QtCore )
+
+# Component(s) in the package:
+atlas_add_library( VP1UtilsBase src/*.cxx
+ PUBLIC_HEADERS VP1UtilsBase
+ PRIVATE_LINK_LIBRARIES Qt4::QtCore )
diff --git a/graphics/VP1/VP1UtilsBase/VP1UtilsBase/VP1FileUtilities.h b/graphics/VP1/VP1UtilsBase/VP1UtilsBase/VP1FileUtilities.h
old mode 100755
new mode 100644
index a9dfc312eaec..2f76de8d3229
--- a/graphics/VP1/VP1UtilsBase/VP1UtilsBase/VP1FileUtilities.h
+++ b/graphics/VP1/VP1UtilsBase/VP1UtilsBase/VP1FileUtilities.h
@@ -10,6 +10,9 @@
// //
// Initial version: October 2007 //
// //
+// Updates: //
+// Riccardo Maria BIANCHI <rbianchi@cern.ch> //
+// //
/////////////////////////////////////////////////////////////
#ifndef VP1FILEUTILITIES_H
@@ -32,7 +35,7 @@ public:
// * Writing: maximum number of files to keep in 'inputDirectory'
// * Reading: maximum number of files to keep in 'copyDirectory' (see below)
VP1FileUtilities(const std::string& inputDirectory,
- unsigned int fileLimit);
+ unsigned int fileLimit, const std::string& outputDir = "", bool forceMakeOutputDir = false, bool removeInputFile = true);
virtual ~VP1FileUtilities();
//
@@ -62,7 +65,10 @@ private:
void cleanUp();
std::string m_inputDirectory;
- unsigned int m_fileLimit;
+ int m_fileLimit;
+ std::string m_outputDirectory;
+ bool m_forceMakeOutputDir;
+ bool m_removeInputFile;
};
#endif
diff --git a/graphics/VP1/VP1UtilsBase/src/VP1FileUtilities.cxx b/graphics/VP1/VP1UtilsBase/src/VP1FileUtilities.cxx
old mode 100755
new mode 100644
index 5bb42f00e995..6ee33209a750
--- a/graphics/VP1/VP1UtilsBase/src/VP1FileUtilities.cxx
+++ b/graphics/VP1/VP1UtilsBase/src/VP1FileUtilities.cxx
@@ -25,11 +25,16 @@
#include <iomanip>
#include <iostream>
#include <stdexcept>
+#include <thread> // std::this_thread::sleep_for // C++11
+#include <chrono> // std::chrono::seconds // C++11
VP1FileUtilities::VP1FileUtilities(const std::string& inputDirectory,
- unsigned int fileLimit):
+ unsigned int fileLimit, const std::string& outputDir, bool forceMakeOutputDir, bool removeInputFile):
m_inputDirectory(inputDirectory),
- m_fileLimit(fileLimit)
+ m_fileLimit(fileLimit),
+ m_outputDirectory(outputDir),
+ m_forceMakeOutputDir(forceMakeOutputDir),
+ m_removeInputFile(removeInputFile)
{
// Check if the input directory exists and is writable
QFileInfo inpDir(m_inputDirectory.c_str());
@@ -37,6 +42,21 @@ VP1FileUtilities::VP1FileUtilities(const std::string& inputDirectory,
std::string errMessage = std::string("VP1FileUtilities: ERROR!! The directory ") + inputDirectory + std::string(" either does not exist or is not writable");
throw std::runtime_error(errMessage.c_str());
}
+
+ // Check if the output directory exists and is writable
+ if (m_outputDirectory != "") {
+ QFileInfo inpDir(m_outputDirectory.c_str());
+ if(!inpDir.exists()||!inpDir.isDir()||!inpDir.isReadable()||!inpDir.isWritable()) {
+ std::string errMessage = std::string("VP1FileUtilities: ERROR!! The directory ") + m_outputDirectory + std::string(" does not exist.");
+ if (m_forceMakeOutputDir) {
+ errMessage += "\nforceMakeOutputDir=True --> Creating the output folder now...";
+ QDir().mkdir(m_outputDirectory.c_str());
+ } else {
+ throw std::runtime_error(errMessage.c_str());
+ }
+ }
+ }
+
}
VP1FileUtilities::~VP1FileUtilities()
@@ -56,30 +76,85 @@ void VP1FileUtilities::produceNewFile(const std::string& sourceFile,
throw std::runtime_error("Source file does not exist!");
QString inpDirName(m_inputDirectory.c_str());
+ QString outDirName(m_outputDirectory.c_str());
// Construct new filename
QString newFileName = inpDirName;
std::ostringstream newFileStr;
// if input directory name is empty don't add / to the file name
- // also take into account a possibility od trailing slash in directory name
+ // also take into account a possibility of trailing slash in directory name
if(m_inputDirectory!="" && m_inputDirectory.rfind("/")!=m_inputDirectory.size()-1)
newFileStr << "/";
QString latestEventFileName = inpDirName + QString(newFileStr.str().c_str()) + QString("latest_vp1event");
if (textLabel != "" ) {
- newFileStr << "vp1_" << runNumber << "_" << eventNumber << "_" << timeStamp << "_" << textLabel << ".pool.root";
+ newFileStr << "vp1_r" << runNumber << "_ev" << eventNumber << "_u" << timeStamp << "_t" << textLabel << ".pool.root";
} else {
- newFileStr << "vp1_" << runNumber << "_" << eventNumber << "_" << timeStamp << ".pool.root";
+ newFileStr << "vp1_r" << runNumber << "_ev" << eventNumber << "_u" << timeStamp << ".pool.root";
}
newFileName += QString(newFileStr.str().c_str());
+
+ // adding the output folder, if not empty
+ if (outDirName != "")
+ newFileName = outDirName + QDir::separator() + newFileName;
+
// do copy
+ std::cout << "copying '" << (srcFile.fileName()).toStdString() << "' to: '" << newFileName.toStdString() << "'..." << std::endl;
+
+
+ // get size of the file to be copied
+ qint64 inputSize = srcFile.size();
+ std::cout << "Size of the input file to be copied: " << inputSize << std::endl;
+
+
if(!srcFile.copy(newFileName))
throw std::runtime_error("Unable to produce new vp1 event file");
+
+
+ // the operation to time (for elapsed time)
+ char ch; std::cout << '?' && std::cin >> ch ;
+
+
+
+ // remove the input file (if not disabled by user)
+ if (m_removeInputFile) {
+ bool copyDone = false;
+ std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now() ;
+ typedef std::chrono::duration<int,std::milli> millisecs_t ;
+ while (!copyDone) {
+
+ // get handle on new file
+ QFileInfo checkFile(newFileName);
+
+
+ // check if file exists (and it is a file, and not a directory)
+ if (checkFile.exists() && checkFile.isFile() && (checkFile.size() == inputSize) )
+ {
+ std::cout << "Size of the copied file: " << checkFile.size() << std::endl;
+ copyDone = true;
+ if(!srcFile.remove())
+ std::cerr << "VP1FileUtilities WARNING! Unable to delete " << sourceFile << std::endl;
+ }
+ else
+ {
+ std::cout << "I could not find the output file, so probably the copy action is not finished yet. I'll wait for a short while and I will retry..." << std::endl;
+ std::this_thread::sleep_for(std::chrono::milliseconds(500)); //make the program waiting for 0.5 seconds
+ std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now() ;
+ millisecs_t duration( std::chrono::duration_cast<millisecs_t>(end-start) ) ;
+ if (duration.count() > 2000.0 )
+ {
+ std::cout << "WARNING!!! " << duration.count() << " milliseconds passed and still I cannot find the output file. Probably there was a problem. Giving up with the removal of the source file...\n" ;
+ copyDone = true;
+ }
+ }
+ }
+ }
+
// create/update the latest event file
QFile latestEvent(latestEventFileName);
if(latestEvent.exists() && !latestEvent.remove())
@@ -90,25 +165,34 @@ void VP1FileUtilities::produceNewFile(const std::string& sourceFile,
latestEvent.write(newFileName.toStdString().c_str());
latestEvent.close();
- // do cleanup if needed
- cleanUp();
+ // do cleanup if requested. '-1' means 'KEEP ALL FILES'
+ if (m_fileLimit != -1) {
+ std::cout << "VP1FileUtilities - cleaning up..." << std::endl;
+ cleanUp();
+ }
}
+// if the user set a "m_fileLimit" then clean the outputDirectory,
+// to only keep this predefined number of output files.
+// Please notice: default behaviours is: keep ALL output events.
void VP1FileUtilities::cleanUp()
{
+ //std::cout << "VP1FileUtilities::cleanUp()" << std::endl;
+
QDir _dir;
- _dir.setPath(QString(m_inputDirectory.c_str()));
+ _dir.setPath(QString(m_outputDirectory.c_str()));
QStringList nameFilters;
nameFilters << "vp1_*.pool.root";
-
_dir.setFilter(QDir::Files);
_dir.setNameFilters(nameFilters);
_dir.setSorting(QDir::Time|QDir::Reversed);
QFileInfoList list = _dir.entryInfoList();
- if(unsigned(list.size()) > m_fileLimit)
+ //std::cout << "m_fileLimit: " << m_fileLimit << " - list.size(): " << list.size() << " - list: " << list << std::endl;
+
+ if(int(list.size()) > m_fileLimit)
{
QFileInfo fileInfo = list.at(0);
@@ -116,6 +200,17 @@ void VP1FileUtilities::cleanUp()
throw std::runtime_error("VP1FileUtilities::cleanup() - WARNING: Unable to do the clean up!");
}
+ QString poolCatalog("PoolFileCatalog.xml");
+ std::string poolCatalogStr = poolCatalog.toStdString();
+ std::cout << "looking for " << poolCatalogStr << " in " << QDir::currentPath().toStdString() << std::endl;
+ if ( QDir::current().entryList().contains( poolCatalog ) ) {
+ std::cout << "VP1FileUtilities::cleanUp() - removing the file '" << poolCatalogStr << "' because it causes problems for subsequent Athena commands opening the copied file." << std::endl;
+ QDir cwd = QDir::currentPath();
+ if ( ! cwd.remove( poolCatalog ) )
+ std::cerr << "VP1FileUtilities WARNING! Unable to delete " << poolCatalogStr << std::endl;
+ }
+
+
return;
}
--
GitLab