From f97e14730cd9d2f9d129a5cc6fd84ee2a256ab99 Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Thu, 16 Jul 2020 11:34:14 -0700
Subject: [PATCH 01/23] add CSV sink
---
src/libDataSink/CMakeLists.txt | 1 +
src/libDataSink/CSVSink.cpp | 213 +++++++++++++++++++++++++++++++++
src/libDataSink/CSVSink.h | 138 +++++++++++++++++++++
src/tools/ps_monitor.cpp | 5 +-
4 files changed, 355 insertions(+), 2 deletions(-)
create mode 100644 src/libDataSink/CSVSink.cpp
create mode 100644 src/libDataSink/CSVSink.h
diff --git a/src/libDataSink/CMakeLists.txt b/src/libDataSink/CMakeLists.txt
index 602e2bd3..fa757053 100644
--- a/src/libDataSink/CMakeLists.txt
+++ b/src/libDataSink/CMakeLists.txt
@@ -3,6 +3,7 @@ target_sources(DataSink
PRIVATE
IDataSink.cpp
ConsoleSink.cpp
+ CSVSink.cpp
DataSinkRegistry.cpp
)
target_link_libraries(DataSink PRIVATE Utils)
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
new file mode 100644
index 00000000..3f9d30dc
--- /dev/null
+++ b/src/libDataSink/CSVSink.cpp
@@ -0,0 +1,213 @@
+#include "CSVSink.h"
+
+#include <algorithm>
+#include <iostream>
+#include <iomanip>
+#include <fstream>
+
+#include "DataSinkRegistry.h"
+REGISTER_DATASINK(CSVSink)
+
+CSVSink::CSVSink(const std::string& name)
+ : IDataSink(name)
+{ }
+
+void CSVSink::setConfiguration(const nlohmann::json& config)
+{
+ for (const auto &kv : config.items())
+ {
+ if(kv.key()=="directory")
+ {
+ m_directory=kv.value();
+ }
+ }
+}
+
+void CSVSink::setTag(const std::string& name, const std::string& value)
+{
+ checkTag(name);
+ m_tagsString[name]=value;
+}
+
+void CSVSink::setTag(const std::string& name, int8_t value)
+{
+ checkTag(name);
+ m_tagsInt8[name]=value;
+}
+
+void CSVSink::setTag(const std::string& name, int32_t value)
+{
+ checkTag(name);
+ m_tagsInt32[name]=value;
+}
+
+void CSVSink::setTag(const std::string& name, int64_t value)
+{
+ checkTag(name);
+ m_tagsInt64[name]=value;
+}
+
+void CSVSink::setTag(const std::string& name, double value)
+{
+ checkTag(name);
+ m_tagsDouble[name]=value;
+}
+
+void CSVSink::startMeasurement(const std::string& measurement, std::chrono::time_point<std::chrono::system_clock> time)
+{
+ // Clean-up last measurement
+ m_fields.clear();
+ m_fieldsString.clear();
+ m_fieldsInt8 .clear();
+ m_fieldsInt32 .clear();
+ m_fieldsInt64 .clear();
+ m_fieldsDouble.clear();
+
+ // State
+ m_finalSchema=false;
+ m_inMeasurement=true;
+
+ m_measurement=measurement;
+}
+
+void CSVSink::setField(const std::string& name, const std::string& value)
+{
+ addField(name);
+ m_fieldsString[name]=value;
+}
+
+void CSVSink::setField(const std::string& name, int8_t value)
+{
+ addField(name);
+ m_fieldsInt8[name]=value;
+}
+
+void CSVSink::setField(const std::string& name, int32_t value)
+{
+ addField(name);
+ m_fieldsInt32[name]=value;
+}
+
+void CSVSink::setField(const std::string& name, int64_t value)
+{
+ addField(name);
+ m_fieldsInt64[name]=value;
+}
+
+void CSVSink::setField(const std::string& name, double value)
+{
+ addField(name);
+ m_fieldsDouble[name]=value;
+}
+
+void CSVSink::recordPoint()
+{
+ bool same=(m_measurement==m_lastMeasurement)&&(m_fields==m_lastFields)&&m_dirty;
+
+ std::ofstream ofile;
+ ofile.open (m_directory+m_measurement+".csv", std::ios_base::app);
+
+ //
+ // Print the header
+ if(!same)
+ { // First time recording point for measurement, print header
+ printHeader(ofile);
+ m_dirty=false;
+ }
+
+ //
+ // Print the fields
+ for(const std::string& field : m_fields)
+ {
+ if(m_fieldsString.count(field)>0)
+ ofile << "," << m_fieldsString[field];
+
+ if(m_fieldsInt8.count(field)>0)
+ ofile << "," << m_fieldsInt8[field];
+
+ if(m_fieldsInt32.count(field)>0)
+ ofile << "," << m_fieldsInt32[field];
+
+ if(m_fieldsInt64.count(field)>0)
+ ofile << "," << m_fieldsInt64[field];
+
+ if(m_fieldsDouble.count(field)>0)
+ ofile << "," << m_fieldsDouble[field];
+ }
+ ofile << "\n";
+
+ ofile.close();
+ // Update last configuration
+ m_lastMeasurement=m_measurement;
+ m_lastFields=m_fields;
+
+ m_finalSchema=true;
+}
+
+void CSVSink::endMeasurement()
+{
+ m_inMeasurement=false;
+}
+
+void CSVSink::checkTag(const std::string& name)
+{
+ if(checkReserved(name))
+ throw std::runtime_error("Tag name \""+name+"\" is reserved.");
+
+
+ if(std::find(m_tags.begin(), m_tags.end(), name)==m_tags.end())
+ {
+ if(m_inMeasurement)
+ throw std::runtime_error("CSVSink: Cannot change tag during measurement.");
+
+ m_tags.push_back(name);
+ }
+
+ m_dirty=true;
+}
+
+void CSVSink::addField(const std::string& name)
+{
+ if(checkReserved(name))
+ throw std::runtime_error("Tag name \""+name+"\" is reserved.");
+
+ if(std::find(m_fields.begin(), m_fields.end(), name)==m_fields.end())
+ {
+ if(m_finalSchema)
+ throw std::runtime_error("CSVSink: Cannot add new fields after the first recordPoint.");
+ m_fields.push_back(name);
+ }
+}
+
+void CSVSink::printHeader(std::ofstream& ofile)
+{
+ // Print out the tags
+ for(const std::string& tag : m_tags)
+ {
+ ofile << tag << ": ";
+
+ if(m_tagsString.count(tag)>0)
+ ofile << m_tagsString[tag];
+
+ if(m_tagsInt8.count(tag)>0)
+ ofile << m_tagsInt8 [tag];
+
+ if(m_tagsInt32.count(tag)>0)
+ ofile << m_tagsInt32 [tag];
+
+ if(m_tagsInt64.count(tag)>0)
+ ofile << m_tagsInt64 [tag];
+
+ if(m_tagsDouble.count(tag)>0)
+ ofile << m_tagsDouble[tag];
+
+ ofile << std::endl;
+ }
+
+ // Print field names
+ for(const std::string& field : m_fields)
+ {
+ ofile << "," << field;
+ }
+ ofile << "\n";
+}
diff --git a/src/libDataSink/CSVSink.h b/src/libDataSink/CSVSink.h
new file mode 100644
index 00000000..3a4de1cc
--- /dev/null
+++ b/src/libDataSink/CSVSink.h
@@ -0,0 +1,138 @@
+#ifndef CSVSINK_H
+#define CSVSINK_H
+
+#include <vector>
+#include <unordered_map>
+
+#include "IDataSink.h"
+
+//! \brief Data sink that saves to CSV files
+/**
+ * When a data point is recorded, all fields are saved separated by a space
+ * The header (tags and the column names) are printed when any of the following happens:
+ * - The name of the measurement changes.
+ * - A tag value changes.
+ * - The set of fields changes.
+ *
+ * The header is not printed just because a previous measurement has been stopped. If
+ * the field definitions and tags are the same, the previous header still applies.
+ */
+class CSVSink : public IDataSink
+{
+public:
+ CSVSink(const std::string& name);
+
+ /** \brief Configure file sink based on JSON object
+ *
+ * Valid keys:
+ * - `directory`: output directory where files are saved
+ *
+ * \param config JSON configuration
+ */
+ virtual void setConfiguration(const nlohmann::json& config);
+
+ virtual void setTag(const std::string& name, const std::string& value);
+ virtual void setTag(const std::string& name, int8_t value);
+ virtual void setTag(const std::string& name, int32_t value);
+ virtual void setTag(const std::string& name, int64_t value);
+ virtual void setTag(const std::string& name, double value);
+
+ virtual void startMeasurement(const std::string& measurement, std::chrono::time_point<std::chrono::system_clock> time);
+ virtual void setField(const std::string& name, const std::string& value);
+ virtual void setField(const std::string& name, int8_t value);
+ virtual void setField(const std::string& name, int32_t value);
+ virtual void setField(const std::string& name, int64_t value);
+ virtual void setField(const std::string& name, double value);
+
+ //! \brief Print data point
+ /**
+ * Prints the fields in the current data points. If the header
+ * needs to be reprinted, it is also done here.
+ */
+ virtual void recordPoint();
+ virtual void endMeasurement();
+
+private:
+ //! \brief Checks whether a tag can be added
+ /**
+ * If tag is not defined, then it is added to the list.
+ *
+ * Marks tags as dirty.
+ *
+ * Error checking is performed. An exception is thrown if
+ * - Currently performing a measurement
+ *
+ * \param name Field name
+ */
+ void checkTag(const std::string& name);
+
+ //! \brief Adds a new field column if it already does not exists.
+ /**
+ * If field is already defined, nothing is done.
+ *
+ * Error checking is performed. An exception is thrown if
+ * - called after the first `recordPoint` in a measurement
+ *
+ * \param name Field name
+ */
+ void addField(const std::string& name);
+
+ //! \brief Print the header
+ /**
+ * Prints the header in the format.
+ * Tag1: value
+ * ...
+ * Tagn: value
+ * FIELD1 FIELD2 FIELD3
+ *
+ * \param ofile file to save the measurement
+ */
+ void printHeader(std::ofstream& ofile);
+
+ //
+ // Directory where files are saved
+ std::string m_directory;
+
+ //
+ // The last state (before endMeasurement)
+
+ // Name of the last measurement
+ std::string m_lastMeasurement;
+
+ // The last columns
+ std::vector<std::string> m_lastFields;
+
+ //
+ // State
+
+ // Name of the current measurement (valid onyl if performing one)
+ std::string m_measurement;
+
+ // Currently performing a measurement
+ bool m_inMeasurement=false;
+
+
+ //
+ // Data store for fields
+ bool m_finalSchema=false; // The measurement schema is locked
+ std::vector<std::string> m_fields;
+ std::unordered_map<std::string, std::string> m_fieldsString;
+ std::unordered_map<std::string, int8_t > m_fieldsInt8 ;
+ std::unordered_map<std::string, int32_t > m_fieldsInt32;
+ std::unordered_map<std::string, int64_t > m_fieldsInt64;
+ std::unordered_map<std::string, double > m_fieldsDouble;
+
+
+ //
+ // Data store for tags
+ bool m_dirty=true; // Tag values have changed, reprint
+ std::vector<std::string> m_tags;
+ std::unordered_map<std::string, std::string> m_tagsString;
+ std::unordered_map<std::string, int8_t > m_tagsInt8 ;
+ std::unordered_map<std::string, int32_t > m_tagsInt32;
+ std::unordered_map<std::string, int64_t > m_tagsInt64;
+ std::unordered_map<std::string, double > m_tagsDouble;
+
+};
+
+#endif // CSVSINK_H
diff --git a/src/tools/ps_monitor.cpp b/src/tools/ps_monitor.cpp
index 3fd3a58b..2adbc3bc 100644
--- a/src/tools/ps_monitor.cpp
+++ b/src/tools/ps_monitor.cpp
@@ -85,8 +85,9 @@ int main(int argc, char*argv[])
logger(logDEBUG) << " Config file: " << configFile;
logger(logDEBUG) << " sink name: " << sinkName;
logger(logDEBUG) << " PS channel: " << channelName;
-
- // Crease sink
+
+ //
+ // Create sink
DataSinkConf ds;
ds.setHardwareConfig(configFile);
std::shared_ptr<IDataSink> sink = ds.getDataSink(sinkName);
--
GitLab
From 37f96715da81aead42d0fbea469956e1aaea8229 Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Thu, 16 Jul 2020 11:36:45 -0700
Subject: [PATCH 02/23] remove space
---
src/tools/ps_monitor.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/src/tools/ps_monitor.cpp b/src/tools/ps_monitor.cpp
index 2adbc3bc..525b0729 100644
--- a/src/tools/ps_monitor.cpp
+++ b/src/tools/ps_monitor.cpp
@@ -86,7 +86,6 @@ int main(int argc, char*argv[])
logger(logDEBUG) << " sink name: " << sinkName;
logger(logDEBUG) << " PS channel: " << channelName;
- //
// Create sink
DataSinkConf ds;
ds.setHardwareConfig(configFile);
--
GitLab
From 4c0641e308410f4d1fd64712e6451db662e96281 Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Thu, 16 Jul 2020 12:59:20 -0700
Subject: [PATCH 03/23] add tags as columns
---
src/libDataSink/CSVSink.cpp | 34 +++++++++++++++++++++++++++++++---
1 file changed, 31 insertions(+), 3 deletions(-)
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index 3f9d30dc..d8e45b23 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -102,7 +102,7 @@ void CSVSink::setField(const std::string& name, double value)
void CSVSink::recordPoint()
{
- bool same=(m_measurement==m_lastMeasurement)&&(m_fields==m_lastFields)&&m_dirty;
+ bool same=(m_measurement==m_lastMeasurement)&&(m_fields==m_lastFields)&&!m_dirty;
std::ofstream ofile;
ofile.open (m_directory+m_measurement+".csv", std::ios_base::app);
@@ -115,6 +115,30 @@ void CSVSink::recordPoint()
m_dirty=false;
}
+ bool first = true;
+ for(const std::string& tag : m_tags)
+ {
+ if (!(first == true)) ofile << ", ";
+
+ if(m_tagsString.count(tag)>0)
+ ofile << m_tagsString[tag];
+
+ if(m_tagsInt8.count(tag)>0)
+ ofile << m_tagsInt8 [tag];
+
+ if(m_tagsInt32.count(tag)>0)
+ ofile << m_tagsInt32 [tag];
+
+ if(m_tagsInt64.count(tag)>0)
+ ofile << m_tagsInt64 [tag];
+
+ if(m_tagsDouble.count(tag)>0)
+ ofile << m_tagsDouble[tag];
+
+ if (first == true) first = false;
+
+ }
+
//
// Print the fields
for(const std::string& field : m_fields)
@@ -181,10 +205,13 @@ void CSVSink::addField(const std::string& name)
void CSVSink::printHeader(std::ofstream& ofile)
{
+
+ bool first = true;
+
// Print out the tags
for(const std::string& tag : m_tags)
{
- ofile << tag << ": ";
+ if (!(first == true)) ofile << ", ";
if(m_tagsString.count(tag)>0)
ofile << m_tagsString[tag];
@@ -201,7 +228,8 @@ void CSVSink::printHeader(std::ofstream& ofile)
if(m_tagsDouble.count(tag)>0)
ofile << m_tagsDouble[tag];
- ofile << std::endl;
+ if (first == true) first = false;
+
}
// Print field names
--
GitLab
From 6b7849973e56d54e434f3d7fa8add4b16a815ef6 Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Thu, 16 Jul 2020 15:10:38 -0700
Subject: [PATCH 04/23] check tags are same within measurement
---
src/libDataSink/CSVSink.cpp | 3 ++-
src/libDataSink/CSVSink.h | 3 +++
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index d8e45b23..fcb7fc0c 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -102,7 +102,7 @@ void CSVSink::setField(const std::string& name, double value)
void CSVSink::recordPoint()
{
- bool same=(m_measurement==m_lastMeasurement)&&(m_fields==m_lastFields)&&!m_dirty;
+ bool same=(m_measurement==m_lastMeasurement)&&(m_fields==m_lastFields)&&(m_tags==m_lastTags)&&!m_dirty;
std::ofstream ofile;
ofile.open (m_directory+m_measurement+".csv", std::ios_base::app);
@@ -164,6 +164,7 @@ void CSVSink::recordPoint()
// Update last configuration
m_lastMeasurement=m_measurement;
m_lastFields=m_fields;
+ m_lastTags = m_tags;
m_finalSchema=true;
}
diff --git a/src/libDataSink/CSVSink.h b/src/libDataSink/CSVSink.h
index 3a4de1cc..ec897157 100644
--- a/src/libDataSink/CSVSink.h
+++ b/src/libDataSink/CSVSink.h
@@ -102,6 +102,9 @@ private:
// The last columns
std::vector<std::string> m_lastFields;
+ // The last tags
+ std::vector<std::string> m_lastTags;
+
//
// State
--
GitLab
From 6cee4076fc34ececb76e5b79372f7e59df07eaf7 Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Fri, 17 Jul 2020 12:02:58 -0700
Subject: [PATCH 05/23] update comments
---
src/libDataSink/CSVSink.h | 19 ++++++++-----------
1 file changed, 8 insertions(+), 11 deletions(-)
diff --git a/src/libDataSink/CSVSink.h b/src/libDataSink/CSVSink.h
index ec897157..a564c8f6 100644
--- a/src/libDataSink/CSVSink.h
+++ b/src/libDataSink/CSVSink.h
@@ -8,13 +8,13 @@
//! \brief Data sink that saves to CSV files
/**
- * When a data point is recorded, all fields are saved separated by a space
- * The header (tags and the column names) are printed when any of the following happens:
+ * When a data point is recorded, all fields are saved separated by comma
+ * The header (tags and the column names) are written to a file when any of the following happens:
* - The name of the measurement changes.
* - A tag value changes.
* - The set of fields changes.
*
- * The header is not printed just because a previous measurement has been stopped. If
+ * The header is not written to file just because a previous measurement has been stopped. If
* the field definitions and tags are the same, the previous header still applies.
*/
class CSVSink : public IDataSink
@@ -46,8 +46,8 @@ public:
//! \brief Print data point
/**
- * Prints the fields in the current data points. If the header
- * needs to be reprinted, it is also done here.
+ * Saves the fields in the current data points in csv file. If the header
+ * needs to be written to file, it is also done here.
*/
virtual void recordPoint();
virtual void endMeasurement();
@@ -77,13 +77,10 @@ private:
*/
void addField(const std::string& name);
- //! \brief Print the header
+ //! \brief write the header
/**
- * Prints the header in the format.
- * Tag1: value
- * ...
- * Tagn: value
- * FIELD1 FIELD2 FIELD3
+ * write the header in the format.
+ * Tag1 Tag 2 FIELD1 FIELD2 FIELD3
*
* \param ofile file to save the measurement
*/
--
GitLab
From 418953c7e745620148647b4b5280d02210e37c28 Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Mon, 27 Jul 2020 13:57:53 -0700
Subject: [PATCH 06/23] add timestamp,fix tag printing in header, add sink
example in config
---
src/configs/input-hw.json | 6 ++-
src/libDataSink/CSVSink.cpp | 74 ++++++++++++++++---------------------
src/libDataSink/CSVSink.h | 13 +++++--
3 files changed, 47 insertions(+), 46 deletions(-)
diff --git a/src/configs/input-hw.json b/src/configs/input-hw.json
index 4f1ebafe..5ac1f503 100644
--- a/src/configs/input-hw.json
+++ b/src/configs/input-hw.json
@@ -6,7 +6,11 @@
"datasinks":{
"Console":{
"sinktype": "ConsoleSink"
- }
+ },
+ "File":{
+ "sinktype": "CSVSink",
+ "directory": "/home"
+ }
},
"devices": {
"PS" : {
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index fcb7fc0c..5e904133 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -3,7 +3,6 @@
#include <algorithm>
#include <iostream>
#include <iomanip>
-#include <fstream>
#include "DataSinkRegistry.h"
REGISTER_DATASINK(CSVSink)
@@ -68,6 +67,14 @@ void CSVSink::startMeasurement(const std::string& measurement, std::chrono::time
m_inMeasurement=true;
m_measurement=measurement;
+
+ // Convert the timestamp to ms
+ m_timestamp = std::chrono::duration_cast<std::chrono::milliseconds>(time.time_since_epoch()).count();
+
+ // open CSV file
+ m_ofile.open (m_directory+"/"+m_measurement+".csv", std::ios_base::app);
+
+
}
void CSVSink::setField(const std::string& name, const std::string& value)
@@ -104,38 +111,37 @@ void CSVSink::recordPoint()
{
bool same=(m_measurement==m_lastMeasurement)&&(m_fields==m_lastFields)&&(m_tags==m_lastTags)&&!m_dirty;
- std::ofstream ofile;
- ofile.open (m_directory+m_measurement+".csv", std::ios_base::app);
-
//
// Print the header
if(!same)
{ // First time recording point for measurement, print header
- printHeader(ofile);
+ printHeader(m_ofile);
m_dirty=false;
}
+
+ // Print timestamp
+ m_ofile << m_timestamp;
- bool first = true;
+ //
+ // Print the tags
for(const std::string& tag : m_tags)
- {
- if (!(first == true)) ofile << ", ";
+ {
+ m_ofile << ", ";
if(m_tagsString.count(tag)>0)
- ofile << m_tagsString[tag];
+ m_ofile << m_tagsString[tag];
if(m_tagsInt8.count(tag)>0)
- ofile << m_tagsInt8 [tag];
+ m_ofile << m_tagsInt8 [tag];
if(m_tagsInt32.count(tag)>0)
- ofile << m_tagsInt32 [tag];
+ m_ofile << m_tagsInt32 [tag];
if(m_tagsInt64.count(tag)>0)
- ofile << m_tagsInt64 [tag];
+ m_ofile << m_tagsInt64 [tag];
if(m_tagsDouble.count(tag)>0)
- ofile << m_tagsDouble[tag];
-
- if (first == true) first = false;
+ m_ofile << m_tagsDouble[tag];
}
@@ -144,23 +150,22 @@ void CSVSink::recordPoint()
for(const std::string& field : m_fields)
{
if(m_fieldsString.count(field)>0)
- ofile << "," << m_fieldsString[field];
+ m_ofile << "," << m_fieldsString[field];
if(m_fieldsInt8.count(field)>0)
- ofile << "," << m_fieldsInt8[field];
+ m_ofile << "," << m_fieldsInt8[field];
if(m_fieldsInt32.count(field)>0)
- ofile << "," << m_fieldsInt32[field];
+ m_ofile << "," << m_fieldsInt32[field];
if(m_fieldsInt64.count(field)>0)
- ofile << "," << m_fieldsInt64[field];
+ m_ofile << "," << m_fieldsInt64[field];
if(m_fieldsDouble.count(field)>0)
- ofile << "," << m_fieldsDouble[field];
+ m_ofile << "," << m_fieldsDouble[field];
}
- ofile << "\n";
+ m_ofile << std::endl;
- ofile.close();
// Update last configuration
m_lastMeasurement=m_measurement;
m_lastFields=m_fields;
@@ -172,6 +177,7 @@ void CSVSink::recordPoint()
void CSVSink::endMeasurement()
{
m_inMeasurement=false;
+ m_ofile.close();
}
void CSVSink::checkTag(const std::string& name)
@@ -209,28 +215,12 @@ void CSVSink::printHeader(std::ofstream& ofile)
bool first = true;
+ ofile << "Time";
+
// Print out the tags
for(const std::string& tag : m_tags)
{
- if (!(first == true)) ofile << ", ";
-
- if(m_tagsString.count(tag)>0)
- ofile << m_tagsString[tag];
-
- if(m_tagsInt8.count(tag)>0)
- ofile << m_tagsInt8 [tag];
-
- if(m_tagsInt32.count(tag)>0)
- ofile << m_tagsInt32 [tag];
-
- if(m_tagsInt64.count(tag)>0)
- ofile << m_tagsInt64 [tag];
-
- if(m_tagsDouble.count(tag)>0)
- ofile << m_tagsDouble[tag];
-
- if (first == true) first = false;
-
+ ofile << "," << tag ;
}
// Print field names
@@ -238,5 +228,5 @@ void CSVSink::printHeader(std::ofstream& ofile)
{
ofile << "," << field;
}
- ofile << "\n";
+ ofile << std::endl;
}
diff --git a/src/libDataSink/CSVSink.h b/src/libDataSink/CSVSink.h
index a564c8f6..99167695 100644
--- a/src/libDataSink/CSVSink.h
+++ b/src/libDataSink/CSVSink.h
@@ -3,12 +3,13 @@
#include <vector>
#include <unordered_map>
+#include <fstream>
#include "IDataSink.h"
//! \brief Data sink that saves to CSV files
/**
- * When a data point is recorded, all fields are saved separated by comma
+ * When a data point is recorded, all fields are saved separated by a comma.
* The header (tags and the column names) are written to a file when any of the following happens:
* - The name of the measurement changes.
* - A tag value changes.
@@ -44,10 +45,11 @@ public:
virtual void setField(const std::string& name, int64_t value);
virtual void setField(const std::string& name, double value);
- //! \brief Print data point
+ //! \brief Save data point to CSV
/**
* Saves the fields in the current data points in csv file. If the header
* needs to be written to file, it is also done here.
+ * The timestamp will be uploaded with milliseconds precision.
*/
virtual void recordPoint();
virtual void endMeasurement();
@@ -105,13 +107,18 @@ private:
//
// State
+ // timestamp
+ unsigned long long m_timestamp;
+
+ // Output CSV file
+ std::ofstream m_ofile;
+
// Name of the current measurement (valid onyl if performing one)
std::string m_measurement;
// Currently performing a measurement
bool m_inMeasurement=false;
-
//
// Data store for fields
bool m_finalSchema=false; // The measurement schema is locked
--
GitLab
From bb7d6e9ab69a276ff97b5b56b8f86a2881ebc3be Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Mon, 27 Jul 2020 15:54:25 -0700
Subject: [PATCH 07/23] print header only first time file is created for the
first measurement
---
src/libDataSink/CSVSink.cpp | 23 ++++++++++++++---------
src/libDataSink/CSVSink.h | 8 +++++++-
2 files changed, 21 insertions(+), 10 deletions(-)
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index 5e904133..5913d621 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -28,7 +28,7 @@ void CSVSink::setTag(const std::string& name, const std::string& value)
m_tagsString[name]=value;
}
-void CSVSink::setTag(const std::string& name, int8_t value)
+ void CSVSink::setTag(const std::string& name, int8_t value)
{
checkTag(name);
m_tagsInt8[name]=value;
@@ -71,10 +71,15 @@ void CSVSink::startMeasurement(const std::string& measurement, std::chrono::time
// Convert the timestamp to ms
m_timestamp = std::chrono::duration_cast<std::chrono::milliseconds>(time.time_since_epoch()).count();
- // open CSV file
- m_ofile.open (m_directory+"/"+m_measurement+".csv", std::ios_base::app);
+ std::string fileName = m_directory+"/"+m_measurement+".csv";
-
+ // Check if file exists
+ m_rfile.open(fileName);
+ m_fileExist = m_rfile.fail();
+ m_rfile.close();
+
+ // Open CSV file
+ m_ofile.open (fileName, std::ios_base::app);
}
void CSVSink::setField(const std::string& name, const std::string& value)
@@ -109,14 +114,15 @@ void CSVSink::setField(const std::string& name, double value)
void CSVSink::recordPoint()
{
- bool same=(m_measurement==m_lastMeasurement)&&(m_fields==m_lastFields)&&(m_tags==m_lastTags)&&!m_dirty;
+ bool same=(m_measurement==m_lastMeasurement)&&(m_fields==m_lastFields)&&(m_tags==m_lastTags);
//
// Print the header
- if(!same)
+ if(!same&&(m_fileExist&&m_first))
{ // First time recording point for measurement, print header
printHeader(m_ofile);
m_dirty=false;
+ m_first=false;
}
// Print timestamp
@@ -169,7 +175,7 @@ void CSVSink::recordPoint()
// Update last configuration
m_lastMeasurement=m_measurement;
m_lastFields=m_fields;
- m_lastTags = m_tags;
+ m_lastTags=m_tags;
m_finalSchema=true;
}
@@ -177,6 +183,7 @@ void CSVSink::recordPoint()
void CSVSink::endMeasurement()
{
m_inMeasurement=false;
+ m_first=false;
m_ofile.close();
}
@@ -193,8 +200,6 @@ void CSVSink::checkTag(const std::string& name)
m_tags.push_back(name);
}
-
- m_dirty=true;
}
void CSVSink::addField(const std::string& name)
diff --git a/src/libDataSink/CSVSink.h b/src/libDataSink/CSVSink.h
index 99167695..3f021431 100644
--- a/src/libDataSink/CSVSink.h
+++ b/src/libDataSink/CSVSink.h
@@ -119,6 +119,13 @@ private:
// Currently performing a measurement
bool m_inMeasurement=false;
+ // First time record point in data file
+ bool m_first = true;
+
+ // File exists
+ std::ifstream m_rfile;
+ bool m_fileExist;
+
//
// Data store for fields
bool m_finalSchema=false; // The measurement schema is locked
@@ -129,7 +136,6 @@ private:
std::unordered_map<std::string, int64_t > m_fieldsInt64;
std::unordered_map<std::string, double > m_fieldsDouble;
-
//
// Data store for tags
bool m_dirty=true; // Tag values have changed, reprint
--
GitLab
From 3a05ab43a32d65c7242a11168a52263842e5d56c Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Mon, 27 Jul 2020 15:58:54 -0700
Subject: [PATCH 08/23] change capitalization
---
src/libDataSink/CSVSink.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index 5913d621..f0bff3eb 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -220,7 +220,7 @@ void CSVSink::printHeader(std::ofstream& ofile)
bool first = true;
- ofile << "Time";
+ ofile << "time";
// Print out the tags
for(const std::string& tag : m_tags)
--
GitLab
From 3683c3b1412ffbf7bf0f50c6013fa96470600a65 Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Mon, 27 Jul 2020 16:00:42 -0700
Subject: [PATCH 09/23] update description
---
src/libDataSink/CSVSink.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/libDataSink/CSVSink.h b/src/libDataSink/CSVSink.h
index 3f021431..2d80813c 100644
--- a/src/libDataSink/CSVSink.h
+++ b/src/libDataSink/CSVSink.h
@@ -7,7 +7,7 @@
#include "IDataSink.h"
-//! \brief Data sink that saves to CSV files
+//! \brief Data sink that saves to CSV files with comma delimiter
/**
* When a data point is recorded, all fields are saved separated by a comma.
* The header (tags and the column names) are written to a file when any of the following happens:
--
GitLab
From 8ece6022cd6e90038c6f4e2e9e62728bd371dd07 Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Mon, 27 Jul 2020 20:23:37 -0700
Subject: [PATCH 10/23] check if string contains comma
---
src/libDataSink/CSVSink.cpp | 30 +++++++++++++++++++++++++-----
src/libDataSink/CSVSink.h | 13 +++++++++++++
2 files changed, 38 insertions(+), 5 deletions(-)
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index f0bff3eb..2304b585 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -25,7 +25,7 @@ void CSVSink::setConfiguration(const nlohmann::json& config)
void CSVSink::setTag(const std::string& name, const std::string& value)
{
checkTag(name);
- m_tagsString[name]=value;
+ m_tagsString[name]=checkCSVEncoding(value);
}
void CSVSink::setTag(const std::string& name, int8_t value)
@@ -85,7 +85,7 @@ void CSVSink::startMeasurement(const std::string& measurement, std::chrono::time
void CSVSink::setField(const std::string& name, const std::string& value)
{
addField(name);
- m_fieldsString[name]=value;
+ m_fieldsString[name]=checkCSVEncoding(value);
}
void CSVSink::setField(const std::string& name, int8_t value)
@@ -114,7 +114,7 @@ void CSVSink::setField(const std::string& name, double value)
void CSVSink::recordPoint()
{
- bool same=(m_measurement==m_lastMeasurement)&&(m_fields==m_lastFields)&&(m_tags==m_lastTags);
+ bool same=(m_measurement==m_lastMeasurement)&&(m_fields==m_lastFields)&&(m_tags==m_lastTags)&&(!m_dirty);
//
// Print the header
@@ -187,8 +187,22 @@ void CSVSink::endMeasurement()
m_ofile.close();
}
+std::string CSVSink::checkCSVEncoding(const std::string& name)
+{
+ std::string newstr = name;
+
+ if (name.find(',') != std::string::npos)
+ {
+ newstr = "\""+name+"\"";
+ }
+ return newstr;
+}
+
+
void CSVSink::checkTag(const std::string& name)
{
+ std::string newname;
+
if(checkReserved(name))
throw std::runtime_error("Tag name \""+name+"\" is reserved.");
@@ -198,12 +212,16 @@ void CSVSink::checkTag(const std::string& name)
if(m_inMeasurement)
throw std::runtime_error("CSVSink: Cannot change tag during measurement.");
- m_tags.push_back(name);
+ newname = checkCSVEncoding(name);
+ m_tags.push_back(newname);
}
+
+ m_dirty=true;
}
void CSVSink::addField(const std::string& name)
{
+ std::string newname;
if(checkReserved(name))
throw std::runtime_error("Tag name \""+name+"\" is reserved.");
@@ -211,7 +229,9 @@ void CSVSink::addField(const std::string& name)
{
if(m_finalSchema)
throw std::runtime_error("CSVSink: Cannot add new fields after the first recordPoint.");
- m_fields.push_back(name);
+
+ newname = checkCSVEncoding(name);
+ m_fields.push_back(newname);
}
}
diff --git a/src/libDataSink/CSVSink.h b/src/libDataSink/CSVSink.h
index 2d80813c..09908e47 100644
--- a/src/libDataSink/CSVSink.h
+++ b/src/libDataSink/CSVSink.h
@@ -55,6 +55,19 @@ public:
virtual void endMeasurement();
private:
+ //! \brief Check if string contains CSV character
+ /**
+ * Check if the string contains a comma or double quote.
+ *
+ * If the string contains a comma, return the string in double-quotes.
+ *
+ * \param name string to be check.
+ *
+ * return encoded string if string contains a comma.
+ * else return original name.
+ */
+ std::string checkCSVEncoding(const std::string& name);
+
//! \brief Checks whether a tag can be added
/**
* If tag is not defined, then it is added to the list.
--
GitLab
From 11a5a6dd592e541eaa266185f8c45ad8d56ae3b5 Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Mon, 27 Jul 2020 20:52:48 -0700
Subject: [PATCH 11/23] fix encoding
---
src/libDataSink/CSVSink.cpp | 61 ++++++++++++++++++++-----------------
1 file changed, 33 insertions(+), 28 deletions(-)
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index 2304b585..d21487da 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -24,32 +24,37 @@ void CSVSink::setConfiguration(const nlohmann::json& config)
void CSVSink::setTag(const std::string& name, const std::string& value)
{
- checkTag(name);
- m_tagsString[name]=checkCSVEncoding(value);
+ std::string newTag = checkCSVEncoding(name);
+ checkTag(newTag);
+ m_tagsString[newTag]=checkCSVEncoding(value);
}
- void CSVSink::setTag(const std::string& name, int8_t value)
+void CSVSink::setTag(const std::string& name, int8_t value)
{
- checkTag(name);
- m_tagsInt8[name]=value;
+ std::string newTag = checkCSVEncoding(name);
+ checkTag(newTag);
+ m_tagsInt8[newTag]=value;
}
void CSVSink::setTag(const std::string& name, int32_t value)
{
- checkTag(name);
- m_tagsInt32[name]=value;
+ std::string newTag = checkCSVEncoding(name);
+ checkTag(newTag);
+ m_tagsInt32[newTag]=value;
}
void CSVSink::setTag(const std::string& name, int64_t value)
{
- checkTag(name);
- m_tagsInt64[name]=value;
+ std::string newTag = checkCSVEncoding(name);
+ checkTag(newTag);
+ m_tagsInt64[newTag]=value;
}
void CSVSink::setTag(const std::string& name, double value)
{
- checkTag(name);
- m_tagsDouble[name]=value;
+ std::string newTag = checkCSVEncoding(name);
+ checkTag(newTag);
+ m_tagsDouble[newTag]=value;
}
void CSVSink::startMeasurement(const std::string& measurement, std::chrono::time_point<std::chrono::system_clock> time)
@@ -84,32 +89,37 @@ void CSVSink::startMeasurement(const std::string& measurement, std::chrono::time
void CSVSink::setField(const std::string& name, const std::string& value)
{
- addField(name);
- m_fieldsString[name]=checkCSVEncoding(value);
+ std::string newField = checkCSVEncoding(name);
+ addField(newField);
+ m_fieldsString[newField]=checkCSVEncoding(value);
}
void CSVSink::setField(const std::string& name, int8_t value)
{
- addField(name);
- m_fieldsInt8[name]=value;
+ std::string newField = checkCSVEncoding(name);
+ addField(newField);
+ m_fieldsInt8[newField]=value;
}
void CSVSink::setField(const std::string& name, int32_t value)
{
- addField(name);
- m_fieldsInt32[name]=value;
+ std::string newField = checkCSVEncoding(name);
+ addField(newField);
+ m_fieldsInt32[newField]=value;
}
void CSVSink::setField(const std::string& name, int64_t value)
{
- addField(name);
- m_fieldsInt64[name]=value;
+ std::string newField = checkCSVEncoding(name);
+ addField(newField);
+ m_fieldsInt64[newField]=value;
}
void CSVSink::setField(const std::string& name, double value)
{
- addField(name);
- m_fieldsDouble[name]=value;
+ std::string newField = checkCSVEncoding(name);
+ addField(newField);
+ m_fieldsDouble[newField]=value;
}
void CSVSink::recordPoint()
@@ -201,8 +211,6 @@ std::string CSVSink::checkCSVEncoding(const std::string& name)
void CSVSink::checkTag(const std::string& name)
{
- std::string newname;
-
if(checkReserved(name))
throw std::runtime_error("Tag name \""+name+"\" is reserved.");
@@ -212,8 +220,7 @@ void CSVSink::checkTag(const std::string& name)
if(m_inMeasurement)
throw std::runtime_error("CSVSink: Cannot change tag during measurement.");
- newname = checkCSVEncoding(name);
- m_tags.push_back(newname);
+ m_tags.push_back(name);
}
m_dirty=true;
@@ -221,7 +228,6 @@ void CSVSink::checkTag(const std::string& name)
void CSVSink::addField(const std::string& name)
{
- std::string newname;
if(checkReserved(name))
throw std::runtime_error("Tag name \""+name+"\" is reserved.");
@@ -230,8 +236,7 @@ void CSVSink::addField(const std::string& name)
if(m_finalSchema)
throw std::runtime_error("CSVSink: Cannot add new fields after the first recordPoint.");
- newname = checkCSVEncoding(name);
- m_fields.push_back(newname);
+ m_fields.push_back(name);
}
}
--
GitLab
From ac89036eaf0e8a05ff658b115e09a0d83eeaed9e Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Mon, 27 Jul 2020 20:57:39 -0700
Subject: [PATCH 12/23] change rfile declaration
---
src/libDataSink/CSVSink.cpp | 7 ++++---
src/libDataSink/CSVSink.h | 1 -
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index d21487da..68fb6026 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -79,9 +79,10 @@ void CSVSink::startMeasurement(const std::string& measurement, std::chrono::time
std::string fileName = m_directory+"/"+m_measurement+".csv";
// Check if file exists
- m_rfile.open(fileName);
- m_fileExist = m_rfile.fail();
- m_rfile.close();
+ std::ifstream rfile;
+ rfile.open(fileName);
+ m_fileExist = rfile.fail();
+ rfile.close();
// Open CSV file
m_ofile.open (fileName, std::ios_base::app);
diff --git a/src/libDataSink/CSVSink.h b/src/libDataSink/CSVSink.h
index 09908e47..a2686a06 100644
--- a/src/libDataSink/CSVSink.h
+++ b/src/libDataSink/CSVSink.h
@@ -136,7 +136,6 @@ private:
bool m_first = true;
// File exists
- std::ifstream m_rfile;
bool m_fileExist;
//
--
GitLab
From 1ec17879c4fef3720ec49237c6ba69130da06f0f Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Mon, 27 Jul 2020 21:04:03 -0700
Subject: [PATCH 13/23] check if output file opens
---
src/libDataSink/CSVSink.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index 68fb6026..4e8849b4 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -86,6 +86,8 @@ void CSVSink::startMeasurement(const std::string& measurement, std::chrono::time
// Open CSV file
m_ofile.open (fileName, std::ios_base::app);
+ if (!(m_ofile.is_open()))
+ throw std::runtime_error("Could not open file: "+fileName);
}
void CSVSink::setField(const std::string& name, const std::string& value)
--
GitLab
From 8f9e231ae929fc4bf51ff03f0bc19ce5b259d311 Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Mon, 27 Jul 2020 21:07:35 -0700
Subject: [PATCH 14/23] remove unnecessary m_first
---
src/libDataSink/CSVSink.cpp | 4 +---
src/libDataSink/CSVSink.h | 3 ---
2 files changed, 1 insertion(+), 6 deletions(-)
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index 4e8849b4..00e53009 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -131,11 +131,10 @@ void CSVSink::recordPoint()
//
// Print the header
- if(!same&&(m_fileExist&&m_first))
+ if(!same&&(m_fileExist))
{ // First time recording point for measurement, print header
printHeader(m_ofile);
m_dirty=false;
- m_first=false;
}
// Print timestamp
@@ -196,7 +195,6 @@ void CSVSink::recordPoint()
void CSVSink::endMeasurement()
{
m_inMeasurement=false;
- m_first=false;
m_ofile.close();
}
diff --git a/src/libDataSink/CSVSink.h b/src/libDataSink/CSVSink.h
index a2686a06..68526eb6 100644
--- a/src/libDataSink/CSVSink.h
+++ b/src/libDataSink/CSVSink.h
@@ -132,9 +132,6 @@ private:
// Currently performing a measurement
bool m_inMeasurement=false;
- // First time record point in data file
- bool m_first = true;
-
// File exists
bool m_fileExist;
--
GitLab
From dc87785d3ee948da333df44b39201023e1a2a51b Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Mon, 27 Jul 2020 21:11:33 -0700
Subject: [PATCH 15/23] change when to print header
---
src/libDataSink/CSVSink.cpp | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index 00e53009..9c1c9f12 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -81,7 +81,7 @@ void CSVSink::startMeasurement(const std::string& measurement, std::chrono::time
// Check if file exists
std::ifstream rfile;
rfile.open(fileName);
- m_fileExist = rfile.fail();
+ m_fileExist = !(rfile.fail());
rfile.close();
// Open CSV file
@@ -127,12 +127,10 @@ void CSVSink::setField(const std::string& name, double value)
void CSVSink::recordPoint()
{
- bool same=(m_measurement==m_lastMeasurement)&&(m_fields==m_lastFields)&&(m_tags==m_lastTags)&&(!m_dirty);
-
//
// Print the header
- if(!same&&(m_fileExist))
- { // First time recording point for measurement, print header
+ if(!(m_fileExist))
+ { // First time recording point in CSV file, print header
printHeader(m_ofile);
m_dirty=false;
}
--
GitLab
From c3ff837f2925e39544dcdcd8aa94afd5e4cfdbbb Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Mon, 27 Jul 2020 21:16:37 -0700
Subject: [PATCH 16/23] update documentation and remove m_dirty
---
src/libDataSink/CSVSink.cpp | 2 --
src/libDataSink/CSVSink.h | 11 ++---------
2 files changed, 2 insertions(+), 11 deletions(-)
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index 9c1c9f12..2efedc33 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -132,7 +132,6 @@ void CSVSink::recordPoint()
if(!(m_fileExist))
{ // First time recording point in CSV file, print header
printHeader(m_ofile);
- m_dirty=false;
}
// Print timestamp
@@ -222,7 +221,6 @@ void CSVSink::checkTag(const std::string& name)
m_tags.push_back(name);
}
- m_dirty=true;
}
void CSVSink::addField(const std::string& name)
diff --git a/src/libDataSink/CSVSink.h b/src/libDataSink/CSVSink.h
index 68526eb6..b7fa6e74 100644
--- a/src/libDataSink/CSVSink.h
+++ b/src/libDataSink/CSVSink.h
@@ -9,14 +9,8 @@
//! \brief Data sink that saves to CSV files with comma delimiter
/**
- * When a data point is recorded, all fields are saved separated by a comma.
- * The header (tags and the column names) are written to a file when any of the following happens:
- * - The name of the measurement changes.
- * - A tag value changes.
- * - The set of fields changes.
- *
- * The header is not written to file just because a previous measurement has been stopped. If
- * the field definitions and tags are the same, the previous header still applies.
+ * When a data point is recorded, a timestamp followed by tags and fields are saved separated by a comma.
+ * The header (tags and field names) are written to a file when a file is first created.
*/
class CSVSink : public IDataSink
{
@@ -147,7 +141,6 @@ private:
//
// Data store for tags
- bool m_dirty=true; // Tag values have changed, reprint
std::vector<std::string> m_tags;
std::unordered_map<std::string, std::string> m_tagsString;
std::unordered_map<std::string, int8_t > m_tagsInt8 ;
--
GitLab
From 144aac6e1a49cd8cfc8522072c9fb4f4d52be243 Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Mon, 27 Jul 2020 21:27:01 -0700
Subject: [PATCH 17/23] update format
---
src/libDataSink/CSVSink.cpp | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index 2efedc33..6db84f76 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -141,7 +141,7 @@ void CSVSink::recordPoint()
// Print the tags
for(const std::string& tag : m_tags)
{
- m_ofile << ", ";
+ m_ofile << ",";
if(m_tagsString.count(tag)>0)
m_ofile << m_tagsString[tag];
@@ -164,20 +164,22 @@ void CSVSink::recordPoint()
// Print the fields
for(const std::string& field : m_fields)
{
+ m_ofile << ",";
+
if(m_fieldsString.count(field)>0)
- m_ofile << "," << m_fieldsString[field];
+ m_ofile << m_fieldsString[field];
if(m_fieldsInt8.count(field)>0)
- m_ofile << "," << m_fieldsInt8[field];
+ m_ofile << m_fieldsInt8[field];
if(m_fieldsInt32.count(field)>0)
- m_ofile << "," << m_fieldsInt32[field];
+ m_ofile << m_fieldsInt32[field];
if(m_fieldsInt64.count(field)>0)
- m_ofile << "," << m_fieldsInt64[field];
+ m_ofile << m_fieldsInt64[field];
if(m_fieldsDouble.count(field)>0)
- m_ofile << "," << m_fieldsDouble[field];
+ m_ofile << m_fieldsDouble[field];
}
m_ofile << std::endl;
--
GitLab
From 0287076d5dc24a5811dc1009ba5e463276e58885 Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Mon, 27 Jul 2020 21:38:06 -0700
Subject: [PATCH 18/23] update documentation
---
src/libDataSink/CSVSink.h | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/src/libDataSink/CSVSink.h b/src/libDataSink/CSVSink.h
index b7fa6e74..8a1518d9 100644
--- a/src/libDataSink/CSVSink.h
+++ b/src/libDataSink/CSVSink.h
@@ -9,8 +9,12 @@
//! \brief Data sink that saves to CSV files with comma delimiter
/**
- * When a data point is recorded, a timestamp followed by tags and fields are saved separated by a comma.
- * The header (tags and field names) are written to a file when a file is first created.
+ * When a data point is recorded, the following are saved in this order, separated by a comma:
+ * - timestamp (under column heading time)
+ * - tags
+ * - fields
+ *
+ * The header (time, tag names, and field names) are written to a file when a file is first created.
*/
class CSVSink : public IDataSink
{
--
GitLab
From 41a4f6f3d6cc762d14287dda52aa38a874bc0e94 Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Mon, 27 Jul 2020 21:46:26 -0700
Subject: [PATCH 19/23] remove unused variable
---
src/libDataSink/CSVSink.cpp | 3 ---
1 file changed, 3 deletions(-)
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index 6db84f76..1a72a691 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -241,9 +241,6 @@ void CSVSink::addField(const std::string& name)
void CSVSink::printHeader(std::ofstream& ofile)
{
-
- bool first = true;
-
ofile << "time";
// Print out the tags
--
GitLab
From b23bb92fe9547f956964f9e37b3c0363bd85a27c Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Mon, 27 Jul 2020 22:08:44 -0700
Subject: [PATCH 20/23] update csv encoding
---
src/libDataSink/CSVSink.cpp | 14 +++++++++++++-
src/libDataSink/CSVSink.h | 1 +
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index 1a72a691..eb706649 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -201,9 +201,21 @@ std::string CSVSink::checkCSVEncoding(const std::string& name)
{
std::string newstr = name;
+ if(name.find("\"") != std::string::npos)
+ {
+ size_t start_pos = 0;
+ std::string sold = "\"";
+ std::string snew = "\"\"";
+
+ while((start_pos = newstr.find(sold, start_pos)) != std::string::npos) {
+ newstr.replace(start_pos, sold.length(), snew);
+ start_pos += snew.length();
+ }
+ }
+
if (name.find(',') != std::string::npos)
{
- newstr = "\""+name+"\"";
+ newstr = "\""+newstr+"\"";
}
return newstr;
}
diff --git a/src/libDataSink/CSVSink.h b/src/libDataSink/CSVSink.h
index 8a1518d9..c5627032 100644
--- a/src/libDataSink/CSVSink.h
+++ b/src/libDataSink/CSVSink.h
@@ -58,6 +58,7 @@ private:
* Check if the string contains a comma or double quote.
*
* If the string contains a comma, return the string in double-quotes.
+ * If the string contains quotes, the quotation mark will be escaped.
*
* \param name string to be check.
*
--
GitLab
From 6bf9411c38a9f61642803745759871ba48916024 Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Tue, 28 Jul 2020 06:28:52 -0700
Subject: [PATCH 21/23] remove unusued variables, updated config, update
encoding
---
src/configs/input-hw.json | 2 +-
src/libDataSink/CSVSink.cpp | 28 ++++++++++++----------------
src/libDataSink/CSVSink.h | 14 +-------------
3 files changed, 14 insertions(+), 30 deletions(-)
diff --git a/src/configs/input-hw.json b/src/configs/input-hw.json
index 5ac1f503..6ec89242 100644
--- a/src/configs/input-hw.json
+++ b/src/configs/input-hw.json
@@ -9,7 +9,7 @@
},
"File":{
"sinktype": "CSVSink",
- "directory": "/home"
+ "directory": "myOutputData"
}
},
"devices": {
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index eb706649..7434d18e 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -183,11 +183,6 @@ void CSVSink::recordPoint()
}
m_ofile << std::endl;
- // Update last configuration
- m_lastMeasurement=m_measurement;
- m_lastFields=m_fields;
- m_lastTags=m_tags;
-
m_finalSchema=true;
}
@@ -201,20 +196,21 @@ std::string CSVSink::checkCSVEncoding(const std::string& name)
{
std::string newstr = name;
- if(name.find("\"") != std::string::npos)
+ if (name.find(',') != std::string::npos)
{
- size_t start_pos = 0;
- std::string sold = "\"";
- std::string snew = "\"\"";
- while((start_pos = newstr.find(sold, start_pos)) != std::string::npos) {
- newstr.replace(start_pos, sold.length(), snew);
- start_pos += snew.length();
- }
- }
+ if(name.find("\"") != std::string::npos)
+ {
+ size_t start_pos = 0;
+ std::string sold = "\"";
+ std::string snew = "\"\"";
+
+ while((start_pos = newstr.find(sold, start_pos)) != std::string::npos) {
+ newstr.replace(start_pos, sold.length(), snew);
+ start_pos += snew.length();
+ }
+ }
- if (name.find(',') != std::string::npos)
- {
newstr = "\""+newstr+"\"";
}
return newstr;
diff --git a/src/libDataSink/CSVSink.h b/src/libDataSink/CSVSink.h
index c5627032..fe795cfc 100644
--- a/src/libDataSink/CSVSink.h
+++ b/src/libDataSink/CSVSink.h
@@ -58,7 +58,7 @@ private:
* Check if the string contains a comma or double quote.
*
* If the string contains a comma, return the string in double-quotes.
- * If the string contains quotes, the quotation mark will be escaped.
+ * If the string contains quotes and a comma, the quotation mark will be escaped.
*
* \param name string to be check.
*
@@ -104,18 +104,6 @@ private:
// Directory where files are saved
std::string m_directory;
- //
- // The last state (before endMeasurement)
-
- // Name of the last measurement
- std::string m_lastMeasurement;
-
- // The last columns
- std::vector<std::string> m_lastFields;
-
- // The last tags
- std::vector<std::string> m_lastTags;
-
//
// State
--
GitLab
From a06b6824bcf628ea405ba38cc3492d924c191793 Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Tue, 28 Jul 2020 07:57:06 -0700
Subject: [PATCH 22/23] remove unnecessary if and add comments
---
src/libDataSink/CSVSink.cpp | 23 +++++++++++------------
1 file changed, 11 insertions(+), 12 deletions(-)
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index 7434d18e..fdb9c413 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -196,21 +196,20 @@ std::string CSVSink::checkCSVEncoding(const std::string& name)
{
std::string newstr = name;
+ // check if the name contains a comma
if (name.find(',') != std::string::npos)
{
-
- if(name.find("\"") != std::string::npos)
- {
- size_t start_pos = 0;
- std::string sold = "\"";
- std::string snew = "\"\"";
-
- while((start_pos = newstr.find(sold, start_pos)) != std::string::npos) {
- newstr.replace(start_pos, sold.length(), snew);
- start_pos += snew.length();
- }
- }
+ size_t start_pos = 0;
+ std::string sold = "\"";
+ std::string snew = "\"\"";
+
+ // check if string has quotation mark, if so replace by double quotation marks
+ while((start_pos = newstr.find(sold, start_pos)) != std::string::npos) {
+ newstr.replace(start_pos, sold.length(), snew);
+ start_pos += snew.length();
+ }
+ // enclose the string in double quotes
newstr = "\""+newstr+"\"";
}
return newstr;
--
GitLab
From 7bef1f33ad230f33b89ebfefcfe9d9ac4282e2f3 Mon Sep 17 00:00:00 2001
From: Elodie Resseguie <elodie.deborah.resseguie@cern.ch>
Date: Tue, 28 Jul 2020 07:58:58 -0700
Subject: [PATCH 23/23] change string type to static const
---
src/libDataSink/CSVSink.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/libDataSink/CSVSink.cpp b/src/libDataSink/CSVSink.cpp
index fdb9c413..414ab539 100644
--- a/src/libDataSink/CSVSink.cpp
+++ b/src/libDataSink/CSVSink.cpp
@@ -200,8 +200,8 @@ std::string CSVSink::checkCSVEncoding(const std::string& name)
if (name.find(',') != std::string::npos)
{
size_t start_pos = 0;
- std::string sold = "\"";
- std::string snew = "\"\"";
+ static const std::string sold = "\"";
+ static const std::string snew = "\"\"";
// check if string has quotation mark, if so replace by double quotation marks
while((start_pos = newstr.find(sold, start_pos)) != std::string::npos) {
--
GitLab