diff --git a/CrestApi/CrestContainer.h b/CrestApi/CrestContainer.h
index ceaa37b6f09342933e27c9fc12066a13467e6791..7afcad913a72be545fb371ae8e4b84bcaaf2feca 100644
--- a/CrestApi/CrestContainer.h
+++ b/CrestApi/CrestContainer.h
@@ -12,7 +12,7 @@ namespace Crest {
Bool, UChar, Int16, UInt16,
Int32, UInt32, UInt63, Int64,
Float, Double, String255, String4k,
- String64k, String16M, Blob64k, Blob16M, TypeIdCount
+ String64k, String16M, String128M, Blob64k, Blob16M, Blob128M, TypeIdCount
};
static std::map<TypeId, std::string> s_typeToString = {
{ Bool, "Bool" },
@@ -29,8 +29,10 @@ namespace Crest {
{ String4k, "String4k" },
{ String64k, "String64k" },
{ String16M, "String16M" },
+ { String128M,"String128M"},
{ Blob64k, "Blob64k" },
- { Blob16M, "Blob16M" }
+ { Blob16M, "Blob16M" },
+ { Blob128M, "Blob128M" }
};
enum ModeId{
@@ -43,10 +45,11 @@ namespace Crest {
nlohmann::json m_row={};
nlohmann::json m_iov_data={};
nlohmann::json m_vector_data=nlohmann::json::array();
+ nlohmann::json m_full_data={};
ModeId m_modeId;
bool m_isVectorPayload=false;
- void validatePayloadSize() const;
+ void validatePayloadSize(const nlohmann::json &data) const;
nlohmann::json createRowArray(const nlohmann::json& data_row) const;
const nlohmann::json& getRow();
@@ -156,11 +159,11 @@ namespace Crest {
/**
* @brief It encodes the input string to base64.
*/
- std::string base64_encode(const uint8_t* bytes_to_encode, unsigned int in_len);
+ static std::string base64_encode(const uint8_t* bytes_to_encode, unsigned int in_len);
/**
* @brief It decodes the input string from base64.
*/
- std::vector<unsigned char> base64_decode(const std::string& encodedData);
+ static std::vector<unsigned char> base64_decode(const std::string& encodedData);
/**
* @brief It returns the index of the column with the given name.
* @param name The name of the column.
@@ -171,6 +174,9 @@ namespace Crest {
* @brief It reinitializes the containers.
*/
void flush();
+ void clear();
+ void selectIov(const uint64_t since);
+ std::vector<std::string> channelIds();
/**
* @brief It returns the json representation of the container.
*/
@@ -190,7 +196,9 @@ namespace Crest {
/**
* @brief It reads a json object to fill the containers.
*/
- void from_json(const nlohmann::json& j);
+ void from_json(const uint64_t since, const nlohmann::json& j);
+
+ void parseOldFormat(std::string& colName, TypeId& typespec,const nlohmann::json & j);
};
}
#endif
diff --git a/CrestApi/CrestModel.h b/CrestApi/CrestModel.h
index a0ef692b92fc720685a96b4c7482a66e1a410914..d12144187bd40331b20d96254cd72414457a2508 100644
--- a/CrestApi/CrestModel.h
+++ b/CrestApi/CrestModel.h
@@ -228,6 +228,7 @@ public:
{
return m_channel_list;
}
+ std::string getFolderDescription(){ return m_node_description;}
size_t getChannelSize() const {return m_channel_list.getSize();}
size_t getColumnSize() const {return m_payload_spec.getSize();}
json to_json() const;
@@ -291,7 +292,7 @@ public:
std::vector<IovDto> resources;
const char* getFormat() const {return "IovSetDto";}
int64_t getSize() const{return resources.size();}
-
+ std::vector<long_t> getListSince();
json to_json() const;
static IovSetDto from_json(const json &j);
diff --git a/src/CrestContainer.cxx b/src/CrestContainer.cxx
index 3b596523bbdcdf058e07abcc42ae8e83caaca64c..2a01c45245a7db2b0cebb6ad30144dc58f54c509 100644
--- a/src/CrestContainer.cxx
+++ b/src/CrestContainer.cxx
@@ -170,11 +170,10 @@ void Crest::CrestContainer::addExternalData(const char *channel_id, const nlohma
//std::cout << "CondContainer::addData: add data for channel " << channel_id << std::endl;
//std::cout << "CondContainer::addData: row " << data.dump() << std::endl;
- validatePayloadSize();
-
+ validatePayloadSize(data);
nlohmann::json arr_data = m_isVectorPayload ? nlohmann::json::array() : nlohmann::json();
- for (const auto &data_row : m_vector_data)
+ for (const auto &data_row : data)//m_vector_data)
{
auto row_arr_data = createRowArray(data_row);
@@ -187,14 +186,15 @@ void Crest::CrestContainer::addExternalData(const char *channel_id, const nlohma
arr_data = row_arr_data;
}
}
-
m_payload[channel_id] = arr_data;
m_vector_data.clear();
m_row.clear();
}
-void Crest::CrestContainer::validatePayloadSize() const
+void Crest::CrestContainer::validatePayloadSize(const nlohmann::json &data) const
{
+ if (!data.is_array())
+ throw CommonCrestException("The format of data is wrong. It should be the vector (size 1 for non vector payload)");
if (!m_isVectorPayload && m_vector_data.size() > 1)
{
throw CommonCrestException("The payload is not a vector, but the size seems to be larger than 1....");
@@ -222,10 +222,23 @@ void Crest::CrestContainer::addIov(const uint64_t since)
//std::cout << "CondContainer::addIov: data:" << m_payload.dump() << std::endl;
m_iov_data["since"] = since;
m_iov_data["data"] = m_payload;
+ m_full_data[std::to_string(since)]=m_payload;
m_payload.clear();
// return m_iov_data;
}
+void Crest::CrestContainer::selectIov(const uint64_t since){
+ m_iov_data["since"]=since;
+ m_iov_data["data"]=m_full_data[std::to_string(since)];
+}
+
+std::vector<std::string> Crest::CrestContainer::channelIds(){
+ std::vector<std::string> chs;
+ for (auto& x : m_iov_data["data"].items()){
+ chs.push_back(x.key());
+ }
+ return chs;
+}
const std::vector<std::pair<std::string, Crest::TypeId>> &Crest::CrestContainer::getMPayloadSpec()
{
return m_payload_spec;
@@ -351,10 +364,10 @@ json Crest::CrestContainer::getPayloadSpec()
for (auto &column : m_payload_spec)
{
json j={};
- std::cout << "CondContainer::getPayloadSpec: name = " << column.first << ", type = " << column.second << std::endl;
+ //std::cout << "CondContainer::getPayloadSpec: name = " << column.first << ", type = " << column.second << std::endl;
std::map<TypeId, std::string>::const_iterator pos = Crest::s_typeToString.find(column.second);
if (pos == Crest::s_typeToString.end()) {
- std::cout << "CondContainer::getPayloadSpec: Type do not exist in the map. Type: name = " << column.first << ", type = " << column.second << std::endl;
+ //std::cout << "CondContainer::getPayloadSpec: Type do not exist in the map. Type: name = " << column.first << ", type = " << column.second << std::endl;
throw CommonCrestException("Type do not exist in the map.");
} else {
j[column.first] = pos->second;
@@ -389,10 +402,19 @@ void Crest::CrestContainer::setPayloadSpec(const nlohmann::json &j)
void Crest::CrestContainer::flush()
{
+// m_payload.clear();
+ m_iov_data.clear();
+ m_row.clear();
+ m_vector_data.clear();
+ m_full_data={};
+}
+
+void Crest::CrestContainer::clear(){
m_payload.clear();
m_iov_data.clear();
m_row.clear();
m_vector_data.clear();
+ m_full_data={};
}
// Function to dump JSON object into a file
@@ -445,9 +467,115 @@ nlohmann::json Crest::CrestContainer::read_json_from_file(const std::string &fil
return j;
}
-void Crest::CrestContainer::from_json(const nlohmann::json &j)
-{
+void Crest::CrestContainer::parseOldFormat(std::string& colName, TypeId& typespec,const nlohmann::json & thisVal){
+ try{
+ if (thisVal.is_null()){
+ m_row[colName] ="NULL";
+ return;
+ }
+ std::string strVal = to_string(thisVal);
+ if(strVal.size()>2&& strVal[0]=='"'&& strVal[strVal.size()-1]=='"')
+ strVal=strVal.substr(1,strVal.size()-2);
+
+ if((strVal.compare("NULL")==0||strVal.compare("null")==0)&&
+ (typespec==Bool || typespec==Int16 || typespec==UInt16
+ || typespec==Int32 || typespec==UInt32
+ || typespec==Int64 || typespec==UInt63
+ || typespec==Float || typespec==Double)){
+ m_row[colName] ="NULL";
+ return;
+ }
+ switch (typespec) {
+ case Bool:
+ {
+ const bool newVal=(strVal == "true");
+ m_row[colName] = newVal;
+ break;
+ }
+ case UChar:
+ {
+ m_row[colName]=std::stoul(strVal);
+ break;
+ }
+ case Int16:
+ {
+ m_row[colName]=std::stol(strVal);
+ break;
+ }
+ case UInt16:
+ {
+ m_row[colName]=std::stoul(strVal);
+ break;
+ }
+ case Int32:
+ {
+ m_row[colName]=std::stoi(strVal);
+ break;
+ }
+ case UInt32:
+ {
+ m_row[colName]=std::stoull(strVal);
+ break;
+ }
+ case UInt63:
+ {
+ m_row[colName]=std::stoull(strVal);
+ break;
+ }
+ case Int64:
+ {
+ m_row[colName]=std::stoll(strVal);
+ break;
+ }
+ case Float:
+ {
+ m_row[colName]=std::stof(strVal);
+ break;
+ }
+ case Double:
+ {
+ m_row[colName]=std::stod(strVal);
+ break;
+ }
+ case String255:
+ case String4k:
+ case String64k:
+ case String16M:
+ case String128M:
+ {
+ m_row[colName]=thisVal.get<std::string>();
+ break;
+ }
+ case Blob128M:
+ case Blob16M:
+ case Blob64k:
+ {
+ m_row[colName]=thisVal.get<std::string>();
+ break;
+ }
+ default:
+ {
+ std::cerr << "UNTREATED TYPE! typespec=" <<typespec<< std::endl;
+ throw std::runtime_error("UNTREATED TYPE!");
+ break;
+ }
+ }
+ }
+ catch(json::exception& e){
+ std::cerr << e.what() << std::endl;
+ throw std::runtime_error(e.what());
+ }
+}
+void Crest::CrestContainer::from_json(const uint64_t since,const nlohmann::json &j_in)
+{
+ nlohmann::json j = j_in;
+ if (j.is_string()){
+ std::istringstream ss(to_string(j));
+ std::string st;
+ ss >> std::quoted(st);
+ j = json::parse(st);
+ }
// Accessing the "data" object
if (j.contains("data") && j["data"].is_object())
{
@@ -457,65 +585,81 @@ void Crest::CrestContainer::from_json(const nlohmann::json &j)
for (const auto &channel : data.items())
{
std::string channelKey = channel.key();
- const auto &values = channel.value();
+ const auto &data_ch = channel.value();
+ nlohmann::json vecJson(json::value_t::array);
+ if(m_isVectorPayload)
+ vecJson=data_ch;
+ else
+ vecJson.push_back(data_ch);
- // std::cout << "Channel " << channelKey << " contains:" << values << std::endl;
+ for (const auto &values : vecJson)
+ {
+ //std::cout << "Channel " << channelKey << " contains:" << values << std::endl;
// TODO: this will not work for vector payloads
// Check if the number of values in the array matches the expected number of columns
- if (values.is_array() && values.size() == m_payload_spec.size())
- {
- for (size_t i = 0; i < values.size(); ++i)
+ if (values.is_array() && values.size() == m_payload_spec.size())
{
- const auto &spec = m_payload_spec[i];
- std::string colName = spec.first;
- TypeId colType = spec.second;
-
- switch (colType)
+ for (size_t i = 0; i < values.size(); ++i)
{
- case Bool:
- m_row[colName] = values[i].get<bool>();
- break;
- case UChar:
- m_row[colName] = values[i].get<unsigned char>();
- break;
- case Int16:
- case Int32:
- m_row[colName] = values[i].get<int>();
- break;
- case UInt16:
- case UInt32:
- m_row[colName] = values[i].get<unsigned int>();
+ const auto &spec = m_payload_spec[i];
+ std::string colName = spec.first;
+ TypeId colType = spec.second;
+ if(true){
+ parseOldFormat(colName,colType,values[i]);
+ continue;
+ }
+ switch (colType)
+ {
+ case Bool:
+ m_row[colName] = values[i].get<bool>();
+ break;
+ case UChar:
+ m_row[colName] = values[i].get<unsigned char>();
+ break;
+ case Int16:
+ case Int32:
+ m_row[colName] = values[i].get<int>();
+ break;
+ case UInt16:
+ case UInt32:
+ m_row[colName] = values[i].get<unsigned int>();
break;
- case UInt63:
- m_row[colName] = values[i].get<uint64_t>();
- break;
- case Int64:
- m_row[colName] = values[i].get<int64_t>();
- break;
- case Float:
- case Double:
- m_row[colName] = values[i].get<double>();
- break;
- case String255:
- case String4k:
- case String64k:
- case String16M:
- m_row[colName] = values[i].get<std::string>();
- break;
- case Blob64k:
- case Blob16M:
- m_row[colName] = values[i].get<std::string>();
- break;
- default:
- throw CommonCrestException("Unsupported column type.");
- }
+ case UInt63:
+ m_row[colName] = values[i].get<uint64_t>();
+ break;
+ case Int64:
+ m_row[colName] = values[i].get<int64_t>();
+ break;
+ case Float:
+ case Double:
+ m_row[colName] = values[i].get<double>();
+ break;
+ case String255:
+ case String4k:
+ case String64k:
+ case String16M:
+ case String128M:
+ m_row[colName] = values[i].get<std::string>();
+ break;
+ case Blob128M:
+ case Blob64k:
+ case Blob16M:
+ m_row[colName] = values[i].get<std::string>();
+ break;
+ default:
+ throw CommonCrestException("Unsupported column type.");
+ }
+ }
}
+ else
+ {
+ std::cerr << "CrestContainer::from_json: Mismatch in number of values for channel " << channelKey << std::endl;
+ }
+ if(m_isVectorPayload)
+ putRow2Vector();
}
- else
- {
- std::cerr << "CondContainer::from_json: Mismatch in number of values for channel " << channelKey << std::endl;
- }
+ //std::cout << " m_row: " << m_row.dump(2) << std::endl;
addData(channelKey.c_str());
// std::cout << " m_row: " << m_row.dump(2) << std::endl;
// std::cout << " m_payload: " << m_payload.dump(2) << std::endl;
@@ -523,9 +667,10 @@ void Crest::CrestContainer::from_json(const nlohmann::json &j)
}
else
{
- std::cerr << "CondContainer::from_json: JSON does not contain a 'data' object or it is not a JSON object." << std::endl;
+ std::cerr << "CrestContainer::from_json: JSON does not contain a 'data' object or it is not a JSON object." << std::endl;
+ std::cerr << "CrestContainer::from_json json:"<<j<<std::endl;
}
- addIov(j["since"]);
- // std::cout << " m_iov_data: " << m_iov_data.dump(2) << std::endl;
+ addIov(since);
+ //std::cout << " m_iov_data: " << m_iov_data.dump(2) << std::endl;
// std::cout << "CondContainer: from json method has filled internal attributes" << std::endl;
}
diff --git a/src/CrestModel.cxx b/src/CrestModel.cxx
index 7ff70303dca740ee185ced5df908f79f374f65a5..2b8e40878b1b791b3b72c6c67d3aecda09aca567 100644
--- a/src/CrestModel.cxx
+++ b/src/CrestModel.cxx
@@ -2,6 +2,7 @@
#include <iostream>
#include <string>
#include <iomanip>
+#include "boost/regex.hpp"
json RespPage::to_json() const
{
@@ -319,6 +320,23 @@ void PayloadSpecDto::add(std::string name, std::string type)
PayloadSpecDto PayloadSpecDto::from_json(const json &j)
{
PayloadSpecDto chDto;
+ //old format
+ if(!j.is_object()&&!j.is_array()){
+ std::string regex=R"delim(([^\s,:]*):\s?([^\s,]*),?)delim";
+ boost::regex expression(regex);
+ boost::smatch what;
+ std::string input=j.dump();
+ input=boost::regex_replace(input,boost::regex("\""),"");
+ bool match=boost::regex_search(input, what, expression);
+ while (match){
+ std::string n(what[1]);
+ std::string t(what[2]);
+ chDto.add(boost::regex_replace(n, boost::regex("(^[ ]+)|([ ]+$)"),""),boost::regex_replace(t, boost::regex("(^[ ]+)|([ ]+$)"),""));
+ input = what.suffix();
+ match=boost::regex_search(input, what, expression);
+ }
+ return chDto;
+ }
for (unsigned int i = 0; i < j.size(); i++)
{
for (auto &el : j[i].items())
@@ -483,6 +501,15 @@ json IovSetDto::to_json() const
return baseJson;
}
+std::vector<long_t> IovSetDto::getListSince(){
+ std::vector<long_t> v;
+ for (IovDto item_iov: resources){
+ long_t since = item_iov.since;
+ v.emplace_back(since);
+ }
+ return v;
+}
+
IovSetDto IovSetDto::from_json(const json &j)
{
IovSetDto iovSet;
@@ -537,6 +564,7 @@ StoreDto StoreDto::from_json(const json &j)
storeDto.hash = j.value("hash", "");
storeDto.since = j.value("since", 0.0);
storeDto.data = j.value("data", "");
+ //do we need get streamerInfo?
return storeDto;
}