diff --git a/CrestApi/CrestApiFs.h b/CrestApi/CrestApiFs.h
index d686fe883406b56dbb1704cdb3c71bbd17b64b56..8677139f91985cffd9f8cc400225917814205cbd 100644
--- a/CrestApi/CrestApiFs.h
+++ b/CrestApi/CrestApiFs.h
@@ -119,8 +119,10 @@ namespace Crest
void createTagMeta(TagMetaDto &tag) override;
void updateTagMeta(TagMetaDto &tag) override;
TagMetaDto findTagMeta(const std::string &name) override;
+
// Iovs
IovSetDto selectIovs(const std::string &name, long since, long until, long snapshot, int size, int page, const std::string &sort) override;
+ // IovSetDto selectIovs(const std::string &name, long since, long until, long snapshot, int size, int page, const std::string & = "") override; // removing unused parameter
IovSetDto selectGroups(const std::string &name, long snapshot, int size, int page, const std::string &sort) override;
// GlobalTagMap methods
@@ -128,6 +130,7 @@ namespace Crest
GlobalTagMapSetDto findGlobalTagMap(const std::string &name, const std::string &xCrestMapMode) override;
void removeGlobalTagMap(const std::string &name, const std::string &record, const std::string &label, const std::string &tagname) override;
+
// Payload methods
void storeData(const std::string &payloadFormat,
const std::string &tag,
@@ -140,14 +143,36 @@ namespace Crest
std::string getPayload(const std::string &hash) override;
PayloadDto getPayloadMeta(const std::string &hash) override;
+ // testing (private methods):
+ // std::vector<std::string> nameList(std::string & folder, bool ascending = true);
+ // nlohmann::json sortJson(nlohmann::json js, const std::string & par, bool order);
+ // nlohmann::json sortIOVJson(nlohmann::json js, bool order);
+ // bool isMatch(std::string word, std::string pattern);
+ // std::vector<std::string> getVectorPage(std::vector<std::string> data, int size, int page);
+
private:
std::string getDateAndTime();
std::string getHash(std::string_view str);
std::string getFirstLetters(const std::string &str);
- // Auxillary methods to build the path to the file.
+ // auxiliary methods to extract a single page of the data:
+ nlohmann::json getPage(nlohmann::json data, int size, int page);
+ std::vector<std::string> getVectorPage(std::vector<std::string> data, int size, int page);
+
+ // auxiliary method to get folder names:
+ std::vector<std::string> nameList(std::string & folder, bool ascending = true);
+
+ // Auxiliary methods to build the path to the file.
std::string buildPath(const std::string &path, const std::string &file);
+ // Auxiliary methods to sort the JSON array by the key value:
+ nlohmann::json sortJson(nlohmann::json js, const std::string & par, bool order);
+ nlohmann::json sortIOVJson(nlohmann::json js, bool order);
+
+ // auxiliary methods to check if the string corresponds to the mask:
+ bool isMatch(std::string word, long unsigned int n, std::string pattern, long unsigned int m);
+ bool isMatch(std::string word, std::string pattern);
+
nlohmann::json findAllIovs(const std::string &tagname);
std::string getHashForFile(const std::string &file);
void storePayloadDump(const std::string &tag,
diff --git a/CrestApi/CrestModel.h b/CrestApi/CrestModel.h
index f123b08a7223a5c17b6266b1dcc0ff9972600cd9..f343917c1b50dca4db52b23db583a3d46c516222 100644
--- a/CrestApi/CrestModel.h
+++ b/CrestApi/CrestModel.h
@@ -375,6 +375,23 @@ public:
}
return tagMapSet;
}
+
+ static GlobalTagMapSetDto from_fs_json(const json &j)
+ {
+ GlobalTagMapSetDto tagMapSet;
+ int n = j.size();
+
+ for (auto it = j.begin(); it != j.end(); ++it)
+ {
+ tagMapSet.resources.push_back(GlobalTagMapDto::from_json(*it));
+ }
+
+ tagMapSet.size = n;
+ tagMapSet.datatype = "maps";
+ tagMapSet.format = "GlobalTagMapSetDto";
+
+ return tagMapSet;
+ }
};
@@ -471,7 +488,7 @@ public:
{
IovDto iov;
iov.tagName = j.value("tagName", "");
- iov.since = j.value("since", 0.0);
+ iov.since = j.value("since", 0.0);
iov.insertionTime = j.value("insertionTime", "");
iov.payloadHash = j.value("payloadHash", "");
return iov;
@@ -494,6 +511,7 @@ public:
baseJson["resources"] = jsonResources;
return baseJson;
}
+
static IovSetDto from_json(const json &j)
{
IovSetDto iovSet;
@@ -516,6 +534,25 @@ public:
// Parse other properties as needed
return iovSet;
}
+
+ static IovSetDto from_fs_json(const json &j)
+ {
+ IovSetDto iovSet;
+ int n = j.size();
+
+ for (auto it = j.begin(); it != j.end(); ++it)
+ {
+ iovSet.resources.push_back(IovDto::from_json(*it));
+ }
+
+ // Deserialize CrestBaseResponse part
+ iovSet.size = n;
+ iovSet.datatype = "iovs";
+ iovSet.format = "IovSetDto";
+
+ // Parse other properties as needed
+ return iovSet;
+ }
};
class StoreDto
@@ -539,7 +576,7 @@ public:
// Function to deserialize the object from JSON
static StoreDto from_json(const json &j)
{
- StoreDto storeDto;
+ StoreDto storeDto;
storeDto.hash = j.value("hash", "");
storeDto.since = j.value("since", 0.0);
storeDto.data = j.value("data", "");
diff --git a/doc/crest_example_fs.cxx b/doc/crest_example_fs.cxx
index a2093d03bf2a5d7b177b69a06b7f089a17210f5f..824b63af0bd457e89d3fe3ea1e8c32cbe5714410 100644
--- a/doc/crest_example_fs.cxx
+++ b/doc/crest_example_fs.cxx
@@ -3,7 +3,7 @@
\brief Main file
This file contains the examples for the CREST C++ Client Library.
- Main part of the examples is commented. Pl testCreateGlobalTagMapFsease uncomment the code you need.
+ Main part of the examples is commented. Please uncomment the code you need.
Check and correct (if it is necessary) the CREST Server parameters in CrestClient instances.
*/
@@ -316,7 +316,7 @@ void testStorePayloads_FS(const std::string& tagname) {
StoreSetDto storeSetDto = StoreSetDto::from_json(js);
try {
- std::cout << "Storing " << storeSetDto.to_json() << std::endl;
+ std::cout << "Storing " << storeSetDto.to_json().dump(4) << std::endl;
myCrestClient.storeData("JSON", tagname, storeSetDto, "test", "test", "1", endtime);
std::cout << std::endl << "test: testStorePayloads FS (success) " << std::endl;
}
@@ -351,7 +351,7 @@ void testStorePayloads_FS(const std::string& tagname) {
try {
- std::cout << "Storing " << storeSetDto2.to_json() << std::endl;
+ std::cout << "Storing " << storeSetDto2.to_json().dump(4) << std::endl;
myCrestClient.storeData("JSON", tagname, storeSetDto2, "test", "test", "1", endtime);
std::cout << std::endl << "test: testStorePayloads FS (success) " << std::endl;
}
@@ -378,6 +378,524 @@ void testGetSize_FS(const std::string& tagname) {
}
}
+// New:
+
+int createFile(std::string filePath) {
+ // Seed the random number generator with the current time
+ sleep(1);
+ // std::cout << "current time = " << static_cast<unsigned>(std::time(0)) << std::endl;
+ // std::cout << "current time = " << std::time(0) << std::endl;
+ // std::cout << "current time = " << std::time(nullptr) << std::endl;
+
+ std::srand(static_cast<unsigned>(std::time(0)));
+
+ // Open the file for writing
+ std::ofstream outFile(filePath);
+
+ // Check if the file is successfully opened
+ if (!outFile.is_open()) {
+ std::cerr << "Error opening file for writing." << std::endl;
+ return 1; // Return an error code
+ }
+
+ // Generate and write some random content to the file
+ for (int i = 0; i < 10; ++i) {
+ int randomValue = std::rand() % 100; // Generate a random number between 0 and 99
+ outFile << "Random value " << i + 1 << ": " << randomValue << std::endl;
+ }
+
+ // Close the file
+ outFile.close();
+
+ std::cout << "File created successfully at: " << filePath << std::endl;
+ return 0;
+}
+
+
+void testStorePayloadsFile_FS(const std::string& tagname) {
+ std::cout << std::endl << "test: testStorePayloadsFile_FS" << std::endl;
+
+ CrestFsClient myCrestClient = CrestFsClient(true, "/tmp/crest");
+
+ createFile("/tmp/pyld_1.txt");
+ createFile("/tmp/pyld_2.txt");
+
+ uint64_t endtime = 0;
+
+ // Your JSON string
+ std::string jsonString = R"(
+ {
+ "size": 2,
+ "datatype": "data",
+ "format": "StoreSetDto",
+ "resources": [
+ {
+ "since": 5000,
+ "data": "file:///tmp/pyld_1.txt",
+ "streamerInfo": "Info123"
+ },
+ {
+ "since": 6000,
+ "data": "file:///tmp/pyld_2.txt",
+ "streamerInfo": "Info456"
+ }
+ ]
+ }
+ )";
+
+ nlohmann::json js = json::parse(jsonString);
+ StoreSetDto storeSetDto = StoreSetDto::from_json(js);
+
+ try {
+ std::cout << "Storing " << std::endl << storeSetDto.to_json().dump(4) << std::endl;
+ myCrestClient.storeData("FILE", tagname, storeSetDto, "test", "test", "1", endtime);
+ std::cout << std::endl << "test: testStorePayloadsFile_FS (success) " << std::endl;
+ }
+ catch (const std::exception& e) {
+ std::cout << std::endl << "test: testStorePayloadsFile_FS (failed)" << std::endl;
+ std::cerr << e.what() << std::endl;
+ }
+
+
+ createFile("/tmp/pyld_3.txt");
+ createFile("/tmp/pyld_4.txt");
+
+ // Your JSON string
+ std::string jsonString2 = R"(
+ {
+ "size": 2,
+ "datatype": "data",
+ "format": "StoreSetDto",
+ "resources": [
+ {
+ "since": 7000,
+ "data": "file:///tmp/pyld_3.txt",
+ "streamerInfo": "Info123"
+ },
+ {
+ "since": 8000,
+ "data": "file:///tmp/pyld_4.txt",
+ "streamerInfo": "Info456"
+ }
+ ]
+ }
+ )";
+
+ nlohmann::json js2 = json::parse(jsonString2);
+ StoreSetDto storeSetDto2 = StoreSetDto::from_json(js2);
+
+ try {
+ std::cout << "Storing " << std::endl << storeSetDto2.to_json().dump(4) << std::endl;
+ myCrestClient.storeData("FILE", tagname, storeSetDto2, "test", "test", "1", endtime);
+ std::cout << std::endl << "test: testStorePayloadsFile_FS (success) " << std::endl;
+ }
+ catch (const std::exception& e) {
+ std::cout << std::endl << "test: testStorePayloadsFile_FS (failed)" << std::endl;
+ std::cerr << e.what() << std::endl;
+ }
+
+}
+
+
+void testGetPayload_FS(const std::string& hash) {
+ std::cout << std::endl << "test: getPayload FS" << std::endl;
+ CrestFsClient myCrestClient = CrestFsClient(true, "/tmp/crest");
+
+ try{
+ std::string payload = myCrestClient.getPayload(hash);
+ std::cout << std::endl << "test: getPayload FS(result) =" << std::endl
+ << payload << std::endl;
+ }
+ catch (const std::exception& e) {
+ std::cout << std::endl << "test: getPayload FS(failed)" << std::endl;
+ std::cerr << e.what() << std::endl;
+ }
+}
+
+void testGetPayloadMeta_FS(const std::string& hash) {
+ std::cout << std::endl << "test: getPayloadMeta FS" << std::endl;
+ // CrestClient myCrestClient = CrestClient(true);
+ CrestFsClient myCrestClient = CrestFsClient(true, "/tmp/crest");
+
+ try {
+ PayloadDto dto = myCrestClient.getPayloadMeta(hash);
+ std::cout << std::endl << "test: getPayloadMeta FS (result) =" << std::endl
+ << dto.to_json().dump(4) << std::endl;
+ }
+ catch (const std::exception& e) {
+ std::cout << std::endl << "test: getPayloadMeta FS (failed)" << std::endl;
+ std::cout << e.what() << std::endl;
+ }
+}
+
+//
+/* test for private method getPage
+void testGetPage_FS()
+{
+ std::cout << std::endl
+ << "test: getPage FS" << std::endl;
+
+ CrestFsClient myCrestClient = CrestFsClient(true, "/tmp/crest");
+
+ nlohmann::json js = nlohmann::json::array();
+ for (int i = 0; i < 35; i++) {
+ nlohmann::json elem =
+ {
+ {"description", "none"},
+ {"timeType", i}
+ };
+ js.push_back(elem);
+ }
+
+ std::cout << "getPage(10,0) = " << std::endl
+ << myCrestClient.getPage(js,10,0).dump(4) << std::endl;
+
+ std::cout << "getPage(10,100) = " << std::endl
+ << myCrestClient.getPage(js,10,100).dump(4) << std::endl;
+
+ std::cout << "getPage(10,3) = " << std::endl
+ << myCrestClient.getPage(js,10,3).dump(4) << std::endl;
+}
+*/
+
+/*
+// test for private method getVectorPage
+void testGetVectorPage_FS()
+{
+ std::cout << std::endl
+ << "test: getVectorPage FS" << std::endl;
+
+ CrestFsClient myCrestClient = CrestFsClient(true, "/tmp/crest");
+
+ std::vector<std::string> res;
+ for (int i = 0; i < 35; i++) {
+ std::string elem = "data_" + std::to_string(i);
+ res.push_back(elem);
+ }
+
+ std::cout << "original vector = " << std::endl;
+ for (std::string i: res)
+ std::cout << i << ' ';
+ std::cout << std::endl;
+
+ std::vector<std::string> result1 = myCrestClient.getVectorPage(res,10,0);
+ std::cout << "getVectorPage(10,0) = " << std::endl;
+ for (std::string i: result1)
+ std::cout << i << ' ';
+ std::cout << std::endl;
+
+ std::vector<std::string> result2 = myCrestClient.getVectorPage(res,10,100);
+ std::cout << "getVectorPage(10,100) = " << std::endl;
+ for (std::string i: result2)
+ std::cout << i << ' ';
+ std::cout << std::endl;
+
+ std::vector<std::string> result3 = myCrestClient.getVectorPage(res,10,3);
+ std::cout << "getVectorPage(10,3) = " << std::endl;
+ for (std::string i: result3)
+ std::cout << i << ' ';
+ std::cout << std::endl;
+
+}
+*/
+
+void testFindGlobalTagMap_FS(const std::string& name) {
+ std::cout << std::endl << "test: findGlobalTagMap FS" << std::endl;
+ CrestFsClient myCrestClient = CrestFsClient(true, "/tmp/crest");
+
+ try {
+ GlobalTagMapSetDto dto = myCrestClient.findGlobalTagMap(name,"Trace");
+ std::cout << std::endl << "test: findGlobalTagMap FS (result) = " << std::endl
+ << dto.to_json().dump(4) << std::endl;
+ }
+ catch (const std::exception& e) {
+ std::cout << std::endl << "test: findGlobalTagMap FS (failed)" << std::endl;
+ std::cout << e.what() << std::endl;
+ }
+}
+
+
+void testListIovs_FS(const std::string& tagname) {
+ std::cout << std::endl << "test: listIovs FS" << std::endl;
+ CrestFsClient myCrestClient = CrestFsClient(true, "/tmp/crest");
+
+ try {
+ IovSetDto dto = myCrestClient.selectIovs(tagname, 0, -1, 0, 1000, 0, "id.since:ASC");
+ std::cout << std::endl << "page size = 1000, page = 0, id.since:ASC" << std::endl;
+ std::cout << std::endl << "test: listIovs FS (result) = " << std::endl
+ << dto.to_json().dump(4) << std::endl;
+
+ IovSetDto dto2 = myCrestClient.selectIovs(tagname, 0, -1, 0, 2, 1, "id.since:ASC");
+ std::cout << std::endl << "page size = 2, page = 1, id.since:ASC" << std::endl;
+ std::cout << std::endl << "test: listIovs FS (result) = " << std::endl
+ << dto2.to_json().dump(4) << std::endl;
+
+ IovSetDto dto3 = myCrestClient.selectIovs(tagname, 0, -1, 0, 1000, 0, "id.since:DESC");
+ std::cout << std::endl << "page size = 1000, page = 0, id.since:DESC" << std::endl;
+ std::cout << std::endl << "test: listIovs FS (result) = " << std::endl
+ << dto3.to_json().dump(4) << std::endl;
+ }
+ catch (const std::exception& e) {
+ std::cout << std::endl << "test: listIovs FS (failed)" << std::endl;
+ std::cout << e.what() << std::endl;
+ }
+}
+
+/*
+// test for private method nameList
+void testNameList_FS() {
+ std::cout << std::endl << "test: nameList FS" << std::endl;
+ std::string workdir = "/tmp/crest";
+ std::string s_FS_TAG_PATH = "/tags";
+
+ CrestFsClient myCrestClient = CrestFsClient(true, workdir);
+
+ std::string folder = workdir + s_FS_TAG_PATH;
+
+ try {
+ std::vector<std::string> taglist = myCrestClient.nameList(folder);
+
+ std::cout << std::endl << "ascending:" << std::endl
+ << "test: nameList FS (result):" << std::endl;
+ for (std::string tag: taglist)
+ std::cout << tag << ' ';
+ std::cout << std::endl << "test: nameList FS (end)" << std::endl;
+
+ taglist = myCrestClient.nameList(folder,false);
+
+ std::cout << std::endl << "descending:" << std::endl
+ << "test: nameList FS (result):" << std::endl;
+ for (std::string tag: taglist)
+ std::cout << tag << ' ';
+ std::cout << std::endl << "test: nameList FS (end)" << std::endl;
+ }
+ catch (const std::exception& e) {
+ std::cout << std::endl << "test: nameList FS (failed)" << std::endl;
+ std::cout << e.what() << std::endl;
+ }
+}
+*/
+
+void testListTags_FS(const std::string& name, int size, int page) {
+ std::cout << std::endl << "test: listTags FS" << std::endl;
+ std::string workdir = "/tmp/crest";
+
+ CrestFsClient myCrestClient = CrestFsClient(true, "/tmp/crest");
+
+ try {
+ TagSetDto dto = myCrestClient.listTags(name,size,page,"name:ASC");
+ std::cout << std::endl << "test: listTags FS (result) = " << std::endl
+ << dto.to_json().dump(4) << std::endl;
+ }
+ catch (const std::exception& e) {
+ std::cout << std::endl << "test: listTags FS (failed)" << std::endl;
+ std::cout << e.what() << std::endl;
+ }
+}
+
+void createNTags_FS(const std::string &tagname, int n)
+{
+ std::cout << std::endl
+ << "createNTags FS" << std::endl;
+
+ CrestFsClient myCrestClient = CrestFsClient(true, "/tmp/crest");
+
+ for (int i = 0; i < n; i++){
+ std::string tg = tagname + "__" + std::to_string(i);
+
+ nlohmann::json js =
+ {
+ {"description", "none"},
+ {"endOfValidity", 0},
+ {"insertionTime", "2018-12-06T11:18:35.641+0000"},
+ {"lastValidatedTime", 0},
+ {"modificationTime", "2018-12-06T11:18:35.641+0000"},
+ {"name", tg},
+ {"payloadSpec", "Json2Cool"},
+ {"synchronization", "none"},
+ {"timeType", "time"}};
+
+ TagDto dto = TagDto();
+ dto = dto.from_json(js);
+
+ try
+ {
+ myCrestClient.createTag(dto);
+ std::cout << std::endl
+ << "tag " << tg << " created" << std::endl;
+ }
+ catch (const std::exception &e)
+ {
+ std::cout << std::endl
+ << "test: createTag FS (failed)" << std::endl;
+ std::cout << e.what() << std::endl;
+ }
+ } // i
+}
+
+/*
+// test for private method
+void testSortJson(const std::string & tagname, int n){
+ std::cout << std::endl << "test: sortJson FS" << std::endl;
+
+ CrestFsClient myCrestClient = CrestFsClient(true, "/tmp/crest");
+
+ nlohmann::json j_list = nlohmann::json::array();
+
+ for (int i = 0; i < n; i++){
+ int randNum = rand() % 100; // 0-99
+ std::string tg = tagname + "__" + std::to_string(randNum);
+ nlohmann::json elem =
+ {
+ {"name", tg},
+ {"validity", 0},
+ {"description", "test"}
+ };
+ j_list.push_back(elem);
+ }
+
+ std::cout << std::endl << "direct order:" << std::endl;
+ nlohmann::json sorted = myCrestClient.sortJson(j_list, "name", true);
+
+ std::cout << "sorted = " << std::endl
+ << sorted.dump(4);
+
+ std::cout << std::endl << "reverse order:" << std::endl;
+ sorted = myCrestClient.sortJson(j_list, "name", false);
+
+ std::cout << "sorted = " << std::endl
+ << sorted.dump(4);
+
+ std::cout << std::endl
+ << "test: sortJson FS (ended)" << std::endl;
+}
+*/
+
+/*
+// test for private method
+void testSortIOVJson(const std::string & tagname, int n){
+ std::cout << std::endl << "test: sortIOVJson FS" << std::endl;
+
+ CrestFsClient myCrestClient = CrestFsClient(true, "/tmp/crest");
+
+ nlohmann::json j_list = nlohmann::json::array();
+
+ for (int i = 0; i < n; i++){
+ int randNum = rand() / 100; // test
+
+ nlohmann::json elem =
+ {
+ {"insertionTime", "2024-02-28 17:08:26"},
+ {"payloadHash", "944aa370272a061513bc68d5f6b2d5f83664b8da8cc0def2fb87cf2cef34fc6f"},
+ {"since", randNum},
+ {"tagName", "test_MvG03"}
+ };
+
+ j_list.push_back(elem);
+ }
+
+ std::cout << "IOV list = " << std::endl
+ << j_list.dump(4) << std::endl;
+
+ std::cout << std::endl << "direct order:" << std::endl;
+ nlohmann::json sorted = myCrestClient.sortIOVJson(j_list, true);
+
+ std::cout << "sorted = " << std::endl
+ << sorted.dump(4);
+
+ std::cout << std::endl << "reverse order:" << std::endl;
+ sorted = myCrestClient.sortIOVJson(j_list, false);
+
+ std::cout << "sorted = " << std::endl
+ << sorted.dump(4);
+
+ std::cout << std::endl
+ << "test: sortJson FS (ended)" << std::endl;
+}
+*/
+
+/*
+// test for private method
+void testIsMatch(){
+ std::cout << std::endl << "test: isMatch FS" << std::endl;
+
+ CrestFsClient myCrestClient = CrestFsClient(true, "/tmp/crest");
+
+ std::cout << "string = " << "XYXZZXY" << " mask = " << "XY%" << " result = "
+ << " result = " << myCrestClient.isMatch("XYXZZXY", "XY%") << std::endl; // true
+ std::cout << "string = " << "XYXZZXY" << " mask = " << "%XZZ%" << " result = "
+ << " result = " << myCrestClient.isMatch("XYXZZXY", "%XZZ%") << std::endl; // true
+ std::cout << "string = " << "XYXZZXY" << " mask = " << "%YZZ%" << " result = "
+ << myCrestClient.isMatch("XYXZZXY", "%YZZ%") << std::endl; // false
+
+ std::cout << std::endl
+ << "test: isMatch FS (ended)" << std::endl;
+}
+*/
+
+void createNGlobalTags_FS(const std::string &tagname, int n)
+{
+ std::cout << std::endl
+ << "createNGlobalTags FS" << std::endl;
+
+ CrestFsClient myCrestClient = CrestFsClient(true, "/tmp/crest");
+
+ for (int i = 0; i < n; i++){
+ std::string tg = tagname + "__" + std::to_string(i);
+
+ nlohmann::json js =
+ {
+ {"name", tg},
+ {"validity", 0},
+ {"description", "test"},
+ {"release", "1"},
+ {"insertionTime", "2018-12-18T11:32:58.081+0000"},
+ {"snapshotTime", "2018-12-18T11:32:57.952+0000"},
+ {"scenario", "test"},
+ {"workflow", "M"},
+ {"type", "t"}
+ };
+
+ GlobalTagDto dto = GlobalTagDto();
+ dto = dto.from_json(js);
+
+ try
+ {
+ myCrestClient.createGlobalTag(dto);
+ std::cout << std::endl
+ << "global tag " << tg << " created" << std::endl;
+ }
+ catch (const std::exception &e)
+ {
+ std::cout << std::endl
+ << "test: createNGlobalTags FS (failed)" << std::endl;
+ std::cout << e.what() << std::endl;
+ }
+ } // i
+}
+
+void testListGlobalTags_FS(const std::string& name, int size, int page) {
+ std::cout << std::endl << "test: listTags FS" << std::endl;
+ std::string workdir = "/tmp/crest";
+
+ std::cout << "Parameters:" << std::endl;
+ std::cout << " name = " << name << std::endl;
+ std::cout << " size = " << size << std::endl;
+ std::cout << " page = " << page << std::endl;
+
+ CrestFsClient myCrestClient = CrestFsClient(true, "/tmp/crest");
+
+ try {
+ GlobalTagSetDto dto = myCrestClient.listGlobalTags(name,size,page,"name:ASC");
+ std::cout << std::endl << "test: listGlobalTags FS (result) = " << std::endl
+ << dto.to_json().dump(4) << std::endl;
+ }
+ catch (const std::exception& e) {
+ std::cout << std::endl << "test: listGlobalTags FS (failed)" << std::endl;
+ std::cout << e.what() << std::endl;
+ }
+}
+
int main(int argc, char *argv[])
{
@@ -389,20 +907,71 @@ int main(int argc, char *argv[])
// (and check parameters!)
std::string tagname = "test_MvG03";
+ std::string globaltag = "MvG_TEST_01";
+ //
// Global Tag Tests:
- testCreateGlobalTag_FS("MvG_TEST_01");
+ testCreateGlobalTag_FS(globaltag);
+ //
+ //
// tag method tests for file storage
testCreateTag_FS(tagname);
testFindTag_FS(tagname);
+ //
+ //
+ testCreateGlobalTagMap_FS(globaltag,tagname);
+ testFindGlobalTagMap_FS(globaltag);
+ //
+
+ //
// tag meta info method tests for file storage
testCreateTag_FS(tagname);
testFindTag_FS(tagname);
testStorePayloads_FS(tagname);
testGetSize_FS(tagname);
+ //
+ /*
+ testCreateTag_FS(tagname);
+ testStorePayloadsFile_FS(tagname);
+ testGetPayload_FS("abdc424af89446400ba4b31ace71538b05eab779449f35cfa3e4a00d9bf53c1b");
+ testGetPayloadMeta_FS("abdc424af89446400ba4b31ace71538b05eab779449f35cfa3e4a00d9bf53c1b");
+ */
+
+ testListIovs_FS(tagname);
+
+
+ // testNameList_FS(); // private method
+
+ /*
+ // test for tag list:
+ createNTags_FS("test_data",10);
+ testListTags_FS("%",10000,0);
+ testListTags_FS("test_MvG%",10000,0);
+ testListTags_FS("%3",10000,0);
+ testListTags_FS("",10000,0);
+ testListTags_FS("",5,0);
+ testListTags_FS("%",4,0);
+ */
+
+ /*
+ // test for global tag list:
+ createNGlobalTags_FS("test_data",10);
+ testListGlobalTags_FS("%",10000,0);
+ testListGlobalTags_FS("test_MvG%",10000,0);
+ testListGlobalTags_FS("%3",10000,0);
+ testListGlobalTags_FS("",10000,0);
+ testListGlobalTags_FS("",5,0);
+ testListGlobalTags_FS("%",4,0);
+ */
+
+ // testSortJson("test_only", 20); // private method
+ // testSortIOVJson("test_only", 20); // private method
+ // testIsMatch(); // private method
+ // testGetVectorPage_FS(); // private method
+
std::cout << "Test ended : check output in /tmp/crest " << std::endl;
return 0;
}
diff --git a/src/CrestApiFs.cxx b/src/CrestApiFs.cxx
index 2a6bd072fe9a5d50a2632448f514143d1c097026..95de2666a1d1cc1a214056d6658b23919ccf38f0 100644
--- a/src/CrestApiFs.cxx
+++ b/src/CrestApiFs.cxx
@@ -18,6 +18,8 @@
#include <ctime>
#include <cstdlib>
+#include <algorithm>
+
namespace Crest
{
@@ -94,6 +96,8 @@ namespace Crest
}
}
+ // Global tag methods:
+
void CrestFsClient::createGlobalTag(GlobalTagDto &globalTag)
{
std::string name = "";
@@ -149,17 +153,68 @@ namespace Crest
return dto;
}
- GlobalTagSetDto CrestFsClient::listGlobalTags(const std::string &name, int size, int page, const std::string &sort)
- {
- checkFsException("CrestFsClient::listGlobalTags");
- return GlobalTagSetDto();
- }
void CrestFsClient::removeGlobalTag(const std::string &name)
{
checkFsException("CrestFsClient::removeGlobalTag");
}
+
+ GlobalTagSetDto CrestFsClient::listGlobalTags(const std::string &name, int size, int page, const std::string &sort){
+
+ std::string folder = m_root_folder + s_FS_GLOBALTAG_PATH;
+
+ GlobalTagSetDto tagSet;
+ int tagNumber = 0;
+
+ bool ascending = true;
+ if (sort == "name:ASC") ascending = true;
+ else if (sort == "name:DESC") ascending = false;
+ else {
+ throw std::runtime_error(
+ "ERROR in CrestFsClient::listTags: wrong sort parameter." + sort);
+ }
+
+ try {
+ std::vector<std::string> taglist = nameList(folder, ascending);
+ std::vector<std::string> clearedTaglist;
+
+ for (std::string tag: taglist){
+ std::string file_name = folder + "/" + tag + "/" + s_FS_GLOBALTAG_FILE;
+
+ if (std::filesystem::exists(file_name)){
+ if (name != ""){
+ if (isMatch(tag, name)){
+ clearedTaglist.push_back(tag);
+ }
+ }
+ else {
+ clearedTaglist.push_back(tag);
+ }
+ }
+ }
+
+ taglist = getVectorPage(clearedTaglist, size, page);
+ for (std::string tag: taglist){
+ std::string file_name = folder + "/" + tag + "/" + s_FS_GLOBALTAG_FILE;
+ tagNumber++;
+ GlobalTagDto dto = findGlobalTag(tag);
+ tagSet.resources.push_back(dto);
+ }
+ tagSet.size = tagNumber;
+ tagSet.datatype = "";
+ tagSet.format = "TagSetDto";
+ }
+ catch (const std::exception& e) {
+ throw std::runtime_error(
+ "ERROR in CrestFsClient::listGlobalTags: cannot get the tag list.");
+ }
+
+ return tagSet;
+ }
+
+ // Tag methods:
+
void CrestFsClient::createTag(TagDto &tag)
{
std::string name = tag.name;
@@ -217,17 +272,65 @@ namespace Crest
return dto;
}
- TagSetDto CrestFsClient::listTags(const std::string &name, int size, int page, const std::string &sort)
- {
- checkFsException("CrestFsClient::listTags");
- return TagSetDto();
- }
void CrestFsClient::removeTag(const std::string &name)
{
checkFsException("CrestFsClient::removeTag");
}
+ TagSetDto CrestFsClient::listTags(const std::string &name, int size, int page, const std::string &sort){
+
+ std::string folder = m_root_folder + s_FS_TAG_PATH;
+
+ TagSetDto tagSet;
+ int tagNumber = 0;
+
+ bool ascending = true;
+ if (sort == "name:ASC") ascending = true;
+ else if (sort == "name:DESC") ascending = false;
+ else {
+ throw std::runtime_error(
+ "ERROR in CrestFsClient::listTags: wrong sort parameter." + sort);
+ }
+
+ try {
+ std::vector<std::string> taglist = nameList(folder, ascending);
+ std::vector<std::string> clearedTaglist;
+
+ for (std::string tag: taglist){
+ std::string file_name = folder + "/" + tag + "/" + s_FS_TAG_FILE;
+
+ if (std::filesystem::exists(file_name)){
+ if (name != ""){
+ if (isMatch(tag, name)){
+ clearedTaglist.push_back(tag);
+ }
+ }
+ else {
+ clearedTaglist.push_back(tag);
+ }
+ }
+ }
+
+ taglist = getVectorPage(clearedTaglist, size, page);
+ for (std::string tag: taglist){
+ std::string file_name = folder + "/" + tag + "/" + s_FS_TAG_FILE;
+ tagNumber++;
+ TagDto dto = findTag(tag);
+ tagSet.resources.push_back(dto);
+ }
+ tagSet.size = tagNumber;
+ tagSet.datatype = "";
+ tagSet.format = "TagSetDto";
+ }
+ catch (const std::exception& e) {
+ throw std::runtime_error(
+ "ERROR in CrestFsClient::listTags: cannot get the tag list.");
+ }
+
+ return tagSet;
+ }
+
void CrestFsClient::flush()
{
@@ -246,6 +349,8 @@ namespace Crest
m_data.clear();
}
+
+
// TagMeta methods
void CrestFsClient::createTagMeta(TagMetaDto &tag)
{
@@ -298,6 +403,7 @@ namespace Crest
}
// GlobalTagMap methods
+
void CrestFsClient::createGlobalTagMap(GlobalTagMapDto &globalTagMap)
{
nlohmann::json js = globalTagMap.to_json();
@@ -385,11 +491,6 @@ namespace Crest
return;
}
- GlobalTagMapSetDto CrestFsClient::findGlobalTagMap(const std::string &name, const std::string &xCrestMapMode)
- {
- checkFsException("CrestFsClient::findGlobalTagMap");
- return GlobalTagMapSetDto();
- }
void CrestFsClient::removeGlobalTagMap(const std::string &name, const std::string &record, const std::string &label, const std::string &tagname)
{
@@ -401,13 +502,134 @@ namespace Crest
throw std::runtime_error("ERROR in " + std::string(method_name) + " This methods is unsupported for FILESYSTEM mode");
}
+
+ GlobalTagMapSetDto CrestFsClient::findGlobalTagMap(const std::string &name, const std::string &xCrestMapMode) {
+
+ nlohmann::json js = nullptr;
+
+ if(xCrestMapMode != "Trace"){
+ throw std::runtime_error(
+ "ERROR in CrestFsClient::getGlobalTagMap: not supported value for the parameter xCrestMapMode = " + xCrestMapMode);
+ }
+
+ std::string workDir = m_root_folder + s_FS_GLOBALTAG_PATH;
+ std::string file_path = workDir;
+ file_path += '/';
+ file_path += name;
+ file_path += s_FS_MAP_FILE;
+
+ try{
+ std::string tag = getFileString(file_path);
+ js = nlohmann::json::parse(tag);
+ }
+ catch (...) {
+ throw std::runtime_error(
+ "ERROR in CrestFsClient::getGlobalTagMap: cannot get the global tag map " + name +
+ " form the file storage.");
+ }
+
+
+ GlobalTagMapSetDto dto;
+ dto = GlobalTagMapSetDto::from_fs_json(js);
+ return dto;
+ }
+
+
// Iovs methods
- IovSetDto CrestFsClient::selectIovs(const std::string &name, long since, long until, long snapshot, int size, int page, const std::string &sort)
+
+ IovSetDto CrestFsClient::selectIovs(const std::string &name, long since, long until, long snapshot, int size, int page, const std::string &sort){
+ // IovSetDto CrestFsClient::selectIovs(const std::string &name, long since, long until, long snapshot, int size, int page, const std::string &){ // removing unused parameter
+
+ IovSetDto dto;
+
+ nlohmann::json js = nlohmann::json::array();
+
+ try{
+ nlohmann::json iovList = findAllIovs(name);
+ int niovs = iovList.size();
+
+ for (int i = 0; i < niovs; i++) {
+ if (iovList[i].find("since") != iovList[i].end()) {
+ long currentS = iovList[i]["since"];
+
+ if (until != -1) {
+ if (currentS >= since && currentS <= until) {
+ js.push_back(iovList[i]);
+ }
+ }
+ else { // until == -1, Infinity
+ if (currentS >= since) {
+ js.push_back(iovList[i]);
+ }
+ } // until == -1
+
+ }
+ } // for
+
+ }
+ catch (...) {
+ throw std::runtime_error("ERROR in CrestClient::selectIovsFS : cannot get the iov list form file storage");
+ }
+
+ bool ascending = true;
+ if (sort == "id.since:ASC") ascending = true;
+ else if (sort == "id.since:DESC") ascending = false;
+ else {
+ throw std::runtime_error(
+ "ERROR in CrestFsClient::selectIovs: wrong sort parameter." + sort);
+ }
+
+ nlohmann::json sorted = sortIOVJson(js, ascending);
+ nlohmann::json ext = getPage(sorted, size, page);
+ dto = IovSetDto::from_fs_json(ext);
+
+ return dto;
+ }
+
+
+ nlohmann::json CrestFsClient::findAllIovs(const std::string &tagname)
{
- checkFsException("CrestFsClient::selectIovs");
- return IovSetDto();
+ nlohmann::json js = nullptr;
+ std::string file_path = m_root_folder;
+ file_path += '/';
+ file_path += s_FS_TAG_PATH;
+ file_path += '/';
+ file_path += tagname;
+ file_path += s_FS_IOV_FILE;
+
+ try
+ {
+ std::string tag = getFileString(file_path);
+ js = nlohmann::json::parse(tag);
+ }
+ catch (const std::exception& e)
+ {
+ throw std::runtime_error("ERROR in CrestFsClient::findAllIovs : cannot get the iov information form file storage "); // MvG
+ }
+
+ return js;
}
+
+ int CrestFsClient::getSize(const std::string &tagname)
+ {
+ int res = 0;
+
+ try
+ {
+ nlohmann::json iovs = findAllIovs(tagname);
+ int length = iovs.size();
+ return length;
+ }
+ catch (...)
+ {
+ return res;
+ }
+
+ return res;
+ }
+
+
IovSetDto CrestFsClient::selectGroups(const std::string &name, long snapshot, int size, int page, const std::string &sort)
{
checkFsException("CrestFsClient::selectGroups");
@@ -430,7 +652,6 @@ namespace Crest
std::string iovFile = workDir + s_FS_IOV_FILE;
nlohmann::json js_data = storeSetJson.to_json();
- std::cout << "storeSetJson = " << js_data.dump(4) << std::endl;
nlohmann::json res;
auto it = js_data.find("resources");
@@ -438,10 +659,6 @@ namespace Crest
{
res = js_data["resources"];
}
- std::cout << "resources = " << res.dump(4) << std::endl;
-
- int N = res.size();
- std::cout << "res size = " << N << std::endl;
try
{
@@ -485,8 +702,7 @@ namespace Crest
return result;
}
- //
- // void CrestFsClient::storePayloadDump(const std::string& tag, uint64_t since, const std::string& js) { // OLD
+
void CrestFsClient::storePayloadDump(const std::string &tag,
uint64_t since,
const std::string &js,
@@ -498,9 +714,23 @@ namespace Crest
{
std::ofstream outFile;
-
+
+ std::string hashCode;
+ std::string payloadLocalFile;
+
// payload file:
- std::string hashCode = getHash(js);
+ if (payloadFormat == "JSON"){
+ hashCode = getHash(js);
+ }
+ else {
+
+ int found_dots = js.find_first_of(':');
+ int word_size = js.size();
+ payloadLocalFile = js.substr(found_dots + 3, word_size);
+
+ hashCode = getHashForFile(payloadLocalFile);
+ }
+
std::string workDir = m_data_folder;
workDir += '/';
workDir += getFirstLetters(hashCode);
@@ -516,9 +746,19 @@ namespace Crest
}
std::string tagFile = workDir + "/payload.json";
- outFile.open(tagFile);
- outFile << js;
- outFile.close();
+ if (payloadFormat == "JSON"){
+ outFile.open(tagFile);
+ outFile << js;
+ outFile.close();
+ }
+ else {
+ try {
+ std::filesystem::copy_file(payloadLocalFile, tagFile);
+ }
+ catch (std::filesystem::filesystem_error& e){
+ throw std::runtime_error("ERROR in CrestFsClient::storePayloadDump cannot not save payload file: " + tagFile + e.what());
+ }
+ }
// Define the meta info for the payload:
nlohmann::json jsn =
@@ -529,6 +769,7 @@ namespace Crest
{"version", version},
{"size", js.size()},
{"streamerInfo", streamerInfo},
+ {"compressionType", compressionType},
{"insertionTime", getDateAndTime()}};
// payload meta info file:
@@ -551,7 +792,6 @@ namespace Crest
{
try
{ // tag exists, but there are no IOVs
- // nlohmann::json tagJS = findTag(tag); //OLD
nlohmann::json tagJS = findTag(tag).to_json();
nlohmann::json jsFree = nlohmann::json::array({});
m_data.insert(std::pair<std::string, nlohmann::json>(tag, jsFree));
@@ -572,45 +812,20 @@ namespace Crest
nlohmann::json obj(nlohmann::json::value_t::object);
obj["tagName"] = tag;
obj["since"] = since;
- obj["insertionTime"] = 0;
+ // obj["insertionTime"] = "0";
+ obj["insertionTime"] = getDateAndTime();
obj["payloadHash"] = link;
iovs.push_back(obj);
m_data[it->first] = iovs;
}
}
- //
-
- nlohmann::json CrestFsClient::findAllIovs(const std::string &tagname)
- {
- nlohmann::json js = nullptr;
- std::string file_path = m_root_folder;
- file_path += '/';
- file_path += s_FS_TAG_PATH;
- file_path += '/';
- file_path += tagname;
- file_path += s_FS_IOV_FILE;
-
- try
- {
- std::string tag = getFileString(file_path);
- js = nlohmann::json::parse(tag);
- }
- catch (...)
- {
- throw std::runtime_error("ERROR in CrestFsClient::findAllIovs : cannot get the iov information form file storage");
- }
-
- return js;
- }
std::string CrestFsClient::getHashForFile(const std::string &file)
{
std::ifstream ifs(file);
if (!ifs)
{
- // std::cout << "file open error" << std::endl;
- // return 0;
throw std::runtime_error(
"ERROR in CrestFsClient::getHashForFile cannot open file \"" + file + "\".");
}
@@ -630,33 +845,231 @@ namespace Crest
return hash;
}
- int CrestFsClient::getSize(const std::string &tagname)
- {
- int res = 0;
- try
- {
- nlohmann::json iovs = findAllIovs(tagname);
- int length = iovs.size();
- return length;
+ // Payload methods
+
+ std::string CrestFsClient::getPayload(const std::string& hash) {
+ std::string workDir = m_data_folder;
+ workDir += '/';
+ workDir += getFirstLetters(hash);
+ workDir += '/';
+ workDir += hash;
+ std::string filePath = workDir + "/payload.json";
+ std::string res = "";
+
+ try{
+ if (std::filesystem::exists(filePath)){
+ res = getFileString(filePath);
+ }
+ else {
+ throw std::runtime_error("payload with hash " + hash + " does not exist.");
+ }
}
- catch (...)
- {
- return res;
+ catch (const std::exception& e) {
+ std::string message = e.what();
+ throw std::runtime_error("ERROR in CrestFsClient::getPayload cannot get the payload form file storage, " + message);
}
return res;
}
- std::string CrestFsClient::getPayload(const std::string &hash)
- {
- checkFsException("CrestFsClient::getPayload");
- return std::string();
+
+ PayloadDto CrestFsClient::getPayloadMeta(const std::string& hash) {
+ nlohmann::json js = nullptr;
+ PayloadDto dto;
+
+ std::string workDir = m_data_folder;
+ workDir += '/';
+ workDir += getFirstLetters(hash);
+ workDir += '/';
+ workDir += hash;
+
+ std::string filePath = workDir;
+ filePath += "/meta.json";
+
+ std::string res = "";
+
+ try{
+ if (std::filesystem::exists(filePath)){
+ res = getFileString(filePath);
+ js = nlohmann::json::parse(res);
+ dto = PayloadDto::from_json(js);
+ }
+ else {
+ throw std::runtime_error("payload meta info with hash " + hash + " does not exist.");
+ }
+ }
+ catch (const std::exception& e) {
+ std::string message = e.what();
+ throw std::runtime_error("ERROR in CrestClient::getPayloadMeta cannot get the payload meta info form file storage, " + message);
+ }
+
+ return dto;
}
- PayloadDto CrestFsClient::getPayloadMeta(const std::string &hash)
- {
- checkFsException("CrestFsClient::getPayloadMeta");
- return PayloadDto();
+
+ nlohmann::json CrestFsClient::getPage(nlohmann::json data, int size, int page) {
+ nlohmann::json js = nlohmann::json::array();
+ int dataSize = data.size();
+
+ if (dataSize == 0) return js; // the data is absent
+
+ // index interval to load the data from JSON array:
+ int kmin = size * page;
+ int kmax = size * (page + 1);
+
+ // check if the interval is correct:
+ if (kmin > dataSize - 1) return js; // out of range
+
+ if (kmax > dataSize - 1) { // this step is not full
+ kmax = dataSize;
+ }
+
+ for (int i = kmin; i < kmax; i++) {
+ js.push_back(data[i]);
+ }
+ return js;
+ }
+
+ std::vector<std::string> CrestFsClient::getVectorPage(std::vector<std::string> data, int size, int page) {
+ std::vector<std::string> res;
+ int dataSize = data.size();
+
+ if (dataSize == 0) return res; // the data is absent
+
+ // index interval to load the data from JSON array:
+ int kmin = size * page;
+ int kmax = size * (page + 1);
+
+ // check if the interval is correct:
+ if (kmin > dataSize - 1) return res; // out of range
+
+ if (kmax > dataSize - 1) { // this step is not full
+ kmax = dataSize;
+ }
+
+ for (int i = kmin; i < kmax; i++) {
+ res.push_back(data[i]);
+ }
+ return res;
+ }
+
+
+ std::vector<std::string> CrestFsClient::nameList(std::string & folder, bool ascending) {
+ std::vector<std::string> tag_list;
+ std::filesystem::path p(folder);
+
+ for (auto i = std::filesystem::directory_iterator(p); i != std::filesystem::directory_iterator(); i++) {
+ std::string file = i->path().filename().string();
+ tag_list.push_back(file);
+ }
+
+ std::sort(tag_list.begin(), tag_list.end());
+
+ if (ascending == false) {
+ std::reverse(tag_list.begin(), tag_list.end());
+ }
+
+ return tag_list;
}
+
+
+ // method to sort JSON array by the key (par) value
+ nlohmann::json CrestFsClient::sortJson(nlohmann::json js, const std::string & par, bool order){
+ nlohmann::json respond = nlohmann::json::array();
+ std::vector<std::string> parlist;
+ std::map<std::string, nlohmann::json> m;
+
+ int size = js.size();
+ for (int i = 0; i < size; i++){
+ nlohmann::json elem = js[i];
+
+ auto res = elem.find(par);
+
+ if (res != elem.end()) {
+ std::string par_val = elem[par];
+ parlist.push_back(par_val);
+ m[par_val] = elem;
+ }
+ }
+
+ std::sort(parlist.begin(), parlist.end());
+
+ if (order == false) {
+ std::reverse(parlist.begin(), parlist.end());
+ }
+
+ for (std::string item: parlist){
+ respond.push_back(m[item]);
+ }
+
+ return respond;
+ }
+
+ // method to sort JSON array with IOVs by the since value
+ nlohmann::json CrestFsClient::sortIOVJson(nlohmann::json js, bool order){
+ std::string par = "since";
+ nlohmann::json respond = nlohmann::json::array();
+ std::vector<double> parlist;
+ std::map<double, nlohmann::json> m;
+
+ int size = js.size();
+ for (int i = 0; i < size; i++){
+ nlohmann::json elem = js[i];
+
+ auto res = elem.find(par);
+
+ if (res != elem.end()) {
+ double par_val = elem[par];
+ parlist.push_back(par_val);
+ m[par_val] = elem;
+ }
+ }
+
+ std::sort(parlist.begin(), parlist.end());
+
+ if (order == false) {
+ std::reverse(parlist.begin(), parlist.end());
+ }
+
+ for (double item: parlist){
+ respond.push_back(m[item]);
+ }
+
+ return respond;
+ }
+
+
+ bool CrestFsClient::isMatch(std::string word, long unsigned int n, std::string pattern, long unsigned int m){
+ if (m == pattern.size()) {
+ return n == word.size();
+ }
+
+ if (n == word.size()) {
+ for (long unsigned int i = m; i < pattern.size(); i++) {
+ if (pattern[i] != '%') {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ if (pattern[m] == '?' || pattern[m] == word[n]) {
+ return isMatch(word, n + 1, pattern, m + 1);
+ }
+
+ if (pattern[m] == '%') {
+ return isMatch(word, n + 1, pattern, m) || isMatch(word, n, pattern, m + 1);
+ }
+
+ return false;
+ }
+
+
+ bool CrestFsClient::isMatch(std::string word, std::string pattern) {
+ return isMatch(word, 0, pattern, 0);
+ }
+
+
} // namespace Crest