From 07091f662ecbd90ce1e3abb610497f06b88eee4d Mon Sep 17 00:00:00 2001
From: Mikhail Mineev <Mikhail.Mineev@cern.ch>
Date: Wed, 20 Mar 2024 18:48:21 +0100
Subject: [PATCH 01/16] initial version

---
 pycrest/api/crest_fs/__init__.py        |   0
 pycrest/api/crest_fs/crest_fs_config.py |  17 ++
 pycrest/api/crest_fs/crest_fs_utils.py  |  31 +++
 pycrest/api/crest_fs_api.py             | 250 ++++++++++++++++++++++++
 4 files changed, 298 insertions(+)
 create mode 100644 pycrest/api/crest_fs/__init__.py
 create mode 100644 pycrest/api/crest_fs/crest_fs_config.py
 create mode 100644 pycrest/api/crest_fs/crest_fs_utils.py
 create mode 100644 pycrest/api/crest_fs_api.py

diff --git a/pycrest/api/crest_fs/__init__.py b/pycrest/api/crest_fs/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/pycrest/api/crest_fs/crest_fs_config.py b/pycrest/api/crest_fs/crest_fs_config.py
new file mode 100644
index 0000000..e515928
--- /dev/null
+++ b/pycrest/api/crest_fs/crest_fs_config.py
@@ -0,0 +1,17 @@
+fs_tag_path = "/tags"
+fs_globaltag_path = "/globaltags"
+fs_data_path = "/data"
+
+fs_tag_file = "/tag.json"
+fs_iov_file = "/iovs.json"
+fs_tagmetainfo_file = "/tagmetainfo.json"
+
+fs_meta_file = "/meta.json"
+fs_payload_file = "/payload.json"
+
+fs_globaltag_file = "/globaltag.json"
+fs_map_file = "/maps.json"
+
+fs_path = ""
+
+fs_prefix_length = 3
diff --git a/pycrest/api/crest_fs/crest_fs_utils.py b/pycrest/api/crest_fs/crest_fs_utils.py
new file mode 100644
index 0000000..ea36e3c
--- /dev/null
+++ b/pycrest/api/crest_fs/crest_fs_utils.py
@@ -0,0 +1,31 @@
+import hashlib
+from pathlib import Path
+
+def read_file(file):
+    txt = Path(file).read_text()
+    return txt
+
+def write_file(file,data):
+    text_file = open(file, "w")
+    text_file.write(data)
+    text_file.close()
+
+def check_directory(path):
+    Path(path).mkdir(parents=True, exist_ok=True)
+
+def get_hash(string):
+    # hash calculating with SHA-256 for a string
+    hash_object = hashlib.sha256(string.encode())
+    hex_dig = hash_object.hexdigest()
+    print(hex_dig)
+    print(type(hex_dig))
+    return hex_dig
+
+def compute_sha256(file_name):
+    # hash calculating with SHA-256 for a file
+    hash_sha256 = hashlib.sha256()
+    with open(file_name, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_sha256.update(chunk)
+    return hash_sha256.hexdigest()
+
diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py
new file mode 100644
index 0000000..75cdf43
--- /dev/null
+++ b/pycrest/api/crest_fs_api.py
@@ -0,0 +1,250 @@
+from typing import Dict, Any, Union, List, Optional
+import json
+import os
+# import requests
+from datetime import datetime
+# from hep.crest.client import Configuration, ApiClient, ApiException
+from hep.crest.client import ApiException
+from hep.crest.client.api import iovs_api, admin_api, globaltags_api, globaltagmaps_api, payloads_api, tags_api, runinfo_api
+from hep.crest.client.models import (
+    IovSetDto, HTTPResponse, TagMetaSetDto, TagMetaDto, TagSetDto, TagDto, GlobalTagDto, 
+    GlobalTagSetDto, GlobalTagMapDto, StoreSetDto, StoreDto, RunLumiInfoDto, RunLumiSetDto
+)
+
+# from pathlib import Path
+import pycrest.api.crest_fs.crest_fs_config as conf
+from pycrest.api.crest_fs.crest_fs_utils import read_file, write_file, check_directory, get_hash, compute_sha256
+import shutil
+from pycrest.cli.commands.utils import datetime_serializer
+from hep.crest.client.model_utils import model_to_dict
+
+import ast
+
+class CrestApiFs():
+    _config = None
+    _dir = "/tmp/crest_dump"
+    _api_client = None
+
+    def __init__(self, dir: str):
+        self._dir = dir
+        #self._config = Configuration(dir=dir)
+        #self._api_client = ApiClient(self._config)
+
+
+    def print_dir(self):
+        print (self._dir)
+
+    @staticmethod
+    def build_params(kwargs: Dict[str, Any]) -> Dict[str, Any]:
+        # Verify your data
+        return kwargs
+        
+    def create_tag(self, name: str, timeType: str, **kwargs) -> Union[TagDto, HTTPResponse]:
+        """
+        Create a new tag.
+
+        :param name: Name of the tag.
+        :param timeType: Time type of the tag.
+        :param kwargs: Additional parameters for tag creation.
+        :return: Created tag or HTTP response if unsuccessful.
+        """
+        criteria = CrestApiFs.build_params(kwargs)
+        params = {
+            'description': '',
+            'synchronization': 'none',
+            'payload_spec': 'JSON',
+            'last_validated_time': -1.,
+            'end_of_validity': -1.,
+        }
+        params.update(criteria)
+        tag = TagDto(
+            name=name,
+            description=params['description'],
+            time_type=timeType,
+            payload_spec=params['payload_spec'],
+            synchronization=params['synchronization'],
+            last_validated_time=params['last_validated_time'],
+            end_of_validity=params['end_of_validity']
+        )
+
+        path = self._dir + conf.fs_tag_path + "/" + name
+        print (path)
+
+        try:
+            check_directory(path);
+            file = path + conf.fs_tag_file
+            tagjs = model_to_dict(tag)
+            str1 = json.dumps(tagjs)
+            write_file(file,str1)
+        except ApiException as e:
+            print(f"Exception when calling TagsApi->create_tag: {e}\n")
+            return HTTPResponse(
+                status_code=e.status,
+                reason=e.reason,
+                code=e.status,
+                message=e.body
+            )
+
+
+    def create_tag_meta(self, name: str, **kwargs) -> Union[TagMetaDto, HTTPResponse]:
+        """
+        Create the meta information for a tag.
+
+        :param name: Name of the tag.
+        :param kwargs: Additional parameters for tag meta creation.
+        :return: Created tag meta or HTTP response if unsuccessful.
+        """
+        criteria = CrestApiFs.build_params(kwargs)
+        params = {
+            'description': '',
+            'chansize': 1,
+            'colsize': 1,
+            'tag_info': '',
+        }
+        params.update(criteria)
+
+        tag_meta_dto = TagMetaDto(
+            tag_name=name,
+            description=params['description'],
+            chansize=params['chansize'],
+            colsize=params['colsize'],
+            tag_info=params['tag_info'],
+        )
+
+        path = self._dir + conf.fs_tag_path + "/" + name
+        print (path)
+        
+        try:
+            check_directory(path);
+            file = path + conf.fs_tagmetainfo_file
+            tag_meta = model_to_dict(tag_meta_dto)
+            str1 = json.dumps(tag_meta)
+            write_file(file,str1)
+        except ApiException as e:
+            print(f"Exception when calling TagsApi->create_tag_meta: {e}\n")
+            return HTTPResponse(
+                status_code=e.status,
+                reason=e.reason,
+                code=e.status,
+                message=e.body
+            )
+
+
+
+
+    def store_data(self, tag_name: str, since: int, filepath: str, filename: str, compression_type: str = 'none', version: str = '1', object_type: str = 'JSON', **kwargs: Dict[str, Any]) -> Any:
+        """
+        Store data: upload a single file to CREST.
+
+        :param tag_name: Name of the tag.
+        :param since: Since value.
+        :param filepath: Path to the file.
+        :param filename: Name of the file.
+        :param compression_type: Compression type (default is 'none').
+        :param version: Version of the data (default is '1').
+        :param object_type: Type of the object (default is 'JSON').
+        :param kwargs: Additional optional parameters.
+        :return: API response.
+        """
+        criteria = CrestApiFs.build_params(kwargs)
+        params = {
+            'compression_type': compression_type,
+            'version': version,
+            'object_type': object_type,
+        }
+        params.update(criteria)
+
+        file_path = os.path.join(filepath, filename)
+        if not os.path.isfile(file_path):
+            raise FileNotFoundError(f"File '{file_path}' not found.")
+
+        streamer_info = params.get('streamerInfo', f'{{"filename":"{filename}"}}')
+        del params['streamerInfo']
+
+        hash = compute_sha256(file_path)
+        print (hash)
+        prefix = hash[:conf.fs_prefix_length]
+        print(prefix)
+        
+        try:
+            payload_path = self._dir + conf.fs_data_path + "/" + prefix + "/" + hash
+            check_directory(payload_path);
+            payload_file = payload_path + conf.fs_payload_file
+            print(payload_file)
+            shutil.copyfile(file_path, payload_file)
+
+            now = datetime.now()
+            date_string = now.strftime("%Y-%m-%d %H:%M:%S")
+            print(date_string)
+
+            file_size = os.path.getsize(file_path)
+            print(file_size)
+            
+            tag_meta = {
+                "checkSum": "SHA-256",
+                "compressionType": "none",
+                "hash": hash,
+                "insertionTime": now,
+                "objectType": object_type,
+                "size": file_size,
+                #"streamerInfo": streamerInfo,
+                "version": version
+            }
+
+  
+            jsres = json.dumps(tag_meta, default=datetime_serializer)
+            print(jsres)
+
+            meta_file = payload_path + conf.fs_meta_file
+            print(meta_file)
+            write_file(meta_file,jsres)
+
+            iov = {
+                "insertionTime": now,
+                "payloadHash": hash,
+                "since": since,
+                "tagName": tag_name
+            }
+
+            siov = json.dumps(iov, default=datetime_serializer)
+            iov = json.loads(siov)
+            print(siov)
+
+            iov_path = self._dir + conf.fs_tag_path + "/" + tag_name
+            iov_file = iov_path + conf.fs_iov_file
+            print(iov_file)
+
+            if os.path.isfile(iov_file):
+                print("iov list exists")
+                iovs = read_file(iov_file)
+                print("iovs = ")
+                print(iovs)
+                # iov_list = ast.literal_eval(iovs)
+                iov_list = json.loads(iovs)
+                for element in iov_list:
+                    if 'since' in element:
+                        current_since = element['since']
+                        if since == current_since:
+                            iov_list.remove(element)
+
+
+                print("iov = ")
+                print(iov)
+                iov_list.append(iov)
+                print("new iov list = ")
+                print(iov_list)
+                iovs = json.dumps(iov_list)
+                write_file(iov_file,iovs)
+            else:
+                print("iov list does not exist")
+
+                iov_list = json.loads("[]")
+                
+                iov_list.append(iov)
+                iovs = json.dumps(iov_list)
+                write_file(iov_file,iovs)
+
+        except ApiException as e:
+            print("Exception when calling PayloadsApi->store_payload_batch: %s\n" % e)
+
+            
-- 
GitLab


From ece4d288d958bf27c607ba6f261d3eea0ccc7563 Mon Sep 17 00:00:00 2001
From: Mikhail Mineev <Mikhail.Mineev@cern.ch>
Date: Fri, 29 Mar 2024 10:55:26 +0100
Subject: [PATCH 02/16] added DTOs in some methods

---
 pycrest/api/crest_fs_api.py | 57 ++++++++-----------------------------
 1 file changed, 12 insertions(+), 45 deletions(-)

diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py
index 75cdf43..c678e73 100644
--- a/pycrest/api/crest_fs_api.py
+++ b/pycrest/api/crest_fs_api.py
@@ -39,34 +39,16 @@ class CrestApiFs():
         # Verify your data
         return kwargs
         
-    def create_tag(self, name: str, timeType: str, **kwargs) -> Union[TagDto, HTTPResponse]:
+
+    def create_tag(self, tag) -> Union[TagDto, HTTPResponse]:
         """
         Create a new tag.
 
-        :param name: Name of the tag.
-        :param timeType: Time type of the tag.
-        :param kwargs: Additional parameters for tag creation.
+        :param tag (TagDto): tag.
         :return: Created tag or HTTP response if unsuccessful.
         """
-        criteria = CrestApiFs.build_params(kwargs)
-        params = {
-            'description': '',
-            'synchronization': 'none',
-            'payload_spec': 'JSON',
-            'last_validated_time': -1.,
-            'end_of_validity': -1.,
-        }
-        params.update(criteria)
-        tag = TagDto(
-            name=name,
-            description=params['description'],
-            time_type=timeType,
-            payload_spec=params['payload_spec'],
-            synchronization=params['synchronization'],
-            last_validated_time=params['last_validated_time'],
-            end_of_validity=params['end_of_validity']
-        )
 
+        name = tag.name
         path = self._dir + conf.fs_tag_path + "/" + name
         print (path)
 
@@ -76,6 +58,7 @@ class CrestApiFs():
             tagjs = model_to_dict(tag)
             str1 = json.dumps(tagjs)
             write_file(file,str1)
+            return tag
         except ApiException as e:
             print(f"Exception when calling TagsApi->create_tag: {e}\n")
             return HTTPResponse(
@@ -86,40 +69,26 @@ class CrestApiFs():
             )
 
 
-    def create_tag_meta(self, name: str, **kwargs) -> Union[TagMetaDto, HTTPResponse]:
+
+    def create_tag_meta(self, tag_meta) -> Union[TagMetaDto, HTTPResponse]:
         """
         Create the meta information for a tag.
 
-        :param name: Name of the tag.
-        :param kwargs: Additional parameters for tag meta creation.
+        :param tag_meta (TagMetaDto): tag meta info.
         :return: Created tag meta or HTTP response if unsuccessful.
         """
-        criteria = CrestApiFs.build_params(kwargs)
-        params = {
-            'description': '',
-            'chansize': 1,
-            'colsize': 1,
-            'tag_info': '',
-        }
-        params.update(criteria)
-
-        tag_meta_dto = TagMetaDto(
-            tag_name=name,
-            description=params['description'],
-            chansize=params['chansize'],
-            colsize=params['colsize'],
-            tag_info=params['tag_info'],
-        )
 
+        name = tag_meta.tag_name
         path = self._dir + conf.fs_tag_path + "/" + name
         print (path)
         
         try:
             check_directory(path);
             file = path + conf.fs_tagmetainfo_file
-            tag_meta = model_to_dict(tag_meta_dto)
+            tag_meta = model_to_dict(tag_meta)
             str1 = json.dumps(tag_meta)
             write_file(file,str1)
+            return tag_meta
         except ApiException as e:
             print(f"Exception when calling TagsApi->create_tag_meta: {e}\n")
             return HTTPResponse(
@@ -128,9 +97,7 @@ class CrestApiFs():
                 code=e.status,
                 message=e.body
             )
-
-
-
+        
 
     def store_data(self, tag_name: str, since: int, filepath: str, filename: str, compression_type: str = 'none', version: str = '1', object_type: str = 'JSON', **kwargs: Dict[str, Any]) -> Any:
         """
-- 
GitLab


From 2fc45a839fe7b2c8e82d197fbc14cc97c52fa2ae Mon Sep 17 00:00:00 2001
From: Mikhail Mineev <Mikhail.Mineev@cern.ch>
Date: Wed, 3 Apr 2024 12:44:06 +0200
Subject: [PATCH 03/16] store data method written

---
 pycrest/api/crest_fs/crest_fs_utils.py |  14 +++-
 pycrest/api/crest_fs_api.py            | 106 ++++++++++++-------------
 2 files changed, 63 insertions(+), 57 deletions(-)

diff --git a/pycrest/api/crest_fs/crest_fs_utils.py b/pycrest/api/crest_fs/crest_fs_utils.py
index ea36e3c..a3fbf52 100644
--- a/pycrest/api/crest_fs/crest_fs_utils.py
+++ b/pycrest/api/crest_fs/crest_fs_utils.py
@@ -17,8 +17,6 @@ def get_hash(string):
     # hash calculating with SHA-256 for a string
     hash_object = hashlib.sha256(string.encode())
     hex_dig = hash_object.hexdigest()
-    print(hex_dig)
-    print(type(hex_dig))
     return hex_dig
 
 def compute_sha256(file_name):
@@ -29,3 +27,15 @@ def compute_sha256(file_name):
             hash_sha256.update(chunk)
     return hash_sha256.hexdigest()
 
+def utf8len(s):
+    return len(s.encode('utf-8'))
+
+def get_value(json, key):
+    if key in json:
+        return json[key]
+    else:
+        message = 'key ' + key + ' not found in JSON'
+        raise Exception(message)
+
+
+
diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py
index c678e73..dd51806 100644
--- a/pycrest/api/crest_fs_api.py
+++ b/pycrest/api/crest_fs_api.py
@@ -13,7 +13,7 @@ from hep.crest.client.models import (
 
 # from pathlib import Path
 import pycrest.api.crest_fs.crest_fs_config as conf
-from pycrest.api.crest_fs.crest_fs_utils import read_file, write_file, check_directory, get_hash, compute_sha256
+from pycrest.api.crest_fs.crest_fs_utils import read_file, write_file, check_directory, get_hash, compute_sha256, utf8len
 import shutil
 from pycrest.cli.commands.utils import datetime_serializer
 from hep.crest.client.model_utils import model_to_dict
@@ -27,8 +27,6 @@ class CrestApiFs():
 
     def __init__(self, dir: str):
         self._dir = dir
-        #self._config = Configuration(dir=dir)
-        #self._api_client = ApiClient(self._config)
 
 
     def print_dir(self):
@@ -50,7 +48,6 @@ class CrestApiFs():
 
         name = tag.name
         path = self._dir + conf.fs_tag_path + "/" + name
-        print (path)
 
         try:
             check_directory(path);
@@ -80,7 +77,6 @@ class CrestApiFs():
 
         name = tag_meta.tag_name
         path = self._dir + conf.fs_tag_path + "/" + name
-        print (path)
         
         try:
             check_directory(path);
@@ -99,54 +95,49 @@ class CrestApiFs():
             )
         
 
-    def store_data(self, tag_name: str, since: int, filepath: str, filename: str, compression_type: str = 'none', version: str = '1', object_type: str = 'JSON', **kwargs: Dict[str, Any]) -> Any:
+
+    def store_payload(self, tag_name: str, since: int,  payload: StoreDto, compression_type: str = 'none', version: str = '1',
+                       object_type: str = 'JSON', payload_format: str = 'JSON', streamer_info: str = 'none', **kwargs: Dict[str, Any]) -> Any:
         """
-        Store data: upload a single file to CREST.
+        store data: upload a single payload as a file/string to CREST.
 
         :param tag_name: Name of the tag.
         :param since: Since value.
-        :param filepath: Path to the file.
-        :param filename: Name of the file.
+        :param payload: payload of file with the payload.
         :param compression_type: Compression type (default is 'none').
         :param version: Version of the data (default is '1').
-        :param object_type: Type of the object (default is 'JSON').
+        :param payload_format: Type of the object (default is 'JSON').
         :param kwargs: Additional optional parameters.
         :return: API response.
         """
-        criteria = CrestApiFs.build_params(kwargs)
-        params = {
-            'compression_type': compression_type,
-            'version': version,
-            'object_type': object_type,
-        }
-        params.update(criteria)
-
-        file_path = os.path.join(filepath, filename)
-        if not os.path.isfile(file_path):
-            raise FileNotFoundError(f"File '{file_path}' not found.")
-
-        streamer_info = params.get('streamerInfo', f'{{"filename":"{filename}"}}')
-        del params['streamerInfo']
-
-        hash = compute_sha256(file_path)
-        print (hash)
-        prefix = hash[:conf.fs_prefix_length]
-        print(prefix)
-        
+
+        if (payload_format == 'FILE') :
+            file_path = payload.replace("file://", "")
+            if not os.path.isfile(file_path):
+                raise FileNotFoundError(f"File '{file_path}' not found.")
+
+
+            hash = compute_sha256(file_path)
+            prefix = hash[:conf.fs_prefix_length]
+        else:
+            hash = get_hash(payload)
+            prefix = hash[:conf.fs_prefix_length]      
+
         try:
             payload_path = self._dir + conf.fs_data_path + "/" + prefix + "/" + hash
-            check_directory(payload_path);
+            check_directory(payload_path)
             payload_file = payload_path + conf.fs_payload_file
-            print(payload_file)
-            shutil.copyfile(file_path, payload_file)
+
+            if (payload_format == 'FILE') :
+                shutil.copyfile(file_path, payload_file)
+                file_size = os.path.getsize(file_path)
+            else:
+                write_file(payload_file,payload)
+                file_size = utf8len(payload)
 
             now = datetime.now()
             date_string = now.strftime("%Y-%m-%d %H:%M:%S")
-            print(date_string)
-
-            file_size = os.path.getsize(file_path)
-            print(file_size)
-            
+  
             tag_meta = {
                 "checkSum": "SHA-256",
                 "compressionType": "none",
@@ -154,16 +145,14 @@ class CrestApiFs():
                 "insertionTime": now,
                 "objectType": object_type,
                 "size": file_size,
-                #"streamerInfo": streamerInfo,
+                "streamerInfo": streamer_info,
                 "version": version
             }
 
   
             jsres = json.dumps(tag_meta, default=datetime_serializer)
-            print(jsres)
 
             meta_file = payload_path + conf.fs_meta_file
-            print(meta_file)
             write_file(meta_file,jsres)
 
             iov = {
@@ -175,18 +164,12 @@ class CrestApiFs():
 
             siov = json.dumps(iov, default=datetime_serializer)
             iov = json.loads(siov)
-            print(siov)
 
             iov_path = self._dir + conf.fs_tag_path + "/" + tag_name
             iov_file = iov_path + conf.fs_iov_file
-            print(iov_file)
 
             if os.path.isfile(iov_file):
-                print("iov list exists")
                 iovs = read_file(iov_file)
-                print("iovs = ")
-                print(iovs)
-                # iov_list = ast.literal_eval(iovs)
                 iov_list = json.loads(iovs)
                 for element in iov_list:
                     if 'since' in element:
@@ -194,24 +177,37 @@ class CrestApiFs():
                         if since == current_since:
                             iov_list.remove(element)
 
-
-                print("iov = ")
-                print(iov)
                 iov_list.append(iov)
-                print("new iov list = ")
-                print(iov_list)
                 iovs = json.dumps(iov_list)
                 write_file(iov_file,iovs)
             else:
-                print("iov list does not exist")
-
                 iov_list = json.loads("[]")
-                
                 iov_list.append(iov)
                 iovs = json.dumps(iov_list)
                 write_file(iov_file,iovs)
 
         except ApiException as e:
             print("Exception when calling PayloadsApi->store_payload_batch: %s\n" % e)
+            
+            
 
+    def store_data(self, tag: str, store_set: StoreSetDto, payload_format: str = 'JSON', compression_type: str = 'none', version: str = '1', object_type: str = 'JSON', **kwargs: Dict[str, Any]) -> Any:
+
+        data = str(store_set)
+        dto = store_set.to_dict()
+
+        res = store_set.resources
+        n_elem = len(res)
+
+        for elem in res:
+            item = elem.to_dict()
             
+            try:
+                payload = elem.data
+                since = int(elem.since)
+                streamer_info = elem.streamer_info
+
+                self.store_payload( tag, since,  payload, payload_format = payload_format, streamer_info = streamer_info)
+                
+            except Exception as e:
+                print('Error' % e)
-- 
GitLab


From b85847770b5b38be04ed73d5685aa44772ce223e Mon Sep 17 00:00:00 2001
From: Mikhail Mineev <Mikhail.Mineev@cern.ch>
Date: Fri, 5 Apr 2024 16:44:04 +0200
Subject: [PATCH 04/16] find tag and tag meta methods added

---
 pycrest/api/crest_fs_api.py | 81 +++++++++++++++++++++++++++++++++++--
 1 file changed, 78 insertions(+), 3 deletions(-)

diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py
index dd51806..0a41307 100644
--- a/pycrest/api/crest_fs_api.py
+++ b/pycrest/api/crest_fs_api.py
@@ -16,7 +16,8 @@ import pycrest.api.crest_fs.crest_fs_config as conf
 from pycrest.api.crest_fs.crest_fs_utils import read_file, write_file, check_directory, get_hash, compute_sha256, utf8len
 import shutil
 from pycrest.cli.commands.utils import datetime_serializer
-from hep.crest.client.model_utils import model_to_dict
+from hep.crest.client.model_utils import model_to_dict, validate_and_convert_types
+# from hep.crest.client.model_utils import validate_and_convert_types
 
 import ast
 
@@ -38,7 +39,7 @@ class CrestApiFs():
         return kwargs
         
 
-    def create_tag(self, tag) -> Union[TagDto, HTTPResponse]:
+    def create_tag(self, tag: TagDto) -> Union[TagDto, HTTPResponse]:
         """
         Create a new tag.
 
@@ -67,7 +68,7 @@ class CrestApiFs():
 
 
 
-    def create_tag_meta(self, tag_meta) -> Union[TagMetaDto, HTTPResponse]:
+    def create_tag_meta(self, tag_meta: TagMetaDto) -> Union[TagMetaDto, HTTPResponse]:
         """
         Create the meta information for a tag.
 
@@ -211,3 +212,77 @@ class CrestApiFs():
                 
             except Exception as e:
                 print('Error' % e)
+
+    def find_tag(self, tag_name: str) -> Union[TagDto, HTTPResponse]:
+        """
+        Create a new tag.
+
+        :param tag_name (str): tag name.
+        :return: tag (as TagDto) or HTTP response if unsuccessful.
+        """
+
+        path = self._dir + conf.fs_tag_path + "/" + tag_name
+
+        try:
+            file = path + conf.fs_tag_file
+            print(file)
+
+            result = read_file(file) # str
+            print(result)
+            result = json.loads(result)
+
+            dto = validate_and_convert_types(
+                result,
+                {dict, TagDto},
+                ['result'],
+                True,
+                True,
+                TagDto
+            )
+            
+            return dto
+        except ApiException as e:
+            print(f"Exception when calling TagsApi->create_tag: {e}\n")
+            return HTTPResponse(
+                status_code=e.status,
+                reason=e.reason,
+                code=e.status,
+                message=e.body
+            )
+
+    def find_tag_meta(self, tag_name: str) -> Union[TagMetaDto, HTTPResponse]:
+        """
+        Create a new tag.
+
+        :param tag_name (str): tag name.
+        :return: tag (as TagDto) or HTTP response if unsuccessful.
+        """
+
+        path = self._dir + conf.fs_tag_path + "/" + tag_name
+
+        try:
+            file = path + conf.fs_tagmetainfo_file
+            print(file)
+
+            result = read_file(file) # str
+            print(result)
+            result = json.loads(result)
+
+            dto = validate_and_convert_types(
+                result,
+                {dict, TagMetaDto},
+                ['result'],
+                True,
+                True,
+                TagMetaDto
+            )
+            
+            return dto
+        except ApiException as e:
+            print(f"Exception when calling TagsApi->create_tag: {e}\n")
+            return HTTPResponse(
+                status_code=e.status,
+                reason=e.reason,
+                code=e.status,
+                message=e.body
+            )
-- 
GitLab


From d26105ae50366f21eec533fbfbbae76bcf1e8eaa Mon Sep 17 00:00:00 2001
From: formica <andrea.formica@cern.ch>
Date: Sat, 6 Apr 2024 10:44:10 +0200
Subject: [PATCH 05/16] change API for server to make it more similar to
 CrestApi ini C++

---
 pycrest/api/crest_api.py                    | 180 ++++++++------------
 pycrest/cli/commands/global_tag_commands.py |  24 ++-
 pycrest/cli/commands/tag_commands.py        |  31 +++-
 tests/test_crestapi_http.py                 |  28 ++-
 4 files changed, 141 insertions(+), 122 deletions(-)

diff --git a/pycrest/api/crest_api.py b/pycrest/api/crest_api.py
index 6aedb30..fe3e126 100644
--- a/pycrest/api/crest_api.py
+++ b/pycrest/api/crest_api.py
@@ -37,9 +37,17 @@ class CrestApi():
         # Verify your data
         return kwargs
 
-    def select_groups(self, tagname: str, snapshot: Optional[float] = None, size: int = 1000, page: int = 0, sort: str = 'id.since:ASC') -> Union[IovSetDto, HTTPResponse]:
+    def select_groups(self, name: str, snapshot: Optional[float] = None, size: int = 1000, page: int = 0, sort: str = 'id.since:ASC') -> Union[IovSetDto, HTTPResponse]:
         """
         List iov groups for a given tagname
+        :param name: Name of the tag
+        :param snapshot: Snapshot time  
+        :param size: Number of results per page
+        :param page: Page number    
+        :param sort: Sorting order
+        :return: IovSetDto if successful, else HTTPResponse 
+
+        This method is the equivalent of CrestApi.selectGroups() in the C++ code.
         """
         criteria = {
             'snapshot': snapshot,
@@ -49,12 +57,12 @@ class CrestApi():
         }
         api_instance = iovs_api.IovsApi(self._api_client)
         try:
-            api_response = api_instance.find_all_iovs(tagname=tagname, method="GROUPS", **criteria)
+            api_response = api_instance.find_all_iovs(tagname=name, method="GROUPS", **criteria)
             return api_response
         except ApiException as e:
             print("Exception when calling IovsApi->find_all_iovs: %s\n" % e)
 
-    def select_iovs(self, tagname: str, since: int = 0, until: int = 0, snapshot: Optional[float] = None,
+    def select_iovs(self, name: str, since: int = 0, until: int = 0, snapshot: Optional[float] = None,
                     timeformat: str = 'NUMBER', size: int = 1000, page: int = 0,
                     sort: str = 'id.since:ASC') -> Union[IovSetDto, HTTPResponse]:
         """
@@ -68,6 +76,9 @@ class CrestApi():
         :param page: Page number
         :param sort: Sorting order
         :return: IovSetDto if successful, else HTTPResponse
+
+        This method is the equivalent of CrestApi.selectIovs() in the C++ code.
+        The field timeformat is not used in the C++ code, as it defaults to 'NUMBER'.
         """
         criteria = {
             'snapshot': snapshot,
@@ -80,7 +91,7 @@ class CrestApi():
         }
         api_instance = iovs_api.IovsApi(self._api_client)
         try:
-            api_response = api_instance.find_all_iovs(tagname=tagname, method="IOVS", **criteria)
+            api_response = api_instance.find_all_iovs(tagname=name, method="IOVS", **criteria)
             return api_response
         except ApiException as e:
             print("Exception when calling IovsApi->find_all_iovs: %s\n" % e)
@@ -100,6 +111,8 @@ class CrestApi():
             - page: number for page.
             - size: number for size of the page.
         :return: IovSetDto if successful, else HTTPResponse.
+
+        This method is not exposed by CrestApi in the C++ code.
         """
         criteria = CrestApi.build_params(kwargs)
         params = {
@@ -117,24 +130,29 @@ class CrestApi():
             print("Exception when calling IovsApi->find_all_iovs: %s\n" % e)
 
 
-    def list_tags(self, **kwargs: Dict[str, Any]) -> Union[TagSetDto, HTTPResponse]:
+    def list_tags(self, name: str, size: int = 1000, page: int = 0, sort: str = 'name:ASC', **kwargs: Dict[str, Any]) -> Union[TagSetDto, HTTPResponse]:
         """
         List all tags.
-
+        :param name: Name of the tag.
+        :param size: Number of results per page.
+        :param page: Page number.
+        :param sort: Sorting order.
         :param kwargs: Additional criteria.
-            - name: string for tag name.
             - description: string for description.
             - timeType: selection for timeType [time, run, run-lumi].
             - objectType: selection for objectType (payloadSpec).
-            - page: number for page.
-            - size: number for size of the page.
         :return: TagSetDto if successful, else HTTPResponse.
+
+        This method is the equivalent of CrestApi.listTags() in the C++ code.
         """
         criteria = CrestApi.build_params(kwargs)
         params = {
-            'page': 0,
-            'size': 1000
+            'page': page,
+            'size': size,
+            'sort': sort
         }
+        if name != 'none':
+            params['name'] = name
         params.update(criteria)
         api_instance = tags_api.TagsApi(self._api_client)
         try:
@@ -149,6 +167,8 @@ class CrestApi():
 
         :param name: The name of the global tag to find.
         :return: GlobalTagSetDto if successful, else HTTPResponse.
+
+        This method is the equivalent of CrestApi.findGlobalTag() in the C++ code.
         """
         api_instance = globaltags_api.GlobaltagsApi(self._api_client)
         try:
@@ -157,24 +177,30 @@ class CrestApi():
         except ApiException as e:
             print("Exception when calling GlobaltagsApi->find_global_tag: %s\n" % e)
 
-    def list_global_tags(self, **kwargs) -> Union[GlobalTagSetDto, HTTPResponse]:
+    def list_global_tags(self, name: str, size: int = 1000, page: int = 0, sort: str = 'name:ASC', **kwargs: Dict[str, Any]) -> Union[GlobalTagSetDto, HTTPResponse]:
         """
         List all global tags.
 
+        :param name: Name of the global tag.
+        :param size: Number of results per page.
+        :param page: Page number.
+        :param sort: Sorting order.
         :param kwargs: Additional criteria for filtering the global tags.
-            - name: string for global tag name
             - description: string for description
             - release: string for release
             - scenario: string for scenario
-            - page: number for page
-            - size: number for size of the page
         :return: GlobalTagSetDto if successful, else HTTPResponse.
+
+        This method is the equivalent of CrestApi.listGlobalTags() in the C++ code.
         """
         criteria = CrestApi.build_params(kwargs)
         params = {
-            'page': 0,
-            'size': 1000
+            'page': page,
+            'size': size,
+            'sort': sort
         }
+        if name != 'none':
+            params['name'] = name
         params.update(criteria)
         api_instance = globaltags_api.GlobaltagsApi(self._api_client)
         try:
@@ -198,36 +224,19 @@ class CrestApi():
         except ApiException as e:
             print(f"Exception when calling GlobaltagmapsApi->find_global_tag_map: {e}\n")
 
-    def create_tag(self, name: str, timeType: str, **kwargs) -> Union[TagDto, HTTPResponse]:
+    def create_tag(self, dto: TagDto) -> Union[TagDto, HTTPResponse]:
         """
         Create a new tag.
 
-        :param name: Name of the tag.
-        :param timeType: Time type of the tag.
-        :param kwargs: Additional parameters for tag creation.
+        :param dto: The TagDto.
         :return: Created tag or HTTP response if unsuccessful.
+
+        This method is the equivalent of CrestApi.createTag() in the C++ code.
         """
-        criteria = CrestApi.build_params(kwargs)
-        params = {
-            'description': '',
-            'synchronization': 'none',
-            'payload_spec': 'JSON',
-            'last_validated_time': -1.,
-            'end_of_validity': -1.,
-        }
-        params.update(criteria)
-        tag = TagDto(
-            name=name,
-            description=params['description'],
-            time_type=timeType,
-            payload_spec=params['payload_spec'],
-            synchronization=params['synchronization'],
-            last_validated_time=params['last_validated_time'],
-            end_of_validity=params['end_of_validity']
-        )
+        
         api_instance = tags_api.TagsApi(self._api_client)
         try:
-            return api_instance.create_tag(tag_dto=tag)
+            return api_instance.create_tag(tag_dto=dto)
         except ApiException as e:
             print(f"Exception when calling TagsApi->create_tag: {e}\n")
             return HTTPResponse(
@@ -237,34 +246,18 @@ class CrestApi():
                 message=e.body
             )
         
-    def create_tag_meta(self, name: str, **kwargs) -> Union[TagMetaDto, HTTPResponse]:
+    def create_tag_meta(self, dto: TagMetaDto) -> Union[TagMetaDto, HTTPResponse]:
         """
         Create the meta information for a tag.
 
-        :param name: Name of the tag.
-        :param kwargs: Additional parameters for tag meta creation.
+        :param dto: The TagMetaDto.
         :return: Created tag meta or HTTP response if unsuccessful.
-        """
-        criteria = CrestApi.build_params(kwargs)
-        params = {
-            'description': '',
-            'chansize': 1,
-            'colsize': 1,
-            'tag_info': '',
-        }
-        params.update(criteria)
-
-        tag_meta_dto = TagMetaDto(
-            tag_name=name,
-            description=params['description'],
-            chansize=params['chansize'],
-            colsize=params['colsize'],
-            tag_info=params['tag_info'],
-        )
 
+        This method is the equivalent of CrestApi.createTagMeta() in the C++ code.
+        """
         api_instance = tags_api.TagsApi(self._api_client)
         try:
-            return api_instance.create_tag_meta(name, tag_meta_dto=tag_meta_dto)
+            return api_instance.create_tag_meta(name=dto.name, tag_meta_dto=dto)
         except ApiException as e:
             print(f"Exception when calling TagsApi->create_tag_meta: {e}\n")
             return HTTPResponse(
@@ -274,34 +267,19 @@ class CrestApi():
                 message=e.body
             )
 
-    def update_tag_meta(self, name: str, **kwargs) -> Union[TagMetaDto, HTTPResponse]:
+    def update_tag_meta(self,  dto: TagMetaDto) -> Union[TagMetaDto, HTTPResponse]:
         """
         Update the meta information for a tag.
 
-        :param name: Name of the tag.
-        :param kwargs: Additional parameters for tag meta update.
+        :param dto: The TagMetaDto for update.
         :return: Updated tag meta or HTTP response if unsuccessful.
-        """
-        criteria = CrestApi.build_params(kwargs)
-        params = {
-            'description': '',
-            'chansize': 1,
-            'colsize': 1,
-            'tag_info': '',
-        }
-        params.update(criteria)
 
-        tag_meta_dto = TagMetaDto(
-            tag_name=name,
-            description=params['description'],
-            chansize=params['chansize'],
-            colsize=params['colsize'],
-            tag_info=params['tag_info'],
-        )
+        This method is the equivalent of CrestApi.updateTagMeta() in the C++ code.        
+        """
 
         api_instance = tags_api.TagsApi(self._api_client)
         try:
-            return api_instance.update_tag_meta(name, tag_meta_dto=tag_meta_dto)
+            return api_instance.update_tag_meta(name=dto.name, tag_meta_dto=dto)
         except ApiException as e:
             print(f"Exception when calling TagsApi->update_tag_meta: {e}\n")
             return HTTPResponse(
@@ -331,38 +309,18 @@ class CrestApi():
                 message=e.body
             )
 
-    def create_global_tag(self, name: str, force: str = "false", **kwargs) -> Union[GlobalTagDto, HTTPResponse]:
+    def create_global_tag(self, dto: GlobalTagDto, force: str) -> Union[GlobalTagDto, HTTPResponse]:
         """
         Create a new global tag.
 
-        :param name: Name of the global tag.
-        :param force: Whether to force creation if the global tag already exists.
-        :param kwargs: Additional parameters for global tag creation.
+        :param dto: The GlobalTagDto.
         :return: Created global tag or HTTP response if unsuccessful.
-        """
-        criteria = CrestApi.build_params(kwargs)
-        params = {
-            'description': 'none',
-            'validity': 0.0,
-            'release': 'none',
-            'scenario': 'none',
-            'workflow': 'all',
-            'type': 'T',
-        }
-        params.update(criteria)
-        globaltag = GlobalTagDto(
-            name=name,
-            description=params['description'],
-            release=params['release'],
-            scenario=params['scenario'],
-            validity=params['validity'],
-            workflow=params['workflow'],
-            type=params['type']
-        )
-        
+
+        This method is the equivalent of CrestApi.createTagMeta() in the C++ code.        
+        """        
         api_instance = globaltags_api.GlobaltagsApi(self._api_client)
         try:
-            return api_instance.create_global_tag(force=force, global_tag_dto=globaltag)
+            return api_instance.create_global_tag(force=force, global_tag_dto=dto)
         except ApiException as e:
             print(f"Exception when calling GlobaltagsApi->create_global_tag: {e}\n")
             return HTTPResponse(
@@ -464,7 +422,7 @@ class CrestApi():
                 message=e.body
             )
 
-    def get_tag(self, name: str = 'none') -> Union[TagDto, HTTPResponse]:
+    def find_tag(self, name: str = 'none') -> Union[TagDto, HTTPResponse]:
         """
         Get tag.
 
@@ -473,13 +431,13 @@ class CrestApi():
         """
         api_instance = tags_api.TagsApi(self._api_client)   
         try:
-            api_response = api_instance.list_tags(name=name)
+            api_response = api_instance.find_tag(name=name)
             if api_response['size'] == 1:
                 return api_response['resources'][0]
             else:
                 return None
         except ApiException as e:
-            print("Exception when calling TagsApi->list_tags in get_tag: %s\n" % e)
+            print("Exception when calling TagsApi->find_tag: %s\n" % e)
             return HTTPResponse(
                 status_code=e.status,
                 reason=e.reason,
@@ -487,7 +445,7 @@ class CrestApi():
                 message=e.body
             )
 
-    def get_global_tag(self, name: str = 'none') -> Union[GlobalTagDto, HTTPResponse]:
+    def find_global_tag(self, name: str = 'none') -> Union[GlobalTagDto, HTTPResponse]:
         """
         Get global tag.
 
diff --git a/pycrest/cli/commands/global_tag_commands.py b/pycrest/cli/commands/global_tag_commands.py
index 92907a2..edc1a07 100644
--- a/pycrest/cli/commands/global_tag_commands.py
+++ b/pycrest/cli/commands/global_tag_commands.py
@@ -1,9 +1,21 @@
 
 from pycrest.api.crest_api import CrestApi
 from pycrest.cli.commands.utils import print_full_res, print_multiple_res
+from hep.crest.client.models import (
+    IovSetDto, HTTPResponse, TagMetaSetDto, TagMetaDto, TagSetDto, TagDto, GlobalTagDto, 
+    GlobalTagSetDto, GlobalTagMapDto, StoreSetDto, StoreDto, RunLumiInfoDto, RunLumiSetDto
+)
+import sys
 
 def create_global_tag_func(args):
+
+    required_args = ['description']
+    missing_args = [arg for arg in required_args if getattr(args, arg) is None]
+    if missing_args:
+        sys.exit(f"Error: The following parameters are not set: {', '.join(missing_args)}")
+
     params = {
+        'name': args.name,
         'validity': args.validity,
         'description': args.description,
         'release': args.release,
@@ -11,9 +23,19 @@ def create_global_tag_func(args):
         'workflow': args.workflow,
         'type': args.type,
     }
+
+    globaltag = GlobalTagDto(
+        name=params['name'],
+        description=params['description'],
+        release=params['release'],
+        scenario=params['scenario'],
+        validity=params['validity'],
+        workflow=params['workflow'],
+        type=params['type']
+    )
     try:
         crest_api = CrestApi(host=args.crest_host)
-        resp = crest_api.create_global_tag(name=args.name, **params)
+        resp = crest_api.create_global_tag(dto=globaltag)
         # print(f"global tag {args.name} created")
         print_full_res(resp)
     except Exception as e:
diff --git a/pycrest/cli/commands/tag_commands.py b/pycrest/cli/commands/tag_commands.py
index a346755..425eadc 100644
--- a/pycrest/cli/commands/tag_commands.py
+++ b/pycrest/cli/commands/tag_commands.py
@@ -1,7 +1,10 @@
 from pycrest.api.crest_api import CrestApi
 from pycrest.cli.commands.utils import print_multiple_res, print_full_res, print_single_res
 from pycrest.cli.config import crest_host
-
+from hep.crest.client.models import (
+    IovSetDto, HTTPResponse, TagMetaSetDto, TagMetaDto, TagSetDto, TagDto, GlobalTagDto, 
+    GlobalTagSetDto, GlobalTagMapDto, StoreSetDto, StoreDto, RunLumiInfoDto, RunLumiSetDto
+)
 import sys
 
 def get_tag_func(args):
@@ -39,17 +42,24 @@ def create_tag_func(args):
         'last_validated_time': args.last_validated_time,
         'end_of_validity': args.end_of_validity,
     }
-
+    tag = TagDto(
+        name=args.name,
+        description=params['description'],
+        time_type=args.time_type,
+        payload_spec=params['payload_spec'],
+        synchronization=params['synchronization'],
+        last_validated_time=params['last_validated_time'],
+        end_of_validity=params['end_of_validity']
+    )
     try:
         crest_api = CrestApi(host=args.crest_host)
-        resp = crest_api.create_tag(name=args.name, timeType=args.time_type, **params)
+        resp = crest_api.create_tag(dto=tag)
         # print(f"tag {args.name} created")
         print_full_res(resp)
     except Exception as e:
         print("Error: "+ repr(e))
 
 
-
 def remove_tag_func(args):
     # print("GET TAG function")
     # print(f"command = {args.command}")
@@ -65,6 +75,9 @@ def remove_tag_func(args):
         print("Error: "+ repr(e))
 
 def create_tag_meta_info_func(args):
+    """
+    :param args: the parameters to create tag meta information.
+    """
 
     required_args = ['description', 'chansize', 'colsize', 'tag_info']
     missing_args = [arg for arg in required_args if getattr(args, arg) is None]
@@ -78,9 +91,17 @@ def create_tag_meta_info_func(args):
         'tag_info': args.tag_info,
     }
 
+    tag_meta = TagMetaDto(
+        tag_name=args.name,
+        description=params['description'],
+        chansize=params['chansize'],
+        colsize=params['colsize'],
+        tag_info=params['tag_info'],
+    )
+
     try:
         crest_api = CrestApi(host=args.crest_host)
-        resp = crest_api.create_tag_meta(name=args.name, **params)
+        resp = crest_api.create_tag_meta(dto=tag_meta)
         # print(f"tag meta info {args.name} created")
         print_full_res(resp)
     except Exception as e:
diff --git a/tests/test_crestapi_http.py b/tests/test_crestapi_http.py
index 8140004..6744a81 100644
--- a/tests/test_crestapi_http.py
+++ b/tests/test_crestapi_http.py
@@ -18,7 +18,7 @@ from pycrest.api.crest_cond_tag_manager import TagManager
 from pycrest.api.tag_info_container import TagInfoContainer
 from pycrest.api.crest_cond_builder import CrestCondBuilder
 from hep.crest.client.models import (
-    TagMetaDto, GlobalTagDto
+    TagMetaDto, TagDto, GlobalTagDto
 )
 from pycrest.api.crest_api import CrestApi
 
@@ -43,12 +43,21 @@ def test_create_tag(crest_client):
     time_type = 'time'
     try:
         print('Create a tag %s' % name)
+        tag = TagDto(
+            name=name,
+            description=description,
+            time_type=time_type,
+            payload_spec='JSON',
+            synchronization='none',
+            last_validated_time=-1.,
+            end_of_validity=-1.
+        )
         # Create a new tag
-        api_response = api.create_tag(name=name, timeType=time_type, description=description)
+        api_response = api.create_tag(tag)
         assert api_response.name == name
 
         # Find the tag
-        api_response = api.get_tag(name=name)
+        api_response = api.find_tag(name=name)
         assert api_response.name == name
 
         # Delete the tag
@@ -65,11 +74,20 @@ def test_create_global_tag(crest_client):
     description = 'a brand new test global tag'
     try:
         # Create a new global tag
-        api_response = api.create_global_tag(name=global_tag, description=description)
+        globaltag = GlobalTagDto(
+            name=global_tag,
+            description=description,
+            release='1.0',
+            scenario='test',
+            validity=0.,
+            workflow='all',
+            type='T'
+        )
+        api_response = api.create_global_tag(globaltag)
         assert api_response.name == global_tag
 
         # Find the global tag
-        api_response = api.get_global_tag(name=global_tag)
+        api_response = api.find_global_tag(name=global_tag)
         assert api_response.name == global_tag
 
         # Delete the global tag
-- 
GitLab


From 8de2aaf8461eb374238313fe232ec830dfb1bc44 Mon Sep 17 00:00:00 2001
From: formica <andrea.formica@cern.ch>
Date: Sat, 6 Apr 2024 13:29:36 +0200
Subject: [PATCH 06/16] change API for server to make it more similar to
 CrestApi ini C++

---
 pycrest/cli/commands/global_tag_commands.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pycrest/cli/commands/global_tag_commands.py b/pycrest/cli/commands/global_tag_commands.py
index edc1a07..e47576b 100644
--- a/pycrest/cli/commands/global_tag_commands.py
+++ b/pycrest/cli/commands/global_tag_commands.py
@@ -22,6 +22,7 @@ def create_global_tag_func(args):
         'scenario': args.scenario,
         'workflow': args.workflow,
         'type': args.type,
+        'force': False
     }
 
     globaltag = GlobalTagDto(
@@ -35,7 +36,7 @@ def create_global_tag_func(args):
     )
     try:
         crest_api = CrestApi(host=args.crest_host)
-        resp = crest_api.create_global_tag(dto=globaltag)
+        resp = crest_api.create_global_tag(dto=globaltag, force=params['force'])
         # print(f"global tag {args.name} created")
         print_full_res(resp)
     except Exception as e:
-- 
GitLab


From 34970f77cb071325b0bafb09af1a366568230fd0 Mon Sep 17 00:00:00 2001
From: formica <andrea.formica@cern.ch>
Date: Sat, 6 Apr 2024 14:39:06 +0200
Subject: [PATCH 07/16] change API for server to make it more similar to
 CrestApi ini C++

---
 tests/test_crestapi_http.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_crestapi_http.py b/tests/test_crestapi_http.py
index 6744a81..c2a1504 100644
--- a/tests/test_crestapi_http.py
+++ b/tests/test_crestapi_http.py
@@ -83,7 +83,7 @@ def test_create_global_tag(crest_client):
             workflow='all',
             type='T'
         )
-        api_response = api.create_global_tag(globaltag)
+        api_response = api.create_global_tag(globaltag, force=False)
         assert api_response.name == global_tag
 
         # Find the global tag
-- 
GitLab


From 5aa3745a20ae0f9262f941e571d77719176cc123 Mon Sep 17 00:00:00 2001
From: formica <andrea.formica@cern.ch>
Date: Sat, 6 Apr 2024 14:50:12 +0200
Subject: [PATCH 08/16] fix method

---
 pycrest/cli/commands/global_tag_commands.py | 2 +-
 tests/test_crestapi_http.py                 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pycrest/cli/commands/global_tag_commands.py b/pycrest/cli/commands/global_tag_commands.py
index e47576b..123ae9b 100644
--- a/pycrest/cli/commands/global_tag_commands.py
+++ b/pycrest/cli/commands/global_tag_commands.py
@@ -22,7 +22,7 @@ def create_global_tag_func(args):
         'scenario': args.scenario,
         'workflow': args.workflow,
         'type': args.type,
-        'force': False
+        'force': 'false'
     }
 
     globaltag = GlobalTagDto(
diff --git a/tests/test_crestapi_http.py b/tests/test_crestapi_http.py
index c2a1504..ab5fea8 100644
--- a/tests/test_crestapi_http.py
+++ b/tests/test_crestapi_http.py
@@ -83,7 +83,7 @@ def test_create_global_tag(crest_client):
             workflow='all',
             type='T'
         )
-        api_response = api.create_global_tag(globaltag, force=False)
+        api_response = api.create_global_tag(globaltag, force='false')
         assert api_response.name == global_tag
 
         # Find the global tag
-- 
GitLab


From c09303efc267413b9e2f482093fde886857b33b3 Mon Sep 17 00:00:00 2001
From: formica <andrea.formica@cern.ch>
Date: Sun, 7 Apr 2024 22:01:56 +0200
Subject: [PATCH 09/16] update methods

---
 pycrest/api/crest_api.py                    | 26 +++++++++------------
 pycrest/cli/commands/global_tag_commands.py | 14 ++++++++++-
 2 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/pycrest/api/crest_api.py b/pycrest/api/crest_api.py
index fe3e126..48b5385 100644
--- a/pycrest/api/crest_api.py
+++ b/pycrest/api/crest_api.py
@@ -216,6 +216,8 @@ class CrestApi():
         :param name: Name of the global tag or tag.
         :param mode: Map mode [Trace, BackTrace].
         :return: Response from the API.
+
+        This method is the equivalent of CrestApi.findGlobalTagMap() in the C++ code.
         """
         headers = {'x_crest_map_mode': mode}
         api_instance = globaltagmaps_api.GlobaltagmapsApi(self._api_client)
@@ -296,6 +298,8 @@ class CrestApi():
 
         :param name: Name of the tag to delete.
         :return: None if successful, or HTTP response if unsuccessful.
+
+        This method is the equivalent of CrestApi.removeTag() in the C++ code.
         """
         api_instance = admin_api.AdminApi(self._api_client)
         try:
@@ -336,6 +340,8 @@ class CrestApi():
 
         :param name: Name of the global tag to delete.
         :return: None if successful, or HTTP response if unsuccessful.
+
+        This method is the equivalent of CrestApi.removeGlobalTag() in the C++ code.
         """
         api_instance = admin_api.AdminApi(self._api_client)
         try:
@@ -349,7 +355,7 @@ class CrestApi():
                 message=e.body
             )
 
-    def create_global_tag_map(self, globaltagname: str, tagname: str, **kwargs) -> Union[GlobalTagMapDto, HTTPResponse]:
+    def create_global_tag_map(self, dto: GlobalTagMapDto) -> Union[GlobalTagMapDto, HTTPResponse]:
         """
         Create a new global tag map.
 
@@ -357,23 +363,13 @@ class CrestApi():
         :param tagname: Name of the tag.
         :param kwargs: Additional parameters for global tag map creation.
         :return: Created global tag map or HTTP response if unsuccessful.
+
+        This method is the equivalent of CrestApi.createGlobalTagMap() in the C++ code.
         """
-        criteria = CrestApi.build_params(kwargs)
-        params = {
-            'record': 'none',
-            'label': 'none',
-        }
-        params.update(criteria)
-        globaltagmap = GlobalTagMapDto(
-            global_tag_name=globaltagname,
-            tag_name=tagname,
-            record=params['record'],
-            label=params['label']
-        )
-        
+       
         api_instance = globaltagmaps_api.GlobaltagmapsApi(self._api_client)
         try:
-            return api_instance.create_global_tag_map(global_tag_map_dto=globaltagmap)
+            return api_instance.create_global_tag_map(global_tag_map_dto=dto)
         except ApiException as e:
             print(f"Exception when calling GlobaltagmapsApi->create_global_tag_map: {e}\n")
             return HTTPResponse(
diff --git a/pycrest/cli/commands/global_tag_commands.py b/pycrest/cli/commands/global_tag_commands.py
index 123ae9b..a8b63b1 100644
--- a/pycrest/cli/commands/global_tag_commands.py
+++ b/pycrest/cli/commands/global_tag_commands.py
@@ -104,14 +104,26 @@ def get_global_tag_map_func(args):
 
 
 def create_global_tag_map_func(args):
+    required_args = ['record', 'label', 'tag_name', 'name']
+    missing_args = [arg for arg in required_args if getattr(args, arg) is None]
+    if missing_args:
+        sys.exit(f"Error: The following parameters are not set: {', '.join(missing_args)}")
 
     params = {
         'record': args.record,
         'label': args.label,
     }
+
+    globaltagmap = GlobalTagMapDto(
+        global_tag_name=args.name,
+        tag_name=args.tag_name,
+        record=params['record'],
+        label=params['label']
+    )
+
     try:
         crest_api = CrestApi(host=args.crest_host)
-        resp = crest_api.create_global_tag_map(globaltagname=args.name, tagname=args.tag_name, **params)
+        resp = crest_api.create_global_tag_map(globaltagmap)
         # print(f"global tag map {args.name} - {args.tag_name} created")
         print_full_res(resp)
     except Exception as e:
-- 
GitLab


From 355a15c8673c79c93cc4577ef36141245b6c5f2f Mon Sep 17 00:00:00 2001
From: Mikhail Mineev <Mikhail.Mineev@cern.ch>
Date: Wed, 10 Apr 2024 17:40:16 +0200
Subject: [PATCH 10/16] added methods to read data from file storage

---
 pycrest/api/crest_fs_api.py | 139 +++++++++++++++++++++++++++++++++---
 1 file changed, 130 insertions(+), 9 deletions(-)

diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py
index 0a41307..0d68794 100644
--- a/pycrest/api/crest_fs_api.py
+++ b/pycrest/api/crest_fs_api.py
@@ -1,23 +1,24 @@
 from typing import Dict, Any, Union, List, Optional
 import json
 import os
-# import requests
+
 from datetime import datetime
-# from hep.crest.client import Configuration, ApiClient, ApiException
+
 from hep.crest.client import ApiException
 from hep.crest.client.api import iovs_api, admin_api, globaltags_api, globaltagmaps_api, payloads_api, tags_api, runinfo_api
 from hep.crest.client.models import (
     IovSetDto, HTTPResponse, TagMetaSetDto, TagMetaDto, TagSetDto, TagDto, GlobalTagDto, 
-    GlobalTagSetDto, GlobalTagMapDto, StoreSetDto, StoreDto, RunLumiInfoDto, RunLumiSetDto
+    GlobalTagSetDto, GlobalTagMapDto, StoreSetDto, StoreDto, RunLumiInfoDto, RunLumiSetDto,
+    PayloadDto, IovDto
 )
 
-# from pathlib import Path
+
 import pycrest.api.crest_fs.crest_fs_config as conf
 from pycrest.api.crest_fs.crest_fs_utils import read_file, write_file, check_directory, get_hash, compute_sha256, utf8len
 import shutil
 from pycrest.cli.commands.utils import datetime_serializer
 from hep.crest.client.model_utils import model_to_dict, validate_and_convert_types
-# from hep.crest.client.model_utils import validate_and_convert_types
+
 
 import ast
 
@@ -100,7 +101,7 @@ class CrestApiFs():
     def store_payload(self, tag_name: str, since: int,  payload: StoreDto, compression_type: str = 'none', version: str = '1',
                        object_type: str = 'JSON', payload_format: str = 'JSON', streamer_info: str = 'none', **kwargs: Dict[str, Any]) -> Any:
         """
-        store data: upload a single payload as a file/string to CREST.
+        store data: uploads a single payload as a file/string to CREST. The auxiliary method.
 
         :param tag_name: Name of the tag.
         :param since: Since value.
@@ -146,7 +147,6 @@ class CrestApiFs():
                 "insertionTime": now,
                 "objectType": object_type,
                 "size": file_size,
-                "streamerInfo": streamer_info,
                 "version": version
             }
 
@@ -193,7 +193,20 @@ class CrestApiFs():
             
 
     def store_data(self, tag: str, store_set: StoreSetDto, payload_format: str = 'JSON', compression_type: str = 'none', version: str = '1', object_type: str = 'JSON', **kwargs: Dict[str, Any]) -> Any:
+        """
+        This method stores a set of the payloads (with additional parameters: since and streamer info) on the file storage. 
+        The payloads and parameters have to be in the StoreSetDto object.
 
+        :param tag: Name of the tag.
+        :param since: Since value.
+        :param store_set (StoreSetDto): StoreSetDto object, which contains payloads (in form of strings or files), since parameters for them
+        :param payload_format: payload format (default is 'JSON').
+        :param compression_type: Compression type (default is 'none').
+        :param version: Version of the data (default is '1').
+        :param payload_format: Type of the object (default is 'JSON').
+        :param kwargs: Additional optional parameters.
+        :return: API response
+        """
         data = str(store_set)
         dto = store_set.to_dict()
 
@@ -215,7 +228,7 @@ class CrestApiFs():
 
     def find_tag(self, tag_name: str) -> Union[TagDto, HTTPResponse]:
         """
-        Create a new tag.
+        This method returns a tag for the tag name.
 
         :param tag_name (str): tag name.
         :return: tag (as TagDto) or HTTP response if unsuccessful.
@@ -252,7 +265,7 @@ class CrestApiFs():
 
     def find_tag_meta(self, tag_name: str) -> Union[TagMetaDto, HTTPResponse]:
         """
-        Create a new tag.
+        This method returns a tag meta info for the tag name.
 
         :param tag_name (str): tag name.
         :return: tag (as TagDto) or HTTP response if unsuccessful.
@@ -286,3 +299,111 @@ class CrestApiFs():
                 code=e.status,
                 message=e.body
             )
+
+    def get_payload(self, hash: str) -> Union[str, HTTPResponse]:
+        """
+        This methods returns a payload by the hash.
+
+        :param hash     (str): hash    .
+        :return: payload (as string) or HTTP response if unsuccessful.
+        """
+        
+        try:
+            prefix = hash[:conf.fs_prefix_length] 
+            payload_path = self._dir + conf.fs_data_path + "/" + prefix + "/" + hash
+            payload_file = payload_path + conf.fs_payload_file
+
+            result = read_file(payload_file) # str
+            
+            return result
+        except ApiException as e:
+            print(f"Exception when calling TagsApi->create_tag: {e}\n")
+            return HTTPResponse(
+                status_code=e.status,
+                reason=e.reason,
+                code=e.status,
+                message=e.body
+            )
+
+
+    def get_payload_meta(self, hash: str) -> Union[PayloadDto, HTTPResponse]:
+        """
+        This method returns a payload meta info for the hash.
+
+        :param hash     (str): hash    .
+        :return: payload meta (as PayloadDto) or HTTP response if unsuccessful.
+        """
+        
+        try:
+            prefix = hash[:conf.fs_prefix_length] 
+            payload_path = self._dir + conf.fs_data_path + "/" + prefix + "/" + hash
+            meta_file = payload_path + conf.fs_meta_file
+
+            result = read_file(meta_file) # str
+            result = json.loads(result)
+
+            dto = validate_and_convert_types(
+                result,
+                {json, dict, PayloadDto},
+                ['result'],
+                True,
+                True,
+                PayloadDto
+            )
+            
+            return dto
+        except ApiException as e:
+            print(f"Exception when calling TagsApi->create_tag: {e}\n")
+            return HTTPResponse(
+                status_code=e.status,
+                reason=e.reason,
+                code=e.status,
+                message=e.body
+            )
+
+
+    
+    def select_iovs(self, tag_name: str) -> Union[ IovSetDto, HTTPResponse]:
+        """
+        Find an IOV list by the tag name.
+
+        :param tag_name (str): tag name.
+        :return: tag (as  IovSetDto) or HTTP response if unsuccessful.
+        """
+
+        path = self._dir + conf.fs_tag_path + "/" + tag_name
+
+        try:
+            file = path + conf.fs_iov_file
+
+            result = read_file(file) # str
+            result = json.loads(result)
+
+            iov_list = IovSetDto(
+                size = len(result),
+                format = "IovSetDto",
+                datatype = "iovs",
+                resources= []
+            )
+            
+            for element in result:
+                dto = validate_and_convert_types(
+                    element,
+                    {IovDto},
+                    ['element'],
+                    True,
+                    True,
+                    IovDto
+                 )
+
+                iov_list.resources.append(dto)
+
+            return iov_list
+        except ApiException as e:
+            print(f"Exception when calling TagsApi->create_tag: {e}\n")
+            return HTTPResponse(
+                status_code=e.status,
+                reason=e.reason,
+                code=e.status,
+                message=e.body
+            )
-- 
GitLab


From 54c7e36b6e484dab295add4c3251cfc50713ea9e Mon Sep 17 00:00:00 2001
From: Mikhail Mineev <Mikhail.Mineev@cern.ch>
Date: Thu, 11 Apr 2024 17:42:28 +0200
Subject: [PATCH 11/16] select_iovs corrected

---
 pycrest/api/crest_fs_api.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py
index 0d68794..18a91f8 100644
--- a/pycrest/api/crest_fs_api.py
+++ b/pycrest/api/crest_fs_api.py
@@ -363,15 +363,15 @@ class CrestApiFs():
 
 
     
-    def select_iovs(self, tag_name: str) -> Union[ IovSetDto, HTTPResponse]:
+    def select_iovs(self, name: str) -> Union[ IovSetDto, HTTPResponse]:
         """
         Find an IOV list by the tag name.
 
-        :param tag_name (str): tag name.
+        :param name (str): tag name.
         :return: tag (as  IovSetDto) or HTTP response if unsuccessful.
         """
 
-        path = self._dir + conf.fs_tag_path + "/" + tag_name
+        path = self._dir + conf.fs_tag_path + "/" + name
 
         try:
             file = path + conf.fs_iov_file
-- 
GitLab


From 16bad16ab66c197cc08c2e11888fc3a0bd525e97 Mon Sep 17 00:00:00 2001
From: formica <andrea.formica@cern.ch>
Date: Mon, 15 Apr 2024 16:19:53 +0200
Subject: [PATCH 12/16] new create tag example

---
 examples/create_tag.py | 76 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 76 insertions(+)
 create mode 100644 examples/create_tag.py

diff --git a/examples/create_tag.py b/examples/create_tag.py
new file mode 100644
index 0000000..f8f3c6c
--- /dev/null
+++ b/examples/create_tag.py
@@ -0,0 +1,76 @@
+from pycrest.api.crest_api import CrestApi
+from hep.crest.client.models import (
+    IovSetDto, HTTPResponse, TagMetaSetDto, TagMetaDto, TagSetDto, TagDto, GlobalTagDto, 
+    GlobalTagSetDto, GlobalTagMapDto, StoreSetDto, StoreDto, RunLumiInfoDto, RunLumiSetDto
+)
+# create an instance of the API class
+def socks(proxy_host):
+    SOCKS5_PROXY_HOST = proxy_host
+    SOCKS5_PROXY_PORT = 3129
+    try:
+        import socket
+        import socks  # you need to install pysocks (use the command: pip install pysocks)
+        # Configuration
+
+        # Remove this if you don't plan to "deactivate" the proxy later
+        #        default_socket = socket.socket
+        # Set up a proxy
+        #            if self.useSocks:
+        socks.set_default_proxy(socks.SOCKS5, SOCKS5_PROXY_HOST, SOCKS5_PROXY_PORT)
+        socket.socket = socks.socksocket
+        print('Activated socks proxy on %s:%s' % (SOCKS5_PROXY_HOST, SOCKS5_PROXY_PORT))
+    except:
+        print('Error activating socks...%s %s' % (SOCKS5_PROXY_HOST, SOCKS5_PROXY_PORT))
+
+socks('localhost')
+api_instance = CrestApi(host='http://crest-undertow-api.web.cern.ch/api-v4.0')
+
+# tag name
+name = 'test_tag'
+# tag description
+description = 'a brand new test tag'
+# tag time type
+time_type = 'time'
+
+params = {
+    'payload_spec': 'ascii',
+    'description': description,
+    'synchronization': 'none',
+    'last_validated_time': -1.,
+    'end_of_validity': -1.,
+}
+tagdto = TagDto(
+    name=name,
+    description=params['description'],
+    time_type=time_type,
+    payload_spec=params['payload_spec'],
+    synchronization=params['synchronization'],
+    last_validated_time=params['last_validated_time'],
+    end_of_validity=params['end_of_validity']
+)
+
+print('Create a tag %s' % name)
+try:
+    # activate socks
+    # Create a new tag
+    api_response = api_instance.create_tag(tagdto)
+    print(api_response)
+except Exception as e:
+    print("Exception when calling CrestApi->create_tag: %s\n" % e)
+
+print('Find a tag %s' % name)
+try:
+    # activate socks
+    # Create a new tag
+    api_response = api_instance.find_tag(name=name)
+    print(api_response)
+except Exception as e:
+    print("Exception when calling CrestApi->create_tag: %s\n" % e)
+
+# Remove the tag
+print('Delete tag %s' % name)
+try:
+    api_response = api_instance.remove_tag(name=name)
+    print('Done')
+except Exception as e:
+    print("Exception when calling CrestApi->remove_tag: %s\n" % e)
-- 
GitLab


From eff52c95b98a834ee1ebcfe8f896bffb6c12dae4 Mon Sep 17 00:00:00 2001
From: mavogel <mavogel@cern.ch>
Date: Thu, 18 Apr 2024 13:00:39 +0200
Subject: [PATCH 13/16] fixed incorrect reference to TagMetaDto.tag_name

---
 pycrest/api/crest_api.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pycrest/api/crest_api.py b/pycrest/api/crest_api.py
index 48b5385..c491099 100644
--- a/pycrest/api/crest_api.py
+++ b/pycrest/api/crest_api.py
@@ -259,7 +259,7 @@ class CrestApi():
         """
         api_instance = tags_api.TagsApi(self._api_client)
         try:
-            return api_instance.create_tag_meta(name=dto.name, tag_meta_dto=dto)
+            return api_instance.create_tag_meta(name=dto.tag_name, tag_meta_dto=dto)
         except ApiException as e:
             print(f"Exception when calling TagsApi->create_tag_meta: {e}\n")
             return HTTPResponse(
@@ -281,7 +281,7 @@ class CrestApi():
 
         api_instance = tags_api.TagsApi(self._api_client)
         try:
-            return api_instance.update_tag_meta(name=dto.name, tag_meta_dto=dto)
+            return api_instance.update_tag_meta(name=dto.tag_name, tag_meta_dto=dto)
         except ApiException as e:
             print(f"Exception when calling TagsApi->update_tag_meta: {e}\n")
             return HTTPResponse(
@@ -691,4 +691,4 @@ class CrestApi():
             return api_response
         except ApiException as e:
             print("Exception when calling RuninfoApi->list_run_info: %s\n" % e)
-            return HTTPResponse(status_code=e.status, reason=e.reason, code=e.status, message=e.body)
\ No newline at end of file
+            return HTTPResponse(status_code=e.status, reason=e.reason, code=e.status, message=e.body)
-- 
GitLab


From 171be65c0ceacc6cf083fcda95b90f86511f2c2e Mon Sep 17 00:00:00 2001
From: Mikhail Mineev <Mikhail.Mineev@cern.ch>
Date: Tue, 23 Apr 2024 19:17:12 +0200
Subject: [PATCH 14/16] added methods for global tags and global tag maps for
 file storage

---
 pycrest/api/crest_fs_api.py | 352 ++++++++++++++++++++++++++++++++++--
 1 file changed, 340 insertions(+), 12 deletions(-)

diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py
index 18a91f8..f729a5d 100644
--- a/pycrest/api/crest_fs_api.py
+++ b/pycrest/api/crest_fs_api.py
@@ -8,8 +8,8 @@ from hep.crest.client import ApiException
 from hep.crest.client.api import iovs_api, admin_api, globaltags_api, globaltagmaps_api, payloads_api, tags_api, runinfo_api
 from hep.crest.client.models import (
     IovSetDto, HTTPResponse, TagMetaSetDto, TagMetaDto, TagSetDto, TagDto, GlobalTagDto, 
-    GlobalTagSetDto, GlobalTagMapDto, StoreSetDto, StoreDto, RunLumiInfoDto, RunLumiSetDto,
-    PayloadDto, IovDto
+    GlobalTagSetDto, GlobalTagMapDto, GlobalTagMapSetDto, StoreSetDto, StoreDto,
+    RunLumiInfoDto, RunLumiSetDto, PayloadDto, IovDto
 )
 
 
@@ -59,7 +59,7 @@ class CrestApiFs():
             write_file(file,str1)
             return tag
         except ApiException as e:
-            print(f"Exception when calling TagsApi->create_tag: {e}\n")
+            print(f"Exception in method CrestApiFs->create_tag: {e}\n")
             return HTTPResponse(
                 status_code=e.status,
                 reason=e.reason,
@@ -88,7 +88,7 @@ class CrestApiFs():
             write_file(file,str1)
             return tag_meta
         except ApiException as e:
-            print(f"Exception when calling TagsApi->create_tag_meta: {e}\n")
+            print(f"Exception in method CrestApiFs->create_tag_meta: {e}\n")
             return HTTPResponse(
                 status_code=e.status,
                 reason=e.reason,
@@ -188,7 +188,7 @@ class CrestApiFs():
                 write_file(iov_file,iovs)
 
         except ApiException as e:
-            print("Exception when calling PayloadsApi->store_payload_batch: %s\n" % e)
+            print("Exception in method CrestApiFs->store_payload: %s\n" % e)
             
             
 
@@ -226,6 +226,7 @@ class CrestApiFs():
             except Exception as e:
                 print('Error' % e)
 
+                
     def find_tag(self, tag_name: str) -> Union[TagDto, HTTPResponse]:
         """
         This method returns a tag for the tag name.
@@ -255,7 +256,7 @@ class CrestApiFs():
             
             return dto
         except ApiException as e:
-            print(f"Exception when calling TagsApi->create_tag: {e}\n")
+            print(f"Exception in method CrestApiFs->find_tag: {e}\n")
             return HTTPResponse(
                 status_code=e.status,
                 reason=e.reason,
@@ -292,7 +293,7 @@ class CrestApiFs():
             
             return dto
         except ApiException as e:
-            print(f"Exception when calling TagsApi->create_tag: {e}\n")
+            print(f"Exception in method CrestApiFs->find_tag_meta: {e}\n")
             return HTTPResponse(
                 status_code=e.status,
                 reason=e.reason,
@@ -317,7 +318,7 @@ class CrestApiFs():
             
             return result
         except ApiException as e:
-            print(f"Exception when calling TagsApi->create_tag: {e}\n")
+            print(f"Exception in method CrestApiFs->get_payload: {e}\n")
             return HTTPResponse(
                 status_code=e.status,
                 reason=e.reason,
@@ -353,7 +354,7 @@ class CrestApiFs():
             
             return dto
         except ApiException as e:
-            print(f"Exception when calling TagsApi->create_tag: {e}\n")
+            print(f"Exception in method CrestApiFs->get_payload_meta: {e}\n")
             return HTTPResponse(
                 status_code=e.status,
                 reason=e.reason,
@@ -362,12 +363,129 @@ class CrestApiFs():
             )
 
 
+
+    def sort_iov_json(self, data: json, order: bool):
+        """
+        The auxiliary method to sort the JSON array with the IOVs by the since parameters.
+
+        :param json  : JSON array with the IOVs (IOV list).
+        :param order : sorting order ( True - ascending order, False - descending)
+        :return: sorted JSON array with the IOV list.
+        """
+        
+        res = json.loads("[]")
+        unsorted = {}
+
+        for element in data:
+            # print(element)
+            if element.get('since') is not None:
+                # print('since exists')
+                since = element['since']
+                # print(since)
+                unsorted[since] = element
+            else:
+                raise Exception("JSON has no since parameter")
+        if order:
+            sorted_dict = dict(sorted(unsorted.items(),reverse = False))
+        else:
+            sorted_dict = dict(sorted(unsorted.items(),reverse = True))
+        # print(sorted_dict)
+
+        for key in sorted_dict:
+            # print(key, '->', sorted_dict[key])
+            res.append(sorted_dict[key])
+
+        # print(res)
+        return res
+
+
     
-    def select_iovs(self, name: str) -> Union[ IovSetDto, HTTPResponse]:
+    def get_page(self, data: json, size: int, page: int):
+        """
+        The auxiliary method to extract a subarray from JSON array.
+
+        :param json  : JSON array to extract subarray.
+        :param size: Number of item per page
+        :param page: Page number
+        :return: JSON subarray.
+        """
+        
+        res = json.loads("[]")
+        data_size = len(data)
+        # print (data_size)
+
+        if data_size == 0:
+            # no elements to return
+            return res
+
+        # index interval to load the data from JSON array:
+        kmin = size * page
+        kmax = size * (page + 1)
+
+        # check if the interval is correct:
+        if kmin > (data_size - 1): # out of range
+            return res
+
+        if kmax > (data_size - 1):
+            kmax = data_size
+
+        for i in range (kmin , kmax):
+            # print (i)
+            res.append(data[i])
+    
+        return res
+
+
+
+    def extract_interval(self, data: json, since: int, until: int):
+        """
+        The auxiliary method to extract an IOV list in the time interval since-until.
+
+        :param json  : JSON array with the IOV list.
+        :param since: Start time
+        :param until: End time
+        :return: IOV list (JSON array).
+        """
+        res = json.loads("[]")
+        # print(since)
+        # print(until)
+
+        for element in data:
+            # print(element)
+            if element.get('since') is not None:
+                # print('since exists')
+                time = element['since']
+                # print(time)
+            
+                if until == -1:  # infinity
+                    if time >= since:
+                        # print("b")
+                        res.append(element)
+                else:
+                    if (time >= since) and (time <= until):
+                        # print("a")
+                        res.append(element)
+            else:
+                raise Exception("JSON has no since parameter")
+    
+        return res    
+
+
+    
+    # def select_iovs(self, name: str) -> Union[ IovSetDto, HTTPResponse]: # OLD
+    def select_iovs(self, name: str,
+                    since: int = 0, until: int = 0,
+                    size: int = 1000, page: int = 0,
+                    sort: str = 'id.since:ASC') -> Union[IovSetDto, HTTPResponse]:
         """
         Find an IOV list by the tag name.
 
-        :param name (str): tag name.
+        :param name (str): Tag name.
+        :param since: Start time
+        :param until: End time
+        :param size: Number of results per page
+        :param page: Page number
+        :param sort: Sorting order ('id.since:ASC' or 'id.since:DESC')
         :return: tag (as  IovSetDto) or HTTP response if unsuccessful.
         """
 
@@ -379,6 +497,18 @@ class CrestApiFs():
             result = read_file(file) # str
             result = json.loads(result)
 
+            # IOV list ordering:
+            if sort == 'id.since:ASC':
+                result = self.sort_iov_json(result, True)
+            else:
+                result = self.sort_iov_json(result, False)
+
+            # IOV list time interval:
+            result = self.extract_interval(result, since=since, until=until)
+
+            # page extracting:
+            result = self.get_page(result, size=size, page=page)
+
             iov_list = IovSetDto(
                 size = len(result),
                 format = "IovSetDto",
@@ -400,7 +530,205 @@ class CrestApiFs():
 
             return iov_list
         except ApiException as e:
-            print(f"Exception when calling TagsApi->create_tag: {e}\n")
+            print(f"Exception in method CrestApiFs->select_iovs: {e}\n")
+            return HTTPResponse(
+                status_code=e.status,
+                reason=e.reason,
+                code=e.status,
+                message=e.body
+            )
+
+    
+
+    def create_global_tag(self, dto: GlobalTagDto) -> Union[GlobalTagDto, HTTPResponse]:
+        """
+        Create the global tag.
+
+        :param dto (GlobalTagDto): global tag.
+        :return: Created global tag (GlobalTagDto) or HTTP response if unsuccessful.
+        """
+
+        name = dto.name
+        path = self._dir + conf.fs_globaltag_path + "/" + name
+        
+        try:
+            check_directory(path);
+            file = path + conf.fs_globaltag_file
+            gtag = model_to_dict(dto)
+            str1 = json.dumps(gtag)
+            write_file(file,str1)
+            return dto
+        except ApiException as e:
+            print(f"Exception in method CrestApiFs->create_global_tag: {e}\n")
+            return HTTPResponse(
+                status_code=e.status,
+                reason=e.reason,
+                code=e.status,
+                message=e.body
+            )
+
+        
+    def find_global_tag(self, name: str) -> Union[GlobalTagDto, HTTPResponse]:
+        """
+        This metod returns the global tag for the given global tag name.
+
+        :param name : global tag name.
+        :return: global tag (GlobalTagDto) or HTTP response if unsuccessful.
+        """
+        path = self._dir + conf.fs_globaltag_path + "/" + name
+        
+        try:
+
+            file = path + conf.fs_globaltag_file
+            result = read_file(file) # str
+            result = json.loads(result)
+
+            dto = validate_and_convert_types(
+                result,
+                {dict, GlobalTagDto},
+                ['result'],
+                True,
+                True,
+                GlobalTagDto
+            )
+            
+            return dto
+        except ApiException as e:
+            print(f"Exception in method CrestApiFs->find_global_tag: {e}\n")
+            return HTTPResponse(
+                status_code=e.status,
+                reason=e.reason,
+                code=e.status,
+                message=e.body
+            )
+
+
+    def check_global_tag_map(self, dto: GlobalTagMapDto):
+        """
+        The auxiliary method to check if the global tag and tag exist.
+
+        :param dto (GlobalTagMapDto): global tag map.
+        """
+
+        if dto.get('globalTagName') is not None:
+            global_tag = dto['globalTagName']
+            global_tag_path = self._dir + conf.fs_globaltag_path + "/" + global_tag + conf.fs_globaltag_file
+            # print(global_tag_path)
+            if not os.path.isfile(global_tag_path):
+                raise FileNotFoundError(f"Global tag '{global_tag}' does not exist.")
+        else : 
+            raise Exception("Error: globalTagName not found in JSON.")
+
+
+        if dto.get('tagName') is not None:
+            tag = dto['tagName']
+            tag_path = self._dir + conf.fs_tag_path + "/" + tag + conf.fs_tag_file
+            # print(tag_path)
+            if not os.path.isfile(tag_path):
+                raise FileNotFoundError(f"Tag '{tag}' does not exist.")
+        else :
+            raise Exception("Error: tagName not found in JSON.")
+        
+        
+        
+    def create_global_tag_map(self, dto: GlobalTagMapDto) -> Union[GlobalTagMapDto, HTTPResponse]:
+        """
+        This method creates a new global tag map on the file storage.
+
+        :param dto (GlobalTagMapDto): global tag.
+        :return: Created global tag map or HTTP response if unsuccessful.
+        """
+        inserted = False
+        
+        try:
+            self.check_global_tag_map(dto=dto)
+            global_tag = dto['globalTagName']
+            global_tagmap_path = self._dir + conf.fs_globaltag_path + "/" + global_tag + conf.fs_map_file
+            # print(global_tagmap_path)
+            if os.path.isfile(global_tagmap_path):
+                # print("exists")
+                tag = dto['tagName']
+                tag_map = read_file(global_tagmap_path)
+                tagmap = json.loads(tag_map)
+                for element in tagmap:
+                    current_tag = element['tagName']
+                    if current_tag == tag:
+                        tagmap.remove(element)
+                        map = model_to_dict(dto)
+                        # print(map)
+                        tagmap.append(map)
+                        map_string = json.dumps(tagmap)
+                        write_file(global_tagmap_path,map_string)
+                        inserted = True
+                if inserted == False:
+                    map = model_to_dict(dto)
+                    # print(map)
+                    tagmap.append(map)
+                    map_string = json.dumps(tagmap)
+                    write_file(global_tagmap_path,map_string)
+                
+            else:
+                # print("does not exist")
+                tagmap = json.loads("[]")
+                map = model_to_dict(dto)
+                # print(map)
+                tagmap.append(map)
+                map_string = json.dumps(tagmap)
+                write_file(global_tagmap_path,map_string)           
+
+            return dto
+        
+        except ApiException as e:
+            print(f"Exception in method CrestApiFs->create_global_tag_map: {e}\n")
+            return HTTPResponse(
+                status_code=e.status,
+                reason=e.reason,
+                code=e.status,
+                message=e.body
+            )
+
+
+
+    def find_global_tag_map(self, name: str) -> Union[GlobalTagMapSetDto, HTTPResponse]:
+        """
+        This method returns a tag for the tag name.
+
+        :param ame (str): global tag name.
+        :return: tag (as TagDto) or HTTP response if unsuccessful.
+        """
+
+        file = self._dir + conf.fs_globaltag_path + "/" + name + conf.fs_map_file
+        # print(file)
+        
+        try:
+        
+            result = read_file(file) # str
+            # print(result)
+            result = json.loads(result)
+
+            dto = GlobalTagMapSetDto(
+                size = len(result),
+                format = 'GlobalTagMapSetDto',
+                datatype = 'maps',
+                resources= []
+            )
+
+            for element in result:
+                map = validate_and_convert_types(
+                    element,
+                    {GlobalTagMapDto},
+                    ['element'],
+                    True,
+                    True,
+                    GlobalTagMapDto
+                 )
+
+                dto.resources.append(map)
+            
+            return dto
+
+        except ApiException as e:
+            print(f"Exception in method CrestApiFs->find_global_tag_map: {e}\n")
             return HTTPResponse(
                 status_code=e.status,
                 reason=e.reason,
-- 
GitLab


From 7a25e7dbe5389fda792bdb08270cfb3d88af34c7 Mon Sep 17 00:00:00 2001
From: Mikhail Mineev <Mikhail.Mineev@cern.ch>
Date: Thu, 25 Apr 2024 15:51:07 +0200
Subject: [PATCH 15/16] error in select_iovs corrected

---
 pycrest/api/crest_fs_api.py | 39 ++++++-------------------------------
 1 file changed, 6 insertions(+), 33 deletions(-)

diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py
index f729a5d..c305fd9 100644
--- a/pycrest/api/crest_fs_api.py
+++ b/pycrest/api/crest_fs_api.py
@@ -239,10 +239,8 @@ class CrestApiFs():
 
         try:
             file = path + conf.fs_tag_file
-            print(file)
 
             result = read_file(file) # str
-            print(result)
             result = json.loads(result)
 
             dto = validate_and_convert_types(
@@ -276,10 +274,8 @@ class CrestApiFs():
 
         try:
             file = path + conf.fs_tagmetainfo_file
-            print(file)
 
             result = read_file(file) # str
-            print(result)
             result = json.loads(result)
 
             dto = validate_and_convert_types(
@@ -377,11 +373,8 @@ class CrestApiFs():
         unsorted = {}
 
         for element in data:
-            # print(element)
             if element.get('since') is not None:
-                # print('since exists')
                 since = element['since']
-                # print(since)
                 unsorted[since] = element
             else:
                 raise Exception("JSON has no since parameter")
@@ -389,13 +382,10 @@ class CrestApiFs():
             sorted_dict = dict(sorted(unsorted.items(),reverse = False))
         else:
             sorted_dict = dict(sorted(unsorted.items(),reverse = True))
-        # print(sorted_dict)
 
         for key in sorted_dict:
-            # print(key, '->', sorted_dict[key])
             res.append(sorted_dict[key])
 
-        # print(res)
         return res
 
 
@@ -412,7 +402,6 @@ class CrestApiFs():
         
         res = json.loads("[]")
         data_size = len(data)
-        # print (data_size)
 
         if data_size == 0:
             # no elements to return
@@ -430,7 +419,6 @@ class CrestApiFs():
             kmax = data_size
 
         for i in range (kmin , kmax):
-            # print (i)
             res.append(data[i])
     
         return res
@@ -443,27 +431,20 @@ class CrestApiFs():
 
         :param json  : JSON array with the IOV list.
         :param since: Start time
-        :param until: End time
+        :param until: End time, (value -1 means "infinity")
         :return: IOV list (JSON array).
         """
         res = json.loads("[]")
-        # print(since)
-        # print(until)
 
         for element in data:
-            # print(element)
             if element.get('since') is not None:
-                # print('since exists')
                 time = element['since']
-                # print(time)
             
                 if until == -1:  # infinity
                     if time >= since:
-                        # print("b")
                         res.append(element)
                 else:
                     if (time >= since) and (time <= until):
-                        # print("a")
                         res.append(element)
             else:
                 raise Exception("JSON has no since parameter")
@@ -472,9 +453,8 @@ class CrestApiFs():
 
 
     
-    # def select_iovs(self, name: str) -> Union[ IovSetDto, HTTPResponse]: # OLD
     def select_iovs(self, name: str,
-                    since: int = 0, until: int = 0,
+                    since: int = 0, until: int = -1,
                     size: int = 1000, page: int = 0,
                     sort: str = 'id.since:ASC') -> Union[IovSetDto, HTTPResponse]:
         """
@@ -502,7 +482,7 @@ class CrestApiFs():
                 result = self.sort_iov_json(result, True)
             else:
                 result = self.sort_iov_json(result, False)
-
+                
             # IOV list time interval:
             result = self.extract_interval(result, since=since, until=until)
 
@@ -613,7 +593,7 @@ class CrestApiFs():
         if dto.get('globalTagName') is not None:
             global_tag = dto['globalTagName']
             global_tag_path = self._dir + conf.fs_globaltag_path + "/" + global_tag + conf.fs_globaltag_file
-            # print(global_tag_path)
+
             if not os.path.isfile(global_tag_path):
                 raise FileNotFoundError(f"Global tag '{global_tag}' does not exist.")
         else : 
@@ -623,7 +603,7 @@ class CrestApiFs():
         if dto.get('tagName') is not None:
             tag = dto['tagName']
             tag_path = self._dir + conf.fs_tag_path + "/" + tag + conf.fs_tag_file
-            # print(tag_path)
+
             if not os.path.isfile(tag_path):
                 raise FileNotFoundError(f"Tag '{tag}' does not exist.")
         else :
@@ -644,9 +624,8 @@ class CrestApiFs():
             self.check_global_tag_map(dto=dto)
             global_tag = dto['globalTagName']
             global_tagmap_path = self._dir + conf.fs_globaltag_path + "/" + global_tag + conf.fs_map_file
-            # print(global_tagmap_path)
+
             if os.path.isfile(global_tagmap_path):
-                # print("exists")
                 tag = dto['tagName']
                 tag_map = read_file(global_tagmap_path)
                 tagmap = json.loads(tag_map)
@@ -655,23 +634,19 @@ class CrestApiFs():
                     if current_tag == tag:
                         tagmap.remove(element)
                         map = model_to_dict(dto)
-                        # print(map)
                         tagmap.append(map)
                         map_string = json.dumps(tagmap)
                         write_file(global_tagmap_path,map_string)
                         inserted = True
                 if inserted == False:
                     map = model_to_dict(dto)
-                    # print(map)
                     tagmap.append(map)
                     map_string = json.dumps(tagmap)
                     write_file(global_tagmap_path,map_string)
                 
             else:
-                # print("does not exist")
                 tagmap = json.loads("[]")
                 map = model_to_dict(dto)
-                # print(map)
                 tagmap.append(map)
                 map_string = json.dumps(tagmap)
                 write_file(global_tagmap_path,map_string)           
@@ -698,12 +673,10 @@ class CrestApiFs():
         """
 
         file = self._dir + conf.fs_globaltag_path + "/" + name + conf.fs_map_file
-        # print(file)
         
         try:
         
             result = read_file(file) # str
-            # print(result)
             result = json.loads(result)
 
             dto = GlobalTagMapSetDto(
-- 
GitLab


From 3bd6e120146fe37c8b8f609b5745781caec04b75 Mon Sep 17 00:00:00 2001
From: Mikhail Mineev <Mikhail.Mineev@cern.ch>
Date: Mon, 6 May 2024 16:30:39 +0200
Subject: [PATCH 16/16] correction in command get iovList

---
 pycrest/cli/commands/iovs_commands.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/pycrest/cli/commands/iovs_commands.py b/pycrest/cli/commands/iovs_commands.py
index f77e199..e3ea100 100644
--- a/pycrest/cli/commands/iovs_commands.py
+++ b/pycrest/cli/commands/iovs_commands.py
@@ -6,6 +6,8 @@ def get_iov_list_func(args):
     # print("GET IOV LIST function")
     # print(f"command = {args.command}")
     # print(f"name = {args.name}")
+    # print(f"since = {args.since}")
+    # print(f"until = {args.until}")
     # print(f"host = {crest_host}")
     # tag name is mandatory
     name = args.name
@@ -15,12 +17,15 @@ def get_iov_list_func(args):
     params = {}
     for arg in args.__dict__.keys():
         if arg in optional_args:
-            params[arg] = arg_dict[arg]
+            if ((arg=='until') and (args.until=='-1')):
+                params[arg] = 'INF'
+            elif (arg_dict[arg]!=None):
+                params[arg] =arg_dict[arg]
 
     try:
         crest_api = CrestApi(host=args.crest_host)
         resp = crest_api.find_all_iovs(tagname=name, **params)
-        # print_json(resp) # OLD MvG
+
         print_multiple_res(resp)
     except Exception as e:
         print("Error: "+ repr(e))
-- 
GitLab