From 07091f662ecbd90ce1e3abb610497f06b88eee4d Mon Sep 17 00:00:00 2001 From: Mikhail Mineev <Mikhail.Mineev@cern.ch> Date: Wed, 20 Mar 2024 18:48:21 +0100 Subject: [PATCH 01/16] initial version --- pycrest/api/crest_fs/__init__.py | 0 pycrest/api/crest_fs/crest_fs_config.py | 17 ++ pycrest/api/crest_fs/crest_fs_utils.py | 31 +++ pycrest/api/crest_fs_api.py | 250 ++++++++++++++++++++++++ 4 files changed, 298 insertions(+) create mode 100644 pycrest/api/crest_fs/__init__.py create mode 100644 pycrest/api/crest_fs/crest_fs_config.py create mode 100644 pycrest/api/crest_fs/crest_fs_utils.py create mode 100644 pycrest/api/crest_fs_api.py diff --git a/pycrest/api/crest_fs/__init__.py b/pycrest/api/crest_fs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pycrest/api/crest_fs/crest_fs_config.py b/pycrest/api/crest_fs/crest_fs_config.py new file mode 100644 index 0000000..e515928 --- /dev/null +++ b/pycrest/api/crest_fs/crest_fs_config.py @@ -0,0 +1,17 @@ +fs_tag_path = "/tags" +fs_globaltag_path = "/globaltags" +fs_data_path = "/data" + +fs_tag_file = "/tag.json" +fs_iov_file = "/iovs.json" +fs_tagmetainfo_file = "/tagmetainfo.json" + +fs_meta_file = "/meta.json" +fs_payload_file = "/payload.json" + +fs_globaltag_file = "/globaltag.json" +fs_map_file = "/maps.json" + +fs_path = "" + +fs_prefix_length = 3 diff --git a/pycrest/api/crest_fs/crest_fs_utils.py b/pycrest/api/crest_fs/crest_fs_utils.py new file mode 100644 index 0000000..ea36e3c --- /dev/null +++ b/pycrest/api/crest_fs/crest_fs_utils.py @@ -0,0 +1,31 @@ +import hashlib +from pathlib import Path + +def read_file(file): + txt = Path(file).read_text() + return txt + +def write_file(file,data): + text_file = open(file, "w") + text_file.write(data) + text_file.close() + +def check_directory(path): + Path(path).mkdir(parents=True, exist_ok=True) + +def get_hash(string): + # hash calculating with SHA-256 for a string + hash_object = hashlib.sha256(string.encode()) + hex_dig = hash_object.hexdigest() + print(hex_dig) + print(type(hex_dig)) + return hex_dig + +def compute_sha256(file_name): + # hash calculating with SHA-256 for a file + hash_sha256 = hashlib.sha256() + with open(file_name, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_sha256.update(chunk) + return hash_sha256.hexdigest() + diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py new file mode 100644 index 0000000..75cdf43 --- /dev/null +++ b/pycrest/api/crest_fs_api.py @@ -0,0 +1,250 @@ +from typing import Dict, Any, Union, List, Optional +import json +import os +# import requests +from datetime import datetime +# from hep.crest.client import Configuration, ApiClient, ApiException +from hep.crest.client import ApiException +from hep.crest.client.api import iovs_api, admin_api, globaltags_api, globaltagmaps_api, payloads_api, tags_api, runinfo_api +from hep.crest.client.models import ( + IovSetDto, HTTPResponse, TagMetaSetDto, TagMetaDto, TagSetDto, TagDto, GlobalTagDto, + GlobalTagSetDto, GlobalTagMapDto, StoreSetDto, StoreDto, RunLumiInfoDto, RunLumiSetDto +) + +# from pathlib import Path +import pycrest.api.crest_fs.crest_fs_config as conf +from pycrest.api.crest_fs.crest_fs_utils import read_file, write_file, check_directory, get_hash, compute_sha256 +import shutil +from pycrest.cli.commands.utils import datetime_serializer +from hep.crest.client.model_utils import model_to_dict + +import ast + +class CrestApiFs(): + _config = None + _dir = "/tmp/crest_dump" + _api_client = None + + def __init__(self, dir: str): + self._dir = dir + #self._config = Configuration(dir=dir) + #self._api_client = ApiClient(self._config) + + + def print_dir(self): + print (self._dir) + + @staticmethod + def build_params(kwargs: Dict[str, Any]) -> Dict[str, Any]: + # Verify your data + return kwargs + + def create_tag(self, name: str, timeType: str, **kwargs) -> Union[TagDto, HTTPResponse]: + """ + Create a new tag. + + :param name: Name of the tag. + :param timeType: Time type of the tag. + :param kwargs: Additional parameters for tag creation. + :return: Created tag or HTTP response if unsuccessful. + """ + criteria = CrestApiFs.build_params(kwargs) + params = { + 'description': '', + 'synchronization': 'none', + 'payload_spec': 'JSON', + 'last_validated_time': -1., + 'end_of_validity': -1., + } + params.update(criteria) + tag = TagDto( + name=name, + description=params['description'], + time_type=timeType, + payload_spec=params['payload_spec'], + synchronization=params['synchronization'], + last_validated_time=params['last_validated_time'], + end_of_validity=params['end_of_validity'] + ) + + path = self._dir + conf.fs_tag_path + "/" + name + print (path) + + try: + check_directory(path); + file = path + conf.fs_tag_file + tagjs = model_to_dict(tag) + str1 = json.dumps(tagjs) + write_file(file,str1) + except ApiException as e: + print(f"Exception when calling TagsApi->create_tag: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + + def create_tag_meta(self, name: str, **kwargs) -> Union[TagMetaDto, HTTPResponse]: + """ + Create the meta information for a tag. + + :param name: Name of the tag. + :param kwargs: Additional parameters for tag meta creation. + :return: Created tag meta or HTTP response if unsuccessful. + """ + criteria = CrestApiFs.build_params(kwargs) + params = { + 'description': '', + 'chansize': 1, + 'colsize': 1, + 'tag_info': '', + } + params.update(criteria) + + tag_meta_dto = TagMetaDto( + tag_name=name, + description=params['description'], + chansize=params['chansize'], + colsize=params['colsize'], + tag_info=params['tag_info'], + ) + + path = self._dir + conf.fs_tag_path + "/" + name + print (path) + + try: + check_directory(path); + file = path + conf.fs_tagmetainfo_file + tag_meta = model_to_dict(tag_meta_dto) + str1 = json.dumps(tag_meta) + write_file(file,str1) + except ApiException as e: + print(f"Exception when calling TagsApi->create_tag_meta: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + + + + def store_data(self, tag_name: str, since: int, filepath: str, filename: str, compression_type: str = 'none', version: str = '1', object_type: str = 'JSON', **kwargs: Dict[str, Any]) -> Any: + """ + Store data: upload a single file to CREST. + + :param tag_name: Name of the tag. + :param since: Since value. + :param filepath: Path to the file. + :param filename: Name of the file. + :param compression_type: Compression type (default is 'none'). + :param version: Version of the data (default is '1'). + :param object_type: Type of the object (default is 'JSON'). + :param kwargs: Additional optional parameters. + :return: API response. + """ + criteria = CrestApiFs.build_params(kwargs) + params = { + 'compression_type': compression_type, + 'version': version, + 'object_type': object_type, + } + params.update(criteria) + + file_path = os.path.join(filepath, filename) + if not os.path.isfile(file_path): + raise FileNotFoundError(f"File '{file_path}' not found.") + + streamer_info = params.get('streamerInfo', f'{{"filename":"{filename}"}}') + del params['streamerInfo'] + + hash = compute_sha256(file_path) + print (hash) + prefix = hash[:conf.fs_prefix_length] + print(prefix) + + try: + payload_path = self._dir + conf.fs_data_path + "/" + prefix + "/" + hash + check_directory(payload_path); + payload_file = payload_path + conf.fs_payload_file + print(payload_file) + shutil.copyfile(file_path, payload_file) + + now = datetime.now() + date_string = now.strftime("%Y-%m-%d %H:%M:%S") + print(date_string) + + file_size = os.path.getsize(file_path) + print(file_size) + + tag_meta = { + "checkSum": "SHA-256", + "compressionType": "none", + "hash": hash, + "insertionTime": now, + "objectType": object_type, + "size": file_size, + #"streamerInfo": streamerInfo, + "version": version + } + + + jsres = json.dumps(tag_meta, default=datetime_serializer) + print(jsres) + + meta_file = payload_path + conf.fs_meta_file + print(meta_file) + write_file(meta_file,jsres) + + iov = { + "insertionTime": now, + "payloadHash": hash, + "since": since, + "tagName": tag_name + } + + siov = json.dumps(iov, default=datetime_serializer) + iov = json.loads(siov) + print(siov) + + iov_path = self._dir + conf.fs_tag_path + "/" + tag_name + iov_file = iov_path + conf.fs_iov_file + print(iov_file) + + if os.path.isfile(iov_file): + print("iov list exists") + iovs = read_file(iov_file) + print("iovs = ") + print(iovs) + # iov_list = ast.literal_eval(iovs) + iov_list = json.loads(iovs) + for element in iov_list: + if 'since' in element: + current_since = element['since'] + if since == current_since: + iov_list.remove(element) + + + print("iov = ") + print(iov) + iov_list.append(iov) + print("new iov list = ") + print(iov_list) + iovs = json.dumps(iov_list) + write_file(iov_file,iovs) + else: + print("iov list does not exist") + + iov_list = json.loads("[]") + + iov_list.append(iov) + iovs = json.dumps(iov_list) + write_file(iov_file,iovs) + + except ApiException as e: + print("Exception when calling PayloadsApi->store_payload_batch: %s\n" % e) + + -- GitLab From ece4d288d958bf27c607ba6f261d3eea0ccc7563 Mon Sep 17 00:00:00 2001 From: Mikhail Mineev <Mikhail.Mineev@cern.ch> Date: Fri, 29 Mar 2024 10:55:26 +0100 Subject: [PATCH 02/16] added DTOs in some methods --- pycrest/api/crest_fs_api.py | 57 ++++++++----------------------------- 1 file changed, 12 insertions(+), 45 deletions(-) diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py index 75cdf43..c678e73 100644 --- a/pycrest/api/crest_fs_api.py +++ b/pycrest/api/crest_fs_api.py @@ -39,34 +39,16 @@ class CrestApiFs(): # Verify your data return kwargs - def create_tag(self, name: str, timeType: str, **kwargs) -> Union[TagDto, HTTPResponse]: + + def create_tag(self, tag) -> Union[TagDto, HTTPResponse]: """ Create a new tag. - :param name: Name of the tag. - :param timeType: Time type of the tag. - :param kwargs: Additional parameters for tag creation. + :param tag (TagDto): tag. :return: Created tag or HTTP response if unsuccessful. """ - criteria = CrestApiFs.build_params(kwargs) - params = { - 'description': '', - 'synchronization': 'none', - 'payload_spec': 'JSON', - 'last_validated_time': -1., - 'end_of_validity': -1., - } - params.update(criteria) - tag = TagDto( - name=name, - description=params['description'], - time_type=timeType, - payload_spec=params['payload_spec'], - synchronization=params['synchronization'], - last_validated_time=params['last_validated_time'], - end_of_validity=params['end_of_validity'] - ) + name = tag.name path = self._dir + conf.fs_tag_path + "/" + name print (path) @@ -76,6 +58,7 @@ class CrestApiFs(): tagjs = model_to_dict(tag) str1 = json.dumps(tagjs) write_file(file,str1) + return tag except ApiException as e: print(f"Exception when calling TagsApi->create_tag: {e}\n") return HTTPResponse( @@ -86,40 +69,26 @@ class CrestApiFs(): ) - def create_tag_meta(self, name: str, **kwargs) -> Union[TagMetaDto, HTTPResponse]: + + def create_tag_meta(self, tag_meta) -> Union[TagMetaDto, HTTPResponse]: """ Create the meta information for a tag. - :param name: Name of the tag. - :param kwargs: Additional parameters for tag meta creation. + :param tag_meta (TagMetaDto): tag meta info. :return: Created tag meta or HTTP response if unsuccessful. """ - criteria = CrestApiFs.build_params(kwargs) - params = { - 'description': '', - 'chansize': 1, - 'colsize': 1, - 'tag_info': '', - } - params.update(criteria) - - tag_meta_dto = TagMetaDto( - tag_name=name, - description=params['description'], - chansize=params['chansize'], - colsize=params['colsize'], - tag_info=params['tag_info'], - ) + name = tag_meta.tag_name path = self._dir + conf.fs_tag_path + "/" + name print (path) try: check_directory(path); file = path + conf.fs_tagmetainfo_file - tag_meta = model_to_dict(tag_meta_dto) + tag_meta = model_to_dict(tag_meta) str1 = json.dumps(tag_meta) write_file(file,str1) + return tag_meta except ApiException as e: print(f"Exception when calling TagsApi->create_tag_meta: {e}\n") return HTTPResponse( @@ -128,9 +97,7 @@ class CrestApiFs(): code=e.status, message=e.body ) - - - + def store_data(self, tag_name: str, since: int, filepath: str, filename: str, compression_type: str = 'none', version: str = '1', object_type: str = 'JSON', **kwargs: Dict[str, Any]) -> Any: """ -- GitLab From 2fc45a839fe7b2c8e82d197fbc14cc97c52fa2ae Mon Sep 17 00:00:00 2001 From: Mikhail Mineev <Mikhail.Mineev@cern.ch> Date: Wed, 3 Apr 2024 12:44:06 +0200 Subject: [PATCH 03/16] store data method written --- pycrest/api/crest_fs/crest_fs_utils.py | 14 +++- pycrest/api/crest_fs_api.py | 106 ++++++++++++------------- 2 files changed, 63 insertions(+), 57 deletions(-) diff --git a/pycrest/api/crest_fs/crest_fs_utils.py b/pycrest/api/crest_fs/crest_fs_utils.py index ea36e3c..a3fbf52 100644 --- a/pycrest/api/crest_fs/crest_fs_utils.py +++ b/pycrest/api/crest_fs/crest_fs_utils.py @@ -17,8 +17,6 @@ def get_hash(string): # hash calculating with SHA-256 for a string hash_object = hashlib.sha256(string.encode()) hex_dig = hash_object.hexdigest() - print(hex_dig) - print(type(hex_dig)) return hex_dig def compute_sha256(file_name): @@ -29,3 +27,15 @@ def compute_sha256(file_name): hash_sha256.update(chunk) return hash_sha256.hexdigest() +def utf8len(s): + return len(s.encode('utf-8')) + +def get_value(json, key): + if key in json: + return json[key] + else: + message = 'key ' + key + ' not found in JSON' + raise Exception(message) + + + diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py index c678e73..dd51806 100644 --- a/pycrest/api/crest_fs_api.py +++ b/pycrest/api/crest_fs_api.py @@ -13,7 +13,7 @@ from hep.crest.client.models import ( # from pathlib import Path import pycrest.api.crest_fs.crest_fs_config as conf -from pycrest.api.crest_fs.crest_fs_utils import read_file, write_file, check_directory, get_hash, compute_sha256 +from pycrest.api.crest_fs.crest_fs_utils import read_file, write_file, check_directory, get_hash, compute_sha256, utf8len import shutil from pycrest.cli.commands.utils import datetime_serializer from hep.crest.client.model_utils import model_to_dict @@ -27,8 +27,6 @@ class CrestApiFs(): def __init__(self, dir: str): self._dir = dir - #self._config = Configuration(dir=dir) - #self._api_client = ApiClient(self._config) def print_dir(self): @@ -50,7 +48,6 @@ class CrestApiFs(): name = tag.name path = self._dir + conf.fs_tag_path + "/" + name - print (path) try: check_directory(path); @@ -80,7 +77,6 @@ class CrestApiFs(): name = tag_meta.tag_name path = self._dir + conf.fs_tag_path + "/" + name - print (path) try: check_directory(path); @@ -99,54 +95,49 @@ class CrestApiFs(): ) - def store_data(self, tag_name: str, since: int, filepath: str, filename: str, compression_type: str = 'none', version: str = '1', object_type: str = 'JSON', **kwargs: Dict[str, Any]) -> Any: + + def store_payload(self, tag_name: str, since: int, payload: StoreDto, compression_type: str = 'none', version: str = '1', + object_type: str = 'JSON', payload_format: str = 'JSON', streamer_info: str = 'none', **kwargs: Dict[str, Any]) -> Any: """ - Store data: upload a single file to CREST. + store data: upload a single payload as a file/string to CREST. :param tag_name: Name of the tag. :param since: Since value. - :param filepath: Path to the file. - :param filename: Name of the file. + :param payload: payload of file with the payload. :param compression_type: Compression type (default is 'none'). :param version: Version of the data (default is '1'). - :param object_type: Type of the object (default is 'JSON'). + :param payload_format: Type of the object (default is 'JSON'). :param kwargs: Additional optional parameters. :return: API response. """ - criteria = CrestApiFs.build_params(kwargs) - params = { - 'compression_type': compression_type, - 'version': version, - 'object_type': object_type, - } - params.update(criteria) - - file_path = os.path.join(filepath, filename) - if not os.path.isfile(file_path): - raise FileNotFoundError(f"File '{file_path}' not found.") - - streamer_info = params.get('streamerInfo', f'{{"filename":"{filename}"}}') - del params['streamerInfo'] - - hash = compute_sha256(file_path) - print (hash) - prefix = hash[:conf.fs_prefix_length] - print(prefix) - + + if (payload_format == 'FILE') : + file_path = payload.replace("file://", "") + if not os.path.isfile(file_path): + raise FileNotFoundError(f"File '{file_path}' not found.") + + + hash = compute_sha256(file_path) + prefix = hash[:conf.fs_prefix_length] + else: + hash = get_hash(payload) + prefix = hash[:conf.fs_prefix_length] + try: payload_path = self._dir + conf.fs_data_path + "/" + prefix + "/" + hash - check_directory(payload_path); + check_directory(payload_path) payload_file = payload_path + conf.fs_payload_file - print(payload_file) - shutil.copyfile(file_path, payload_file) + + if (payload_format == 'FILE') : + shutil.copyfile(file_path, payload_file) + file_size = os.path.getsize(file_path) + else: + write_file(payload_file,payload) + file_size = utf8len(payload) now = datetime.now() date_string = now.strftime("%Y-%m-%d %H:%M:%S") - print(date_string) - - file_size = os.path.getsize(file_path) - print(file_size) - + tag_meta = { "checkSum": "SHA-256", "compressionType": "none", @@ -154,16 +145,14 @@ class CrestApiFs(): "insertionTime": now, "objectType": object_type, "size": file_size, - #"streamerInfo": streamerInfo, + "streamerInfo": streamer_info, "version": version } jsres = json.dumps(tag_meta, default=datetime_serializer) - print(jsres) meta_file = payload_path + conf.fs_meta_file - print(meta_file) write_file(meta_file,jsres) iov = { @@ -175,18 +164,12 @@ class CrestApiFs(): siov = json.dumps(iov, default=datetime_serializer) iov = json.loads(siov) - print(siov) iov_path = self._dir + conf.fs_tag_path + "/" + tag_name iov_file = iov_path + conf.fs_iov_file - print(iov_file) if os.path.isfile(iov_file): - print("iov list exists") iovs = read_file(iov_file) - print("iovs = ") - print(iovs) - # iov_list = ast.literal_eval(iovs) iov_list = json.loads(iovs) for element in iov_list: if 'since' in element: @@ -194,24 +177,37 @@ class CrestApiFs(): if since == current_since: iov_list.remove(element) - - print("iov = ") - print(iov) iov_list.append(iov) - print("new iov list = ") - print(iov_list) iovs = json.dumps(iov_list) write_file(iov_file,iovs) else: - print("iov list does not exist") - iov_list = json.loads("[]") - iov_list.append(iov) iovs = json.dumps(iov_list) write_file(iov_file,iovs) except ApiException as e: print("Exception when calling PayloadsApi->store_payload_batch: %s\n" % e) + + + def store_data(self, tag: str, store_set: StoreSetDto, payload_format: str = 'JSON', compression_type: str = 'none', version: str = '1', object_type: str = 'JSON', **kwargs: Dict[str, Any]) -> Any: + + data = str(store_set) + dto = store_set.to_dict() + + res = store_set.resources + n_elem = len(res) + + for elem in res: + item = elem.to_dict() + try: + payload = elem.data + since = int(elem.since) + streamer_info = elem.streamer_info + + self.store_payload( tag, since, payload, payload_format = payload_format, streamer_info = streamer_info) + + except Exception as e: + print('Error' % e) -- GitLab From b85847770b5b38be04ed73d5685aa44772ce223e Mon Sep 17 00:00:00 2001 From: Mikhail Mineev <Mikhail.Mineev@cern.ch> Date: Fri, 5 Apr 2024 16:44:04 +0200 Subject: [PATCH 04/16] find tag and tag meta methods added --- pycrest/api/crest_fs_api.py | 81 +++++++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 3 deletions(-) diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py index dd51806..0a41307 100644 --- a/pycrest/api/crest_fs_api.py +++ b/pycrest/api/crest_fs_api.py @@ -16,7 +16,8 @@ import pycrest.api.crest_fs.crest_fs_config as conf from pycrest.api.crest_fs.crest_fs_utils import read_file, write_file, check_directory, get_hash, compute_sha256, utf8len import shutil from pycrest.cli.commands.utils import datetime_serializer -from hep.crest.client.model_utils import model_to_dict +from hep.crest.client.model_utils import model_to_dict, validate_and_convert_types +# from hep.crest.client.model_utils import validate_and_convert_types import ast @@ -38,7 +39,7 @@ class CrestApiFs(): return kwargs - def create_tag(self, tag) -> Union[TagDto, HTTPResponse]: + def create_tag(self, tag: TagDto) -> Union[TagDto, HTTPResponse]: """ Create a new tag. @@ -67,7 +68,7 @@ class CrestApiFs(): - def create_tag_meta(self, tag_meta) -> Union[TagMetaDto, HTTPResponse]: + def create_tag_meta(self, tag_meta: TagMetaDto) -> Union[TagMetaDto, HTTPResponse]: """ Create the meta information for a tag. @@ -211,3 +212,77 @@ class CrestApiFs(): except Exception as e: print('Error' % e) + + def find_tag(self, tag_name: str) -> Union[TagDto, HTTPResponse]: + """ + Create a new tag. + + :param tag_name (str): tag name. + :return: tag (as TagDto) or HTTP response if unsuccessful. + """ + + path = self._dir + conf.fs_tag_path + "/" + tag_name + + try: + file = path + conf.fs_tag_file + print(file) + + result = read_file(file) # str + print(result) + result = json.loads(result) + + dto = validate_and_convert_types( + result, + {dict, TagDto}, + ['result'], + True, + True, + TagDto + ) + + return dto + except ApiException as e: + print(f"Exception when calling TagsApi->create_tag: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + def find_tag_meta(self, tag_name: str) -> Union[TagMetaDto, HTTPResponse]: + """ + Create a new tag. + + :param tag_name (str): tag name. + :return: tag (as TagDto) or HTTP response if unsuccessful. + """ + + path = self._dir + conf.fs_tag_path + "/" + tag_name + + try: + file = path + conf.fs_tagmetainfo_file + print(file) + + result = read_file(file) # str + print(result) + result = json.loads(result) + + dto = validate_and_convert_types( + result, + {dict, TagMetaDto}, + ['result'], + True, + True, + TagMetaDto + ) + + return dto + except ApiException as e: + print(f"Exception when calling TagsApi->create_tag: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) -- GitLab From d26105ae50366f21eec533fbfbbae76bcf1e8eaa Mon Sep 17 00:00:00 2001 From: formica <andrea.formica@cern.ch> Date: Sat, 6 Apr 2024 10:44:10 +0200 Subject: [PATCH 05/16] change API for server to make it more similar to CrestApi ini C++ --- pycrest/api/crest_api.py | 180 ++++++++------------ pycrest/cli/commands/global_tag_commands.py | 24 ++- pycrest/cli/commands/tag_commands.py | 31 +++- tests/test_crestapi_http.py | 28 ++- 4 files changed, 141 insertions(+), 122 deletions(-) diff --git a/pycrest/api/crest_api.py b/pycrest/api/crest_api.py index 6aedb30..fe3e126 100644 --- a/pycrest/api/crest_api.py +++ b/pycrest/api/crest_api.py @@ -37,9 +37,17 @@ class CrestApi(): # Verify your data return kwargs - def select_groups(self, tagname: str, snapshot: Optional[float] = None, size: int = 1000, page: int = 0, sort: str = 'id.since:ASC') -> Union[IovSetDto, HTTPResponse]: + def select_groups(self, name: str, snapshot: Optional[float] = None, size: int = 1000, page: int = 0, sort: str = 'id.since:ASC') -> Union[IovSetDto, HTTPResponse]: """ List iov groups for a given tagname + :param name: Name of the tag + :param snapshot: Snapshot time + :param size: Number of results per page + :param page: Page number + :param sort: Sorting order + :return: IovSetDto if successful, else HTTPResponse + + This method is the equivalent of CrestApi.selectGroups() in the C++ code. """ criteria = { 'snapshot': snapshot, @@ -49,12 +57,12 @@ class CrestApi(): } api_instance = iovs_api.IovsApi(self._api_client) try: - api_response = api_instance.find_all_iovs(tagname=tagname, method="GROUPS", **criteria) + api_response = api_instance.find_all_iovs(tagname=name, method="GROUPS", **criteria) return api_response except ApiException as e: print("Exception when calling IovsApi->find_all_iovs: %s\n" % e) - def select_iovs(self, tagname: str, since: int = 0, until: int = 0, snapshot: Optional[float] = None, + def select_iovs(self, name: str, since: int = 0, until: int = 0, snapshot: Optional[float] = None, timeformat: str = 'NUMBER', size: int = 1000, page: int = 0, sort: str = 'id.since:ASC') -> Union[IovSetDto, HTTPResponse]: """ @@ -68,6 +76,9 @@ class CrestApi(): :param page: Page number :param sort: Sorting order :return: IovSetDto if successful, else HTTPResponse + + This method is the equivalent of CrestApi.selectIovs() in the C++ code. + The field timeformat is not used in the C++ code, as it defaults to 'NUMBER'. """ criteria = { 'snapshot': snapshot, @@ -80,7 +91,7 @@ class CrestApi(): } api_instance = iovs_api.IovsApi(self._api_client) try: - api_response = api_instance.find_all_iovs(tagname=tagname, method="IOVS", **criteria) + api_response = api_instance.find_all_iovs(tagname=name, method="IOVS", **criteria) return api_response except ApiException as e: print("Exception when calling IovsApi->find_all_iovs: %s\n" % e) @@ -100,6 +111,8 @@ class CrestApi(): - page: number for page. - size: number for size of the page. :return: IovSetDto if successful, else HTTPResponse. + + This method is not exposed by CrestApi in the C++ code. """ criteria = CrestApi.build_params(kwargs) params = { @@ -117,24 +130,29 @@ class CrestApi(): print("Exception when calling IovsApi->find_all_iovs: %s\n" % e) - def list_tags(self, **kwargs: Dict[str, Any]) -> Union[TagSetDto, HTTPResponse]: + def list_tags(self, name: str, size: int = 1000, page: int = 0, sort: str = 'name:ASC', **kwargs: Dict[str, Any]) -> Union[TagSetDto, HTTPResponse]: """ List all tags. - + :param name: Name of the tag. + :param size: Number of results per page. + :param page: Page number. + :param sort: Sorting order. :param kwargs: Additional criteria. - - name: string for tag name. - description: string for description. - timeType: selection for timeType [time, run, run-lumi]. - objectType: selection for objectType (payloadSpec). - - page: number for page. - - size: number for size of the page. :return: TagSetDto if successful, else HTTPResponse. + + This method is the equivalent of CrestApi.listTags() in the C++ code. """ criteria = CrestApi.build_params(kwargs) params = { - 'page': 0, - 'size': 1000 + 'page': page, + 'size': size, + 'sort': sort } + if name != 'none': + params['name'] = name params.update(criteria) api_instance = tags_api.TagsApi(self._api_client) try: @@ -149,6 +167,8 @@ class CrestApi(): :param name: The name of the global tag to find. :return: GlobalTagSetDto if successful, else HTTPResponse. + + This method is the equivalent of CrestApi.findGlobalTag() in the C++ code. """ api_instance = globaltags_api.GlobaltagsApi(self._api_client) try: @@ -157,24 +177,30 @@ class CrestApi(): except ApiException as e: print("Exception when calling GlobaltagsApi->find_global_tag: %s\n" % e) - def list_global_tags(self, **kwargs) -> Union[GlobalTagSetDto, HTTPResponse]: + def list_global_tags(self, name: str, size: int = 1000, page: int = 0, sort: str = 'name:ASC', **kwargs: Dict[str, Any]) -> Union[GlobalTagSetDto, HTTPResponse]: """ List all global tags. + :param name: Name of the global tag. + :param size: Number of results per page. + :param page: Page number. + :param sort: Sorting order. :param kwargs: Additional criteria for filtering the global tags. - - name: string for global tag name - description: string for description - release: string for release - scenario: string for scenario - - page: number for page - - size: number for size of the page :return: GlobalTagSetDto if successful, else HTTPResponse. + + This method is the equivalent of CrestApi.listGlobalTags() in the C++ code. """ criteria = CrestApi.build_params(kwargs) params = { - 'page': 0, - 'size': 1000 + 'page': page, + 'size': size, + 'sort': sort } + if name != 'none': + params['name'] = name params.update(criteria) api_instance = globaltags_api.GlobaltagsApi(self._api_client) try: @@ -198,36 +224,19 @@ class CrestApi(): except ApiException as e: print(f"Exception when calling GlobaltagmapsApi->find_global_tag_map: {e}\n") - def create_tag(self, name: str, timeType: str, **kwargs) -> Union[TagDto, HTTPResponse]: + def create_tag(self, dto: TagDto) -> Union[TagDto, HTTPResponse]: """ Create a new tag. - :param name: Name of the tag. - :param timeType: Time type of the tag. - :param kwargs: Additional parameters for tag creation. + :param dto: The TagDto. :return: Created tag or HTTP response if unsuccessful. + + This method is the equivalent of CrestApi.createTag() in the C++ code. """ - criteria = CrestApi.build_params(kwargs) - params = { - 'description': '', - 'synchronization': 'none', - 'payload_spec': 'JSON', - 'last_validated_time': -1., - 'end_of_validity': -1., - } - params.update(criteria) - tag = TagDto( - name=name, - description=params['description'], - time_type=timeType, - payload_spec=params['payload_spec'], - synchronization=params['synchronization'], - last_validated_time=params['last_validated_time'], - end_of_validity=params['end_of_validity'] - ) + api_instance = tags_api.TagsApi(self._api_client) try: - return api_instance.create_tag(tag_dto=tag) + return api_instance.create_tag(tag_dto=dto) except ApiException as e: print(f"Exception when calling TagsApi->create_tag: {e}\n") return HTTPResponse( @@ -237,34 +246,18 @@ class CrestApi(): message=e.body ) - def create_tag_meta(self, name: str, **kwargs) -> Union[TagMetaDto, HTTPResponse]: + def create_tag_meta(self, dto: TagMetaDto) -> Union[TagMetaDto, HTTPResponse]: """ Create the meta information for a tag. - :param name: Name of the tag. - :param kwargs: Additional parameters for tag meta creation. + :param dto: The TagMetaDto. :return: Created tag meta or HTTP response if unsuccessful. - """ - criteria = CrestApi.build_params(kwargs) - params = { - 'description': '', - 'chansize': 1, - 'colsize': 1, - 'tag_info': '', - } - params.update(criteria) - - tag_meta_dto = TagMetaDto( - tag_name=name, - description=params['description'], - chansize=params['chansize'], - colsize=params['colsize'], - tag_info=params['tag_info'], - ) + This method is the equivalent of CrestApi.createTagMeta() in the C++ code. + """ api_instance = tags_api.TagsApi(self._api_client) try: - return api_instance.create_tag_meta(name, tag_meta_dto=tag_meta_dto) + return api_instance.create_tag_meta(name=dto.name, tag_meta_dto=dto) except ApiException as e: print(f"Exception when calling TagsApi->create_tag_meta: {e}\n") return HTTPResponse( @@ -274,34 +267,19 @@ class CrestApi(): message=e.body ) - def update_tag_meta(self, name: str, **kwargs) -> Union[TagMetaDto, HTTPResponse]: + def update_tag_meta(self, dto: TagMetaDto) -> Union[TagMetaDto, HTTPResponse]: """ Update the meta information for a tag. - :param name: Name of the tag. - :param kwargs: Additional parameters for tag meta update. + :param dto: The TagMetaDto for update. :return: Updated tag meta or HTTP response if unsuccessful. - """ - criteria = CrestApi.build_params(kwargs) - params = { - 'description': '', - 'chansize': 1, - 'colsize': 1, - 'tag_info': '', - } - params.update(criteria) - tag_meta_dto = TagMetaDto( - tag_name=name, - description=params['description'], - chansize=params['chansize'], - colsize=params['colsize'], - tag_info=params['tag_info'], - ) + This method is the equivalent of CrestApi.updateTagMeta() in the C++ code. + """ api_instance = tags_api.TagsApi(self._api_client) try: - return api_instance.update_tag_meta(name, tag_meta_dto=tag_meta_dto) + return api_instance.update_tag_meta(name=dto.name, tag_meta_dto=dto) except ApiException as e: print(f"Exception when calling TagsApi->update_tag_meta: {e}\n") return HTTPResponse( @@ -331,38 +309,18 @@ class CrestApi(): message=e.body ) - def create_global_tag(self, name: str, force: str = "false", **kwargs) -> Union[GlobalTagDto, HTTPResponse]: + def create_global_tag(self, dto: GlobalTagDto, force: str) -> Union[GlobalTagDto, HTTPResponse]: """ Create a new global tag. - :param name: Name of the global tag. - :param force: Whether to force creation if the global tag already exists. - :param kwargs: Additional parameters for global tag creation. + :param dto: The GlobalTagDto. :return: Created global tag or HTTP response if unsuccessful. - """ - criteria = CrestApi.build_params(kwargs) - params = { - 'description': 'none', - 'validity': 0.0, - 'release': 'none', - 'scenario': 'none', - 'workflow': 'all', - 'type': 'T', - } - params.update(criteria) - globaltag = GlobalTagDto( - name=name, - description=params['description'], - release=params['release'], - scenario=params['scenario'], - validity=params['validity'], - workflow=params['workflow'], - type=params['type'] - ) - + + This method is the equivalent of CrestApi.createTagMeta() in the C++ code. + """ api_instance = globaltags_api.GlobaltagsApi(self._api_client) try: - return api_instance.create_global_tag(force=force, global_tag_dto=globaltag) + return api_instance.create_global_tag(force=force, global_tag_dto=dto) except ApiException as e: print(f"Exception when calling GlobaltagsApi->create_global_tag: {e}\n") return HTTPResponse( @@ -464,7 +422,7 @@ class CrestApi(): message=e.body ) - def get_tag(self, name: str = 'none') -> Union[TagDto, HTTPResponse]: + def find_tag(self, name: str = 'none') -> Union[TagDto, HTTPResponse]: """ Get tag. @@ -473,13 +431,13 @@ class CrestApi(): """ api_instance = tags_api.TagsApi(self._api_client) try: - api_response = api_instance.list_tags(name=name) + api_response = api_instance.find_tag(name=name) if api_response['size'] == 1: return api_response['resources'][0] else: return None except ApiException as e: - print("Exception when calling TagsApi->list_tags in get_tag: %s\n" % e) + print("Exception when calling TagsApi->find_tag: %s\n" % e) return HTTPResponse( status_code=e.status, reason=e.reason, @@ -487,7 +445,7 @@ class CrestApi(): message=e.body ) - def get_global_tag(self, name: str = 'none') -> Union[GlobalTagDto, HTTPResponse]: + def find_global_tag(self, name: str = 'none') -> Union[GlobalTagDto, HTTPResponse]: """ Get global tag. diff --git a/pycrest/cli/commands/global_tag_commands.py b/pycrest/cli/commands/global_tag_commands.py index 92907a2..edc1a07 100644 --- a/pycrest/cli/commands/global_tag_commands.py +++ b/pycrest/cli/commands/global_tag_commands.py @@ -1,9 +1,21 @@ from pycrest.api.crest_api import CrestApi from pycrest.cli.commands.utils import print_full_res, print_multiple_res +from hep.crest.client.models import ( + IovSetDto, HTTPResponse, TagMetaSetDto, TagMetaDto, TagSetDto, TagDto, GlobalTagDto, + GlobalTagSetDto, GlobalTagMapDto, StoreSetDto, StoreDto, RunLumiInfoDto, RunLumiSetDto +) +import sys def create_global_tag_func(args): + + required_args = ['description'] + missing_args = [arg for arg in required_args if getattr(args, arg) is None] + if missing_args: + sys.exit(f"Error: The following parameters are not set: {', '.join(missing_args)}") + params = { + 'name': args.name, 'validity': args.validity, 'description': args.description, 'release': args.release, @@ -11,9 +23,19 @@ def create_global_tag_func(args): 'workflow': args.workflow, 'type': args.type, } + + globaltag = GlobalTagDto( + name=params['name'], + description=params['description'], + release=params['release'], + scenario=params['scenario'], + validity=params['validity'], + workflow=params['workflow'], + type=params['type'] + ) try: crest_api = CrestApi(host=args.crest_host) - resp = crest_api.create_global_tag(name=args.name, **params) + resp = crest_api.create_global_tag(dto=globaltag) # print(f"global tag {args.name} created") print_full_res(resp) except Exception as e: diff --git a/pycrest/cli/commands/tag_commands.py b/pycrest/cli/commands/tag_commands.py index a346755..425eadc 100644 --- a/pycrest/cli/commands/tag_commands.py +++ b/pycrest/cli/commands/tag_commands.py @@ -1,7 +1,10 @@ from pycrest.api.crest_api import CrestApi from pycrest.cli.commands.utils import print_multiple_res, print_full_res, print_single_res from pycrest.cli.config import crest_host - +from hep.crest.client.models import ( + IovSetDto, HTTPResponse, TagMetaSetDto, TagMetaDto, TagSetDto, TagDto, GlobalTagDto, + GlobalTagSetDto, GlobalTagMapDto, StoreSetDto, StoreDto, RunLumiInfoDto, RunLumiSetDto +) import sys def get_tag_func(args): @@ -39,17 +42,24 @@ def create_tag_func(args): 'last_validated_time': args.last_validated_time, 'end_of_validity': args.end_of_validity, } - + tag = TagDto( + name=args.name, + description=params['description'], + time_type=args.time_type, + payload_spec=params['payload_spec'], + synchronization=params['synchronization'], + last_validated_time=params['last_validated_time'], + end_of_validity=params['end_of_validity'] + ) try: crest_api = CrestApi(host=args.crest_host) - resp = crest_api.create_tag(name=args.name, timeType=args.time_type, **params) + resp = crest_api.create_tag(dto=tag) # print(f"tag {args.name} created") print_full_res(resp) except Exception as e: print("Error: "+ repr(e)) - def remove_tag_func(args): # print("GET TAG function") # print(f"command = {args.command}") @@ -65,6 +75,9 @@ def remove_tag_func(args): print("Error: "+ repr(e)) def create_tag_meta_info_func(args): + """ + :param args: the parameters to create tag meta information. + """ required_args = ['description', 'chansize', 'colsize', 'tag_info'] missing_args = [arg for arg in required_args if getattr(args, arg) is None] @@ -78,9 +91,17 @@ def create_tag_meta_info_func(args): 'tag_info': args.tag_info, } + tag_meta = TagMetaDto( + tag_name=args.name, + description=params['description'], + chansize=params['chansize'], + colsize=params['colsize'], + tag_info=params['tag_info'], + ) + try: crest_api = CrestApi(host=args.crest_host) - resp = crest_api.create_tag_meta(name=args.name, **params) + resp = crest_api.create_tag_meta(dto=tag_meta) # print(f"tag meta info {args.name} created") print_full_res(resp) except Exception as e: diff --git a/tests/test_crestapi_http.py b/tests/test_crestapi_http.py index 8140004..6744a81 100644 --- a/tests/test_crestapi_http.py +++ b/tests/test_crestapi_http.py @@ -18,7 +18,7 @@ from pycrest.api.crest_cond_tag_manager import TagManager from pycrest.api.tag_info_container import TagInfoContainer from pycrest.api.crest_cond_builder import CrestCondBuilder from hep.crest.client.models import ( - TagMetaDto, GlobalTagDto + TagMetaDto, TagDto, GlobalTagDto ) from pycrest.api.crest_api import CrestApi @@ -43,12 +43,21 @@ def test_create_tag(crest_client): time_type = 'time' try: print('Create a tag %s' % name) + tag = TagDto( + name=name, + description=description, + time_type=time_type, + payload_spec='JSON', + synchronization='none', + last_validated_time=-1., + end_of_validity=-1. + ) # Create a new tag - api_response = api.create_tag(name=name, timeType=time_type, description=description) + api_response = api.create_tag(tag) assert api_response.name == name # Find the tag - api_response = api.get_tag(name=name) + api_response = api.find_tag(name=name) assert api_response.name == name # Delete the tag @@ -65,11 +74,20 @@ def test_create_global_tag(crest_client): description = 'a brand new test global tag' try: # Create a new global tag - api_response = api.create_global_tag(name=global_tag, description=description) + globaltag = GlobalTagDto( + name=global_tag, + description=description, + release='1.0', + scenario='test', + validity=0., + workflow='all', + type='T' + ) + api_response = api.create_global_tag(globaltag) assert api_response.name == global_tag # Find the global tag - api_response = api.get_global_tag(name=global_tag) + api_response = api.find_global_tag(name=global_tag) assert api_response.name == global_tag # Delete the global tag -- GitLab From 8de2aaf8461eb374238313fe232ec830dfb1bc44 Mon Sep 17 00:00:00 2001 From: formica <andrea.formica@cern.ch> Date: Sat, 6 Apr 2024 13:29:36 +0200 Subject: [PATCH 06/16] change API for server to make it more similar to CrestApi ini C++ --- pycrest/cli/commands/global_tag_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pycrest/cli/commands/global_tag_commands.py b/pycrest/cli/commands/global_tag_commands.py index edc1a07..e47576b 100644 --- a/pycrest/cli/commands/global_tag_commands.py +++ b/pycrest/cli/commands/global_tag_commands.py @@ -22,6 +22,7 @@ def create_global_tag_func(args): 'scenario': args.scenario, 'workflow': args.workflow, 'type': args.type, + 'force': False } globaltag = GlobalTagDto( @@ -35,7 +36,7 @@ def create_global_tag_func(args): ) try: crest_api = CrestApi(host=args.crest_host) - resp = crest_api.create_global_tag(dto=globaltag) + resp = crest_api.create_global_tag(dto=globaltag, force=params['force']) # print(f"global tag {args.name} created") print_full_res(resp) except Exception as e: -- GitLab From 34970f77cb071325b0bafb09af1a366568230fd0 Mon Sep 17 00:00:00 2001 From: formica <andrea.formica@cern.ch> Date: Sat, 6 Apr 2024 14:39:06 +0200 Subject: [PATCH 07/16] change API for server to make it more similar to CrestApi ini C++ --- tests/test_crestapi_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_crestapi_http.py b/tests/test_crestapi_http.py index 6744a81..c2a1504 100644 --- a/tests/test_crestapi_http.py +++ b/tests/test_crestapi_http.py @@ -83,7 +83,7 @@ def test_create_global_tag(crest_client): workflow='all', type='T' ) - api_response = api.create_global_tag(globaltag) + api_response = api.create_global_tag(globaltag, force=False) assert api_response.name == global_tag # Find the global tag -- GitLab From 5aa3745a20ae0f9262f941e571d77719176cc123 Mon Sep 17 00:00:00 2001 From: formica <andrea.formica@cern.ch> Date: Sat, 6 Apr 2024 14:50:12 +0200 Subject: [PATCH 08/16] fix method --- pycrest/cli/commands/global_tag_commands.py | 2 +- tests/test_crestapi_http.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pycrest/cli/commands/global_tag_commands.py b/pycrest/cli/commands/global_tag_commands.py index e47576b..123ae9b 100644 --- a/pycrest/cli/commands/global_tag_commands.py +++ b/pycrest/cli/commands/global_tag_commands.py @@ -22,7 +22,7 @@ def create_global_tag_func(args): 'scenario': args.scenario, 'workflow': args.workflow, 'type': args.type, - 'force': False + 'force': 'false' } globaltag = GlobalTagDto( diff --git a/tests/test_crestapi_http.py b/tests/test_crestapi_http.py index c2a1504..ab5fea8 100644 --- a/tests/test_crestapi_http.py +++ b/tests/test_crestapi_http.py @@ -83,7 +83,7 @@ def test_create_global_tag(crest_client): workflow='all', type='T' ) - api_response = api.create_global_tag(globaltag, force=False) + api_response = api.create_global_tag(globaltag, force='false') assert api_response.name == global_tag # Find the global tag -- GitLab From c09303efc267413b9e2f482093fde886857b33b3 Mon Sep 17 00:00:00 2001 From: formica <andrea.formica@cern.ch> Date: Sun, 7 Apr 2024 22:01:56 +0200 Subject: [PATCH 09/16] update methods --- pycrest/api/crest_api.py | 26 +++++++++------------ pycrest/cli/commands/global_tag_commands.py | 14 ++++++++++- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/pycrest/api/crest_api.py b/pycrest/api/crest_api.py index fe3e126..48b5385 100644 --- a/pycrest/api/crest_api.py +++ b/pycrest/api/crest_api.py @@ -216,6 +216,8 @@ class CrestApi(): :param name: Name of the global tag or tag. :param mode: Map mode [Trace, BackTrace]. :return: Response from the API. + + This method is the equivalent of CrestApi.findGlobalTagMap() in the C++ code. """ headers = {'x_crest_map_mode': mode} api_instance = globaltagmaps_api.GlobaltagmapsApi(self._api_client) @@ -296,6 +298,8 @@ class CrestApi(): :param name: Name of the tag to delete. :return: None if successful, or HTTP response if unsuccessful. + + This method is the equivalent of CrestApi.removeTag() in the C++ code. """ api_instance = admin_api.AdminApi(self._api_client) try: @@ -336,6 +340,8 @@ class CrestApi(): :param name: Name of the global tag to delete. :return: None if successful, or HTTP response if unsuccessful. + + This method is the equivalent of CrestApi.removeGlobalTag() in the C++ code. """ api_instance = admin_api.AdminApi(self._api_client) try: @@ -349,7 +355,7 @@ class CrestApi(): message=e.body ) - def create_global_tag_map(self, globaltagname: str, tagname: str, **kwargs) -> Union[GlobalTagMapDto, HTTPResponse]: + def create_global_tag_map(self, dto: GlobalTagMapDto) -> Union[GlobalTagMapDto, HTTPResponse]: """ Create a new global tag map. @@ -357,23 +363,13 @@ class CrestApi(): :param tagname: Name of the tag. :param kwargs: Additional parameters for global tag map creation. :return: Created global tag map or HTTP response if unsuccessful. + + This method is the equivalent of CrestApi.createGlobalTagMap() in the C++ code. """ - criteria = CrestApi.build_params(kwargs) - params = { - 'record': 'none', - 'label': 'none', - } - params.update(criteria) - globaltagmap = GlobalTagMapDto( - global_tag_name=globaltagname, - tag_name=tagname, - record=params['record'], - label=params['label'] - ) - + api_instance = globaltagmaps_api.GlobaltagmapsApi(self._api_client) try: - return api_instance.create_global_tag_map(global_tag_map_dto=globaltagmap) + return api_instance.create_global_tag_map(global_tag_map_dto=dto) except ApiException as e: print(f"Exception when calling GlobaltagmapsApi->create_global_tag_map: {e}\n") return HTTPResponse( diff --git a/pycrest/cli/commands/global_tag_commands.py b/pycrest/cli/commands/global_tag_commands.py index 123ae9b..a8b63b1 100644 --- a/pycrest/cli/commands/global_tag_commands.py +++ b/pycrest/cli/commands/global_tag_commands.py @@ -104,14 +104,26 @@ def get_global_tag_map_func(args): def create_global_tag_map_func(args): + required_args = ['record', 'label', 'tag_name', 'name'] + missing_args = [arg for arg in required_args if getattr(args, arg) is None] + if missing_args: + sys.exit(f"Error: The following parameters are not set: {', '.join(missing_args)}") params = { 'record': args.record, 'label': args.label, } + + globaltagmap = GlobalTagMapDto( + global_tag_name=args.name, + tag_name=args.tag_name, + record=params['record'], + label=params['label'] + ) + try: crest_api = CrestApi(host=args.crest_host) - resp = crest_api.create_global_tag_map(globaltagname=args.name, tagname=args.tag_name, **params) + resp = crest_api.create_global_tag_map(globaltagmap) # print(f"global tag map {args.name} - {args.tag_name} created") print_full_res(resp) except Exception as e: -- GitLab From 355a15c8673c79c93cc4577ef36141245b6c5f2f Mon Sep 17 00:00:00 2001 From: Mikhail Mineev <Mikhail.Mineev@cern.ch> Date: Wed, 10 Apr 2024 17:40:16 +0200 Subject: [PATCH 10/16] added methods to read data from file storage --- pycrest/api/crest_fs_api.py | 139 +++++++++++++++++++++++++++++++++--- 1 file changed, 130 insertions(+), 9 deletions(-) diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py index 0a41307..0d68794 100644 --- a/pycrest/api/crest_fs_api.py +++ b/pycrest/api/crest_fs_api.py @@ -1,23 +1,24 @@ from typing import Dict, Any, Union, List, Optional import json import os -# import requests + from datetime import datetime -# from hep.crest.client import Configuration, ApiClient, ApiException + from hep.crest.client import ApiException from hep.crest.client.api import iovs_api, admin_api, globaltags_api, globaltagmaps_api, payloads_api, tags_api, runinfo_api from hep.crest.client.models import ( IovSetDto, HTTPResponse, TagMetaSetDto, TagMetaDto, TagSetDto, TagDto, GlobalTagDto, - GlobalTagSetDto, GlobalTagMapDto, StoreSetDto, StoreDto, RunLumiInfoDto, RunLumiSetDto + GlobalTagSetDto, GlobalTagMapDto, StoreSetDto, StoreDto, RunLumiInfoDto, RunLumiSetDto, + PayloadDto, IovDto ) -# from pathlib import Path + import pycrest.api.crest_fs.crest_fs_config as conf from pycrest.api.crest_fs.crest_fs_utils import read_file, write_file, check_directory, get_hash, compute_sha256, utf8len import shutil from pycrest.cli.commands.utils import datetime_serializer from hep.crest.client.model_utils import model_to_dict, validate_and_convert_types -# from hep.crest.client.model_utils import validate_and_convert_types + import ast @@ -100,7 +101,7 @@ class CrestApiFs(): def store_payload(self, tag_name: str, since: int, payload: StoreDto, compression_type: str = 'none', version: str = '1', object_type: str = 'JSON', payload_format: str = 'JSON', streamer_info: str = 'none', **kwargs: Dict[str, Any]) -> Any: """ - store data: upload a single payload as a file/string to CREST. + store data: uploads a single payload as a file/string to CREST. The auxiliary method. :param tag_name: Name of the tag. :param since: Since value. @@ -146,7 +147,6 @@ class CrestApiFs(): "insertionTime": now, "objectType": object_type, "size": file_size, - "streamerInfo": streamer_info, "version": version } @@ -193,7 +193,20 @@ class CrestApiFs(): def store_data(self, tag: str, store_set: StoreSetDto, payload_format: str = 'JSON', compression_type: str = 'none', version: str = '1', object_type: str = 'JSON', **kwargs: Dict[str, Any]) -> Any: + """ + This method stores a set of the payloads (with additional parameters: since and streamer info) on the file storage. + The payloads and parameters have to be in the StoreSetDto object. + :param tag: Name of the tag. + :param since: Since value. + :param store_set (StoreSetDto): StoreSetDto object, which contains payloads (in form of strings or files), since parameters for them + :param payload_format: payload format (default is 'JSON'). + :param compression_type: Compression type (default is 'none'). + :param version: Version of the data (default is '1'). + :param payload_format: Type of the object (default is 'JSON'). + :param kwargs: Additional optional parameters. + :return: API response + """ data = str(store_set) dto = store_set.to_dict() @@ -215,7 +228,7 @@ class CrestApiFs(): def find_tag(self, tag_name: str) -> Union[TagDto, HTTPResponse]: """ - Create a new tag. + This method returns a tag for the tag name. :param tag_name (str): tag name. :return: tag (as TagDto) or HTTP response if unsuccessful. @@ -252,7 +265,7 @@ class CrestApiFs(): def find_tag_meta(self, tag_name: str) -> Union[TagMetaDto, HTTPResponse]: """ - Create a new tag. + This method returns a tag meta info for the tag name. :param tag_name (str): tag name. :return: tag (as TagDto) or HTTP response if unsuccessful. @@ -286,3 +299,111 @@ class CrestApiFs(): code=e.status, message=e.body ) + + def get_payload(self, hash: str) -> Union[str, HTTPResponse]: + """ + This methods returns a payload by the hash. + + :param hash (str): hash . + :return: payload (as string) or HTTP response if unsuccessful. + """ + + try: + prefix = hash[:conf.fs_prefix_length] + payload_path = self._dir + conf.fs_data_path + "/" + prefix + "/" + hash + payload_file = payload_path + conf.fs_payload_file + + result = read_file(payload_file) # str + + return result + except ApiException as e: + print(f"Exception when calling TagsApi->create_tag: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + + def get_payload_meta(self, hash: str) -> Union[PayloadDto, HTTPResponse]: + """ + This method returns a payload meta info for the hash. + + :param hash (str): hash . + :return: payload meta (as PayloadDto) or HTTP response if unsuccessful. + """ + + try: + prefix = hash[:conf.fs_prefix_length] + payload_path = self._dir + conf.fs_data_path + "/" + prefix + "/" + hash + meta_file = payload_path + conf.fs_meta_file + + result = read_file(meta_file) # str + result = json.loads(result) + + dto = validate_and_convert_types( + result, + {json, dict, PayloadDto}, + ['result'], + True, + True, + PayloadDto + ) + + return dto + except ApiException as e: + print(f"Exception when calling TagsApi->create_tag: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + + + def select_iovs(self, tag_name: str) -> Union[ IovSetDto, HTTPResponse]: + """ + Find an IOV list by the tag name. + + :param tag_name (str): tag name. + :return: tag (as IovSetDto) or HTTP response if unsuccessful. + """ + + path = self._dir + conf.fs_tag_path + "/" + tag_name + + try: + file = path + conf.fs_iov_file + + result = read_file(file) # str + result = json.loads(result) + + iov_list = IovSetDto( + size = len(result), + format = "IovSetDto", + datatype = "iovs", + resources= [] + ) + + for element in result: + dto = validate_and_convert_types( + element, + {IovDto}, + ['element'], + True, + True, + IovDto + ) + + iov_list.resources.append(dto) + + return iov_list + except ApiException as e: + print(f"Exception when calling TagsApi->create_tag: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) -- GitLab From 54c7e36b6e484dab295add4c3251cfc50713ea9e Mon Sep 17 00:00:00 2001 From: Mikhail Mineev <Mikhail.Mineev@cern.ch> Date: Thu, 11 Apr 2024 17:42:28 +0200 Subject: [PATCH 11/16] select_iovs corrected --- pycrest/api/crest_fs_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py index 0d68794..18a91f8 100644 --- a/pycrest/api/crest_fs_api.py +++ b/pycrest/api/crest_fs_api.py @@ -363,15 +363,15 @@ class CrestApiFs(): - def select_iovs(self, tag_name: str) -> Union[ IovSetDto, HTTPResponse]: + def select_iovs(self, name: str) -> Union[ IovSetDto, HTTPResponse]: """ Find an IOV list by the tag name. - :param tag_name (str): tag name. + :param name (str): tag name. :return: tag (as IovSetDto) or HTTP response if unsuccessful. """ - path = self._dir + conf.fs_tag_path + "/" + tag_name + path = self._dir + conf.fs_tag_path + "/" + name try: file = path + conf.fs_iov_file -- GitLab From 16bad16ab66c197cc08c2e11888fc3a0bd525e97 Mon Sep 17 00:00:00 2001 From: formica <andrea.formica@cern.ch> Date: Mon, 15 Apr 2024 16:19:53 +0200 Subject: [PATCH 12/16] new create tag example --- examples/create_tag.py | 76 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 examples/create_tag.py diff --git a/examples/create_tag.py b/examples/create_tag.py new file mode 100644 index 0000000..f8f3c6c --- /dev/null +++ b/examples/create_tag.py @@ -0,0 +1,76 @@ +from pycrest.api.crest_api import CrestApi +from hep.crest.client.models import ( + IovSetDto, HTTPResponse, TagMetaSetDto, TagMetaDto, TagSetDto, TagDto, GlobalTagDto, + GlobalTagSetDto, GlobalTagMapDto, StoreSetDto, StoreDto, RunLumiInfoDto, RunLumiSetDto +) +# create an instance of the API class +def socks(proxy_host): + SOCKS5_PROXY_HOST = proxy_host + SOCKS5_PROXY_PORT = 3129 + try: + import socket + import socks # you need to install pysocks (use the command: pip install pysocks) + # Configuration + + # Remove this if you don't plan to "deactivate" the proxy later + # default_socket = socket.socket + # Set up a proxy + # if self.useSocks: + socks.set_default_proxy(socks.SOCKS5, SOCKS5_PROXY_HOST, SOCKS5_PROXY_PORT) + socket.socket = socks.socksocket + print('Activated socks proxy on %s:%s' % (SOCKS5_PROXY_HOST, SOCKS5_PROXY_PORT)) + except: + print('Error activating socks...%s %s' % (SOCKS5_PROXY_HOST, SOCKS5_PROXY_PORT)) + +socks('localhost') +api_instance = CrestApi(host='http://crest-undertow-api.web.cern.ch/api-v4.0') + +# tag name +name = 'test_tag' +# tag description +description = 'a brand new test tag' +# tag time type +time_type = 'time' + +params = { + 'payload_spec': 'ascii', + 'description': description, + 'synchronization': 'none', + 'last_validated_time': -1., + 'end_of_validity': -1., +} +tagdto = TagDto( + name=name, + description=params['description'], + time_type=time_type, + payload_spec=params['payload_spec'], + synchronization=params['synchronization'], + last_validated_time=params['last_validated_time'], + end_of_validity=params['end_of_validity'] +) + +print('Create a tag %s' % name) +try: + # activate socks + # Create a new tag + api_response = api_instance.create_tag(tagdto) + print(api_response) +except Exception as e: + print("Exception when calling CrestApi->create_tag: %s\n" % e) + +print('Find a tag %s' % name) +try: + # activate socks + # Create a new tag + api_response = api_instance.find_tag(name=name) + print(api_response) +except Exception as e: + print("Exception when calling CrestApi->create_tag: %s\n" % e) + +# Remove the tag +print('Delete tag %s' % name) +try: + api_response = api_instance.remove_tag(name=name) + print('Done') +except Exception as e: + print("Exception when calling CrestApi->remove_tag: %s\n" % e) -- GitLab From eff52c95b98a834ee1ebcfe8f896bffb6c12dae4 Mon Sep 17 00:00:00 2001 From: mavogel <mavogel@cern.ch> Date: Thu, 18 Apr 2024 13:00:39 +0200 Subject: [PATCH 13/16] fixed incorrect reference to TagMetaDto.tag_name --- pycrest/api/crest_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pycrest/api/crest_api.py b/pycrest/api/crest_api.py index 48b5385..c491099 100644 --- a/pycrest/api/crest_api.py +++ b/pycrest/api/crest_api.py @@ -259,7 +259,7 @@ class CrestApi(): """ api_instance = tags_api.TagsApi(self._api_client) try: - return api_instance.create_tag_meta(name=dto.name, tag_meta_dto=dto) + return api_instance.create_tag_meta(name=dto.tag_name, tag_meta_dto=dto) except ApiException as e: print(f"Exception when calling TagsApi->create_tag_meta: {e}\n") return HTTPResponse( @@ -281,7 +281,7 @@ class CrestApi(): api_instance = tags_api.TagsApi(self._api_client) try: - return api_instance.update_tag_meta(name=dto.name, tag_meta_dto=dto) + return api_instance.update_tag_meta(name=dto.tag_name, tag_meta_dto=dto) except ApiException as e: print(f"Exception when calling TagsApi->update_tag_meta: {e}\n") return HTTPResponse( @@ -691,4 +691,4 @@ class CrestApi(): return api_response except ApiException as e: print("Exception when calling RuninfoApi->list_run_info: %s\n" % e) - return HTTPResponse(status_code=e.status, reason=e.reason, code=e.status, message=e.body) \ No newline at end of file + return HTTPResponse(status_code=e.status, reason=e.reason, code=e.status, message=e.body) -- GitLab From 171be65c0ceacc6cf083fcda95b90f86511f2c2e Mon Sep 17 00:00:00 2001 From: Mikhail Mineev <Mikhail.Mineev@cern.ch> Date: Tue, 23 Apr 2024 19:17:12 +0200 Subject: [PATCH 14/16] added methods for global tags and global tag maps for file storage --- pycrest/api/crest_fs_api.py | 352 ++++++++++++++++++++++++++++++++++-- 1 file changed, 340 insertions(+), 12 deletions(-) diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py index 18a91f8..f729a5d 100644 --- a/pycrest/api/crest_fs_api.py +++ b/pycrest/api/crest_fs_api.py @@ -8,8 +8,8 @@ from hep.crest.client import ApiException from hep.crest.client.api import iovs_api, admin_api, globaltags_api, globaltagmaps_api, payloads_api, tags_api, runinfo_api from hep.crest.client.models import ( IovSetDto, HTTPResponse, TagMetaSetDto, TagMetaDto, TagSetDto, TagDto, GlobalTagDto, - GlobalTagSetDto, GlobalTagMapDto, StoreSetDto, StoreDto, RunLumiInfoDto, RunLumiSetDto, - PayloadDto, IovDto + GlobalTagSetDto, GlobalTagMapDto, GlobalTagMapSetDto, StoreSetDto, StoreDto, + RunLumiInfoDto, RunLumiSetDto, PayloadDto, IovDto ) @@ -59,7 +59,7 @@ class CrestApiFs(): write_file(file,str1) return tag except ApiException as e: - print(f"Exception when calling TagsApi->create_tag: {e}\n") + print(f"Exception in method CrestApiFs->create_tag: {e}\n") return HTTPResponse( status_code=e.status, reason=e.reason, @@ -88,7 +88,7 @@ class CrestApiFs(): write_file(file,str1) return tag_meta except ApiException as e: - print(f"Exception when calling TagsApi->create_tag_meta: {e}\n") + print(f"Exception in method CrestApiFs->create_tag_meta: {e}\n") return HTTPResponse( status_code=e.status, reason=e.reason, @@ -188,7 +188,7 @@ class CrestApiFs(): write_file(iov_file,iovs) except ApiException as e: - print("Exception when calling PayloadsApi->store_payload_batch: %s\n" % e) + print("Exception in method CrestApiFs->store_payload: %s\n" % e) @@ -226,6 +226,7 @@ class CrestApiFs(): except Exception as e: print('Error' % e) + def find_tag(self, tag_name: str) -> Union[TagDto, HTTPResponse]: """ This method returns a tag for the tag name. @@ -255,7 +256,7 @@ class CrestApiFs(): return dto except ApiException as e: - print(f"Exception when calling TagsApi->create_tag: {e}\n") + print(f"Exception in method CrestApiFs->find_tag: {e}\n") return HTTPResponse( status_code=e.status, reason=e.reason, @@ -292,7 +293,7 @@ class CrestApiFs(): return dto except ApiException as e: - print(f"Exception when calling TagsApi->create_tag: {e}\n") + print(f"Exception in method CrestApiFs->find_tag_meta: {e}\n") return HTTPResponse( status_code=e.status, reason=e.reason, @@ -317,7 +318,7 @@ class CrestApiFs(): return result except ApiException as e: - print(f"Exception when calling TagsApi->create_tag: {e}\n") + print(f"Exception in method CrestApiFs->get_payload: {e}\n") return HTTPResponse( status_code=e.status, reason=e.reason, @@ -353,7 +354,7 @@ class CrestApiFs(): return dto except ApiException as e: - print(f"Exception when calling TagsApi->create_tag: {e}\n") + print(f"Exception in method CrestApiFs->get_payload_meta: {e}\n") return HTTPResponse( status_code=e.status, reason=e.reason, @@ -362,12 +363,129 @@ class CrestApiFs(): ) + + def sort_iov_json(self, data: json, order: bool): + """ + The auxiliary method to sort the JSON array with the IOVs by the since parameters. + + :param json : JSON array with the IOVs (IOV list). + :param order : sorting order ( True - ascending order, False - descending) + :return: sorted JSON array with the IOV list. + """ + + res = json.loads("[]") + unsorted = {} + + for element in data: + # print(element) + if element.get('since') is not None: + # print('since exists') + since = element['since'] + # print(since) + unsorted[since] = element + else: + raise Exception("JSON has no since parameter") + if order: + sorted_dict = dict(sorted(unsorted.items(),reverse = False)) + else: + sorted_dict = dict(sorted(unsorted.items(),reverse = True)) + # print(sorted_dict) + + for key in sorted_dict: + # print(key, '->', sorted_dict[key]) + res.append(sorted_dict[key]) + + # print(res) + return res + + - def select_iovs(self, name: str) -> Union[ IovSetDto, HTTPResponse]: + def get_page(self, data: json, size: int, page: int): + """ + The auxiliary method to extract a subarray from JSON array. + + :param json : JSON array to extract subarray. + :param size: Number of item per page + :param page: Page number + :return: JSON subarray. + """ + + res = json.loads("[]") + data_size = len(data) + # print (data_size) + + if data_size == 0: + # no elements to return + return res + + # index interval to load the data from JSON array: + kmin = size * page + kmax = size * (page + 1) + + # check if the interval is correct: + if kmin > (data_size - 1): # out of range + return res + + if kmax > (data_size - 1): + kmax = data_size + + for i in range (kmin , kmax): + # print (i) + res.append(data[i]) + + return res + + + + def extract_interval(self, data: json, since: int, until: int): + """ + The auxiliary method to extract an IOV list in the time interval since-until. + + :param json : JSON array with the IOV list. + :param since: Start time + :param until: End time + :return: IOV list (JSON array). + """ + res = json.loads("[]") + # print(since) + # print(until) + + for element in data: + # print(element) + if element.get('since') is not None: + # print('since exists') + time = element['since'] + # print(time) + + if until == -1: # infinity + if time >= since: + # print("b") + res.append(element) + else: + if (time >= since) and (time <= until): + # print("a") + res.append(element) + else: + raise Exception("JSON has no since parameter") + + return res + + + + # def select_iovs(self, name: str) -> Union[ IovSetDto, HTTPResponse]: # OLD + def select_iovs(self, name: str, + since: int = 0, until: int = 0, + size: int = 1000, page: int = 0, + sort: str = 'id.since:ASC') -> Union[IovSetDto, HTTPResponse]: """ Find an IOV list by the tag name. - :param name (str): tag name. + :param name (str): Tag name. + :param since: Start time + :param until: End time + :param size: Number of results per page + :param page: Page number + :param sort: Sorting order ('id.since:ASC' or 'id.since:DESC') :return: tag (as IovSetDto) or HTTP response if unsuccessful. """ @@ -379,6 +497,18 @@ class CrestApiFs(): result = read_file(file) # str result = json.loads(result) + # IOV list ordering: + if sort == 'id.since:ASC': + result = self.sort_iov_json(result, True) + else: + result = self.sort_iov_json(result, False) + + # IOV list time interval: + result = self.extract_interval(result, since=since, until=until) + + # page extracting: + result = self.get_page(result, size=size, page=page) + iov_list = IovSetDto( size = len(result), format = "IovSetDto", @@ -400,7 +530,205 @@ class CrestApiFs(): return iov_list except ApiException as e: - print(f"Exception when calling TagsApi->create_tag: {e}\n") + print(f"Exception in method CrestApiFs->select_iovs: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + + + def create_global_tag(self, dto: GlobalTagDto) -> Union[GlobalTagDto, HTTPResponse]: + """ + Create the global tag. + + :param dto (GlobalTagDto): global tag. + :return: Created global tag (GlobalTagDto) or HTTP response if unsuccessful. + """ + + name = dto.name + path = self._dir + conf.fs_globaltag_path + "/" + name + + try: + check_directory(path); + file = path + conf.fs_globaltag_file + gtag = model_to_dict(dto) + str1 = json.dumps(gtag) + write_file(file,str1) + return dto + except ApiException as e: + print(f"Exception in method CrestApiFs->create_global_tag: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + + def find_global_tag(self, name: str) -> Union[GlobalTagDto, HTTPResponse]: + """ + This metod returns the global tag for the given global tag name. + + :param name : global tag name. + :return: global tag (GlobalTagDto) or HTTP response if unsuccessful. + """ + path = self._dir + conf.fs_globaltag_path + "/" + name + + try: + + file = path + conf.fs_globaltag_file + result = read_file(file) # str + result = json.loads(result) + + dto = validate_and_convert_types( + result, + {dict, GlobalTagDto}, + ['result'], + True, + True, + GlobalTagDto + ) + + return dto + except ApiException as e: + print(f"Exception in method CrestApiFs->find_global_tag: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + + def check_global_tag_map(self, dto: GlobalTagMapDto): + """ + The auxiliary method to check if the global tag and tag exist. + + :param dto (GlobalTagMapDto): global tag map. + """ + + if dto.get('globalTagName') is not None: + global_tag = dto['globalTagName'] + global_tag_path = self._dir + conf.fs_globaltag_path + "/" + global_tag + conf.fs_globaltag_file + # print(global_tag_path) + if not os.path.isfile(global_tag_path): + raise FileNotFoundError(f"Global tag '{global_tag}' does not exist.") + else : + raise Exception("Error: globalTagName not found in JSON.") + + + if dto.get('tagName') is not None: + tag = dto['tagName'] + tag_path = self._dir + conf.fs_tag_path + "/" + tag + conf.fs_tag_file + # print(tag_path) + if not os.path.isfile(tag_path): + raise FileNotFoundError(f"Tag '{tag}' does not exist.") + else : + raise Exception("Error: tagName not found in JSON.") + + + + def create_global_tag_map(self, dto: GlobalTagMapDto) -> Union[GlobalTagMapDto, HTTPResponse]: + """ + This method creates a new global tag map on the file storage. + + :param dto (GlobalTagMapDto): global tag. + :return: Created global tag map or HTTP response if unsuccessful. + """ + inserted = False + + try: + self.check_global_tag_map(dto=dto) + global_tag = dto['globalTagName'] + global_tagmap_path = self._dir + conf.fs_globaltag_path + "/" + global_tag + conf.fs_map_file + # print(global_tagmap_path) + if os.path.isfile(global_tagmap_path): + # print("exists") + tag = dto['tagName'] + tag_map = read_file(global_tagmap_path) + tagmap = json.loads(tag_map) + for element in tagmap: + current_tag = element['tagName'] + if current_tag == tag: + tagmap.remove(element) + map = model_to_dict(dto) + # print(map) + tagmap.append(map) + map_string = json.dumps(tagmap) + write_file(global_tagmap_path,map_string) + inserted = True + if inserted == False: + map = model_to_dict(dto) + # print(map) + tagmap.append(map) + map_string = json.dumps(tagmap) + write_file(global_tagmap_path,map_string) + + else: + # print("does not exist") + tagmap = json.loads("[]") + map = model_to_dict(dto) + # print(map) + tagmap.append(map) + map_string = json.dumps(tagmap) + write_file(global_tagmap_path,map_string) + + return dto + + except ApiException as e: + print(f"Exception in method CrestApiFs->create_global_tag_map: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + + + def find_global_tag_map(self, name: str) -> Union[GlobalTagMapSetDto, HTTPResponse]: + """ + This method returns a tag for the tag name. + + :param ame (str): global tag name. + :return: tag (as TagDto) or HTTP response if unsuccessful. + """ + + file = self._dir + conf.fs_globaltag_path + "/" + name + conf.fs_map_file + # print(file) + + try: + + result = read_file(file) # str + # print(result) + result = json.loads(result) + + dto = GlobalTagMapSetDto( + size = len(result), + format = 'GlobalTagMapSetDto', + datatype = 'maps', + resources= [] + ) + + for element in result: + map = validate_and_convert_types( + element, + {GlobalTagMapDto}, + ['element'], + True, + True, + GlobalTagMapDto + ) + + dto.resources.append(map) + + return dto + + except ApiException as e: + print(f"Exception in method CrestApiFs->find_global_tag_map: {e}\n") return HTTPResponse( status_code=e.status, reason=e.reason, -- GitLab From 7a25e7dbe5389fda792bdb08270cfb3d88af34c7 Mon Sep 17 00:00:00 2001 From: Mikhail Mineev <Mikhail.Mineev@cern.ch> Date: Thu, 25 Apr 2024 15:51:07 +0200 Subject: [PATCH 15/16] error in select_iovs corrected --- pycrest/api/crest_fs_api.py | 39 ++++++------------------------------- 1 file changed, 6 insertions(+), 33 deletions(-) diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py index f729a5d..c305fd9 100644 --- a/pycrest/api/crest_fs_api.py +++ b/pycrest/api/crest_fs_api.py @@ -239,10 +239,8 @@ class CrestApiFs(): try: file = path + conf.fs_tag_file - print(file) result = read_file(file) # str - print(result) result = json.loads(result) dto = validate_and_convert_types( @@ -276,10 +274,8 @@ class CrestApiFs(): try: file = path + conf.fs_tagmetainfo_file - print(file) result = read_file(file) # str - print(result) result = json.loads(result) dto = validate_and_convert_types( @@ -377,11 +373,8 @@ class CrestApiFs(): unsorted = {} for element in data: - # print(element) if element.get('since') is not None: - # print('since exists') since = element['since'] - # print(since) unsorted[since] = element else: raise Exception("JSON has no since parameter") @@ -389,13 +382,10 @@ class CrestApiFs(): sorted_dict = dict(sorted(unsorted.items(),reverse = False)) else: sorted_dict = dict(sorted(unsorted.items(),reverse = True)) - # print(sorted_dict) for key in sorted_dict: - # print(key, '->', sorted_dict[key]) res.append(sorted_dict[key]) - # print(res) return res @@ -412,7 +402,6 @@ class CrestApiFs(): res = json.loads("[]") data_size = len(data) - # print (data_size) if data_size == 0: # no elements to return @@ -430,7 +419,6 @@ class CrestApiFs(): kmax = data_size for i in range (kmin , kmax): - # print (i) res.append(data[i]) return res @@ -443,27 +431,20 @@ class CrestApiFs(): :param json : JSON array with the IOV list. :param since: Start time - :param until: End time + :param until: End time, (value -1 means "infinity") :return: IOV list (JSON array). """ res = json.loads("[]") - # print(since) - # print(until) for element in data: - # print(element) if element.get('since') is not None: - # print('since exists') time = element['since'] - # print(time) if until == -1: # infinity if time >= since: - # print("b") res.append(element) else: if (time >= since) and (time <= until): - # print("a") res.append(element) else: raise Exception("JSON has no since parameter") @@ -472,9 +453,8 @@ class CrestApiFs(): - # def select_iovs(self, name: str) -> Union[ IovSetDto, HTTPResponse]: # OLD def select_iovs(self, name: str, - since: int = 0, until: int = 0, + since: int = 0, until: int = -1, size: int = 1000, page: int = 0, sort: str = 'id.since:ASC') -> Union[IovSetDto, HTTPResponse]: """ @@ -502,7 +482,7 @@ class CrestApiFs(): result = self.sort_iov_json(result, True) else: result = self.sort_iov_json(result, False) - + # IOV list time interval: result = self.extract_interval(result, since=since, until=until) @@ -613,7 +593,7 @@ class CrestApiFs(): if dto.get('globalTagName') is not None: global_tag = dto['globalTagName'] global_tag_path = self._dir + conf.fs_globaltag_path + "/" + global_tag + conf.fs_globaltag_file - # print(global_tag_path) + if not os.path.isfile(global_tag_path): raise FileNotFoundError(f"Global tag '{global_tag}' does not exist.") else : @@ -623,7 +603,7 @@ class CrestApiFs(): if dto.get('tagName') is not None: tag = dto['tagName'] tag_path = self._dir + conf.fs_tag_path + "/" + tag + conf.fs_tag_file - # print(tag_path) + if not os.path.isfile(tag_path): raise FileNotFoundError(f"Tag '{tag}' does not exist.") else : @@ -644,9 +624,8 @@ class CrestApiFs(): self.check_global_tag_map(dto=dto) global_tag = dto['globalTagName'] global_tagmap_path = self._dir + conf.fs_globaltag_path + "/" + global_tag + conf.fs_map_file - # print(global_tagmap_path) + if os.path.isfile(global_tagmap_path): - # print("exists") tag = dto['tagName'] tag_map = read_file(global_tagmap_path) tagmap = json.loads(tag_map) @@ -655,23 +634,19 @@ class CrestApiFs(): if current_tag == tag: tagmap.remove(element) map = model_to_dict(dto) - # print(map) tagmap.append(map) map_string = json.dumps(tagmap) write_file(global_tagmap_path,map_string) inserted = True if inserted == False: map = model_to_dict(dto) - # print(map) tagmap.append(map) map_string = json.dumps(tagmap) write_file(global_tagmap_path,map_string) else: - # print("does not exist") tagmap = json.loads("[]") map = model_to_dict(dto) - # print(map) tagmap.append(map) map_string = json.dumps(tagmap) write_file(global_tagmap_path,map_string) @@ -698,12 +673,10 @@ class CrestApiFs(): """ file = self._dir + conf.fs_globaltag_path + "/" + name + conf.fs_map_file - # print(file) try: result = read_file(file) # str - # print(result) result = json.loads(result) dto = GlobalTagMapSetDto( -- GitLab From 3bd6e120146fe37c8b8f609b5745781caec04b75 Mon Sep 17 00:00:00 2001 From: Mikhail Mineev <Mikhail.Mineev@cern.ch> Date: Mon, 6 May 2024 16:30:39 +0200 Subject: [PATCH 16/16] correction in command get iovList --- pycrest/cli/commands/iovs_commands.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pycrest/cli/commands/iovs_commands.py b/pycrest/cli/commands/iovs_commands.py index f77e199..e3ea100 100644 --- a/pycrest/cli/commands/iovs_commands.py +++ b/pycrest/cli/commands/iovs_commands.py @@ -6,6 +6,8 @@ def get_iov_list_func(args): # print("GET IOV LIST function") # print(f"command = {args.command}") # print(f"name = {args.name}") + # print(f"since = {args.since}") + # print(f"until = {args.until}") # print(f"host = {crest_host}") # tag name is mandatory name = args.name @@ -15,12 +17,15 @@ def get_iov_list_func(args): params = {} for arg in args.__dict__.keys(): if arg in optional_args: - params[arg] = arg_dict[arg] + if ((arg=='until') and (args.until=='-1')): + params[arg] = 'INF' + elif (arg_dict[arg]!=None): + params[arg] =arg_dict[arg] try: crest_api = CrestApi(host=args.crest_host) resp = crest_api.find_all_iovs(tagname=name, **params) - # print_json(resp) # OLD MvG + print_multiple_res(resp) except Exception as e: print("Error: "+ repr(e)) -- GitLab