diff --git a/examples/create_tag.py b/examples/create_tag.py new file mode 100644 index 0000000000000000000000000000000000000000..f8f3c6ccd02454c09c35708cf333483b65214033 --- /dev/null +++ b/examples/create_tag.py @@ -0,0 +1,76 @@ +from pycrest.api.crest_api import CrestApi +from hep.crest.client.models import ( + IovSetDto, HTTPResponse, TagMetaSetDto, TagMetaDto, TagSetDto, TagDto, GlobalTagDto, + GlobalTagSetDto, GlobalTagMapDto, StoreSetDto, StoreDto, RunLumiInfoDto, RunLumiSetDto +) +# create an instance of the API class +def socks(proxy_host): + SOCKS5_PROXY_HOST = proxy_host + SOCKS5_PROXY_PORT = 3129 + try: + import socket + import socks # you need to install pysocks (use the command: pip install pysocks) + # Configuration + + # Remove this if you don't plan to "deactivate" the proxy later + # default_socket = socket.socket + # Set up a proxy + # if self.useSocks: + socks.set_default_proxy(socks.SOCKS5, SOCKS5_PROXY_HOST, SOCKS5_PROXY_PORT) + socket.socket = socks.socksocket + print('Activated socks proxy on %s:%s' % (SOCKS5_PROXY_HOST, SOCKS5_PROXY_PORT)) + except: + print('Error activating socks...%s %s' % (SOCKS5_PROXY_HOST, SOCKS5_PROXY_PORT)) + +socks('localhost') +api_instance = CrestApi(host='http://crest-undertow-api.web.cern.ch/api-v4.0') + +# tag name +name = 'test_tag' +# tag description +description = 'a brand new test tag' +# tag time type +time_type = 'time' + +params = { + 'payload_spec': 'ascii', + 'description': description, + 'synchronization': 'none', + 'last_validated_time': -1., + 'end_of_validity': -1., +} +tagdto = TagDto( + name=name, + description=params['description'], + time_type=time_type, + payload_spec=params['payload_spec'], + synchronization=params['synchronization'], + last_validated_time=params['last_validated_time'], + end_of_validity=params['end_of_validity'] +) + +print('Create a tag %s' % name) +try: + # activate socks + # Create a new tag + api_response = api_instance.create_tag(tagdto) + print(api_response) +except Exception as e: + print("Exception when calling CrestApi->create_tag: %s\n" % e) + +print('Find a tag %s' % name) +try: + # activate socks + # Create a new tag + api_response = api_instance.find_tag(name=name) + print(api_response) +except Exception as e: + print("Exception when calling CrestApi->create_tag: %s\n" % e) + +# Remove the tag +print('Delete tag %s' % name) +try: + api_response = api_instance.remove_tag(name=name) + print('Done') +except Exception as e: + print("Exception when calling CrestApi->remove_tag: %s\n" % e) diff --git a/pycrest/api/crest_api.py b/pycrest/api/crest_api.py index 6aedb302ddd9a21a2a6c3c5e1bdefaff9b93f0ed..c491099c5dafaafd5956d73fc2d38b9904f3d071 100644 --- a/pycrest/api/crest_api.py +++ b/pycrest/api/crest_api.py @@ -37,9 +37,17 @@ class CrestApi(): # Verify your data return kwargs - def select_groups(self, tagname: str, snapshot: Optional[float] = None, size: int = 1000, page: int = 0, sort: str = 'id.since:ASC') -> Union[IovSetDto, HTTPResponse]: + def select_groups(self, name: str, snapshot: Optional[float] = None, size: int = 1000, page: int = 0, sort: str = 'id.since:ASC') -> Union[IovSetDto, HTTPResponse]: """ List iov groups for a given tagname + :param name: Name of the tag + :param snapshot: Snapshot time + :param size: Number of results per page + :param page: Page number + :param sort: Sorting order + :return: IovSetDto if successful, else HTTPResponse + + This method is the equivalent of CrestApi.selectGroups() in the C++ code. """ criteria = { 'snapshot': snapshot, @@ -49,12 +57,12 @@ class CrestApi(): } api_instance = iovs_api.IovsApi(self._api_client) try: - api_response = api_instance.find_all_iovs(tagname=tagname, method="GROUPS", **criteria) + api_response = api_instance.find_all_iovs(tagname=name, method="GROUPS", **criteria) return api_response except ApiException as e: print("Exception when calling IovsApi->find_all_iovs: %s\n" % e) - def select_iovs(self, tagname: str, since: int = 0, until: int = 0, snapshot: Optional[float] = None, + def select_iovs(self, name: str, since: int = 0, until: int = 0, snapshot: Optional[float] = None, timeformat: str = 'NUMBER', size: int = 1000, page: int = 0, sort: str = 'id.since:ASC') -> Union[IovSetDto, HTTPResponse]: """ @@ -68,6 +76,9 @@ class CrestApi(): :param page: Page number :param sort: Sorting order :return: IovSetDto if successful, else HTTPResponse + + This method is the equivalent of CrestApi.selectIovs() in the C++ code. + The field timeformat is not used in the C++ code, as it defaults to 'NUMBER'. """ criteria = { 'snapshot': snapshot, @@ -80,7 +91,7 @@ class CrestApi(): } api_instance = iovs_api.IovsApi(self._api_client) try: - api_response = api_instance.find_all_iovs(tagname=tagname, method="IOVS", **criteria) + api_response = api_instance.find_all_iovs(tagname=name, method="IOVS", **criteria) return api_response except ApiException as e: print("Exception when calling IovsApi->find_all_iovs: %s\n" % e) @@ -100,6 +111,8 @@ class CrestApi(): - page: number for page. - size: number for size of the page. :return: IovSetDto if successful, else HTTPResponse. + + This method is not exposed by CrestApi in the C++ code. """ criteria = CrestApi.build_params(kwargs) params = { @@ -117,24 +130,29 @@ class CrestApi(): print("Exception when calling IovsApi->find_all_iovs: %s\n" % e) - def list_tags(self, **kwargs: Dict[str, Any]) -> Union[TagSetDto, HTTPResponse]: + def list_tags(self, name: str, size: int = 1000, page: int = 0, sort: str = 'name:ASC', **kwargs: Dict[str, Any]) -> Union[TagSetDto, HTTPResponse]: """ List all tags. - + :param name: Name of the tag. + :param size: Number of results per page. + :param page: Page number. + :param sort: Sorting order. :param kwargs: Additional criteria. - - name: string for tag name. - description: string for description. - timeType: selection for timeType [time, run, run-lumi]. - objectType: selection for objectType (payloadSpec). - - page: number for page. - - size: number for size of the page. :return: TagSetDto if successful, else HTTPResponse. + + This method is the equivalent of CrestApi.listTags() in the C++ code. """ criteria = CrestApi.build_params(kwargs) params = { - 'page': 0, - 'size': 1000 + 'page': page, + 'size': size, + 'sort': sort } + if name != 'none': + params['name'] = name params.update(criteria) api_instance = tags_api.TagsApi(self._api_client) try: @@ -149,6 +167,8 @@ class CrestApi(): :param name: The name of the global tag to find. :return: GlobalTagSetDto if successful, else HTTPResponse. + + This method is the equivalent of CrestApi.findGlobalTag() in the C++ code. """ api_instance = globaltags_api.GlobaltagsApi(self._api_client) try: @@ -157,24 +177,30 @@ class CrestApi(): except ApiException as e: print("Exception when calling GlobaltagsApi->find_global_tag: %s\n" % e) - def list_global_tags(self, **kwargs) -> Union[GlobalTagSetDto, HTTPResponse]: + def list_global_tags(self, name: str, size: int = 1000, page: int = 0, sort: str = 'name:ASC', **kwargs: Dict[str, Any]) -> Union[GlobalTagSetDto, HTTPResponse]: """ List all global tags. + :param name: Name of the global tag. + :param size: Number of results per page. + :param page: Page number. + :param sort: Sorting order. :param kwargs: Additional criteria for filtering the global tags. - - name: string for global tag name - description: string for description - release: string for release - scenario: string for scenario - - page: number for page - - size: number for size of the page :return: GlobalTagSetDto if successful, else HTTPResponse. + + This method is the equivalent of CrestApi.listGlobalTags() in the C++ code. """ criteria = CrestApi.build_params(kwargs) params = { - 'page': 0, - 'size': 1000 + 'page': page, + 'size': size, + 'sort': sort } + if name != 'none': + params['name'] = name params.update(criteria) api_instance = globaltags_api.GlobaltagsApi(self._api_client) try: @@ -190,6 +216,8 @@ class CrestApi(): :param name: Name of the global tag or tag. :param mode: Map mode [Trace, BackTrace]. :return: Response from the API. + + This method is the equivalent of CrestApi.findGlobalTagMap() in the C++ code. """ headers = {'x_crest_map_mode': mode} api_instance = globaltagmaps_api.GlobaltagmapsApi(self._api_client) @@ -198,36 +226,19 @@ class CrestApi(): except ApiException as e: print(f"Exception when calling GlobaltagmapsApi->find_global_tag_map: {e}\n") - def create_tag(self, name: str, timeType: str, **kwargs) -> Union[TagDto, HTTPResponse]: + def create_tag(self, dto: TagDto) -> Union[TagDto, HTTPResponse]: """ Create a new tag. - :param name: Name of the tag. - :param timeType: Time type of the tag. - :param kwargs: Additional parameters for tag creation. + :param dto: The TagDto. :return: Created tag or HTTP response if unsuccessful. + + This method is the equivalent of CrestApi.createTag() in the C++ code. """ - criteria = CrestApi.build_params(kwargs) - params = { - 'description': '', - 'synchronization': 'none', - 'payload_spec': 'JSON', - 'last_validated_time': -1., - 'end_of_validity': -1., - } - params.update(criteria) - tag = TagDto( - name=name, - description=params['description'], - time_type=timeType, - payload_spec=params['payload_spec'], - synchronization=params['synchronization'], - last_validated_time=params['last_validated_time'], - end_of_validity=params['end_of_validity'] - ) + api_instance = tags_api.TagsApi(self._api_client) try: - return api_instance.create_tag(tag_dto=tag) + return api_instance.create_tag(tag_dto=dto) except ApiException as e: print(f"Exception when calling TagsApi->create_tag: {e}\n") return HTTPResponse( @@ -237,34 +248,18 @@ class CrestApi(): message=e.body ) - def create_tag_meta(self, name: str, **kwargs) -> Union[TagMetaDto, HTTPResponse]: + def create_tag_meta(self, dto: TagMetaDto) -> Union[TagMetaDto, HTTPResponse]: """ Create the meta information for a tag. - :param name: Name of the tag. - :param kwargs: Additional parameters for tag meta creation. + :param dto: The TagMetaDto. :return: Created tag meta or HTTP response if unsuccessful. - """ - criteria = CrestApi.build_params(kwargs) - params = { - 'description': '', - 'chansize': 1, - 'colsize': 1, - 'tag_info': '', - } - params.update(criteria) - - tag_meta_dto = TagMetaDto( - tag_name=name, - description=params['description'], - chansize=params['chansize'], - colsize=params['colsize'], - tag_info=params['tag_info'], - ) + This method is the equivalent of CrestApi.createTagMeta() in the C++ code. + """ api_instance = tags_api.TagsApi(self._api_client) try: - return api_instance.create_tag_meta(name, tag_meta_dto=tag_meta_dto) + return api_instance.create_tag_meta(name=dto.tag_name, tag_meta_dto=dto) except ApiException as e: print(f"Exception when calling TagsApi->create_tag_meta: {e}\n") return HTTPResponse( @@ -274,34 +269,19 @@ class CrestApi(): message=e.body ) - def update_tag_meta(self, name: str, **kwargs) -> Union[TagMetaDto, HTTPResponse]: + def update_tag_meta(self, dto: TagMetaDto) -> Union[TagMetaDto, HTTPResponse]: """ Update the meta information for a tag. - :param name: Name of the tag. - :param kwargs: Additional parameters for tag meta update. + :param dto: The TagMetaDto for update. :return: Updated tag meta or HTTP response if unsuccessful. - """ - criteria = CrestApi.build_params(kwargs) - params = { - 'description': '', - 'chansize': 1, - 'colsize': 1, - 'tag_info': '', - } - params.update(criteria) - tag_meta_dto = TagMetaDto( - tag_name=name, - description=params['description'], - chansize=params['chansize'], - colsize=params['colsize'], - tag_info=params['tag_info'], - ) + This method is the equivalent of CrestApi.updateTagMeta() in the C++ code. + """ api_instance = tags_api.TagsApi(self._api_client) try: - return api_instance.update_tag_meta(name, tag_meta_dto=tag_meta_dto) + return api_instance.update_tag_meta(name=dto.tag_name, tag_meta_dto=dto) except ApiException as e: print(f"Exception when calling TagsApi->update_tag_meta: {e}\n") return HTTPResponse( @@ -318,6 +298,8 @@ class CrestApi(): :param name: Name of the tag to delete. :return: None if successful, or HTTP response if unsuccessful. + + This method is the equivalent of CrestApi.removeTag() in the C++ code. """ api_instance = admin_api.AdminApi(self._api_client) try: @@ -331,38 +313,18 @@ class CrestApi(): message=e.body ) - def create_global_tag(self, name: str, force: str = "false", **kwargs) -> Union[GlobalTagDto, HTTPResponse]: + def create_global_tag(self, dto: GlobalTagDto, force: str) -> Union[GlobalTagDto, HTTPResponse]: """ Create a new global tag. - :param name: Name of the global tag. - :param force: Whether to force creation if the global tag already exists. - :param kwargs: Additional parameters for global tag creation. + :param dto: The GlobalTagDto. :return: Created global tag or HTTP response if unsuccessful. - """ - criteria = CrestApi.build_params(kwargs) - params = { - 'description': 'none', - 'validity': 0.0, - 'release': 'none', - 'scenario': 'none', - 'workflow': 'all', - 'type': 'T', - } - params.update(criteria) - globaltag = GlobalTagDto( - name=name, - description=params['description'], - release=params['release'], - scenario=params['scenario'], - validity=params['validity'], - workflow=params['workflow'], - type=params['type'] - ) - + + This method is the equivalent of CrestApi.createTagMeta() in the C++ code. + """ api_instance = globaltags_api.GlobaltagsApi(self._api_client) try: - return api_instance.create_global_tag(force=force, global_tag_dto=globaltag) + return api_instance.create_global_tag(force=force, global_tag_dto=dto) except ApiException as e: print(f"Exception when calling GlobaltagsApi->create_global_tag: {e}\n") return HTTPResponse( @@ -378,6 +340,8 @@ class CrestApi(): :param name: Name of the global tag to delete. :return: None if successful, or HTTP response if unsuccessful. + + This method is the equivalent of CrestApi.removeGlobalTag() in the C++ code. """ api_instance = admin_api.AdminApi(self._api_client) try: @@ -391,7 +355,7 @@ class CrestApi(): message=e.body ) - def create_global_tag_map(self, globaltagname: str, tagname: str, **kwargs) -> Union[GlobalTagMapDto, HTTPResponse]: + def create_global_tag_map(self, dto: GlobalTagMapDto) -> Union[GlobalTagMapDto, HTTPResponse]: """ Create a new global tag map. @@ -399,23 +363,13 @@ class CrestApi(): :param tagname: Name of the tag. :param kwargs: Additional parameters for global tag map creation. :return: Created global tag map or HTTP response if unsuccessful. + + This method is the equivalent of CrestApi.createGlobalTagMap() in the C++ code. """ - criteria = CrestApi.build_params(kwargs) - params = { - 'record': 'none', - 'label': 'none', - } - params.update(criteria) - globaltagmap = GlobalTagMapDto( - global_tag_name=globaltagname, - tag_name=tagname, - record=params['record'], - label=params['label'] - ) - + api_instance = globaltagmaps_api.GlobaltagmapsApi(self._api_client) try: - return api_instance.create_global_tag_map(global_tag_map_dto=globaltagmap) + return api_instance.create_global_tag_map(global_tag_map_dto=dto) except ApiException as e: print(f"Exception when calling GlobaltagmapsApi->create_global_tag_map: {e}\n") return HTTPResponse( @@ -464,7 +418,7 @@ class CrestApi(): message=e.body ) - def get_tag(self, name: str = 'none') -> Union[TagDto, HTTPResponse]: + def find_tag(self, name: str = 'none') -> Union[TagDto, HTTPResponse]: """ Get tag. @@ -473,13 +427,13 @@ class CrestApi(): """ api_instance = tags_api.TagsApi(self._api_client) try: - api_response = api_instance.list_tags(name=name) + api_response = api_instance.find_tag(name=name) if api_response['size'] == 1: return api_response['resources'][0] else: return None except ApiException as e: - print("Exception when calling TagsApi->list_tags in get_tag: %s\n" % e) + print("Exception when calling TagsApi->find_tag: %s\n" % e) return HTTPResponse( status_code=e.status, reason=e.reason, @@ -487,7 +441,7 @@ class CrestApi(): message=e.body ) - def get_global_tag(self, name: str = 'none') -> Union[GlobalTagDto, HTTPResponse]: + def find_global_tag(self, name: str = 'none') -> Union[GlobalTagDto, HTTPResponse]: """ Get global tag. @@ -737,4 +691,4 @@ class CrestApi(): return api_response except ApiException as e: print("Exception when calling RuninfoApi->list_run_info: %s\n" % e) - return HTTPResponse(status_code=e.status, reason=e.reason, code=e.status, message=e.body) \ No newline at end of file + return HTTPResponse(status_code=e.status, reason=e.reason, code=e.status, message=e.body) diff --git a/pycrest/api/crest_fs/__init__.py b/pycrest/api/crest_fs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/pycrest/api/crest_fs/crest_fs_config.py b/pycrest/api/crest_fs/crest_fs_config.py new file mode 100644 index 0000000000000000000000000000000000000000..e5159281b719d5991410762da4a147257d0b65a8 --- /dev/null +++ b/pycrest/api/crest_fs/crest_fs_config.py @@ -0,0 +1,17 @@ +fs_tag_path = "/tags" +fs_globaltag_path = "/globaltags" +fs_data_path = "/data" + +fs_tag_file = "/tag.json" +fs_iov_file = "/iovs.json" +fs_tagmetainfo_file = "/tagmetainfo.json" + +fs_meta_file = "/meta.json" +fs_payload_file = "/payload.json" + +fs_globaltag_file = "/globaltag.json" +fs_map_file = "/maps.json" + +fs_path = "" + +fs_prefix_length = 3 diff --git a/pycrest/api/crest_fs/crest_fs_utils.py b/pycrest/api/crest_fs/crest_fs_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a3fbf524f78582aec699a52b053bd1db3c4ac9bd --- /dev/null +++ b/pycrest/api/crest_fs/crest_fs_utils.py @@ -0,0 +1,41 @@ +import hashlib +from pathlib import Path + +def read_file(file): + txt = Path(file).read_text() + return txt + +def write_file(file,data): + text_file = open(file, "w") + text_file.write(data) + text_file.close() + +def check_directory(path): + Path(path).mkdir(parents=True, exist_ok=True) + +def get_hash(string): + # hash calculating with SHA-256 for a string + hash_object = hashlib.sha256(string.encode()) + hex_dig = hash_object.hexdigest() + return hex_dig + +def compute_sha256(file_name): + # hash calculating with SHA-256 for a file + hash_sha256 = hashlib.sha256() + with open(file_name, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_sha256.update(chunk) + return hash_sha256.hexdigest() + +def utf8len(s): + return len(s.encode('utf-8')) + +def get_value(json, key): + if key in json: + return json[key] + else: + message = 'key ' + key + ' not found in JSON' + raise Exception(message) + + + diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py new file mode 100644 index 0000000000000000000000000000000000000000..c305fd9ba90a7dbb9cdcad186cc6e8558f8f16e1 --- /dev/null +++ b/pycrest/api/crest_fs_api.py @@ -0,0 +1,710 @@ +from typing import Dict, Any, Union, List, Optional +import json +import os + +from datetime import datetime + +from hep.crest.client import ApiException +from hep.crest.client.api import iovs_api, admin_api, globaltags_api, globaltagmaps_api, payloads_api, tags_api, runinfo_api +from hep.crest.client.models import ( + IovSetDto, HTTPResponse, TagMetaSetDto, TagMetaDto, TagSetDto, TagDto, GlobalTagDto, + GlobalTagSetDto, GlobalTagMapDto, GlobalTagMapSetDto, StoreSetDto, StoreDto, + RunLumiInfoDto, RunLumiSetDto, PayloadDto, IovDto +) + + +import pycrest.api.crest_fs.crest_fs_config as conf +from pycrest.api.crest_fs.crest_fs_utils import read_file, write_file, check_directory, get_hash, compute_sha256, utf8len +import shutil +from pycrest.cli.commands.utils import datetime_serializer +from hep.crest.client.model_utils import model_to_dict, validate_and_convert_types + + +import ast + +class CrestApiFs(): + _config = None + _dir = "/tmp/crest_dump" + _api_client = None + + def __init__(self, dir: str): + self._dir = dir + + + def print_dir(self): + print (self._dir) + + @staticmethod + def build_params(kwargs: Dict[str, Any]) -> Dict[str, Any]: + # Verify your data + return kwargs + + + def create_tag(self, tag: TagDto) -> Union[TagDto, HTTPResponse]: + """ + Create a new tag. + + :param tag (TagDto): tag. + :return: Created tag or HTTP response if unsuccessful. + """ + + name = tag.name + path = self._dir + conf.fs_tag_path + "/" + name + + try: + check_directory(path); + file = path + conf.fs_tag_file + tagjs = model_to_dict(tag) + str1 = json.dumps(tagjs) + write_file(file,str1) + return tag + except ApiException as e: + print(f"Exception in method CrestApiFs->create_tag: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + + + def create_tag_meta(self, tag_meta: TagMetaDto) -> Union[TagMetaDto, HTTPResponse]: + """ + Create the meta information for a tag. + + :param tag_meta (TagMetaDto): tag meta info. + :return: Created tag meta or HTTP response if unsuccessful. + """ + + name = tag_meta.tag_name + path = self._dir + conf.fs_tag_path + "/" + name + + try: + check_directory(path); + file = path + conf.fs_tagmetainfo_file + tag_meta = model_to_dict(tag_meta) + str1 = json.dumps(tag_meta) + write_file(file,str1) + return tag_meta + except ApiException as e: + print(f"Exception in method CrestApiFs->create_tag_meta: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + + + def store_payload(self, tag_name: str, since: int, payload: StoreDto, compression_type: str = 'none', version: str = '1', + object_type: str = 'JSON', payload_format: str = 'JSON', streamer_info: str = 'none', **kwargs: Dict[str, Any]) -> Any: + """ + store data: uploads a single payload as a file/string to CREST. The auxiliary method. + + :param tag_name: Name of the tag. + :param since: Since value. + :param payload: payload of file with the payload. + :param compression_type: Compression type (default is 'none'). + :param version: Version of the data (default is '1'). + :param payload_format: Type of the object (default is 'JSON'). + :param kwargs: Additional optional parameters. + :return: API response. + """ + + if (payload_format == 'FILE') : + file_path = payload.replace("file://", "") + if not os.path.isfile(file_path): + raise FileNotFoundError(f"File '{file_path}' not found.") + + + hash = compute_sha256(file_path) + prefix = hash[:conf.fs_prefix_length] + else: + hash = get_hash(payload) + prefix = hash[:conf.fs_prefix_length] + + try: + payload_path = self._dir + conf.fs_data_path + "/" + prefix + "/" + hash + check_directory(payload_path) + payload_file = payload_path + conf.fs_payload_file + + if (payload_format == 'FILE') : + shutil.copyfile(file_path, payload_file) + file_size = os.path.getsize(file_path) + else: + write_file(payload_file,payload) + file_size = utf8len(payload) + + now = datetime.now() + date_string = now.strftime("%Y-%m-%d %H:%M:%S") + + tag_meta = { + "checkSum": "SHA-256", + "compressionType": "none", + "hash": hash, + "insertionTime": now, + "objectType": object_type, + "size": file_size, + "version": version + } + + + jsres = json.dumps(tag_meta, default=datetime_serializer) + + meta_file = payload_path + conf.fs_meta_file + write_file(meta_file,jsres) + + iov = { + "insertionTime": now, + "payloadHash": hash, + "since": since, + "tagName": tag_name + } + + siov = json.dumps(iov, default=datetime_serializer) + iov = json.loads(siov) + + iov_path = self._dir + conf.fs_tag_path + "/" + tag_name + iov_file = iov_path + conf.fs_iov_file + + if os.path.isfile(iov_file): + iovs = read_file(iov_file) + iov_list = json.loads(iovs) + for element in iov_list: + if 'since' in element: + current_since = element['since'] + if since == current_since: + iov_list.remove(element) + + iov_list.append(iov) + iovs = json.dumps(iov_list) + write_file(iov_file,iovs) + else: + iov_list = json.loads("[]") + iov_list.append(iov) + iovs = json.dumps(iov_list) + write_file(iov_file,iovs) + + except ApiException as e: + print("Exception in method CrestApiFs->store_payload: %s\n" % e) + + + + def store_data(self, tag: str, store_set: StoreSetDto, payload_format: str = 'JSON', compression_type: str = 'none', version: str = '1', object_type: str = 'JSON', **kwargs: Dict[str, Any]) -> Any: + """ + This method stores a set of the payloads (with additional parameters: since and streamer info) on the file storage. + The payloads and parameters have to be in the StoreSetDto object. + + :param tag: Name of the tag. + :param since: Since value. + :param store_set (StoreSetDto): StoreSetDto object, which contains payloads (in form of strings or files), since parameters for them + :param payload_format: payload format (default is 'JSON'). + :param compression_type: Compression type (default is 'none'). + :param version: Version of the data (default is '1'). + :param payload_format: Type of the object (default is 'JSON'). + :param kwargs: Additional optional parameters. + :return: API response + """ + data = str(store_set) + dto = store_set.to_dict() + + res = store_set.resources + n_elem = len(res) + + for elem in res: + item = elem.to_dict() + + try: + payload = elem.data + since = int(elem.since) + streamer_info = elem.streamer_info + + self.store_payload( tag, since, payload, payload_format = payload_format, streamer_info = streamer_info) + + except Exception as e: + print('Error' % e) + + + def find_tag(self, tag_name: str) -> Union[TagDto, HTTPResponse]: + """ + This method returns a tag for the tag name. + + :param tag_name (str): tag name. + :return: tag (as TagDto) or HTTP response if unsuccessful. + """ + + path = self._dir + conf.fs_tag_path + "/" + tag_name + + try: + file = path + conf.fs_tag_file + + result = read_file(file) # str + result = json.loads(result) + + dto = validate_and_convert_types( + result, + {dict, TagDto}, + ['result'], + True, + True, + TagDto + ) + + return dto + except ApiException as e: + print(f"Exception in method CrestApiFs->find_tag: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + def find_tag_meta(self, tag_name: str) -> Union[TagMetaDto, HTTPResponse]: + """ + This method returns a tag meta info for the tag name. + + :param tag_name (str): tag name. + :return: tag (as TagDto) or HTTP response if unsuccessful. + """ + + path = self._dir + conf.fs_tag_path + "/" + tag_name + + try: + file = path + conf.fs_tagmetainfo_file + + result = read_file(file) # str + result = json.loads(result) + + dto = validate_and_convert_types( + result, + {dict, TagMetaDto}, + ['result'], + True, + True, + TagMetaDto + ) + + return dto + except ApiException as e: + print(f"Exception in method CrestApiFs->find_tag_meta: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + def get_payload(self, hash: str) -> Union[str, HTTPResponse]: + """ + This methods returns a payload by the hash. + + :param hash (str): hash . + :return: payload (as string) or HTTP response if unsuccessful. + """ + + try: + prefix = hash[:conf.fs_prefix_length] + payload_path = self._dir + conf.fs_data_path + "/" + prefix + "/" + hash + payload_file = payload_path + conf.fs_payload_file + + result = read_file(payload_file) # str + + return result + except ApiException as e: + print(f"Exception in method CrestApiFs->get_payload: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + + def get_payload_meta(self, hash: str) -> Union[PayloadDto, HTTPResponse]: + """ + This method returns a payload meta info for the hash. + + :param hash (str): hash . + :return: payload meta (as PayloadDto) or HTTP response if unsuccessful. + """ + + try: + prefix = hash[:conf.fs_prefix_length] + payload_path = self._dir + conf.fs_data_path + "/" + prefix + "/" + hash + meta_file = payload_path + conf.fs_meta_file + + result = read_file(meta_file) # str + result = json.loads(result) + + dto = validate_and_convert_types( + result, + {json, dict, PayloadDto}, + ['result'], + True, + True, + PayloadDto + ) + + return dto + except ApiException as e: + print(f"Exception in method CrestApiFs->get_payload_meta: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + + + def sort_iov_json(self, data: json, order: bool): + """ + The auxiliary method to sort the JSON array with the IOVs by the since parameters. + + :param json : JSON array with the IOVs (IOV list). + :param order : sorting order ( True - ascending order, False - descending) + :return: sorted JSON array with the IOV list. + """ + + res = json.loads("[]") + unsorted = {} + + for element in data: + if element.get('since') is not None: + since = element['since'] + unsorted[since] = element + else: + raise Exception("JSON has no since parameter") + if order: + sorted_dict = dict(sorted(unsorted.items(),reverse = False)) + else: + sorted_dict = dict(sorted(unsorted.items(),reverse = True)) + + for key in sorted_dict: + res.append(sorted_dict[key]) + + return res + + + + def get_page(self, data: json, size: int, page: int): + """ + The auxiliary method to extract a subarray from JSON array. + + :param json : JSON array to extract subarray. + :param size: Number of item per page + :param page: Page number + :return: JSON subarray. + """ + + res = json.loads("[]") + data_size = len(data) + + if data_size == 0: + # no elements to return + return res + + # index interval to load the data from JSON array: + kmin = size * page + kmax = size * (page + 1) + + # check if the interval is correct: + if kmin > (data_size - 1): # out of range + return res + + if kmax > (data_size - 1): + kmax = data_size + + for i in range (kmin , kmax): + res.append(data[i]) + + return res + + + + def extract_interval(self, data: json, since: int, until: int): + """ + The auxiliary method to extract an IOV list in the time interval since-until. + + :param json : JSON array with the IOV list. + :param since: Start time + :param until: End time, (value -1 means "infinity") + :return: IOV list (JSON array). + """ + res = json.loads("[]") + + for element in data: + if element.get('since') is not None: + time = element['since'] + + if until == -1: # infinity + if time >= since: + res.append(element) + else: + if (time >= since) and (time <= until): + res.append(element) + else: + raise Exception("JSON has no since parameter") + + return res + + + + def select_iovs(self, name: str, + since: int = 0, until: int = -1, + size: int = 1000, page: int = 0, + sort: str = 'id.since:ASC') -> Union[IovSetDto, HTTPResponse]: + """ + Find an IOV list by the tag name. + + :param name (str): Tag name. + :param since: Start time + :param until: End time + :param size: Number of results per page + :param page: Page number + :param sort: Sorting order ('id.since:ASC' or 'id.since:DESC') + :return: tag (as IovSetDto) or HTTP response if unsuccessful. + """ + + path = self._dir + conf.fs_tag_path + "/" + name + + try: + file = path + conf.fs_iov_file + + result = read_file(file) # str + result = json.loads(result) + + # IOV list ordering: + if sort == 'id.since:ASC': + result = self.sort_iov_json(result, True) + else: + result = self.sort_iov_json(result, False) + + # IOV list time interval: + result = self.extract_interval(result, since=since, until=until) + + # page extracting: + result = self.get_page(result, size=size, page=page) + + iov_list = IovSetDto( + size = len(result), + format = "IovSetDto", + datatype = "iovs", + resources= [] + ) + + for element in result: + dto = validate_and_convert_types( + element, + {IovDto}, + ['element'], + True, + True, + IovDto + ) + + iov_list.resources.append(dto) + + return iov_list + except ApiException as e: + print(f"Exception in method CrestApiFs->select_iovs: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + + + def create_global_tag(self, dto: GlobalTagDto) -> Union[GlobalTagDto, HTTPResponse]: + """ + Create the global tag. + + :param dto (GlobalTagDto): global tag. + :return: Created global tag (GlobalTagDto) or HTTP response if unsuccessful. + """ + + name = dto.name + path = self._dir + conf.fs_globaltag_path + "/" + name + + try: + check_directory(path); + file = path + conf.fs_globaltag_file + gtag = model_to_dict(dto) + str1 = json.dumps(gtag) + write_file(file,str1) + return dto + except ApiException as e: + print(f"Exception in method CrestApiFs->create_global_tag: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + + def find_global_tag(self, name: str) -> Union[GlobalTagDto, HTTPResponse]: + """ + This metod returns the global tag for the given global tag name. + + :param name : global tag name. + :return: global tag (GlobalTagDto) or HTTP response if unsuccessful. + """ + path = self._dir + conf.fs_globaltag_path + "/" + name + + try: + + file = path + conf.fs_globaltag_file + result = read_file(file) # str + result = json.loads(result) + + dto = validate_and_convert_types( + result, + {dict, GlobalTagDto}, + ['result'], + True, + True, + GlobalTagDto + ) + + return dto + except ApiException as e: + print(f"Exception in method CrestApiFs->find_global_tag: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + + def check_global_tag_map(self, dto: GlobalTagMapDto): + """ + The auxiliary method to check if the global tag and tag exist. + + :param dto (GlobalTagMapDto): global tag map. + """ + + if dto.get('globalTagName') is not None: + global_tag = dto['globalTagName'] + global_tag_path = self._dir + conf.fs_globaltag_path + "/" + global_tag + conf.fs_globaltag_file + + if not os.path.isfile(global_tag_path): + raise FileNotFoundError(f"Global tag '{global_tag}' does not exist.") + else : + raise Exception("Error: globalTagName not found in JSON.") + + + if dto.get('tagName') is not None: + tag = dto['tagName'] + tag_path = self._dir + conf.fs_tag_path + "/" + tag + conf.fs_tag_file + + if not os.path.isfile(tag_path): + raise FileNotFoundError(f"Tag '{tag}' does not exist.") + else : + raise Exception("Error: tagName not found in JSON.") + + + + def create_global_tag_map(self, dto: GlobalTagMapDto) -> Union[GlobalTagMapDto, HTTPResponse]: + """ + This method creates a new global tag map on the file storage. + + :param dto (GlobalTagMapDto): global tag. + :return: Created global tag map or HTTP response if unsuccessful. + """ + inserted = False + + try: + self.check_global_tag_map(dto=dto) + global_tag = dto['globalTagName'] + global_tagmap_path = self._dir + conf.fs_globaltag_path + "/" + global_tag + conf.fs_map_file + + if os.path.isfile(global_tagmap_path): + tag = dto['tagName'] + tag_map = read_file(global_tagmap_path) + tagmap = json.loads(tag_map) + for element in tagmap: + current_tag = element['tagName'] + if current_tag == tag: + tagmap.remove(element) + map = model_to_dict(dto) + tagmap.append(map) + map_string = json.dumps(tagmap) + write_file(global_tagmap_path,map_string) + inserted = True + if inserted == False: + map = model_to_dict(dto) + tagmap.append(map) + map_string = json.dumps(tagmap) + write_file(global_tagmap_path,map_string) + + else: + tagmap = json.loads("[]") + map = model_to_dict(dto) + tagmap.append(map) + map_string = json.dumps(tagmap) + write_file(global_tagmap_path,map_string) + + return dto + + except ApiException as e: + print(f"Exception in method CrestApiFs->create_global_tag_map: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) + + + + def find_global_tag_map(self, name: str) -> Union[GlobalTagMapSetDto, HTTPResponse]: + """ + This method returns a tag for the tag name. + + :param ame (str): global tag name. + :return: tag (as TagDto) or HTTP response if unsuccessful. + """ + + file = self._dir + conf.fs_globaltag_path + "/" + name + conf.fs_map_file + + try: + + result = read_file(file) # str + result = json.loads(result) + + dto = GlobalTagMapSetDto( + size = len(result), + format = 'GlobalTagMapSetDto', + datatype = 'maps', + resources= [] + ) + + for element in result: + map = validate_and_convert_types( + element, + {GlobalTagMapDto}, + ['element'], + True, + True, + GlobalTagMapDto + ) + + dto.resources.append(map) + + return dto + + except ApiException as e: + print(f"Exception in method CrestApiFs->find_global_tag_map: {e}\n") + return HTTPResponse( + status_code=e.status, + reason=e.reason, + code=e.status, + message=e.body + ) diff --git a/pycrest/cli/commands/global_tag_commands.py b/pycrest/cli/commands/global_tag_commands.py index 92907a2b858bd2e252896b0717ab61c27cd40f05..a8b63b1a75a278eb906caf4ed1765ad8d77c987a 100644 --- a/pycrest/cli/commands/global_tag_commands.py +++ b/pycrest/cli/commands/global_tag_commands.py @@ -1,19 +1,42 @@ from pycrest.api.crest_api import CrestApi from pycrest.cli.commands.utils import print_full_res, print_multiple_res +from hep.crest.client.models import ( + IovSetDto, HTTPResponse, TagMetaSetDto, TagMetaDto, TagSetDto, TagDto, GlobalTagDto, + GlobalTagSetDto, GlobalTagMapDto, StoreSetDto, StoreDto, RunLumiInfoDto, RunLumiSetDto +) +import sys def create_global_tag_func(args): + + required_args = ['description'] + missing_args = [arg for arg in required_args if getattr(args, arg) is None] + if missing_args: + sys.exit(f"Error: The following parameters are not set: {', '.join(missing_args)}") + params = { + 'name': args.name, 'validity': args.validity, 'description': args.description, 'release': args.release, 'scenario': args.scenario, 'workflow': args.workflow, 'type': args.type, + 'force': 'false' } + + globaltag = GlobalTagDto( + name=params['name'], + description=params['description'], + release=params['release'], + scenario=params['scenario'], + validity=params['validity'], + workflow=params['workflow'], + type=params['type'] + ) try: crest_api = CrestApi(host=args.crest_host) - resp = crest_api.create_global_tag(name=args.name, **params) + resp = crest_api.create_global_tag(dto=globaltag, force=params['force']) # print(f"global tag {args.name} created") print_full_res(resp) except Exception as e: @@ -81,14 +104,26 @@ def get_global_tag_map_func(args): def create_global_tag_map_func(args): + required_args = ['record', 'label', 'tag_name', 'name'] + missing_args = [arg for arg in required_args if getattr(args, arg) is None] + if missing_args: + sys.exit(f"Error: The following parameters are not set: {', '.join(missing_args)}") params = { 'record': args.record, 'label': args.label, } + + globaltagmap = GlobalTagMapDto( + global_tag_name=args.name, + tag_name=args.tag_name, + record=params['record'], + label=params['label'] + ) + try: crest_api = CrestApi(host=args.crest_host) - resp = crest_api.create_global_tag_map(globaltagname=args.name, tagname=args.tag_name, **params) + resp = crest_api.create_global_tag_map(globaltagmap) # print(f"global tag map {args.name} - {args.tag_name} created") print_full_res(resp) except Exception as e: diff --git a/pycrest/cli/commands/iovs_commands.py b/pycrest/cli/commands/iovs_commands.py index f77e19931ba050b698b8dbb9e717e8860fd0865b..e3ea10072e1b63ef1a25f06f3c31f12081bb06be 100644 --- a/pycrest/cli/commands/iovs_commands.py +++ b/pycrest/cli/commands/iovs_commands.py @@ -6,6 +6,8 @@ def get_iov_list_func(args): # print("GET IOV LIST function") # print(f"command = {args.command}") # print(f"name = {args.name}") + # print(f"since = {args.since}") + # print(f"until = {args.until}") # print(f"host = {crest_host}") # tag name is mandatory name = args.name @@ -15,12 +17,15 @@ def get_iov_list_func(args): params = {} for arg in args.__dict__.keys(): if arg in optional_args: - params[arg] = arg_dict[arg] + if ((arg=='until') and (args.until=='-1')): + params[arg] = 'INF' + elif (arg_dict[arg]!=None): + params[arg] =arg_dict[arg] try: crest_api = CrestApi(host=args.crest_host) resp = crest_api.find_all_iovs(tagname=name, **params) - # print_json(resp) # OLD MvG + print_multiple_res(resp) except Exception as e: print("Error: "+ repr(e)) diff --git a/pycrest/cli/commands/tag_commands.py b/pycrest/cli/commands/tag_commands.py index a346755bdd4d82ec9a7c442d548ada86f92cf704..425eadc9ae8bd55fe2e1fb9040696d7a7f5061a4 100644 --- a/pycrest/cli/commands/tag_commands.py +++ b/pycrest/cli/commands/tag_commands.py @@ -1,7 +1,10 @@ from pycrest.api.crest_api import CrestApi from pycrest.cli.commands.utils import print_multiple_res, print_full_res, print_single_res from pycrest.cli.config import crest_host - +from hep.crest.client.models import ( + IovSetDto, HTTPResponse, TagMetaSetDto, TagMetaDto, TagSetDto, TagDto, GlobalTagDto, + GlobalTagSetDto, GlobalTagMapDto, StoreSetDto, StoreDto, RunLumiInfoDto, RunLumiSetDto +) import sys def get_tag_func(args): @@ -39,17 +42,24 @@ def create_tag_func(args): 'last_validated_time': args.last_validated_time, 'end_of_validity': args.end_of_validity, } - + tag = TagDto( + name=args.name, + description=params['description'], + time_type=args.time_type, + payload_spec=params['payload_spec'], + synchronization=params['synchronization'], + last_validated_time=params['last_validated_time'], + end_of_validity=params['end_of_validity'] + ) try: crest_api = CrestApi(host=args.crest_host) - resp = crest_api.create_tag(name=args.name, timeType=args.time_type, **params) + resp = crest_api.create_tag(dto=tag) # print(f"tag {args.name} created") print_full_res(resp) except Exception as e: print("Error: "+ repr(e)) - def remove_tag_func(args): # print("GET TAG function") # print(f"command = {args.command}") @@ -65,6 +75,9 @@ def remove_tag_func(args): print("Error: "+ repr(e)) def create_tag_meta_info_func(args): + """ + :param args: the parameters to create tag meta information. + """ required_args = ['description', 'chansize', 'colsize', 'tag_info'] missing_args = [arg for arg in required_args if getattr(args, arg) is None] @@ -78,9 +91,17 @@ def create_tag_meta_info_func(args): 'tag_info': args.tag_info, } + tag_meta = TagMetaDto( + tag_name=args.name, + description=params['description'], + chansize=params['chansize'], + colsize=params['colsize'], + tag_info=params['tag_info'], + ) + try: crest_api = CrestApi(host=args.crest_host) - resp = crest_api.create_tag_meta(name=args.name, **params) + resp = crest_api.create_tag_meta(dto=tag_meta) # print(f"tag meta info {args.name} created") print_full_res(resp) except Exception as e: diff --git a/tests/test_crestapi_http.py b/tests/test_crestapi_http.py index 81400049b9dd670683d7582f19d8cc2b92160e78..ab5fea85ee1aa1c28d485267d2baf5a2d0e852ec 100644 --- a/tests/test_crestapi_http.py +++ b/tests/test_crestapi_http.py @@ -18,7 +18,7 @@ from pycrest.api.crest_cond_tag_manager import TagManager from pycrest.api.tag_info_container import TagInfoContainer from pycrest.api.crest_cond_builder import CrestCondBuilder from hep.crest.client.models import ( - TagMetaDto, GlobalTagDto + TagMetaDto, TagDto, GlobalTagDto ) from pycrest.api.crest_api import CrestApi @@ -43,12 +43,21 @@ def test_create_tag(crest_client): time_type = 'time' try: print('Create a tag %s' % name) + tag = TagDto( + name=name, + description=description, + time_type=time_type, + payload_spec='JSON', + synchronization='none', + last_validated_time=-1., + end_of_validity=-1. + ) # Create a new tag - api_response = api.create_tag(name=name, timeType=time_type, description=description) + api_response = api.create_tag(tag) assert api_response.name == name # Find the tag - api_response = api.get_tag(name=name) + api_response = api.find_tag(name=name) assert api_response.name == name # Delete the tag @@ -65,11 +74,20 @@ def test_create_global_tag(crest_client): description = 'a brand new test global tag' try: # Create a new global tag - api_response = api.create_global_tag(name=global_tag, description=description) + globaltag = GlobalTagDto( + name=global_tag, + description=description, + release='1.0', + scenario='test', + validity=0., + workflow='all', + type='T' + ) + api_response = api.create_global_tag(globaltag, force='false') assert api_response.name == global_tag # Find the global tag - api_response = api.get_global_tag(name=global_tag) + api_response = api.find_global_tag(name=global_tag) assert api_response.name == global_tag # Delete the global tag