diff --git a/examples/create_tag_store_data.py b/examples/create_tag_store_data.py index 3aec40b1fbeaebaadb94b0726ca38b89947be9e9..579bad3be622bba24af59211cbeac96a741ec779 100644 --- a/examples/create_tag_store_data.py +++ b/examples/create_tag_store_data.py @@ -60,7 +60,7 @@ resources = [ ] print('Store data in tag %s' % name) try: - api_response = api_instance.store_json_data(tagname=name, json_array=resources) + api_response = api_instance.store_data(tagname=name, json_array=resources) print(api_response) except Exception as e: print("Exception when calling CrestApi->store_json_data: %s\n" % e) diff --git a/examples/crest-read-lar.py b/examples/crest-read-lar.py index 0eea080db9d6b1a5d3527c303d04daf80e407507..2732636a81761d3d617879dbf9f279a47e580833 100644 --- a/examples/crest-read-lar.py +++ b/examples/crest-read-lar.py @@ -54,5 +54,11 @@ try: api_response = api_instance.get_payload(hash=sel_hash) print(api_response) + for iov in iovs: + print(f'Found iov : {iov.since} - {iov.payload_hash}') + # get payload for this since + api_response = api_instance.get_payload(hash=iov['payload_hash']) + print(f'Data : {api_response}') + except Exception as e: print("Exception when calling CrestApi for iov load or payload access: %s\n" % e) diff --git a/examples/crest-read.py b/examples/crest-read.py index 47145b6bcb88f3270672d0d11c6fa54bc413ecce..fa069b216f863de35f65c4fa4d4f0d33fe904a27 100644 --- a/examples/crest-read.py +++ b/examples/crest-read.py @@ -22,7 +22,7 @@ desttag='TestCrest-REL-01' # Check tag print('Check tag %s' % desttag) try: - tag = api_instance.get_tag(name=desttag) + tag = api_instance.find_tag(name=desttag) if tag is not None and isinstance(tag, TagDto): tagname = tag.name description = tag.description diff --git a/examples/crest-store-lar.py b/examples/crest-store-lar.py index 3632a53525f65a27c5d4e793a994911ffd2fc116..24a93ebf42650259b2df295cdec9f32792aeb738 100644 --- a/examples/crest-store-lar.py +++ b/examples/crest-store-lar.py @@ -125,7 +125,8 @@ try: larcond_payload = generate_since_data(since, larcond_payload) print(f'Add data for since {since}') # Store the data in one go - api_response = api_instance.store_json_data(tag_name=desttag, json_array=larcond_payload.flush_iovs()) + store_set_dto = larcond_payload.get_store_set() + api_response = api_instance.store_data(tag=desttag, store_set=store_set_dto) print(f'Stored data in tag {desttag}') end_time = time.time() # Calculate the elapsed time diff --git a/examples/crest-store-tile.py b/examples/crest-store-tile.py index 6f6ba1421c46038e8bdc9bec1dac03537986b730..74af9f17656cf396e2cc2092eb57a5630f691ced 100644 --- a/examples/crest-store-tile.py +++ b/examples/crest-store-tile.py @@ -122,7 +122,8 @@ try: for since in range(0, 10000, 1000): crest_payload = generate_since_data(since=since, crest_payload=crest_payload) print(f'Add data for since {since}') - api_response = api_instance.store_json_data(tag_name=desttag, json_array=crest_payload.flush_iovs()) + store_set_dto = crest_payload.get_store_set() + api_response = api_instance.store_data(tag=desttag, store_set=store_set_dto) print('Stored data in tag %s' % desttag) end_time = time.time() # Calculate the elapsed time diff --git a/examples/crest-store.py b/examples/crest-store.py index e07d134a1d866a5b2ebe8790da9c6553155aa9d5..40f004add688e36dd0685daec94643e786ec507f 100644 --- a/examples/crest-store.py +++ b/examples/crest-store.py @@ -39,7 +39,7 @@ desttag='TestCrest-REL-01' # Remove the tag print('Delete tag %s if exists' % desttag) try: - tag = api_instance.get_tag(name=desttag) + tag = api_instance.find_tag(name=desttag) if tag is not None and isinstance(tag, TagDto): tagname = tag.name description = tag.description @@ -71,15 +71,17 @@ try: # Create a new tag print(f'Creating tag {desttag} with time type {time_type} and description {description}') - api_response = api_instance.create_tag(name=desttag, timeType=time_type, description=description) + tag_dto = builder.build_tag(tag_description=description, time_type=time_type) + + api_response = api_instance.create_tag(tag_dto) if isinstance(api_response, TagDto): print(f'Created tag {api_response.name}') else: print(f'Error creating tag {desttag}: {api_response}') # Add tag info - params = builder.get_tag_info().get_tag_info_params() - print(f'Creating tag meta info for {desttag} with {params}') - api_response = api_instance.create_tag_meta(name=desttag, **params) + tag_info_dto = builder.get_tag_info().get_tag_info_dto() + print(f'Creating tag meta info for {desttag} with {tag_info_dto}') + api_response = api_instance.create_tag_meta(tag_info_dto) if isinstance(api_response, TagMetaDto): print(f'Created tag meta info for {api_response.tag_name}') else: @@ -99,7 +101,8 @@ try: crest_payload.add_iov(since=since, data=crest_payload.get_payload(), streamer_info=payload_streamer) # Insert data into the tag - api_response = api_instance.store_json_data(tag_name=desttag, json_array=crest_payload.flush_iovs()) + store_set_dto = crest_payload.get_store_set() + api_response = api_instance.store_data(tag=desttag, store_set=store_set_dto) print(f'Stored data in tag {desttag}') end_time = time.time() print(f'Elapsed time: {end_time - start_time} seconds') diff --git a/examples/lumi_cron_simulator.py b/examples/lumi_cron_simulator.py index f7369258f1d978be68c78c3b6adcfcc3ca58efaa..54ab39bccb50b13a15127e1729a03f647f17d306 100644 --- a/examples/lumi_cron_simulator.py +++ b/examples/lumi_cron_simulator.py @@ -77,7 +77,7 @@ while True: ] print(f'Store data {resources} in tag {name} at time {xlumi} with lumi counter = {lumi_counter}') try: - api_response = api_instance.store_json_data(tagname=name, json_array=resources) + api_response = api_instance.store_data(tagname=name, json_array=resources) print(api_response) except Exception as e: print("Exception when calling CrestApi->store_json_data: %s\n" % e) diff --git a/pycrest/api/crest_api.py b/pycrest/api/crest_api.py index c491099c5dafaafd5956d73fc2d38b9904f3d071..9e7361e59868fccdeb8846d2ca5e720032cc18b4 100644 --- a/pycrest/api/crest_api.py +++ b/pycrest/api/crest_api.py @@ -536,7 +536,7 @@ class CrestApi(): else: print(f"Error: HTTP GET request failed for payload {hash}") - def store_data(self, tag_name: str, since: int, filepath: str, filename: str, + def store_payload(self, tag_name: str, since: int, filepath: str, filename: str, compression_type: str = 'none', version: str = '1', object_type: str = 'JSON', **kwargs: Dict[str, Any]) -> Any: """ @@ -577,61 +577,77 @@ class CrestApi(): format="StoreSetDto", resources=[store] ) + self.store_data(payload_format=x_crest_payload_format, tag=tag_name, store_set=storeset, **params) - json_body = json.dumps(storeset.to_dict()) - headers = {'X-Crest-PayloadFormat': x_crest_payload_format} - - api_instance = payloads_api.PayloadsApi(self._api_client) - try: - return api_instance.store_payload_batch(tag=tag_name, storeset=json_body, - x_crest_payload_format=x_crest_payload_format, **params, - files=files) - except ApiException as e: - print("Exception when calling PayloadsApi->store_payload_batch: %s\n" % e) - - def store_json_data(self, tag_name: str, json_array: List[Dict[str, Any]] = [], - compression_type: str = 'none', version: str = '1', - object_type: str = 'JSON', **kwargs: Dict[str, Any]) -> Any: + def store_data(self, payload_format: str = 'JSON', tag: str = None, store_set: StoreSetDto = None, object_type: str = 'JSON', compression_type: str = 'none', version: str = '1', **kwargs: Dict[str, Any]) -> Any: """ - Store JSON data. + This method stores a set of the payloads (with additional parameters: since and streamer info). + The payloads and parameters have to be in the StoreSetDto object. - :param tag_name: Name of the tag. - :param json_array: Array of JSON StoreDto objects. + :param payload_format: payload format (default is 'JSON'). + :param tag: Name of the tag. + :param since: Since value. + :param store_set (StoreSetDto): StoreSetDto object, which contains payloads (in form of strings or files), since parameters for them :param compression_type: Compression type (default is 'none'). :param version: Version of the data (default is '1'). - :param object_type: Type of the object (default is 'JSON'). :param kwargs: Additional optional parameters. - :return: API response. + end_time: a number (double) in the same format as since representing an end of validity propagated to the tag. + :return: API response """ + if tag is None: + raise ValueError("Tag name is required.") + if store_set is None: + raise ValueError("StoreSetDto object is required.") + criteria = CrestApi.build_params(kwargs) params = { 'compression_type': compression_type, 'version': version, 'object_type': object_type, } - params.update(criteria) - - storeset = StoreSetDto( - size=len(json_array), - datatype="iovs", - format="StoreSetDto", - resources=[] - ) - for store in json_array: - storeset["resources"].append(store) - - temp_file = f".{tag_name}.json" - with open(temp_file, "w") as outfile: - json.dump(storeset.to_dict(), outfile) - - api_instance = payloads_api.PayloadsApi(self._api_client) - try: - return api_instance.upload_json(tag=tag_name, storeset=open(temp_file, 'rb')) - except ApiException as e: - print("Exception when calling PayloadsApi->upload_json: %s\n" % e) - finally: - print(f"Removing temporary file {temp_file}") - os.remove(temp_file) + if 'end_time' in criteria: + params['endtime'] = criteria['end_time'] + if payload_format == 'FILE': + # Store as external file + x_crest_payload_format = "FILE" + files = [] + for item in store_set.resources: + if item.data.startswith('file://'): + file_path = item.data[7:] + if not os.path.isfile(file_path): + raise FileNotFoundError(f"File '{file_path}' not found.") + files.append(open(file_path, 'rb')) + break + else: + raise ValueError("No file found in the store set.") + openapi_model = self._api_client.sanitize_for_serialization(store_set) + # print(f'Model after sanitize: {openapi_model}') + json_body = json.dumps(openapi_model) + headers = {'X-Crest-PayloadFormat': x_crest_payload_format} + api_instance = payloads_api.PayloadsApi(self._api_client) + try: + return api_instance.store_payload_batch(tag=tag, storeset=json_body, + x_crest_payload_format=x_crest_payload_format, **params, + files=files) + except ApiException as e: + print("Exception when calling PayloadsApi->store_payload_batch: %s\n" % e) + + elif payload_format == 'JSON': + openapi_model = self._api_client.sanitize_for_serialization(store_set) + ## print(f'Model after sanitize: {openapi_model}') + + temp_file = f".{tag}.json" + with open(temp_file, "w") as outfile: + json.dump(openapi_model, outfile) + + api_instance = payloads_api.PayloadsApi(self._api_client) + try: + return api_instance.upload_json(tag=tag, storeset=open(temp_file, 'rb'), **params) + except ApiException as e: + print("Exception when calling PayloadsApi->upload_json: %s\n" % e) + finally: + print(f"Removing temporary file {temp_file}") + os.remove(temp_file) def create_run_lumi(self, run_number: float, lumiblock_number: float, start_date: float, end_date: float) -> Any: diff --git a/pycrest/api/crest_cond_builder.py b/pycrest/api/crest_cond_builder.py index 199c709ed2117aa9c463788005504ade8d5a6af6..83cb69a2abe0e05126b97e4548a4019a6dd9a3d9 100644 --- a/pycrest/api/crest_cond_builder.py +++ b/pycrest/api/crest_cond_builder.py @@ -82,7 +82,7 @@ class CrestCondBuilder: """ params = { - 'payload_spec': 'none', + 'payload_spec': 'crest-json-single-iov', 'synchronization': 'none', 'last_validated_time': -1., 'end_of_validity': -1., diff --git a/pycrest/api/crest_cond_container.py b/pycrest/api/crest_cond_container.py index 66789b30527416c2dac91a7cc7caed49467fd1a9..f6187e4d165d3c6c8f488b6192afec51a5c67804 100644 --- a/pycrest/api/crest_cond_container.py +++ b/pycrest/api/crest_cond_container.py @@ -1,7 +1,7 @@ import json import base64 from typing import Dict, List, Any -from hep.crest.client.models import StoreDto +from hep.crest.client.models import StoreDto, StoreSetDto class CrestCondContainer: """ @@ -97,10 +97,28 @@ class CrestCondContainer: out_iov_list = [] # convert to StoreDto objects for iov in self._iov_list: - out_iov_list.append(iov) + storedto = StoreDto( + since=float(iov['since']), + data=iov['data'], + streamer_info=iov['streamerInfo'] + ) + out_iov_list.append(storedto) self._iov_list = [] return out_iov_list + def get_store_set(self) -> StoreSetDto: + """ + Create a storeset for payload uploads + """ + storedto_list = self.flush_iovs() + storeset = StoreSetDto( + size=len(storedto_list), + datatype="iovs", + format="StoreSetDto", + resources=storedto_list + ) + return storeset + def get_iov_list_size(self) -> int: """ Get the size of the list of IOVs. diff --git a/pycrest/api/crest_fs_api.py b/pycrest/api/crest_fs_api.py index c305fd9ba90a7dbb9cdcad186cc6e8558f8f16e1..b0e466064c6f17d2e88d1a72daa713e7c77d0c52 100644 --- a/pycrest/api/crest_fs_api.py +++ b/pycrest/api/crest_fs_api.py @@ -203,7 +203,6 @@ class CrestApiFs(): :param payload_format: payload format (default is 'JSON'). :param compression_type: Compression type (default is 'none'). :param version: Version of the data (default is '1'). - :param payload_format: Type of the object (default is 'JSON'). :param kwargs: Additional optional parameters. :return: API response """ diff --git a/pycrest/cli/commands/payload_commands.py b/pycrest/cli/commands/payload_commands.py index b450cbadcefbed161da7d678f3ad8553342455ec..0cb90c2e9b1dd9b42a212c4dba9c2c0a7a3454a0 100644 --- a/pycrest/cli/commands/payload_commands.py +++ b/pycrest/cli/commands/payload_commands.py @@ -35,7 +35,7 @@ def store_payload_func(args): try: crest_api = CrestApi(host=args.crest_host) - resp = crest_api.store_json_data(tag_name=name, json_array=resources) + resp = crest_api.store_data(tag=name, json_array=resources) print(f"payload for tag {name} stored in CREST from file {file}") except Exception as e: print("Error: "+ repr(e)) diff --git a/tests/test_crestapi_containers.py b/tests/test_crestapi_containers.py index 53f6ecba3f9846a80fec453da417d7807a2adc37..0b9f47aa935a16a1323aa8629cd79ba6feac0761 100644 --- a/tests/test_crestapi_containers.py +++ b/tests/test_crestapi_containers.py @@ -18,7 +18,7 @@ from pycrest.api.crest_cond_tag_manager import TagManager from pycrest.api.tag_info_container import TagInfoContainer from pycrest.api.crest_cond_builder import CrestCondBuilder from hep.crest.client.models import ( - TagMetaDto, GlobalTagDto + TagMetaDto, GlobalTagDto, TagDto, StoreDto, StoreSetDto, IovSetDto, IovDto ) from pycrest.api.crest_api import CrestApi @@ -46,7 +46,12 @@ def test_cond_container(): container.add_iov(since=1000, data=container.get_payload(), streamer_info=payload_streamer) json_array=container.flush_iovs() print(json_array) - out_json = [{'since': 1000, 'data': '{"data": {"1": ["MTIzNDU2Nzg5MA=="]}}', 'streamerInfo': '{"author": "myname"}'}] + storedto = StoreDto( + since=float(1000), + data='{"data": {"1": ["MTIzNDU2Nzg5MA=="]}}', + streamer_info='{"author": "myname"}' + ) + out_json = [storedto] assert len(json_array) == 1 assert json_array[0] == out_json[0]