diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5c248dad5f3f1e08156415db59e6b9dc2d420525..70aed2d9a5bee3a7d8495b72b7bd308895dc7572 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -81,13 +81,9 @@ test_reception_upload:
- curl ${CONFIGDB_API_URI}/health
# upload a blank template runkey - creates a runkey in the database
- #- python3 scripts/main_configDB.py --host ${ENV_API_HOSTNAME} --port ${ENV_API_PORT} --upload-template
- # TODO: we currently use a hack: build a blank template and add a new stave with it,
- # because we cannot just re-upload a runkey in ConfigDB now.
- # So, there is no need to upload a preliminary template.
- # Uncomment this, when re-uploading a runkey works in ConfigDB.
+ - python3 scripts/main_configDB.py --host ${ENV_API_HOSTNAME} --port ${ENV_API_PORT} --upload-template
- # upload a new stave to the "latest" runkey
+ ## upload a new stave to the "latest" runkey
- python3 scripts/main_configDB.py --host ${ENV_API_HOSTNAME} --port ${ENV_API_PORT} --upload-stave NewStaveName examples/sr1-barrel-st-4staves/CERNSR1_stave0_testing_setup_small.json
# test uploading a new stave directly from Python
- python3 scripts/test_upload_a_stave.py ${ENV_API_HOSTNAME} ${ENV_API_PORT}
diff --git a/scripts/main_configDB.py b/scripts/main_configDB.py
index 485cfdf9337b71dda0fd8bd4d6a75253bebc9291..e9bdbca318c70c4e786455f0d12c651af9fd4b8c 100755
--- a/scripts/main_configDB.py
+++ b/scripts/main_configDB.py
@@ -442,7 +442,7 @@ if __name__ == "__main__":
runkey_dict_template['children'].append(staves_node)
runkey_id = configDB.stage_create(data=runkey_dict_template, comment="upload an blank runkey")
logging.info(f"Staged id: {runkey_id}")
- ids = configDB.stage_commit(runkey_id, new_name=configDBname, comment=comment)
+ ids = configDB.stage_commit(runkey_id, new_name=configDBname+"_blankTemplate", comment=comment)
logging.info(f"Committed ids: {ids}")
if args.upload:
@@ -473,54 +473,80 @@ if __name__ == "__main__":
# Create RunkeyStave
stave_name, stave_fpath = args.upload_stave
- a_stave_runkey = fxt.build_stave_node_from_file(stave_name, stave_fpath)
+ stave_node = fxt.build_stave_node_from_file(stave_name, stave_fpath)
# pretty-print the stave runkey to the trace log
- logging.debug(f"stave runkey {stave_name}: {json.dumps(a_stave_runkey, indent=2)}")
-
- # TODO: the problem is that ConfigDB does not accept a runkey that you've just downloaded.
- # This must be fixed on the ConfigDB side.
- # In our standard basic example, we always translate the runkey into files
- # before it is re-uploaded.
-
- # TODO: we also cannot dump the runkey into files and read them back
- # because we work only with the standard SR1 files.
- # We do not have more general routines now.
- # TODO: Make the more general routines.
-
- #configDB = pyconfigdb.ConfigDB(dbKey=CONFIGDB_KEY, srUrl=configdb_url)
- #logging.info(f"reading a runkey: {args.runkey_ref}")
- #runkey_dict = configDB.tag_tree(args.runkey_ref, stage=False, payload_data=True)
- ## TODO: check what's "stage=False" here?
-
- ## add it to the staves subtree
- #runkey_root = runkey_dict['objects'][0]
- ##logging.info(runkey_root)
- #assert runkey_root['type'] == 'Root'
- #assert any(tree['type'] == 'Staves' for tree in runkey_root['children'])
-
- ## translate the not-uploadable ConfigDB runkey into files
- #temp_translate_dir = '.temp_translate/'
- #yarr_connectivity = runkey_fiber_to_yarr_connectivity(get_subnode_of_type(runkey_root, 'Felix'))
- #write_cfg_files(yarr_connectivity, temp_translate_dir)
- # load the files back into an uploadble dictionary:
-
- # So, for now, let's just upload a blank template with a new stave added.
- logging.info(f"Uploading a template runkey with an added new stave as: {args.runkey_ref}")
- staves_node = {
- "type": "Staves",
- "payloads": [],
- "children": []
- }
-
- staves_node['children'].append(a_stave_runkey)
- comment = f"added stave {stave_name}"
-
- runkey_dict_template['children'].append(staves_node)
- runkey_id = configDB.stage_create(data=runkey_dict_template, comment="upload an blank runkey with a new stave")
- logging.info(f"Staged id: {runkey_id}")
- ids = configDB.stage_commit(runkey_id, new_name=configDBname, comment=comment)
- logging.info(f"Committed ids: {ids}")
+ logging.debug(f"stave runkey {stave_name}: {json.dumps(stave_node, indent=2)}")
+
+ #
+ # TODO: `search` finds _payloads_ not the nodes???
+ # we need nodes -- the functions `add_node` need IDs of parent nodes
+ #staves_tree_id = configDB.search(runkey_ref, object_type="Staves")
+ staves_subtree = configDB.search_subtree(args.runkey_ref, object_type="Staves")
+ logging.debug(f'Found Staves subtree: {staves_subtree}')
+ staves_tree_id = staves_subtree[0]['id']
+ logging.info(f'Got Staves subtree ID: {staves_tree_id}')
+ assert len(staves_tree_id) > 0
+
+ # and add a new node under it
+ # ConfigDB does not provide an API to just upload a dictionary
+ # with payloads and children together.
+ # you have to create nodes one by one.
+ # Hence, our helper function:
+ def add_subtree(configdb, parent_id, subtree: dict, new_id_map={}, to_stage=False):
+ assert isinstance(subtree, dict), f'Not a dict: {subtree}'
+
+ if 'reuse_id' in subtree:
+ # skip it
+ return
+
+ if 'type' not in subtree:
+ raise RuntimeError(f'No "type" in {subtree}')
+
+ # TODO: what to do with "reuse_id" nodes?
+ # TODO: or else, how to set node ID in add_node?
+
+ # create the root node of this subtree
+ subtree_id = configdb.add_node(subtree['type'],
+ payloads=subtree['payloads'],
+ parents=[{'id': parent_id}],
+ children=[],
+ stage=to_stage)
+
+ if 'children' not in subtree:
+ raise RuntimeError(f'No "children" in {subtree}')
+
+ # create children nodes and save their IDs
+ children_ids = []
+ for child_node in subtree['children']:
+ add_subtree(configdb, subtree_id, child_node, new_id_map)
+
+ # update the children IDs in the root node
+ configdb.add_to_node(subtree_id, children=children_ids, stage=to_stage)
+
+ # update the ID
+ current_id = subtree.get('id')
+ new_id_map[current_id] = subtree_id
+ subtree['id'] = subtree_id
+
+ def update_references(configdb, parent_id, subtree: dict, new_id_map={}, to_stage=False):
+ if 'reuse_id' in subtree:
+ # find the new id of the reference
+ new_id = new_id_map[subtree['reuse_id']]
+ # add it to the node's children
+ configdb.add_to_node(parent_id, children=[new_id], stage=to_stage)
+ # and done
+ return
+
+ uptodate_id = subtree['id']
+ for child_node in subtree['children']:
+ update_references(configdb, uptodate_id, child_node, new_id_map, to_stage)
+
+ id_mapping = {}
+ add_subtree(configDB, staves_tree_id, stave_node, id_mapping)
+ update_references(configDB, staves_tree_id, stave_node, id_mapping)
+
+ logging.info(f'Uploaded a stave and updated references: {stave_name}')
else:
logging.info(" Not committing to configDB")
diff --git a/scripts/test_upload_a_stave.py b/scripts/test_upload_a_stave.py
index dea988362582e559fc351f3802c24a01d2f6cf39..5632fea6c3e57f1d5e1c4823ee8ea300fabdd495 100644
--- a/scripts/test_upload_a_stave.py
+++ b/scripts/test_upload_a_stave.py
@@ -4,7 +4,7 @@ import pyconfigdb
import functions_configDB as fxt
import logging
import datetime
-from configdb_stripsRunkeys import get_subnode_of_type
+from configdb_stripsRunkeys import get_subnode_of_type, get_node_payloads_data
logging.basicConfig(level=logging.INFO)
@@ -26,7 +26,7 @@ stave_name = 'AnotherNewStave' # or use the stave serial number
with open(full_stave_cfg_fname, 'r') as cfg_file:
stave_cfg = json.load(cfg_file)
-a_stave_runkey = fxt.build_stave_node_from_dict(stave_name, stave_cfg)
+stave_node = fxt.build_stave_node_from_dict(stave_name, stave_cfg)
# ConfigDB connection:
_, hostname, port = argv
@@ -41,61 +41,89 @@ configdb_url = f'http://{hostname}:{port}/api'
CONFIGDB_KEY = "demi/default/itk-demo-configdb/api/url"
configDB = pyconfigdb.ConfigDB(dbKey=CONFIGDB_KEY, srUrl=configdb_url)
-
-# TODO: here we try to download a runkey from ConfigDB, add something to it and upload.
-# It does not work in ConfigDB currently. You cannot re-upload a just downloaded runkey.
-# So, as a patch, we upload a template with a new stave added.
-# This must be fixed.
-#logging.info(f"reading a runkey: {runkey_ref}")
-#runkey_dict = configDB.tag_tree(runkey_ref, stage=False, payload_data=True)
-#
-## add it to the staves subtree
-#runkey_root = runkey_dict['objects'][0]
-#assert runkey_root['type'] == 'Root', f"Didn't find Root node: {runkey_root['type']} among {len(runkey_dict['objects'])}\n{runkey_dict}"
-#assert any(tree['type'] == 'Staves' for tree in runkey_root['children'])
-#staves_tree = get_subnode_of_type(runkey_root, "Staves")
-#
-## check if a stave with the same name already exists
-#if any(stv.get('name') == stave_name for stv in staves_tree['children']):
-# # TODO: is stave name a "name" attribute or payload?
-# # TODO: raise or substitute the previous stave node?
-# # if you substitute the node - update all reference nodes?
-# raise RuntimeError(f'Stave by name {stave_name} already exists in runkey {runkey_ref}')
-
-staves_tree = {
- "type": "Staves",
- "payloads": [],
- "children": []
-}
-
-staves_tree['children'].append(a_stave_runkey)
-
-runkey_template = {
- "type": "Root",
- "children": [
- {
- "type": "Felix",
- "payloads": [
- {"type": "FID_DETECTOR_ID", "data": "0"}
- ],
- "children": []
- },
-
- {
- "type": "Geometry",
- "payloads": [],
- "children": [ ]
- },
-
- staves_tree
- ]
-}
-
-# upload the runkey back
-logging.info(f"uploading a runkey with stave {stave_name}: {runkey_ref}")
-comment = f"added stave {stave_name}"
-runkey_id = configDB.stage_create(data=runkey_template, comment=comment)
-
-configDBname = "stripsRunkeyTestPython_" + datetime.datetime.now().strftime("%Y-%m-%d_%Hh%Mm%Ss")
-ids = configDB.stage_commit(runkey_id, new_name=configDBname, comment=comment)
-logging.info(f"uploaded a runkey with the new stave {stave_name}")
+logging.info(f'ConfigDB is UP: {configDB.health()}')
+
+# TODO: `search` finds _payloads_ not the nodes???
+# we need nodes -- the functions `add_node` need IDs of parent nodes
+#staves_tree_id = configDB.search(runkey_ref, object_type="Staves")
+staves_subtree = configDB.search_subtree(runkey_ref, object_type="Staves")
+logging.debug(f'Found Staves subtree: {staves_subtree}')
+staves_tree_id = staves_subtree[0]['id']
+logging.info(f'Got Staves subtree ID: {staves_tree_id}')
+assert len(staves_tree_id) > 0
+
+# and add a new node under it
+# ConfigDB does not provide an API to just upload a dictionary
+# with payloads and children together.
+# you have to create nodes one by one.
+# Hence, our helper function:
+def add_subtree(configdb, parent_id, subtree: dict, new_id_map={}, to_stage=False):
+ assert isinstance(subtree, dict), f'Not a dict: {subtree}'
+
+ if 'reuse_id' in subtree:
+ # skip it
+ # the references are done with the special workaround: new_id_map
+ return
+
+ if 'type' not in subtree:
+ raise RuntimeError(f'No "type" in {subtree}')
+
+ # create the root node of this subtree
+ subtree_id = configdb.add_node(subtree['type'],
+ payloads=subtree['payloads'],
+ parents=[{'id': parent_id}],
+ children=[],
+ stage=to_stage)
+
+ if 'children' not in subtree:
+ raise RuntimeError(f'No "children" in {subtree}')
+
+ # create children nodes and save their IDs
+ children_ids = []
+ for child_node in subtree['children']:
+ add_subtree(configdb, subtree_id, child_node, new_id_map)
+
+ # update the children IDs in the root node
+ configdb.add_to_node(subtree_id, children=children_ids, stage=to_stage)
+
+ # update the ID
+ current_id = subtree.get('id')
+ new_id_map[current_id] = subtree_id
+ subtree['id'] = subtree_id
+
+def update_references(configdb, parent_id, subtree: dict, new_id_map={}, to_stage=False):
+ if 'reuse_id' in subtree:
+ # find the new id of the reference
+ new_id = new_id_map[subtree['reuse_id']]
+ # add it to the node's children
+ configdb.add_to_node(parent_id, children=[new_id], stage=to_stage)
+ # and done
+ return
+
+ uptodate_id = subtree['id']
+ for child_node in subtree['children']:
+ update_references(configdb, uptodate_id, child_node, new_id_map, to_stage)
+
+id_mapping = {}
+add_subtree(configDB, staves_tree_id, stave_node, id_mapping)
+update_references(configDB, staves_tree_id, stave_node, id_mapping)
+
+# download the runkey and confirm the stave is there
+tagTree = configDB.tag_tree(runkey_ref, stage=False, payload_data=True)
+logging.info(f'Got full tag tree: {json.dumps(tagTree, indent=2)}')
+assert tagTree['objects'][0]['type'] == 'Root'
+assert any(ch['type'] == "Staves" for ch in tagTree['objects'][0]['children'])
+
+runkey_root = tagTree['objects'][0]
+staves_subtree = get_subnode_of_type(runkey_root, "Staves")
+
+stave_names = []
+for ch in staves_subtree['children']:
+ assert ch['type'] == "Stave"
+ st_names = get_node_payloads_data(ch, 'staveName')
+ assert len(st_names) == 1
+ stave_names.append(st_names[0])
+
+logging.info(f'Found the following staves in the {runkey_ref} runkey: {stave_names}')
+logging.info(f'And our stave {stave_name} is there: {stave_name in stave_names}')
+assert stave_name in stave_names
\ No newline at end of file