prefix git repo json filenames with ann/, default to master instead of HEAD revision

Hlt/HltServices/src/GitANNSvc.cpp

@@ -82,7 +82,7 @@ class GitANNSvc : public ANNSvcBase {
   Gaudi::Property<std::map<unsigned int, std::string>> m_key2commit{this, "KeyMapping", {}};
   Gaudi::Property<std::map<unsigned int, std::string>> m_key2JSON{this, "Overrule", {}};
   Gaudi::Property<std::string>                         m_tag{this, "Version", "master"};
-  Gaudi::Property<std::string> m_fmt{this, "RefFormatter", "{0}:json/{1:.2}/{1}.json"}; // 0=tag, 1=key, 2=label
+  Gaudi::Property<std::string> m_fmt{this, "RefFormatter", "{0}:ann/json/{1:.2}/{1}.json"}; // 0=tag, 1=key, 2=label
 
   std::optional<std::string> fetch_from_repos( std::string const& oid ) const {
     for ( const auto& repo : m_repo.value() ) {

PyConf/python/PyConf/application.py

@@ -13,7 +13,6 @@ import datetime
 import logging
 import math
 import os
-import tempfile
 import re
 from collections import OrderedDict
 from itertools import chain
@@ -69,93 +68,108 @@ ROOT_KEY = 'ROOT'
 #: Valid input/output filetypes
 FILE_TYPES = {MDF_KEY, ROOT_KEY}
 
-
-
 import contextlib
 
+
 @contextlib.contextmanager
-def _cwd( d ):
-    cwd= os.getcwd()
+def _cwd(d):
+    cwd = os.getcwd()
     os.chdir(d)
-    try: yield
-    finally: os.chdir(cwd)
+    try:
+        yield
+    finally:
+        os.chdir(cwd)
 
 
 @contextlib.contextmanager
-def _lockfile( name, attempts = 120, sleeptime = 1 ):
-    for i in range(attempts): 
-        try: 
+def _lockfile(name, attempts=120, sleeptime=1):
+    for i in range(attempts):
+        try:
             # open with  O_EXCL should be atomic for Linux > 2.6.5 and NFS v3... (see http://nfs.sourceforge.net/#faq_d10)
-            with open(name,'+x') as f : 
-                f.write('locked by pid = {}, host = {}'.format(os.getpid(),os.uname()[1] ) )
+            with open(name, '+x') as f:
+                f.write('locked by pid = {}, host = {}'.format(
+                    os.getpid(),
+                    os.uname()[1]))
                 f.flush()
-                try: yield
-                finally: os.remove(name)
+                try:
+                    yield
+                finally:
+                    os.remove(name)
             return
         except OSError:
-            print('waiting in order to acquire lock ',name,)
+            print(
+                'waiting in order to acquire lock ',
+                name,
+            )
             # time.sleep(min(1.4**i,20))
             time.sleep(sleeptime)
-    raise RuntimeError("failed to acquire lockfile {} after {} attempts".format(name,attempts))
-
-def _get_commit_for_branch( path, branch ) :
-    return subprocess.run( ['git', '--git-dir', path, 'rev-parse', branch ] ,
-        capture_output=True, check=False, text=True).stdout.strip()
-
-def _get_hash_for_text( txt, write_to_odb = False, path = None ) :
-    if write_to_odb and path is None : raise RuntimeError("write_to_odb is True, but no path given")
-    cmd = ['git'] + ( [ '--git-dir',path  ] if write_to_odb else [] )+ [ 'hash-object' ] + ( ['-w'] if write_to_odb else [] ) + ['--stdin']
-    oid = subprocess.run(cmd, text=True, capture_output=True, input=txt).stdout.strip()
+    raise RuntimeError(
+        "failed to acquire lockfile {} after {} attempts".format(
+            name, attempts))
+
+
+def _get_commit_for_branch(path, branch):
+    return subprocess.run(['git', '--git-dir', path, 'rev-parse', branch],
+                          capture_output=True,
+                          check=False,
+                          text=True).stdout.strip()
+
+
+def _get_hash_for_text(txt, write_to_odb=False, path=None):
+    if write_to_odb and path is None:
+        raise RuntimeError("write_to_odb is True, but no path given")
+    cmd = ['git'] + (['--git-dir', path] if write_to_odb else []) + [
+        'hash-object'
+    ] + (['-w'] if write_to_odb else []) + ['--stdin']
+    oid = subprocess.run(
+        cmd, text=True, capture_output=True, input=txt).stdout.strip()
     assert len(oid) == 40, "invalid oid: {}".format(oid)
     return oid
 
-def _commit_oid_to_branch( path, branch, branch_name, oid ) :
+
+def _commit_oid_to_branch(path, branch, branch_name, oid):
     key = oid[:8]
-    commit = _get_commit_for_branch( path, branch )
-    assert len(commit) == 40, "invalid commit id : {} for branch {}".format(commit,branch)
+    commit = _get_commit_for_branch(path, branch)
+    assert len(commit) == 40, "invalid commit id : {} for branch {}".format(
+        commit, branch)
     with _cwd(path.removesuffix('/.git')):
         # check that the target branch does _not_ exist
-        result = subprocess.run(['git','--git-dir',path,'rev-parse','--quiet','--verify',branch_name],check=False)
-        if result.returncode == 0 : # we want the above rev-parse to _fail_
-            raise RuntimeError("requested branch {} already exists -- aborting",branch_name)
+        result = subprocess.run([
+            'git', '--git-dir', path, 'rev-parse', '--quiet', '--verify',
+            branch_name
+        ],
+                                check=False)
+        if result.returncode == 0:  # we want the above rev-parse to _fail_
+            raise RuntimeError(
+                "requested branch {} already exists -- aborting", branch_name)
         # make sure we are in a detached head by checking out an explicit commit
-        subprocess.run(['git','--git-dir',path,'checkout','-b',branch_name,commit],capture_output=True,text=True,check=True)
-        # grab the content of the specified oid and create the corresponding file 
-        result = subprocess.run(['git','--git-dir',path,'cat-file','-p',oid],capture_output=True,text=True,check=True)
-        if result.returncode !=0 : raise RuntimeError("could not get content for oid {} ".format(oid))
+        subprocess.run(
+            ['git', '--git-dir', path, 'checkout', '-b', branch_name, commit],
+            capture_output=True,
+            text=True,
+            check=True)
+        # grab the content of the specified oid and create the corresponding file
+        result = subprocess.run(
+            ['git', '--git-dir', path, 'cat-file', '-p', oid],
+            capture_output=True,
+            text=True,
+            check=True)
+        if result.returncode != 0:
+            raise RuntimeError("could not get content for oid {} ".format(oid))
         fname = _key_to_file(key)
-        os.makedirs( os.path.dirname(fname),exist_ok=True) 
-        with open( fname, 'w') as f: f.write( result.stdout )
-        subprocess.run(['git','--git-dir',path,'add',fname],capture_output=True,check=True)
-        subprocess.run(['git','--git-dir',path,'commit','-m','automated commit to define key {}'.format(key),fname],capture_output=True,check=True)
+        os.makedirs(os.path.dirname(fname), exist_ok=True)
+        with open(fname, 'w') as f:
+            f.write(result.stdout)
+        subprocess.run(['git', '--git-dir', path, 'add', fname],
+                       capture_output=True,
+                       check=True)
+        subprocess.run([
+            'git', '--git-dir', path, 'commit', '-m',
+            'automated commit to define key {}'.format(key), fname
+        ],
+                       capture_output=True,
+                       check=True)
 
-def _commit_oid_to_branch2( path, branch, oid ) :
-    key = oid[:8]
-    commit = _get_commit_for_branch( path, branch )
-    assert len(commit) == 40, "invalid commit id : {} for branch {}".format(commit,branch)
-    ### use GIT_INDEX_FILE in env to not interfere with other process which could be writing as well
-    my_env = os.environ.copy()
-    my_env["GIT_INDEX_FILE"] = "{}/index-file-{}-{}-{}".format( tempfile.gettempdir(), os.getpid(),commit, oid)
-    subprocess.check_output([ 'git', '--git-dir', path, 'reset', commit], env=my_env)
-    files = subprocess.run([ 'git', '--git-dir', path, 'ls-tree','--full-tree','--name-only','-r',commit], env=my_env,
-        capture_output=True,check=True,text=True).stdout
-    subprocess.check_output([ 'git', '--git-dir', path, 'update-index','--skip-worktree','--stdin'], env=my_env,
-        input=files,text=True)
-    subprocess.check_output([ 'git', '--git-dir', path, 'update-index', '--add', '--cacheinfo', '100644', oid, _key_to_file(key) ], env=my_env)
-    result = subprocess.run(['git', '--git-dir', path, 'write-tree'],
-                          capture_output=True, check=False, text=True,
-                          env=my_env)
-    if result.returncode != 0 : raise RuntimeError("failed to write-tree: " + result.stderr)
-    tree = result.stdout.strip()
-    assert len(tree) == 40, "invalid tree id: {}".format(tree)
-    new_commit = subprocess.run(
-        ['git', '--git-dir', path, 'commit-tree', tree, '-p', commit],
-        text=True, check=True,
-        input='automated commit to define key ' + key,
-        capture_output=True).stdout.strip()
-    assert len(new_commit) == 40, "invalid commit id: {}".format(commit)
-
-    return new_commit
 
 def get_metainfo_repos():
     repo = os.environ.get('LHCbFileContentMetaDataRepo', None)
@@ -181,13 +195,12 @@ def get_metainfo_repos():
 
 def _key_to_file(key):
     if not isinstance(key, str): key = '{:08x}'.format(key)
-    return "json/{0:.2}/{0}.json".format(key)
+    return "ann/json/{0:.2}/{0}.json".format(key)
 
 
 @configurable
-def retrieve_encoding_dictionary(key, tag='HEAD', fmt='{0}:{1}'):
+def retrieve_encoding_dictionary(key, tag='master', fmt='{0}:{1}'):
     for path in get_metainfo_repos():
-        # TODO: check the entire _stack_ of repos
         result = subprocess.run([
             'git', '--git-dir', path, 'show',
             fmt.format(tag, _key_to_file(key))
@@ -204,7 +217,6 @@ def retrieve_encoding_dictionary(key, tag='HEAD', fmt='{0}:{1}'):
             ]:
                 d[k] = {int(i): j for i, j in d[k].items()}
             return d
-    # print(" could not resolve ",key," -> ",_key_to_file(key)," in ",get_metainfo_repos())
     return None