Frequent test issues related to git metadata repo
Here are a few examples from recent tests:
- https://lhcb-nightlies.web.cern.ch/nightly/lhcb-master-mr/5711/Moore/x86_64_v3-centos7-gcc11-opt+g/tests
- https://lhcb-nightlies.web.cern.ch/nightly/lhcb-master-mr/5705/Moore/x86_64_v2-centos7-gcc11-opt/tests
- https://lhcb-nightlies.web.cern.ch/nightly/lhcb-master-mr/5708/Moore/x86_64_v2-centos7-gcc11-dbg/tests
Most tests fail with exit code 247. Whenever there is a stderr, it looks something like this
Traceback (most recent call last):
File "/workspace/build/LHCb/InstallArea/x86_64_v3-centos7-gcc11-opt+g/python/PyConf/filecontent_metadata.py", line 228, in _lockfile
yield
File "/workspace/build/LHCb/InstallArea/x86_64_v3-centos7-gcc11-opt+g/python/PyConf/filecontent_metadata.py", line 108, in _flush_keys_to_git
_merge_branch_to_master(path, branch_name)
File "/workspace/build/LHCb/InstallArea/x86_64_v3-centos7-gcc11-opt+g/python/PyConf/filecontent_metadata.py", line 438, in _merge_branch_to_master
subprocess.run(['git', '--git-dir', path, 'reset', '--hard'],
File "/cvmfs/lhcb.cern.ch/lib/lcg/releases/Python/3.9.6-b0f98/x86_64-centos7-gcc11-opt/lib/python3.9/subprocess.py", line 505, in run
with Popen(*popenargs, **kwargs) as process:
File "/cvmfs/lhcb.cern.ch/lib/lcg/releases/Python/3.9.6-b0f98/x86_64-centos7-gcc11-opt/lib/python3.9/subprocess.py", line 951, in __init__
self._execute_child(args, executable, preexec_fn, close_fds,
File "/cvmfs/lhcb.cern.ch/lib/lcg/releases/Python/3.9.6-b0f98/x86_64-centos7-gcc11-opt/lib/python3.9/subprocess.py", line 1754, in _execute_child
self.pid = _posixsubprocess.fork_exec(
OSError: [Errno 12] Cannot allocate memory
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/workspace/build/Gaudi/InstallArea/x86_64_v3-centos7-gcc11-opt+g/bin/gaudirun.py", line 584, in <module>
exec(o, g, l)
File "<string>", line 1, in <module>
File "/workspace/build/Gaudi/InstallArea/x86_64_v3-centos7-gcc11-opt+g/bin/gaudirun.py", line 543, in __call__
importOptions(arg)
File "/workspace/build/Gaudi/InstallArea/x86_64_v3-centos7-gcc11-opt+g/python/GaudiKernel/ProcessJobOptions.py", line 552, in importOptions
_import_function_mapping[ext](optsfile)
File "/workspace/build/Gaudi/InstallArea/x86_64_v3-centos7-gcc11-opt+g/python/GaudiKernel/ProcessJobOptions.py", line 486, in _import_python
exec(code, {})
File "/workspace/build/Moore/Hlt/Hlt2Conf/options/hlt2_all_lines_with_reco_for_config.py", line 59, in <module>
config = run_moore(options, make_lines, public_tools)
File "/workspace/build/Moore/Hlt/Moore/python/Moore/config.py", line 680, in run_moore
configure(
File "/workspace/build/LHCb/InstallArea/x86_64_v3-centos7-gcc11-opt+g/python/PyConf/application.py", line 1021, in configure
if options.write_decoding_keys_to_git: key_registry.flush_to_git()
File "/workspace/build/LHCb/InstallArea/x86_64_v3-centos7-gcc11-opt+g/python/PyConf/filecontent_metadata.py", line 191, in flush_to_git
_flush_keys_to_git(
File "/workspace/build/LHCb/InstallArea/x86_64_v3-centos7-gcc11-opt+g/python/PyConf/filecontent_metadata.py", line 121, in _flush_keys_to_git
raise RuntimeError('readback not identical to input???')
File "/cvmfs/lhcb.cern.ch/lib/lcg/releases/Python/3.9.6-b0f98/x86_64-centos7-gcc11-opt/lib/python3.9/contextlib.py", line 166, in __exit__
raise RuntimeError("generator didn't stop after throw()")
RuntimeError: generator didn't stop after throw()
Traceback (most recent call last):
File "/workspace/build/Gaudi/InstallArea/x86_64_v2-centos7-gcc11-opt/bin/gaudirun.py", line 584, in <module>
exec(o, g, l)
File "<string>", line 1, in <module>
File "/workspace/build/Gaudi/InstallArea/x86_64_v2-centos7-gcc11-opt/bin/gaudirun.py", line 543, in __call__
importOptions(arg)
File "/workspace/build/Gaudi/InstallArea/x86_64_v2-centos7-gcc11-opt/python/GaudiKernel/ProcessJobOptions.py", line 552, in importOptions
_import_function_mapping[ext](optsfile)
File "/workspace/build/Gaudi/InstallArea/x86_64_v2-centos7-gcc11-opt/python/GaudiKernel/ProcessJobOptions.py", line 486, in _import_python
exec(code, {})
File "/workspace/build/Moore/Hlt/Hlt2Conf/options/hlt2_ganga.py", line 39, in <module>
run_moore(options, make_lines, public_tools)
File "/workspace/build/Moore/Hlt/Moore/python/Moore/config.py", line 696, in run_moore
configure(
File "/workspace/build/LHCb/InstallArea/x86_64_v2-centos7-gcc11-opt/python/PyConf/application.py", line 1021, in configure
if options.write_decoding_keys_to_git: key_registry.flush_to_git()
File "/workspace/build/LHCb/InstallArea/x86_64_v2-centos7-gcc11-opt/python/PyConf/filecontent_metadata.py", line 191, in flush_to_git
_flush_keys_to_git(
File "/workspace/build/LHCb/InstallArea/x86_64_v2-centos7-gcc11-opt/python/PyConf/filecontent_metadata.py", line 79, in _flush_keys_to_git
with _lockfile(os.path.join(path, 'PyConf.lock')):
File "/cvmfs/lhcb.cern.ch/lib/lcg/releases/Python/3.9.6-b0f98/x86_64-centos7-gcc11-opt/lib/python3.9/contextlib.py", line 117, in __enter__
return next(self.gen)
File "/workspace/build/LHCb/InstallArea/x86_64_v2-centos7-gcc11-opt/python/PyConf/filecontent_metadata.py", line 236, in _lockfile
raise RuntimeError(
RuntimeError: failed to acquire lockfile /workspace/build/Moore/build/file-content-metadata//.git/PyConf.lock after 120 attempts