Commit a63bfd7b authored by Maciej Pawel Szymanski's avatar Maciej Pawel Szymanski
Browse files

Merge branch 'prepare-for-new-scripts' into 'master'

Add "-legacy" aliases for lbn-{checkout,build,test}

See merge request lhcb-core/LbNightlyTools!295
parents 397dcf4e f622277c
......@@ -19,6 +19,9 @@
.DS_Store
._.DS_Store
x86_64-*-*-*/
/venv/
/venv2/
/venv3/
# setuptools artifacts
dist/
......
......@@ -850,3 +850,7 @@ def genBuildReport(build_log,
write_json(jreport, join(output_dir, 'report.json'))
write_report_index(jreport, output_dir)
return jreport
def run():
return Script().run()
......@@ -357,3 +357,7 @@ class Script(BaseScript):
donetime - self.starttime)
return 0
def run():
return Script().run()
......@@ -361,3 +361,7 @@ class Poll(BaseScript):
d.update('ready-builds', strip_removed_entries)
self.log.info('triggered %d tests, %d pending', len(triggered),
len(entries) - len(to_drop))
def run():
return Script().run()
This diff is collapsed.
#!/usr/bin/env python
###############################################################################
# (c) Copyright 2013 CERN #
# (c) Copyright 2013-2020 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
......@@ -9,182 +8,180 @@
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
'''
Run a LHCbPR job
'''
__author__ = 'Ben Couturier <ben.couturier@cern.ch>'
# We first try to import from LbCommon, then revert to the old package (LbUtils)
# if needed
try:
from LbCommon.Script import PlainScript as _PlainScript
except:
from LbUtils.Script import PlainScript as _PlainScript
import sys
import os
from datetime import datetime
from LbPR.LbPRJobManager import JobManager
from LbNightlyTools import Configuration
def getProjectVersionFromConfig(config, project):
''' Look up the project version for SetupProject '''
retval = None
for proj in config[u'projects']:
if proj['name'].lower() == project.lower():
retval = proj['version']
break
return retval
def getSlotDateFromConfig(config):
completed = config["completed"]
return datetime.strptime(completed, '%Y-%m-%dT%H:%M:%S.%f')
class Script(_PlainScript):
def get_command():
'''
Script to create the commands to run a LHCbPR Job
Run a LHCbPR job
'''
__usage__ = '%prog <command> <project> <version> <platform>' \
' <options> <setup-options> <slot config>'
__version__ = ''
def defineOpts(self):
'''Define options.'''
from LbNightlyTools.Scripts.Common import addBasicOptions
self.parser.add_option(
'-u',
'--url',
action='store',
help='URL for the LHCbPR REST service')
self.parser.add_option(
'-c',
'--check-ssl',
action='store_true',
help='Check SSL certificate.')
self.parser.add_option(
'-o',
'--output',
action='store',
help='output file name '
'[default: runlhcbpr.sh]')
addBasicOptions(self.parser)
@staticmethod
def interpretOptions(options, executable, app_name, app_version, platform):
if not executable:
return options["content"]
return executable["content"].format(
app_name=app_name,
app_version=app_version,
platform=platform,
build="$(pwd)/../build",
options=options["content"])
@staticmethod
def interpretSetup(setup, app_name, app_version):
if not setup:
return ''
return setup["content"].format(
app_name=app_name, app_version=app_version)
@staticmethod
def interpretSetupArgs(setupcontent):
if setupcontent:
return "--setup-name='{setup[description]}' --setup-content='{setup[content]}'".format(
setup=setupcontent)
return ""
@staticmethod
def interpretExecArgs(options):
if not options:
return ""
return (
"--exec-name='{ex[name]}' --exec-content='{ex[content]}'".format(
ex=options))
def main(self):
__author__ = 'Ben Couturier <ben.couturier@cern.ch>'
# We first try to import from LbCommon, then revert to the old package (LbUtils)
# if needed
try:
from LbCommon.Script import PlainScript as _PlainScript
except:
from LbUtils.Script import PlainScript as _PlainScript
import sys
import os
from datetime import datetime
from LbPR.LbPRJobManager import JobManager
from LbNightlyTools import Configuration
def getProjectVersionFromConfig(config, project):
''' Look up the project version for SetupProject '''
retval = None
for proj in config[u'projects']:
if proj['name'].lower() == project.lower():
retval = proj['version']
break
return retval
def getSlotDateFromConfig(config):
completed = config["completed"]
return datetime.strptime(completed, '%Y-%m-%dT%H:%M:%S.%f')
class Script(_PlainScript):
'''
Main function of the script.
Script to create the commands to run a LHCbPR Job
'''
__usage__ = '%prog <command> <project> <version> <platform>' \
' <options> <setup-options> <slot config>'
__version__ = ''
def defineOpts(self):
'''Define options.'''
from LbNightlyTools.Scripts.Common import addBasicOptions
self.parser.add_option(
'-u',
'--url',
action='store',
help='URL for the LHCbPR REST service')
self.parser.add_option(
'-c',
'--check-ssl',
action='store_true',
help='Check SSL certificate.')
self.parser.add_option(
'-o',
'--output',
action='store',
help='output file name '
'[default: runlhcbpr.sh]')
addBasicOptions(self.parser)
@staticmethod
def interpretOptions(options, executable, app_name, app_version,
platform):
if not executable:
return options["content"]
return executable["content"].format(
app_name=app_name,
app_version=app_version,
platform=platform,
build="$(pwd)/../build",
options=options["content"])
@staticmethod
def interpretSetup(setup, app_name, app_version):
if not setup:
return ''
return setup["content"].format(
app_name=app_name, app_version=app_version)
@staticmethod
def interpretSetupArgs(setupcontent):
if setupcontent:
return "--setup-name='{setup[description]}' --setup-content='{setup[content]}'".format(
setup=setupcontent)
return ""
# Checking the arguments
if len(self.args) != 6:
self.parser.error('Please specify <project> <version> <platform> '
'<options> <setup-options> <slot config>')
exit(1)
application = self.args[0]
version = self.args[1]
platform = self.args[2]
job_options = self.args[3]
# Handler | Executable
# Handler | Executable | Setup
testenv = self.args[4].split("|")
executuable_options = testenv[0]
handlers = testenv[1]
setup_options = None if len(testenv) < 3 else testenv[2]
slot_config = self.args[5]
# Parsing the slot config to extract relevant information
self.log.info("Parsing config file %s" % slot_config)
config = Configuration.load(slot_config)
project_version = getProjectVersionFromConfig(config, application)
slot_completed = getSlotDateFromConfig(config)
# print("SASHA", slot_completed)
self.log.info("Using %s %s", application, project_version)
# Now create the interface with LHCbPR
manager = JobManager(self.options.url, self.options.check_ssl)
try:
# Get the actual options based on description
optionscontent = manager.getJobOptions(job_options)
# Get the actual executable
executablecontent = manager.getExecutableOptions(
executuable_options)
# Get the actual options based on description
setupcontent = manager.getSetupOptions(setup_options)
except Exception as exc:
self.log.error(
"Could not get job description options "
"from LHCbPR service %s: %s", self.options.url, str(exc))
raise exc
runfilename = "runlhcbpr.sh"
if self.options.output != None:
runfilename = self.options.output
self.log.warning("Writing file: %s", runfilename)
try:
with open(runfilename, 'w') as runfile:
# runfile.write("\n")
# lblogin_cmd = ". LbLogin.sh -c %s" % platform
setup_cmd = self.interpretSetup(
setup=setupcontent,
app_name=application,
app_version=project_version)
run_cmd = self.interpretOptions(
options=optionscontent,
executable=executablecontent,
app_name=application,
app_version=project_version,
platform=platform)
print(run_cmd, optionscontent)
setup_args = self.interpretSetupArgs(setupcontent)
exec_args = self.interpretExecArgs(executablecontent)
runfile.write('''
@staticmethod
def interpretExecArgs(options):
if not options:
return ""
return ("--exec-name='{ex[name]}' --exec-content='{ex[content]}'".
format(ex=options))
def main(self):
'''
Main function of the script.
'''
# Checking the arguments
if len(self.args) != 6:
self.parser.error(
'Please specify <project> <version> <platform> '
'<options> <setup-options> <slot config>')
exit(1)
application = self.args[0]
version = self.args[1]
platform = self.args[2]
job_options = self.args[3]
# Handler | Executable
# Handler | Executable | Setup
testenv = self.args[4].split("|")
executuable_options = testenv[0]
handlers = testenv[1]
setup_options = None if len(testenv) < 3 else testenv[2]
slot_config = self.args[5]
# Parsing the slot config to extract relevant information
self.log.info("Parsing config file %s" % slot_config)
config = Configuration.load(slot_config)
project_version = getProjectVersionFromConfig(config, application)
slot_completed = getSlotDateFromConfig(config)
# print("SASHA", slot_completed)
self.log.info("Using %s %s", application, project_version)
# Now create the interface with LHCbPR
manager = JobManager(self.options.url, self.options.check_ssl)
try:
# Get the actual options based on description
optionscontent = manager.getJobOptions(job_options)
# Get the actual executable
executablecontent = manager.getExecutableOptions(
executuable_options)
# Get the actual options based on description
setupcontent = manager.getSetupOptions(setup_options)
except Exception as exc:
self.log.error(
"Could not get job description options "
"from LHCbPR service %s: %s", self.options.url, str(exc))
raise exc
runfilename = "runlhcbpr.sh"
if self.options.output != None:
runfilename = self.options.output
self.log.warning("Writing file: %s", runfilename)
try:
with open(runfilename, 'w') as runfile:
# runfile.write("\n")
# lblogin_cmd = ". LbLogin.sh -c %s" % platform
setup_cmd = self.interpretSetup(
setup=setupcontent,
app_name=application,
app_version=project_version)
run_cmd = self.interpretOptions(
options=optionscontent,
executable=executablecontent,
app_name=application,
app_version=project_version,
platform=platform)
print(run_cmd, optionscontent)
setup_args = self.interpretSetupArgs(setupcontent)
exec_args = self.interpretExecArgs(executablecontent)
runfile.write('''
#!/usr/bin/env bash
#
# File generated by lbpr-get-command to run the LHCbPR Job
......@@ -194,7 +191,7 @@ set -v
# Setting the CMTPROJECTPATH for the software installed locally
if [ -f build/setupSearchPath.sh ]; then
. build/setupSearchPath.sh
. build/setupSearchPath.sh
fi
export User_release_area=$(pwd)/build
# Setting the environment and cleanup
......@@ -205,12 +202,12 @@ echo $CMTCONFIG > platform.txt
# Add valgrind from AFS to the PATH
if [ -f /afs/cern.ch/lhcb/group/rich/vol4/jonrob/scripts/new-valgrind.sh ]; then
source /afs/cern.ch/lhcb/group/rich/vol4/jonrob/scripts/new-valgrind.sh
source /afs/cern.ch/lhcb/group/rich/vol4/jonrob/scripts/new-valgrind.sh
fi
# Adding jemalloc for the perf jobs
if [ -d /opt/dirac/lhcbpr/sw/jemalloc-3.6.0/lib ]; then
export LD_LIBRARY_PATH=/opt/dirac/lhcbpr/sw/jemalloc-3.6.0/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/opt/dirac/lhcbpr/sw/jemalloc-3.6.0/lib:$LD_LIBRARY_PATH
fi
# Get kerberos ticket
......@@ -222,7 +219,7 @@ set -o pipefail
date +"%Y-%m-%d %T %z" > start.txt
time_start=`cat start.txt`
if [ -d output ]; then
rm -rf output
rm -rf output
fi
mkdir output
pushd $(pwd)
......@@ -237,10 +234,10 @@ import logging
from LbNightlyTools.Utils import Dashboard
logging.basicConfig(level=logging.DEBUG)
dash = Dashboard(credentials=None,
flavour='periodic')
flavour='periodic')
build_id = sys.argv[2] + "_" + sys.argv[3] + "_" + sys.argv[4]
if 'BUILD_ID' in os.environ:
build_id = os.environ.get('BUILD_ID')
build_id = os.environ.get('BUILD_ID')
doc_name = build_id + "." + sys.argv[1]
data_dict = dict()
if 'BUILD_URL' in os.environ:
......@@ -256,16 +253,16 @@ END`
# Set environment for throughput tests
if [[ '{handlers}' == *"Throughput"* ]] ; then
source /cvmfs/projects.cern.ch/intelsw/psxe/linux/all-setup.sh
if [ -d ../lhcb-benchmark-scripts ] ; then
source /cvmfs/projects.cern.ch/intelsw/psxe/linux/all-setup.sh
if [ -d ../lhcb-benchmark-scripts ] ; then
(cd ../lhcb-benchmark-scripts
git fetch --all
git reset --hard origin/master
git fetch --all
git reset --hard origin/master
)
else
else
git clone --quiet ssh://git@gitlab.cern.ch:7999/lhcb-core/lhcb-benchmark-scripts.git ../lhcb-benchmark-scripts
fi
ln -sfv ../lhcb-benchmark-scripts/* .
fi
ln -sfv ../lhcb-benchmark-scripts/* .
fi
if [ -z "$LHCBPR_MOCK_OUTPUT_PATH" ]
......@@ -319,10 +316,10 @@ import subprocess
from LbNightlyTools.Utils import Dashboard
logging.basicConfig(level=logging.DEBUG)
dash = Dashboard(credentials=None,
flavour='periodic')
flavour='periodic')
build_id = sys.argv[2] + "_" + sys.argv[3] + "_" + sys.argv[4]
if 'BUILD_ID' in os.environ:
build_id = os.environ.get('BUILD_ID')
build_id = os.environ.get('BUILD_ID')
doc_name = build_id + "." + sys.argv[1]
data_dict = dict()
data_dict['time_end'] = sys.argv[5] + "_" + sys.argv[6] + "_" + sys.argv[7]
......@@ -352,21 +349,19 @@ dash.update(doc_name, data_dict)
END`
exit $RETCODE
'''.format(
setup_cmd=setup_cmd,
run_cmd=run_cmd,
options=optionscontent,
setup_args=setup_args,
exec_args=exec_args,
handlers=handlers,
app_name=application,
app_version=version,
app_version_datetime=slot_completed.strftime(
"%Y-%m-%d %H:%M:%S -0400")))
except Exception as exc:
self.log.error("Error generating runfile: %s", str(exc))
raise exc
# __main__
sys.exit(Script().run())
'''.format(
setup_cmd=setup_cmd,
run_cmd=run_cmd,
options=optionscontent,
setup_args=setup_args,
exec_args=exec_args,
handlers=handlers,
app_name=application,
app_version=version,
app_version_datetime=slot_completed.strftime(
"%Y-%m-%d %H:%M:%S -0400")))
except Exception as exc:
self.log.error("Error generating runfile: %s", str(exc))
raise exc
return Script().run()
###############################################################################
# (c) Copyright 2013-2020 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
def check_periodic_tests():
from LbNightlyTools.Utils import JenkinsTest
'''
Simple script to check which tests should be run for a given date
'''
__author__ = 'Ben Couturier <ben.couturier@cern.ch>'
import datetime
import json
import logging
import sys
import re
from LbNightlyTools.Scripts.Common import PlainScript
from LbPeriodicTools.LbPeriodicStarter import PeriodicTestStarter
class Script(PlainScript):
'''
Script to print the list of tests to run on a given day,
based on the config specified
'''
__usage__ = '%prog [options] <config.json>'
__version__ = ''
def defineOpts(self):
'''Define options.'''
from LbNightlyTools.Scripts.Common import addBasicOptions
self.parser.add_option(
'-o',
'--output',
action='store',
help='output file format '
'[default: test-params-{0}.txt]',
default='test-params-{0}.txt')
self.parser.add_option(
'-d',
'--date',
action='store',
help='Date for the tests '
'Format: YYYY-MM-dd HH:MM [default: today]')
self.parser.add_option(
'-i',
'--interval',
action='store',
help='Interval for test checks in seconds '
'[default: 60s]',
default="60")
addBasicOptions(self.parser)
def main(self):
'''
Main function of the script.
'''
# Checking we did pass an argument
if len(self.args) != 1:
self.parser.error('Please specify config file')
config_file = self.args[0]
# Checking the date at which to run
opts = self.options
testdate = datetime.datetime.today()
if opts.date:
testdate = datetime.datetime.strptime(opts.date,
'%Y-%m-%d %H:%M')
testdateend = testdate + datetime.timedelta(
seconds=int(opts.interval))
self.log.warning(
"Running tests from %s for the period %s/%s" %
(config_file, testdate.strftime('%Y-%m-%d %H:%M:%S'),
testdateend.strftime('%Y-%m-%d %H:%M:%S')))
# Checking which jobs to run
starter = PeriodicTestStarter(
config_file, testdate.strftime('%Y-%m-%d %H:%M:%S'),
int(opts.interval))
all_tests = starter.getAllTests()
tests_to_run = []
for (test_template, test_list) in all_tests:
self.log.warning("%s: %d actual tests to run" %
(str(test_template), len(test_list)))
for test_instance in test_list:
#tests_to_run.append(test_instance.__dict__)
tests_to_run.append(test_instance)
for idx, ttr in enumerate(tests_to_run):
jenkins_test = JenkinsTest.fromScheduledTest(ttr)
tests_node = re.