diff --git a/jenkins/periodic-tests/tests-pollqueue.sh b/jenkins/periodic-tests/tests-pollqueue.sh index 4948dd7ea18ad400caaa838ef644378427eeb3f4..dfe6891bca24f2b9d168ddb1cd1df778f3cb0939 100755 --- a/jenkins/periodic-tests/tests-pollqueue.sh +++ b/jenkins/periodic-tests/tests-pollqueue.sh @@ -12,7 +12,15 @@ . $(dirname $0)/../utils.sh -set_common --build +set_common -lbq-getteststorun -q JenkinsPeriodic -j +# ensure that we do not use stale configuration files +# (unless we are testing with jenkins/mock.sh) +if [ "${JENKINS_MOCK}" != true ] ; then + rm -rf configs +fi +# checkout configs only if missing +[ -e configs ] || lbn-get-configs + +lbp-check-periodic-tests-msg configs/test_schedule2.xml -q PeriodicJenkins diff --git a/python/LbMsg/BuildMsg.py b/python/LbMsg/BuildMsg.py index bc27b781f6b88f108afbf4eac20f0eb36a76ba4b..2d1f035fe001f197414f1715bd5adf876e89be70 100644 --- a/python/LbMsg/BuildMsg.py +++ b/python/LbMsg/BuildMsg.py @@ -33,12 +33,14 @@ class NightliesMessenger(Messenger): self._topic_name = "topic.build_ready" - def sendBuildDone(self, slot, project, config, buildId, date=datetime.datetime.now()): + def sendBuildDone(self, slot, project, config, + buildId, date=datetime.datetime.now()): ''' Sends the message that a particular project has been built ''' self._basicPublish(".".join([slot, project, config]), - json.dumps([slot, project, config, buildId])) + json.dumps([{'slot':slot,'project':project, + 'platform':config,'build_id':buildId}])) def getBuildsDone(self, queueName=None, bindingKeys=None): @@ -47,17 +49,18 @@ class NightliesMessenger(Messenger): ''' def callback(ch, method, properties, body): print("%r\t%r" % (method.routing_key, body)) - + buildsDone = [] with self._getConnection() as connection: - (channel, queueName) = self._setupClientChannel(connection.channel(), - queueName, bindingKeys) + (channel,queueName)=self._setupClientChannel(connection.channel(), + queueName,bindingKeys) while True: - method_frame, header_frame, body = channel.basic_get(queue=queueName) + method_frame,head_frame,body=channel.basic_get(queue=queueName) if method_frame == None: break - print method_frame.routing_key, body + print method_frame.routing_key, json.loads(body) + buildsDone.append(json.loads(body)[0]) channel.basic_ack(method_frame.delivery_tag) - + return buildsDone def consumeBuildsDone(self, callback, queueName=None, bindingKeys=None): ''' @@ -68,8 +71,8 @@ class NightliesMessenger(Messenger): ''' with self._getConnection() as connection: - (channel, queueName) = self._setupClientChannel(connection.channel(), - queueName, bindingKeys) + (channel,queueName)=self._setupClientChannel(connection.channel(), + queueName,bindingKeys) channel.basic_consume(callback, queue=queueName, no_ack=True) diff --git a/python/LbNightlyTools/Scripts/Common.py b/python/LbNightlyTools/Scripts/Common.py index 1dcb97cfedecdb4ee471adbc6f97c2032667e626..d1f71ebff1f4cf4477b552d984779fc7e1ac2615 100644 --- a/python/LbNightlyTools/Scripts/Common.py +++ b/python/LbNightlyTools/Scripts/Common.py @@ -523,6 +523,38 @@ class DashboardUpdate2(object): else: self.project_test_started(msg) +class PeriodicTestMsg(object): + ''' + Handles sending the message to the queue + of ready builds for the periodic tests. + ''' + def __init__(self, script): + self.slot = script.slot + self.platform = script.platform + self.log = logging.getLogger('PeriodicTestMsg') + + def builds_ready(self, msg): + ''' + sends the message that builds are ready + ''' + import LbMsg.BuildMsg + self.log.debug('sending the message to the queue that builds ' + 'are ready for slot: %s, project: %s, platform: %s, ' + 'build_id: %s', self.slot.name, msg['project'].name, + self.platform, self.slot.build_id) + build_msg = LbMsg.BuildMsg.NightliesMessenger() + build_msg.sendBuildDone(self.slot.name, msg['project'].name, + self.platform, self.slot.build_id) + + def accept(self, msg): + ''' + Implements the receiver side of the message protocol. + ''' + msg_type = msg.get('type', '') + + if msg_type == 'ready.artifacts.build': + self.builds_ready(msg) + class BaseScript(PlainScript): ''' @@ -550,6 +582,7 @@ class BaseScript(PlainScript): if hasattr(self.options, 'submit'): self.receivers.append(DashboardUpdate(self)) self.receivers.append(DashboardUpdate2(self)) + self.receivers.append(PeriodicTestMsg(self)) def _setup(self, build_dir=None, json_type=None, make_dirs=True): ''' diff --git a/python/LbPeriodicTools/LbPeriodicStarter.py b/python/LbPeriodicTools/LbPeriodicStarter.py index 409ce1c6ecb74ad3c9b09b7a38dc2360e73539bc..32e19cd9445d0a87c8b7494a64c85125189d9fe3 100644 --- a/python/LbPeriodicTools/LbPeriodicStarter.py +++ b/python/LbPeriodicTools/LbPeriodicStarter.py @@ -68,9 +68,12 @@ class PeriodicTestStarter (object): alltests = PeriodicTestSchedule(schedule_filename) # Select the tests to be run at the date - to_run = alltests.getTests(lambda x: - x.isForDate(self._testdatetime, - self._testdatetime_end)) + if testperiod < 0: + to_run = alltests.getTests(None) + else: + to_run = alltests.getTests(lambda x: + x.isForDate(self._testdatetime, + self._testdatetime_end)) __log__.info("Found %d tests schedule to be run." % len(to_run)) for test in to_run: __log__.info(str(test)) diff --git a/scripts/lbp-check-periodic-tests-msg b/scripts/lbp-check-periodic-tests-msg new file mode 100755 index 0000000000000000000000000000000000000000..31888f77ec4659cea2b73c42e1b0f2c081f79877 --- /dev/null +++ b/scripts/lbp-check-periodic-tests-msg @@ -0,0 +1,124 @@ +#!/usr/bin/env python +############################################################################### +# (c) Copyright 2013 CERN # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### +from LbNightlyTools.Utils import JenkinsTest +''' +Simple script to check which tests should be run for a given date + +''' +__author__ = 'Ben Couturier <ben.couturier@cern.ch>' + +import datetime +import json +import logging +import sys +import fnmatch +from LbNightlyTools.Scripts.Common import PlainScript +from LbPeriodicTools.LbPeriodicStarter import PeriodicTestStarter +import LbMsg.BuildMsg + +class Script(PlainScript): + ''' + Script to print the list of tests to run on a given day, + based on the config specified + ''' + __usage__ = '%prog [options] <config.json>' + __version__ = '' + + def defineOpts(self): + '''Define options.''' + from LbNightlyTools.Scripts.Common import addBasicOptions + self.parser.add_option('-o', '--output', action='store', + help='output file format ' + '[default: test-params-{0}.txt]', + default='test-params-{0}.txt') + self.parser.add_option('-d', '--date', action='store', + help='Date for the tests ' + 'Format: YYYY-MM-dd HH:MM [default: today]') + self.parser.add_option('-i', '--interval', action='store', + help='Interval for test checks in seconds ' + '[default: 60s]', default="60") + self.parser.add_option('-q', '--queue', + default=None, + help='Name of the (persistent) queue to ' + 'store the messages') + self.parser.add_option('-b', '--bindings', + default=None, + help='Message bindings for this channel') + self.parser.add_option('-c', '--consume', + action="store_true", + default=False, + help='Wait and loop on all messages coming ' + 'from the server') + + addBasicOptions(self.parser) + + + def main(self): + ''' + Main function of the script. + ''' + + # Checking we did pass an argument + if len(self.args) < 1: + self.parser.error('Please specify config file') + + config_file = self.args[0] + + # # Checking the date at which to run + opts = self.options + testdate = datetime.datetime.today() + + queueName = None + if self.options.queue: + queueName = self.options.queue + + binds = None + if self.options.bindings: + binds = [ self.options.bindings ] + + msg = LbMsg.BuildMsg.NightliesMessenger() + if self.options.consume: + def callback(ch, method, properties, body): + print(" [x] %r:%r" % (method.routing_key, body)) + msg.consumeBuildsDone(callback, queueName, binds) + else: + if queueName == None: + raise Exception('No point in just getting messages ' + 'on a newly created queue. ' + 'Name the queue with -q or use -c instead') + buildsDone = msg.getBuildsDone(queueName, binds) + + # Checking which jobs to run + starter = PeriodicTestStarter(config_file, + testdate.strftime('%Y-%m-%d %H:%M:%S'), + -1) + all_tests = starter.getAllTests() + tests_to_run = [] + idx = 0 + for (test_template, test_list) in all_tests: + for test_tmp in test_list: + for build in buildsDone: + if test_tmp.slot == build['slot']\ + and test_tmp.project == build['project']\ + and fnmatch.fnmatch(build['platform'], + test_tmp.platform): + test_tmp.build_id = build['build_id'] + tests_to_run.append(test_tmp) + print test_tmp + jenkins_test = JenkinsTest.fromScheduledTest(test_tmp) + with open(opts.output.format(idx), 'w') as parfile: + parfile.writelines(jenkins_test.getParameterLines()) + self.log.warning(opts.output.format(idx)) + idx += 1 + +# __main__ +sys.exit(Script().run())