Commit 353825f9 authored by cpomies's avatar cpomies
Browse files

Changed name of all scripts related to readyslots in enabledslots

Changed SlotBuildId.Script in a function getIds
Include getIds in EnabledSlots.py
parent 993ba0d0
......@@ -16,6 +16,4 @@ export HOME=$PWD
# Set common environment
. $(dirname $0)/common.sh
export CMTCONFIG=$platform
lbn-check-ready-slots --verbose 'slot-param-{0}.txt'
\ No newline at end of file
lbn-enabled-slots --verbose 'slot-param-{0}.txt' ${slots}
\ No newline at end of file
#!/bin/bash
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
# hack because of a bug with non-writable home (this script is run by tomcat)
export HOME=$PWD
# Set common environment
. $(dirname $0)/common.sh
export CMTCONFIG=$platform
if [ -z "${slot_build_id}" ] ; then
slot_build_id=$(lbn-slot-build-id --verbose "lhcb-test" "lhcb-dirac" "lhcb-colas")
fi
echo ${slot_build_id}
\ No newline at end of file
......@@ -18,23 +18,23 @@ __author__ = 'Colas Pomies <colas.pomies@cern.ch>'
import glob
import json
from sets import Set
from xml.etree.ElementTree import parse
from LbNightlyTools.Utils import JobParams
from os.path import splitext, basename
from LbNightlyTools.SlotBuildId import getIds
import LbUtils.Script
class Script(LbUtils.Script.PlainScript):
'''
TODO : Explain the script
'''
__usage__ = '%prog [options] <format_output_file.txt>'
__usage__ = '%prog [options] <format_output_file.txt> '
__version__ = ''
def extractFromJson(self, file_format_json):
self.log.info('Extract slots from %s files', file_format_json)
slots = Set()
slots = set()
#get all json files for slot configuration
files = glob.glob(file_format_json)
......@@ -65,24 +65,28 @@ class Script(LbUtils.Script.PlainScript):
def extractFromXml(self, config_file):
self.log.info('Extract slots from %s', config_file)
slots = Set()
try:
xmlParse = parse(config_file)
#Extract all slots name from configuration who doesn't have attribute disabled to true
self.log.debug('Get slot from %s', config_file)
slots = Set(el.get('name') for el in xmlParse.findall("slot") if el not in xmlParse.findall("slot[@disabled='true']"))
self.log.info('%s slots from %s', len(slots), config_file)
except:
self.log.warning('Can''t find or open %s', config_file)
return set()
#Extract all slots name from configuration who doesn't have [attribute disabled to true
self.log.debug('Get slot from %s', config_file)
slots = set(el.get('name')
for el in xmlParse.findall("slot")
if el.attrib.get('disabled', 'false').lower() != 'true')
self.log.info('%s slots from %s', len(slots), config_file)
return slots
def writeFiles(self, slots, output_file):
slot_ids = getIds(slots)
#get_id = lambda slot: get_ids([slot]])[slot]
for slot in slots:
output_file_name = output_file.format(slot)
open(output_file_name, 'w').write(str(JobParams(slot=slot)) + '\n')
open(output_file_name, 'w').write(str(JobParams(slot=slot,slot_build_id=slot_ids[slot])) + '\n')
self.log.debug('%s written', output_file_name)
self.log.info('%s slots to start', len(slots))
......@@ -92,12 +96,14 @@ class Script(LbUtils.Script.PlainScript):
self.log.info('Starting extraction of all enable slot')
if len(self.args) != 1:
if len(self.args) < 1:
self.parser.error('wrong number of arguments')
output_file = self.args[0]
slots = self.args[1:]
slots = self.extractFromJson('configs/lhcb-*.json') | self.extractFromXml('configs/configuration.xml')
if not slots:
slots = self.extractFromJson('configs/lhcb-*.json') | self.extractFromXml('configs/configuration.xml')
#Create a file that contain JobParams for each slot
self.writeFiles(slots, output_file)
......
......@@ -14,55 +14,75 @@ Script to get and set current slot_build_id of a slot
'''
__author__ = 'Colas Pomies <colas.pomies@cern.ch>'
import logging
import os
import sys
from xml.etree.ElementTree import parse, Element
import xml.etree.ElementTree as ET
import LbUtils.Script
class Script(LbUtils.Script.PlainScript):
'''
TODO : Explain the script
'''
__usage__ = '%prog [options] <slot1> <slot2> <slot3> ...'
__version__ = ''
def main(self):
def indent(elem, level=0):
i = "\n" + level*" "
if len(elem):
if not elem.text or not elem.text.strip():
elem.text = i + " "
if not elem.tail or not elem.tail.strip():
elem.tail = i
for elem in elem:
indent(elem, level+1)
if not elem.tail or not elem.tail.strip():
elem.tail = i
else:
if level and (not elem.tail or not elem.tail.strip()):
elem.tail = i
if len(self.args) < 1:
self.parser.error('wrong number of arguments')
def getIds(slots):
#slot_id_dir = os.environ['JENKINS_HOME']+'/nightlies/'+os.environ['flavours']
slot_id_dir = 'configs/'
slot_id_file = os.path.join(slot_id_dir, 'slot_id.xml')
slot_id_file = 'configs/slot_id.xml'
if not os.path.exists(slot_id_dir):
os.makedirs(slot_id_dir)
if os.path.isfile(slot_id_file):
try:
xmlParse = parse(slot_id_file)
xmlParse = ET.parse(slot_id_file)
root = xmlParse.getroot()
except:
self.log.error('Can''t find or open %s', slot_id_file)
logging.error('Can''t find or open %s', slot_id_file)
sys.exit(1)
else:
os.open(slot_id_file, os.O_CREAT, 0644)
root = ET.Element('slot_id')
xmlParse = ET.ElementTree(root)
res = {}
root = xmlParse.getroot()
res = {}
add_slot = False
for slot_name in self.args:
slots = root.findall("slot[@name='"+slot_name+"']")
all_slots = dict((el.get('name'), el) for el in root.findall("slot"))
for slot_name in slots:
slot = all_slots.get(slot_name)
if len(slots):
slot = slots[0]
slot_id = slot.get('current_id')
if not slot_id:
self.log.error('no attribute current_id on the slot %s', slot_name)
sys.exit(2)
slot_id = int(slot_id)+1
slot.set('current_id', str(slot_id))
else:
slot_id = 1
slot = Element('slot')
slot.set('name', slot_name)
slot.set('current_id', str(slot_id))
root.append(slot)
self.log.info('Creation du slot %s dans slot_id.xml', slot_name)
if slot is not None:
slot_id = slot.get('last_id')
if not slot_id:
logging.error('No attribute current_id on the slot %s', slot_name)
sys.exit(2)
slot_id = int(slot_id)+1
slot.set('last_id', str(slot_id))
else:
slot_id = 1
slot = ET.Element('slot')
slot.set('name', slot_name)
slot.set('last_id', str(slot_id))
root.append(slot)
add_slot = True
logging.info('Slot %s created in %s', slot_name, slot_id_file)
res[slot_name] = slot_id
res[slot_name] = slot_id
xmlParse.write(slot_id_file)
if add_slot:
indent(root)
xmlParse.write(slot_id_file)
return res
\ No newline at end of file
return res
......@@ -13,7 +13,7 @@
#__test__ = False
__author__ = 'Colas Pomies <colas.pomies@cern.ch>'
from LbNightlyTools import CheckReadySlots
from LbNightlyTools import EnabledSlots
import os
import re
......@@ -38,7 +38,7 @@ def teardown():
def test_wrong_number_argument():
with TemporaryDir(chdir=True):
try:
CheckReadySlots.Script().run(['slot-param-{0}.txt','other_param'])
EnabledSlots.Script().run(['slot-param-{0}.txt','other_param'])
assert False, 'Script should have exited'
except SystemExit, x:
assert x.code != 0
......@@ -48,7 +48,7 @@ def test_wrong_number_argument():
def test_no_data():
with TemporaryDir(chdir=True):
retval = CheckReadySlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
assert retval == 0
assert len([x for x in os.listdir('.') if re.match(r'^slot-param-.*\.txt', x)]) == 0
......@@ -57,7 +57,7 @@ def test_no_file():
with TemporaryDir(chdir=True):
os.makedirs('./configs')
retval = CheckReadySlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
assert retval == 0
assert len([x for x in os.listdir('.') if re.match(r'^slot-param-.*\.txt', x)]) == 0
......@@ -75,9 +75,9 @@ def test_one_file_json_chmod_111():
with open('configs/lhcb-TEST.json', 'w') as slot_file:
slot_file.write(json.dumps(conf_data))
os.chmod('configs/lhcb-TEST.json', 0111)
slots = CheckReadySlots.Script().extractFromJson('lhcb-*.json')
CheckReadySlots.Script().writeFiles(slots, 'slot-param-{0}.txt')
retval = CheckReadySlots.Script().run(['slot-param-{0}.txt'])
slots = EnabledSlots.Script().extractFromJson('lhcb-*.json')
EnabledSlots.Script().writeFiles(slots, 'slot-param-{0}.txt')
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/lhcb-TEST.json'))
assert len([x for x in os.listdir('.') if re.match(r'^slot-param-.*\.txt', x)]) == 0
......@@ -96,7 +96,7 @@ def test_one_file_json_disabled_flase():
os.makedirs('./configs')
with open('configs/lhcb-TEST.json', 'w') as slot_file:
slot_file.write(json.dumps(conf_data))
retval = CheckReadySlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/lhcb-TEST.json'))
assert json.load(open('configs/lhcb-TEST.json')) == conf_data
......@@ -116,7 +116,7 @@ def test_one_file_json_disabled_true():
os.makedirs('./configs')
with open('configs/lhcb-TEST.json', 'w') as slot_file:
slot_file.write(json.dumps(conf_data))
retval = CheckReadySlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/lhcb-TEST.json'))
assert json.load(open('configs/lhcb-TEST.json')) == conf_data
......@@ -135,7 +135,7 @@ def test_one_file_json_no_disabled():
os.makedirs('./configs')
with open('configs/lhcb-TEST.json', 'w') as slot_file:
slot_file.write(json.dumps(conf_data))
retval = CheckReadySlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/lhcb-TEST.json'))
assert json.load(open('configs/lhcb-TEST.json')) == conf_data
......@@ -154,7 +154,7 @@ def test_one_file_json_no_slot():
os.makedirs('./configs')
with open('configs/lhcb-TEST.json', 'w') as slot_file:
slot_file.write(json.dumps(conf_data))
retval = CheckReadySlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/lhcb-TEST.json'))
assert json.load(open('configs/lhcb-TEST.json')) == conf_data
......@@ -181,7 +181,7 @@ def test_two_file_json():
slot_file.write(json.dumps(conf_data1))
with open('configs/lhcb-TEST2.json', 'w') as slot_file:
slot_file.write(json.dumps(conf_data2))
retval = CheckReadySlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/lhcb-TEST1.json'))
assert exists(join('configs/lhcb-TEST2.json'))
......@@ -202,7 +202,7 @@ def test_one_job_xml_disbaled_false():
os.makedirs('./configs')
with open('configs/configuration.xml', 'w') as cfg_file:
cfg_file.write(test_xml)
retval = CheckReadySlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/configuration.xml'))
assert len([x for x in os.listdir('.') if re.match(r'^slot-param-.*\.txt', x)]) == 1
......@@ -220,7 +220,7 @@ def test_one_job_xml_disabled_true():
os.makedirs('./configs')
with open('configs/configuration.xml', 'w') as cfg_file:
cfg_file.write(test_xml)
retval = CheckReadySlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/configuration.xml'))
assert len([x for x in os.listdir('.') if re.match(r'^slot-param-.*\.txt', x)]) == 0
......@@ -238,7 +238,7 @@ def test_one_job_xml_no_disabled():
os.makedirs('./configs')
with open('configs/configuration.xml', 'w') as cfg_file:
cfg_file.write(test_xml)
retval = CheckReadySlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/configuration.xml'))
assert len([x for x in os.listdir('.') if re.match(r'^slot-param-.*\.txt', x)]) == 1
......@@ -258,7 +258,7 @@ def test_two_job_xml():
os.makedirs('./configs')
with open('configs/configuration.xml', 'w') as cfg_file:
cfg_file.write(test_xml)
retval = CheckReadySlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/configuration.xml'))
assert len([x for x in os.listdir('.') if re.match(r'^slot-param-.*\.txt', x)]) == 2
......@@ -286,7 +286,7 @@ def test_same_job_xml_and_json():
with open('configs/lhcb-TEST.json', 'w') as slot_file:
slot_file.write(json.dumps(conf_data))
retval = CheckReadySlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/configuration.xml'))
assert exists(join('configs/lhcb-TEST.json'))
......@@ -315,7 +315,7 @@ def test_different_job_xml_and_json():
with open('configs/lhcb-TEST.json', 'w') as slot_file:
slot_file.write(json.dumps(conf_data))
retval = CheckReadySlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/configuration.xml'))
assert exists(join('configs/lhcb-TEST.json'))
......
......@@ -12,6 +12,6 @@
import LbUtils.Log
LbUtils.Log._default_log_format = '%(asctime)s:' + LbUtils.Log._default_log_format
from LbNightlyTools.CheckReadySlots import Script
from LbNightlyTools.EnabledSlots import Script
import sys
sys.exit(Script().run())
\ No newline at end of file
#!/usr/bin/env python
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
import LbUtils.Log
LbUtils.Log._default_log_format = '%(asctime)s:' + LbUtils.Log._default_log_format
from LbNightlyTools.SlotBuildId import Script
import sys
res = Script().run()
print res.values()[0]
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment