Commit 713d7fd0 authored by Ben Couturier's avatar Ben Couturier Committed by Marco Clemencic
Browse files

Transform LbNightlyTools into a standalone package

parent 70d17645
stages:
- test
- deploy
centos7:
except:
- tags
image: gitlab-registry.cern.ch/lhcb-docker/python-deployment:centos7
script:
- python --version
- pip install -e .
- python setup.py nosetests --cover-package ${CI_PROJECT_NAME}
- mkdir -p cover_report && mv -f cover cover_report/$CI_JOB_NAME
artifacts:
paths:
- cover_report
when: always
expire_in: 1 week
python2.7:
image: gitlab-registry.cern.ch/lhcb-docker/python-deployment:python-2.7
script:
- python --version
- pip install -e .
- python setup.py nosetests --cover-package ${CI_PROJECT_NAME}
- mkdir -p cover_report && mv -f cover cover_report/$CI_JOB_NAME
- python setup.py bdist_wheel --dist-dir public/${CI_PROJECT_NAME,,}
artifacts:
paths:
- cover_report
- public
when: always
expire_in: 1 week
python3.5:
except:
- tags
image: gitlab-registry.cern.ch/lhcb-docker/python-deployment:python-3.5
script:
- python --version
- pip install -e .
- python setup.py nosetests
- mkdir -p cover_report && mv -f build/lib/cover cover_report/$CI_JOB_NAME
artifacts:
paths:
- cover_report
when: always
expire_in: 1 week
python3.6:
image: gitlab-registry.cern.ch/lhcb-docker/python-deployment:python-3.6
script:
- python --version
- pip install -e .
- python setup.py nosetests
- mkdir -p cover_report && mv -f build/lib/cover cover_report/$CI_JOB_NAME
- python setup.py sdist --dist-dir public/${CI_PROJECT_NAME,,}
- python setup.py bdist_wheel --dist-dir public/${CI_PROJECT_NAME,,}
artifacts:
paths:
- cover_report
- public
when: always
expire_in: 1 week
# see https://gitlab.cern.ch/gitlabci-examples/deploy_eos for the details
# of the configuration
deploy-packages:
stage: deploy
only:
- tags
image: gitlab-registry.cern.ch/ci-tools/ci-web-deployer:latest
script:
- test -z "$EOS_ACCOUNT_USERNAME" -o -z "$EOS_ACCOUNT_PASSWORD" -o -z "$EOS_PATH" && exit 0 || true
# Script that performs the deploy to EOS. Makes use of the variables defined in the project
# It will copy the generated content to the folder in EOS
- deploy-eos
# do not run any globally defined before_script or after_script for this step
before_script: []
after_script: []
LbNightlyTools
===============
Scripts to perform LHCb Nightly builds.
\ No newline at end of file
#!/bin/bash
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
# Record disk usage in the dasboard.
# prepare environment
rootdir=$(dirname $0)/..
cd $rootdir
. /afs/cern.ch/lhcb/software/releases/LBSCRIPTS/dev/InstallArea/scripts/LbLogin.sh --silent
. setup.sh
day=$(date +%a)
# hard-coded because it may point to CVMFS
export LHCBNIGHTLIES=/afs/cern.ch/lhcb/software/nightlies
# get the list of slots
slots_on_afs=$(lbn-slots-by-deployment afs)
logfile=$LHCBNIGHTLIES/www/logs/monitor_disk_usage.log
# install the slots
echo "$(date): checking disk usage ($day)" >> $logfile 2>&1
cd $LHCBNIGHTLIES
for slot in $slots_on_afs ; do
lbn-monitor-disk --slot $slot --build-id $day $slot/$day >> $logfile 2>&1
done
......@@ -81,7 +81,10 @@ class _soft_db_singleton(object):
The singleton instance.
'''
if self._instance is None:
from LbRelease.SoftConfDB.SoftConfDBRO import SoftConfDBRO
try:
from LbSoftConfDB.SoftConfDBRO import SoftConfDBRO
except:
from LbRelease.SoftConfDB.SoftConfDBRO import SoftConfDBRO
self._instance = SoftConfDBRO()
return self._instance
......
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
'''
Class and functions used to collect monitoring data.
'''
__author__ = 'Marco Clemencic <marco.clemencic@cern.ch>'
import os
def getDirInfos(path):
'''
Return occupancy informations about a directory (on a local disk or on AFS)
in form of a dictionary.
'''
from LbUtils.afs.directory import Directory, NotInAFS
from LbUtils.afs.volume import Volume
try:
directory = Directory(path)
volume = Volume(dirname=directory.getParentMountPoint())
infos = dict(path=path,
AFS=True,
name=volume.name(),
mountpoints=[m.name() for m in volume.mountPoints()],
bsize=1024,
blocks=volume.quota(),
bavail=volume.remainingSpace())
except NotInAFS:
# find the root of the filesystem
name = path
while not os.path.ismount(name):
name = os.path.dirname(name)
stats = os.statvfs(name)
infos = dict(path=path,
AFS=False,
name=name,
bsize=stats.f_bsize,
blocks=stats.f_blocks,
bavail=stats.f_bavail)
return infos
......@@ -19,13 +19,22 @@ import logging
from datetime import datetime
from LbNightlyTools.Utils import TaskQueue, recursive_update
from LbNightlyTools.Configuration import findSlot
from LbUtils.Script import PlainScript as _PlainScript
# We first try to import from LbCommon, then revert to the old package (LbUtils)
# if needed
try:
from LbCommon.Script import PlainScript as _PlainScript
except:
from LbUtils.Script import PlainScript as _PlainScript
# change the default log format in LbUtils.Script
import LbUtils.Log
try:
import LbCommon.Log as _lblog
except:
import LbUtils.Log as _lblog
LbUtils.Log._default_log_format = ('%(asctime)s:' +
LbUtils.Log._default_log_format)
_lblog._default_log_format = ('%(asctime)s:' +
_lblog._default_log_format)
class PlainScript(_PlainScript):
......@@ -39,7 +48,7 @@ class PlainScript(_PlainScript):
hdlr.setLevel(self.log.level)
# reset the message format (LbUtils.Script uses a fixed one for DEBUG)
if self.log.level <= logging.DEBUG:
formatter = logging.Formatter(LbUtils.Log._default_log_format)
formatter = logging.Formatter(_lblog._default_log_format)
for h in self.log.handlers:
h.setFormatter(formatter)
......
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
# Uncomment to disable the tests.
#__test__ = False
import os
import nose
from pprint import pprint
from LbNightlyTools import Monitoring
from LbNightlyTools.tests.utils import which
def test_getDirInfos():
if not which('fs'):
raise nose.SkipTest
if os.path.exists('/afs/cern.ch/lhcb/software'):
infos = Monitoring.getDirInfos('/afs/cern.ch/lhcb/software')
pprint(infos)
assert 'path' in infos
assert infos['path'] == '/afs/cern.ch/lhcb/software'
assert 'AFS' in infos and infos['AFS'] is True
assert infos.get('name') == 'p.lhcb.software'
assert 'bsize' in infos
assert 'blocks' in infos
assert 'bavail' in infos
assert 'mountpoints' in infos and len(infos['mountpoints']) > 0
infos = Monitoring.getDirInfos('/usr/lib')
pprint(infos)
assert 'path' in infos
assert infos['path'] == '/usr/lib'
assert 'AFS' in infos and infos['AFS'] is False
assert 'name' in infos
assert 'bsize' in infos
assert 'blocks' in infos
assert 'bavail' in infos
assert 'mountpoints' not in infos
......@@ -16,13 +16,19 @@ Created on Dec 3, 2013
@author: Ben Couturier
'''
import LbUtils.Log
import logging
import os
import unittest
from os.path import normpath, join
from LbPeriodicTools.LbPeriodicTestSchedule import PeriodicTestSchedule
# We first try to import from LbCommon, then revert to the old package (LbUtils)
# if needed
try:
import LbCommon.Log as _lblog
except:
import LbUtils.Log as _lblog
class Test(unittest.TestCase):
''' Test case of the LbPeriodicTestSchedule class '''
......@@ -33,8 +39,8 @@ class Test(unittest.TestCase):
self._data_file = join(self._data_dir, "schedule.xml")
self._data_file_nok = join(self._data_dir, "scheduleIncorrect.xml")
logging.basicConfig()
LbUtils.Log._default_log_format = '%(asctime)s:' \
+ LbUtils.Log._default_log_format
_lblog._default_log_format = '%(asctime)s:' \
+ _lblog._default_log_format
def tearDown(self):
''' Tear down the test '''
......
......@@ -24,9 +24,10 @@ from string import Template
from subprocess import Popen, PIPE
from LHCbRPMSpecBuilder import LHCbBaseRpmSpec
from LbLegacy.Utils import getStatusOutput
from LbUtils.Temporary import TempDir
from LbUtils.CMT.Common import CMTCommand as CMT
try:
from LbCommon.Temporary import TempDir
except:
from LbUtils.Temporary import TempDir
tmpdir = TempDir(prefix="LHCbCompatRpmSpec")
PREFIX="/opt/LHCbSoft"
......
......@@ -22,17 +22,25 @@ import re
import sys
import logging
from string import Template
from subprocess import Popen, PIPE
from subprocess import Popen, PIPE, STDOUT
from LHCbRPMSpecBuilder import LHCbBaseRpmSpec
from LbLegacy.Utils import getStatusOutput
from LbUtils.Temporary import TempDir
from LbUtils.CMT.Common import CMTCommand as CMT
try:
from LbCommon.Temporary import TempDir
except:
from LbUtils.Temporary import TempDir
tmpdir = TempDir(prefix="LHCbExternalsRpmSpec")
__log__ = logging.getLogger(__name__)
def getStatusOutput(cmdline):
p = Popen(cmdline, shell=True, stdout=PIPE, stderr=STDOUT, close_fds=True)
output = p.communicate()[0].rstrip("\n")
status = p.returncode
return status, output
#
# Spec for binary RPMs
#
......
......@@ -24,8 +24,10 @@ from string import Template
from subprocess import Popen, PIPE
from LHCbRPMSpecBuilder import LHCbBaseRpmSpec
from LbLegacy.Utils import getStatusOutput
from LbUtils.Temporary import TempDir
try:
from LbCommon.Temporary import TempDir
except:
from LbUtils.Temporary import TempDir
tmpdir = TempDir(prefix="LHCbGenericRpmSpec")
......
......@@ -25,9 +25,10 @@ from string import Template
from subprocess import Popen, PIPE
from LHCbRPMSpecBuilder import LHCbBaseRpmSpec
from LbLegacy.Utils import getStatusOutput
from LbUtils.Temporary import TempDir
from LbUtils.CMT.Common import CMTCommand as CMT
try:
from LbCommon.Temporary import TempDir
except:
from LbUtils.Temporary import TempDir
tmpdir = TempDir(prefix="LHCbLbScriptsRpmSpec")
PREFIX="/opt/LHCbSoft"
......
......@@ -25,8 +25,10 @@ from string import Template
from subprocess import Popen, PIPE
from LHCbRPMSpecBuilder import LHCbBaseRpmSpec
from LbLegacy.Utils import getStatusOutput
from LbUtils.Temporary import TempDir
try:
from LbCommon.Temporary import TempDir
except:
from LbUtils.Temporary import TempDir
tmpdir = TempDir(prefix="LHCbMetaRpmSpec")
......
......@@ -16,13 +16,19 @@ Created on Feb 27, 2014
@author: Ben Couturier
'''
import LbUtils.Log
import logging
import os
import unittest
from os.path import normpath, join
from LbTools.Manifest import Parser
# We first try to import from LbCommon, then revert to the old package (LbUtils)
# if needed
try:
import LbCommon.Log as _lblog
except:
import LbUtils.Log as _lblog
class Test(unittest.TestCase):
''' Test case of the Manifest parser class '''
......@@ -32,8 +38,8 @@ class Test(unittest.TestCase):
+ ['testdata', 'tools'])))
self._data_file = join(self._data_dir, "manifest.xml")
logging.basicConfig()
LbUtils.Log._default_log_format = '%(asctime)s:' \
+ LbUtils.Log._default_log_format
_lblog._default_log_format = '%(asctime)s:' \
+ _lblog._default_log_format
def tearDown(self):
''' Tear down the test '''
......
#!/usr/bin/env python
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
'''
Publish to the dashboard disk usage statistics for all the directories specified
on the command line.
'''
import sys
import os
import urllib
from LbNightlyTools.Monitoring import getDirInfos
from LbNightlyTools.Utils import Dashboard
from datetime import datetime
from LbNightlyTools.Scripts.Common import PlainScript
class Script(PlainScript):
'''
Collect disk usage statistics and publish them to the Dashboard.
'''
__usage__ = '%prog [options] directory'
def defineOpts(self):
'''
Options specific to this script.
'''
self.parser.add_option('-s', '--slot',
help='name of the slot to add to the JSON data')
self.parser.add_option('-b', '--build-id',
help='build id to add to the JSON data')
def main(self):
'''
Script logic.
'''
dash = Dashboard()
if len(self.args) != 1:
self.parser.error('wrong number of arguments')
# gather infos
path = os.path.realpath(self.args[0])
data = getDirInfos(path)
data['type'] = 'disk-usage'
data['updated'] = datetime.now().isoformat()
# prepare object id
if self.options.slot:
data['slot'] = self.options.slot
if self.options.build_id:
data['build_id'] = self.options.build_id
if not self.options.slot and not self.options.build_id:
data['_id'] = urllib.quote(path, safe='')
# publish
dash.update(data['_id'], data)
return 0
if __name__ == '__main__':
sys.exit(Script().run())
......@@ -15,7 +15,13 @@ Run a LHCbPR job
'''
__author__ = 'Ben Couturier <ben.couturier@cern.ch>'
import LbUtils.Script
# We first try to import from LbCommon, then revert to the old package (LbUtils)
# if needed
try:
from LbCommon.Script import PlainScript as _PlainScript
except:
from LbUtils.Script import PlainScript as _PlainScript
import sys
import os
from datetime import datetime
......@@ -37,7 +43,7 @@ def getSlotDateFromConfig(config):
return datetime.strptime(completed, '%Y-%m-%dT%H:%M:%S.%f')
class Script(LbUtils.Script.PlainScript):
class Script(_PlainScript):
'''
Script to create the commands to run a LHCbPR Job
......
"""A setuptools based setup module.
See:
https://packaging.python.org/en/latest/distributing.html
https://github.com/pypa/sampleproject
"""
# Always prefer setuptools over distutils
from setuptools import setup, find_packages
# To use a consistent encoding
from codecs import open
from os import path
from sys import version_info
here = path.abspath(path.dirname(__file__))
# Get the long description from the README file
with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
long_description = f.read()
# Arguments marked as "Required" below must be included for upload to PyPI.
# Fields marked as "Optional" may be commented out.
setup(
# This is the name of your project. The first time you publish this
# package, this name will be registered for you. It will determine how
# users can install this project, e.g.:
#
# $ pip install sampleproject
#
# And where it will live on PyPI: https://pypi.org/project/sampleproject/
#
# There are some restrictions on what makes a valid project name
# specification here:
# https://packaging.python.org/specifications/core-metadata/#name
name='LbNightlyTools', # Required
# Versions should comply with PEP 440:
# https://www.python.org/dev/peps/pep-0440/
#
# For a discussion on single-sourcing the version across setup.py and the
# project code, see
# https://packaging.python.org/en/latest/single_source_version.html
# version='0.0.1', # Required
use_scm_version=True,
# This is a one-line description or tagline of what your project does. This
# corresponds to the "Summary" metadata field:
# https://packaging.python.org/specifications/core-metadata/#summary
description='LHCb Nightly tools', # Required
# This is an optional longer description of your project that represents
# the body of text which users will see when they visit PyPI.
#
# Often, this is the same as your README, so you can just read it in from
# that file directly (as we have already done above)
#
# This field corresponds to the "Description" metadata field:
# https://packaging.python.org/specifications/core-metadata/#description-optional
long_description=long_description, # Optional
# This should be a valid link to your project's main homepage.
#
# This field corresponds to the "Home-Page" metadata field:
# https://packaging.python.org/specifications/core-metadata/#home-page-optional
url='https://gitlab.cern.ch/lhcb-core/LbNightlyTools', # Optional
# This should be your name or the name of the organization which owns the
# project.
author='CERN - LHCb Core Software', # Optional
# This should be a valid email address corresponding to the author listed
# above.
author_email='lhcb-core-soft@cern.ch', # Optional
# Classifiers help users find your project by categorizing it.
#
# For a list of valid classifiers, see
# https://pypi.python.org/pypi?%3Aaction=list_classifiers
classifiers=[ # Optional
# How mature is this project? Common values are
# 3 - Alpha
# 4 - Beta
# 5 - Production/Stable
'Development Status :: 4 - Beta',
# Indicate who your project is intended for
'Intended Audience :: Developers',
'Topic :: Software Development :: Build Tools',
# Pick your license as you wish
'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',
# Specify the Python versions you support here. In particular, ensure
# that you indicate whether you support Python 2, Python 3 or both.