Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Chris Burr
LbNightlyTools
Commits
b43ce5ab
Commit
b43ce5ab
authored
May 17, 2018
by
Marco Clemencic
Browse files
Transform LbNightlyTools into a standalone package (mr !180)
Closes LBCORE-1584
parents
70d17645
713d7fd0
Changes
17
Hide whitespace changes
Inline
Side-by-side
.gitlab-ci.yml
0 → 100644
View file @
b43ce5ab
stages
:
-
test
-
deploy
centos7
:
except
:
-
tags
image
:
gitlab-registry.cern.ch/lhcb-docker/python-deployment:centos7
script
:
-
python --version
-
pip install -e .
-
python setup.py nosetests --cover-package ${CI_PROJECT_NAME}
-
mkdir -p cover_report && mv -f cover cover_report/$CI_JOB_NAME
artifacts
:
paths
:
-
cover_report
when
:
always
expire_in
:
1 week
python2.7
:
image
:
gitlab-registry.cern.ch/lhcb-docker/python-deployment:python-2.7
script
:
-
python --version
-
pip install -e .
-
python setup.py nosetests --cover-package ${CI_PROJECT_NAME}
-
mkdir -p cover_report && mv -f cover cover_report/$CI_JOB_NAME
-
python setup.py bdist_wheel --dist-dir public/${CI_PROJECT_NAME,,}
artifacts
:
paths
:
-
cover_report
-
public
when
:
always
expire_in
:
1 week
python3.5
:
except
:
-
tags
image
:
gitlab-registry.cern.ch/lhcb-docker/python-deployment:python-3.5
script
:
-
python --version
-
pip install -e .
-
python setup.py nosetests
-
mkdir -p cover_report && mv -f build/lib/cover cover_report/$CI_JOB_NAME
artifacts
:
paths
:
-
cover_report
when
:
always
expire_in
:
1 week
python3.6
:
image
:
gitlab-registry.cern.ch/lhcb-docker/python-deployment:python-3.6
script
:
-
python --version
-
pip install -e .
-
python setup.py nosetests
-
mkdir -p cover_report && mv -f build/lib/cover cover_report/$CI_JOB_NAME
-
python setup.py sdist --dist-dir public/${CI_PROJECT_NAME,,}
-
python setup.py bdist_wheel --dist-dir public/${CI_PROJECT_NAME,,}
artifacts
:
paths
:
-
cover_report
-
public
when
:
always
expire_in
:
1 week
# see https://gitlab.cern.ch/gitlabci-examples/deploy_eos for the details
# of the configuration
deploy-packages
:
stage
:
deploy
only
:
-
tags
image
:
gitlab-registry.cern.ch/ci-tools/ci-web-deployer:latest
script
:
-
test -z "$EOS_ACCOUNT_USERNAME" -o -z "$EOS_ACCOUNT_PASSWORD" -o -z "$EOS_PATH" && exit 0 ||
true
# Script that performs the deploy to EOS. Makes use of the variables defined in the project
# It will copy the generated content to the folder in EOS
-
deploy-eos
# do not run any globally defined before_script or after_script for this step
before_script
:
[]
after_script
:
[]
README.rst
0 → 100644
View file @
b43ce5ab
LbNightlyTools
===============
Scripts to perform LHCb Nightly builds.
\ No newline at end of file
cron/monitor_disk_usage.sh
deleted
100755 → 0
View file @
70d17645
#!/bin/bash
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
# Record disk usage in the dasboard.
# prepare environment
rootdir
=
$(
dirname
$0
)
/..
cd
$rootdir
.
/afs/cern.ch/lhcb/software/releases/LBSCRIPTS/dev/InstallArea/scripts/LbLogin.sh
--silent
.
setup.sh
day
=
$(
date
+%a
)
# hard-coded because it may point to CVMFS
export
LHCBNIGHTLIES
=
/afs/cern.ch/lhcb/software/nightlies
# get the list of slots
slots_on_afs
=
$(
lbn-slots-by-deployment afs
)
logfile
=
$LHCBNIGHTLIES
/www/logs/monitor_disk_usage.log
# install the slots
echo
"
$(
date
)
: checking disk usage (
$day
)"
>>
$logfile
2>&1
cd
$LHCBNIGHTLIES
for
slot
in
$slots_on_afs
;
do
lbn-monitor-disk
--slot
$slot
--build-id
$day
$slot
/
$day
>>
$logfile
2>&1
done
python/LbNightlyTools/CheckoutMethods.py
View file @
b43ce5ab
...
...
@@ -81,7 +81,10 @@ class _soft_db_singleton(object):
The singleton instance.
'''
if
self
.
_instance
is
None
:
from
LbRelease.SoftConfDB.SoftConfDBRO
import
SoftConfDBRO
try
:
from
LbSoftConfDB.SoftConfDBRO
import
SoftConfDBRO
except
:
from
LbRelease.SoftConfDB.SoftConfDBRO
import
SoftConfDBRO
self
.
_instance
=
SoftConfDBRO
()
return
self
.
_instance
...
...
python/LbNightlyTools/Monitoring.py
deleted
100644 → 0
View file @
70d17645
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
'''
Class and functions used to collect monitoring data.
'''
__author__
=
'Marco Clemencic <marco.clemencic@cern.ch>'
import
os
def
getDirInfos
(
path
):
'''
Return occupancy informations about a directory (on a local disk or on AFS)
in form of a dictionary.
'''
from
LbUtils.afs.directory
import
Directory
,
NotInAFS
from
LbUtils.afs.volume
import
Volume
try
:
directory
=
Directory
(
path
)
volume
=
Volume
(
dirname
=
directory
.
getParentMountPoint
())
infos
=
dict
(
path
=
path
,
AFS
=
True
,
name
=
volume
.
name
(),
mountpoints
=
[
m
.
name
()
for
m
in
volume
.
mountPoints
()],
bsize
=
1024
,
blocks
=
volume
.
quota
(),
bavail
=
volume
.
remainingSpace
())
except
NotInAFS
:
# find the root of the filesystem
name
=
path
while
not
os
.
path
.
ismount
(
name
):
name
=
os
.
path
.
dirname
(
name
)
stats
=
os
.
statvfs
(
name
)
infos
=
dict
(
path
=
path
,
AFS
=
False
,
name
=
name
,
bsize
=
stats
.
f_bsize
,
blocks
=
stats
.
f_blocks
,
bavail
=
stats
.
f_bavail
)
return
infos
python/LbNightlyTools/Scripts/Common.py
View file @
b43ce5ab
...
...
@@ -19,13 +19,22 @@ import logging
from
datetime
import
datetime
from
LbNightlyTools.Utils
import
TaskQueue
,
recursive_update
from
LbNightlyTools.Configuration
import
findSlot
from
LbUtils.Script
import
PlainScript
as
_PlainScript
# We first try to import from LbCommon, then revert to the old package (LbUtils)
# if needed
try
:
from
LbCommon.Script
import
PlainScript
as
_PlainScript
except
:
from
LbUtils.Script
import
PlainScript
as
_PlainScript
# change the default log format in LbUtils.Script
import
LbUtils.Log
try
:
import
LbCommon.Log
as
_lblog
except
:
import
LbUtils.Log
as
_lblog
LbUtils
.
L
og
.
_default_log_format
=
(
'%(asctime)s:'
+
LbUtils
.
L
og
.
_default_log_format
)
_lbl
og
.
_default_log_format
=
(
'%(asctime)s:'
+
_lbl
og
.
_default_log_format
)
class
PlainScript
(
_PlainScript
):
...
...
@@ -39,7 +48,7 @@ class PlainScript(_PlainScript):
hdlr
.
setLevel
(
self
.
log
.
level
)
# reset the message format (LbUtils.Script uses a fixed one for DEBUG)
if
self
.
log
.
level
<=
logging
.
DEBUG
:
formatter
=
logging
.
Formatter
(
LbUtils
.
L
og
.
_default_log_format
)
formatter
=
logging
.
Formatter
(
_lbl
og
.
_default_log_format
)
for
h
in
self
.
log
.
handlers
:
h
.
setFormatter
(
formatter
)
...
...
python/LbNightlyTools/tests/test_monitoring.py
deleted
100644 → 0
View file @
70d17645
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
# Uncomment to disable the tests.
#__test__ = False
import
os
import
nose
from
pprint
import
pprint
from
LbNightlyTools
import
Monitoring
from
LbNightlyTools.tests.utils
import
which
def
test_getDirInfos
():
if
not
which
(
'fs'
):
raise
nose
.
SkipTest
if
os
.
path
.
exists
(
'/afs/cern.ch/lhcb/software'
):
infos
=
Monitoring
.
getDirInfos
(
'/afs/cern.ch/lhcb/software'
)
pprint
(
infos
)
assert
'path'
in
infos
assert
infos
[
'path'
]
==
'/afs/cern.ch/lhcb/software'
assert
'AFS'
in
infos
and
infos
[
'AFS'
]
is
True
assert
infos
.
get
(
'name'
)
==
'p.lhcb.software'
assert
'bsize'
in
infos
assert
'blocks'
in
infos
assert
'bavail'
in
infos
assert
'mountpoints'
in
infos
and
len
(
infos
[
'mountpoints'
])
>
0
infos
=
Monitoring
.
getDirInfos
(
'/usr/lib'
)
pprint
(
infos
)
assert
'path'
in
infos
assert
infos
[
'path'
]
==
'/usr/lib'
assert
'AFS'
in
infos
and
infos
[
'AFS'
]
is
False
assert
'name'
in
infos
assert
'bsize'
in
infos
assert
'blocks'
in
infos
assert
'bavail'
in
infos
assert
'mountpoints'
not
in
infos
python/LbPeriodicTools/tests/TestXMLParsing.py
View file @
b43ce5ab
...
...
@@ -16,13 +16,19 @@ Created on Dec 3, 2013
@author: Ben Couturier
'''
import
LbUtils.Log
import
logging
import
os
import
unittest
from
os.path
import
normpath
,
join
from
LbPeriodicTools.LbPeriodicTestSchedule
import
PeriodicTestSchedule
# We first try to import from LbCommon, then revert to the old package (LbUtils)
# if needed
try
:
import
LbCommon.Log
as
_lblog
except
:
import
LbUtils.Log
as
_lblog
class
Test
(
unittest
.
TestCase
):
''' Test case of the LbPeriodicTestSchedule class '''
...
...
@@ -33,8 +39,8 @@ class Test(unittest.TestCase):
self
.
_data_file
=
join
(
self
.
_data_dir
,
"schedule.xml"
)
self
.
_data_file_nok
=
join
(
self
.
_data_dir
,
"scheduleIncorrect.xml"
)
logging
.
basicConfig
()
LbUtils
.
L
og
.
_default_log_format
=
'%(asctime)s:'
\
+
LbUtils
.
L
og
.
_default_log_format
_lbl
og
.
_default_log_format
=
'%(asctime)s:'
\
+
_lbl
og
.
_default_log_format
def
tearDown
(
self
):
''' Tear down the test '''
...
...
python/LbRPMTools/LHCbCompatSpecBuilder.py
View file @
b43ce5ab
...
...
@@ -24,9 +24,10 @@ from string import Template
from
subprocess
import
Popen
,
PIPE
from
LHCbRPMSpecBuilder
import
LHCbBaseRpmSpec
from
LbLegacy.Utils
import
getStatusOutput
from
LbUtils.Temporary
import
TempDir
from
LbUtils.CMT.Common
import
CMTCommand
as
CMT
try
:
from
LbCommon.Temporary
import
TempDir
except
:
from
LbUtils.Temporary
import
TempDir
tmpdir
=
TempDir
(
prefix
=
"LHCbCompatRpmSpec"
)
PREFIX
=
"/opt/LHCbSoft"
...
...
python/LbRPMTools/LHCbExternalsSpecBuilder.py
View file @
b43ce5ab
...
...
@@ -22,17 +22,25 @@ import re
import
sys
import
logging
from
string
import
Template
from
subprocess
import
Popen
,
PIPE
from
subprocess
import
Popen
,
PIPE
,
STDOUT
from
LHCbRPMSpecBuilder
import
LHCbBaseRpmSpec
from
LbLegacy.Utils
import
getStatusOutput
from
LbUtils.Temporary
import
TempDir
from
LbUtils.CMT.Common
import
CMTCommand
as
CMT
try
:
from
LbCommon.Temporary
import
TempDir
except
:
from
LbUtils.Temporary
import
TempDir
tmpdir
=
TempDir
(
prefix
=
"LHCbExternalsRpmSpec"
)
__log__
=
logging
.
getLogger
(
__name__
)
def
getStatusOutput
(
cmdline
):
p
=
Popen
(
cmdline
,
shell
=
True
,
stdout
=
PIPE
,
stderr
=
STDOUT
,
close_fds
=
True
)
output
=
p
.
communicate
()[
0
].
rstrip
(
"
\n
"
)
status
=
p
.
returncode
return
status
,
output
#
# Spec for binary RPMs
#
...
...
python/LbRPMTools/LHCbGenericSpecBuilder.py
View file @
b43ce5ab
...
...
@@ -24,8 +24,10 @@ from string import Template
from
subprocess
import
Popen
,
PIPE
from
LHCbRPMSpecBuilder
import
LHCbBaseRpmSpec
from
LbLegacy.Utils
import
getStatusOutput
from
LbUtils.Temporary
import
TempDir
try
:
from
LbCommon.Temporary
import
TempDir
except
:
from
LbUtils.Temporary
import
TempDir
tmpdir
=
TempDir
(
prefix
=
"LHCbGenericRpmSpec"
)
...
...
python/LbRPMTools/LHCbLbScriptsSpecBuilder.py
View file @
b43ce5ab
...
...
@@ -25,9 +25,10 @@ from string import Template
from
subprocess
import
Popen
,
PIPE
from
LHCbRPMSpecBuilder
import
LHCbBaseRpmSpec
from
LbLegacy.Utils
import
getStatusOutput
from
LbUtils.Temporary
import
TempDir
from
LbUtils.CMT.Common
import
CMTCommand
as
CMT
try
:
from
LbCommon.Temporary
import
TempDir
except
:
from
LbUtils.Temporary
import
TempDir
tmpdir
=
TempDir
(
prefix
=
"LHCbLbScriptsRpmSpec"
)
PREFIX
=
"/opt/LHCbSoft"
...
...
python/LbRPMTools/LHCbMetaSpecBuilder.py
View file @
b43ce5ab
...
...
@@ -25,8 +25,10 @@ from string import Template
from
subprocess
import
Popen
,
PIPE
from
LHCbRPMSpecBuilder
import
LHCbBaseRpmSpec
from
LbLegacy.Utils
import
getStatusOutput
from
LbUtils.Temporary
import
TempDir
try
:
from
LbCommon.Temporary
import
TempDir
except
:
from
LbUtils.Temporary
import
TempDir
tmpdir
=
TempDir
(
prefix
=
"LHCbMetaRpmSpec"
)
...
...
python/LbTools/tests/TestXMLParsing.py
View file @
b43ce5ab
...
...
@@ -16,13 +16,19 @@ Created on Feb 27, 2014
@author: Ben Couturier
'''
import
LbUtils.Log
import
logging
import
os
import
unittest
from
os.path
import
normpath
,
join
from
LbTools.Manifest
import
Parser
# We first try to import from LbCommon, then revert to the old package (LbUtils)
# if needed
try
:
import
LbCommon.Log
as
_lblog
except
:
import
LbUtils.Log
as
_lblog
class
Test
(
unittest
.
TestCase
):
''' Test case of the Manifest parser class '''
...
...
@@ -32,8 +38,8 @@ class Test(unittest.TestCase):
+
[
'testdata'
,
'tools'
])))
self
.
_data_file
=
join
(
self
.
_data_dir
,
"manifest.xml"
)
logging
.
basicConfig
()
LbUtils
.
L
og
.
_default_log_format
=
'%(asctime)s:'
\
+
LbUtils
.
L
og
.
_default_log_format
_lbl
og
.
_default_log_format
=
'%(asctime)s:'
\
+
_lbl
og
.
_default_log_format
def
tearDown
(
self
):
''' Tear down the test '''
...
...
scripts/lbn-monitor-disk
deleted
100755 → 0
View file @
70d17645
#!/usr/bin/env python
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
'''
Publish to the dashboard disk usage statistics for all the directories specified
on the command line.
'''
import
sys
import
os
import
urllib
from
LbNightlyTools.Monitoring
import
getDirInfos
from
LbNightlyTools.Utils
import
Dashboard
from
datetime
import
datetime
from
LbNightlyTools.Scripts.Common
import
PlainScript
class
Script
(
PlainScript
):
'''
Collect disk usage statistics and publish them to the Dashboard.
'''
__usage__
=
'%prog [options] directory'
def
defineOpts
(
self
):
'''
Options specific to this script.
'''
self
.
parser
.
add_option
(
'-s'
,
'--slot'
,
help
=
'name of the slot to add to the JSON data'
)
self
.
parser
.
add_option
(
'-b'
,
'--build-id'
,
help
=
'build id to add to the JSON data'
)
def
main
(
self
):
'''
Script logic.
'''
dash
=
Dashboard
()
if
len
(
self
.
args
)
!=
1
:
self
.
parser
.
error
(
'wrong number of arguments'
)
# gather infos
path
=
os
.
path
.
realpath
(
self
.
args
[
0
])
data
=
getDirInfos
(
path
)
data
[
'type'
]
=
'disk-usage'
data
[
'updated'
]
=
datetime
.
now
().
isoformat
()
# prepare object id
if
self
.
options
.
slot
:
data
[
'slot'
]
=
self
.
options
.
slot
if
self
.
options
.
build_id
:
data
[
'build_id'
]
=
self
.
options
.
build_id
if
not
self
.
options
.
slot
and
not
self
.
options
.
build_id
:
data
[
'_id'
]
=
urllib
.
quote
(
path
,
safe
=
''
)
# publish
dash
.
update
(
data
[
'_id'
],
data
)
return
0
if
__name__
==
'__main__'
:
sys
.
exit
(
Script
().
run
())
scripts/lbpr-get-command
View file @
b43ce5ab
...
...
@@ -15,7 +15,13 @@ Run a LHCbPR job
'''
__author__
=
'Ben Couturier <ben.couturier@cern.ch>'
import
LbUtils.Script
# We first try to import from LbCommon, then revert to the old package (LbUtils)
# if needed
try
:
from
LbCommon.Script
import
PlainScript
as
_PlainScript
except
:
from
LbUtils.Script
import
PlainScript
as
_PlainScript
import
sys
import
os
from
datetime
import
datetime
...
...
@@ -37,7 +43,7 @@ def getSlotDateFromConfig(config):
return
datetime
.
strptime
(
completed
,
'%Y-%m-%dT%H:%M:%S.%f'
)
class
Script
(
LbUtils
.
Script
.
PlainScript
):
class
Script
(
_
PlainScript
):
'''
Script to create the commands to run a LHCbPR Job
...
...
setup.py
0 → 100644
View file @
b43ce5ab
"""A setuptools based setup module.
See:
https://packaging.python.org/en/latest/distributing.html
https://github.com/pypa/sampleproject
"""
# Always prefer setuptools over distutils
from
setuptools
import
setup
,
find_packages
# To use a consistent encoding
from
codecs
import
open
from
os
import
path
from
sys
import
version_info
here
=
path
.
abspath
(
path
.
dirname
(
__file__
))
# Get the long description from the README file
with
open
(
path
.
join
(
here
,
'README.rst'
),
encoding
=
'utf-8'
)
as
f
:
long_description
=
f
.
read
()
# Arguments marked as "Required" below must be included for upload to PyPI.
# Fields marked as "Optional" may be commented out.
setup
(
# This is the name of your project. The first time you publish this
# package, this name will be registered for you. It will determine how
# users can install this project, e.g.:
#
# $ pip install sampleproject
#
# And where it will live on PyPI: https://pypi.org/project/sampleproject/
#
# There are some restrictions on what makes a valid project name
# specification here:
# https://packaging.python.org/specifications/core-metadata/#name
name
=
'LbNightlyTools'
,
# Required
# Versions should comply with PEP 440:
# https://www.python.org/dev/peps/pep-0440/
#
# For a discussion on single-sourcing the version across setup.py and the
# project code, see
# https://packaging.python.org/en/latest/single_source_version.html
# version='0.0.1', # Required
use_scm_version
=
True
,
# This is a one-line description or tagline of what your project does. This
# corresponds to the "Summary" metadata field:
# https://packaging.python.org/specifications/core-metadata/#summary
description
=
'LHCb Nightly tools'
,
# Required
# This is an optional longer description of your project that represents
# the body of text which users will see when they visit PyPI.
#
# Often, this is the same as your README, so you can just read it in from
# that file directly (as we have already done above)
#
# This field corresponds to the "Description" metadata field:
# https://packaging.python.org/specifications/core-metadata/#description-optional
long_description
=
long_description
,
# Optional
# This should be a valid link to your project's main homepage.
#
# This field corresponds to the "Home-Page" metadata field:
# https://packaging.python.org/specifications/core-metadata/#home-page-optional
url
=
'https://gitlab.cern.ch/lhcb-core/LbNightlyTools'
,
# Optional
# This should be your name or the name of the organization which owns the
# project.
author
=
'CERN - LHCb Core Software'
,
# Optional
# This should be a valid email address corresponding to the author listed
# above.
author_email
=
'lhcb-core-soft@cern.ch'
,
# Optional
# Classifiers help users find your project by categorizing it.
#
# For a list of valid classifiers, see
# https://pypi.python.org/pypi?%3Aaction=list_classifiers
classifiers
=
[
# Optional
# How mature is this project? Common values are
# 3 - Alpha
# 4 - Beta
# 5 - Production/Stable
'Development Status :: 4 - Beta'
,
# Indicate who your project is intended for
'Intended Audience :: Developers'
,
'Topic :: Software Development :: Build Tools'
,
# Pick your license as you wish
'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)'
,
# Specify the Python versions you support here. In particular, ensure
# that you indicate whether you support Python 2, Python 3 or both.
'Programming Language :: Python :: 2'