Commit 0ebe8880 authored by Andrea Valassi's avatar Andrea Valassi
Browse files

First complete set of tools for el6, el7 and Ubuntu

parent fea3d975
......@@ -3,11 +3,11 @@
# Enable legacy builds (rpmbuild instead of koji) for 7.1/el7 and 1.0/el6?
LEGACY=false # disabled
LEGACY=true # enabled
###LEGACY=true # enabled
#------------------------------------------------------------------------------
# Is this a (pre-)release-candidate tag (_0-rcXX or _0-prcXX)?
# Is this a (pre-)release-candidate tag (-0.rcXX or -0.prcXX)?
function is_rc_tag()
{
# Check arguments: expect "${rc_var} ${tag}"
......@@ -17,11 +17,17 @@ function is_rc_tag()
fi
local _rva=$1
local _tag=$2
local _rev=$(echo $_tag | cut -d '.' -f3 | cut -d '-' -f2)
if [ "${_rev#0_rc}" != "${_rev}" ] || [ "${_rev#0_prc}" != "${_rev}" ]; then
local _rev=$(echo $_tag | cut -d '.' -f3- | cut -d '-' -f2)
if [ "${_rev#0.rc}" != "${_rev}" ] || [ "${_rev#0.prc}" != "${_rev}" ]; then
echo "WARNING: tag ${tag} is a (pre-)release-candidate"
printf "\n"
eval ${_rva}=true
elif [ "${_rev#0_rc}" != "${_rev}" ] || [ "${_rev#0_prc}" != "${_rev}" ] ||
[ "${_rev#0-rc}" != "${_rev}" ] || [ "${_rev#0-prc}" != "${_rev}" ] ||
[ "${_rev#0rc}" != "${_rev}" ] || [ "${_rev#0prc}" != "${_rev}" ]; then
echo "ERROR! Tag ${tag} is an invalid tag"
echo "ERROR! Use -0.rcXX or -0.prcXX for (pre-)release candidates"
exit 1
else
echo "INFO: tag ${tag} is NOT a (pre-)release-candidate"
printf "\n"
......
# Copyright 2017 CERN. Licensed under LGPLv3+.
sources: ;
......@@ -72,7 +72,7 @@ function usage ()
echo " Signs and installs rpm's in the WLCG repo."
echo " Launches 'koji tag-build AUX-stable' (promotes rpm's to stable),"
echo " which eventually signs and installs rpm's in the linuxsoft repo;"
echo " skip this if VR is (pre-)release candidate tag *-0_prc* or *-0_rc*."
echo " skip this if VR is (pre-)release candidate tag *-0.prc* or *-0.rc*."
echo " Updates and commits README for the new release."
echo " Creates new git tag VR and annotates it with release notes."
echo " Deletes git tags VR-testing and VR-qa."
......@@ -125,7 +125,8 @@ function usage ()
echo " [NB: fails if VR or VR-qa tags already exist]"
echo ""
echo " release"
echo " Commits the deb package to git."
echo " Commits the deb package to git;"
echo " skip this if VR is (pre-)release candidate tag *-0.prc* or *-0.rc*."
echo " Updates and commits README for the new release."
echo " Creates new git tag VR and annotates it with release notes."
echo " Deletes git tags VR-testing and VR-qa."
......@@ -277,8 +278,11 @@ elif [ "$step" == "build" ] || [ "$step" == "rebuild" ]; then
# Is this a legacy RedHat release using rpmbuild in the CI instead of koji?
is_legacy_tag use_rpmbuild ${tag} ${brn}
# No need to check here if this is a (pre-)release-candidate.
# Note also that the script does not prevent tags x.y.z-0 (reserved for rc),
# Is this a (pre)-release-candidate?
# [Just check that the correct -0.rcXX or -0.prcXX syntax is used]
is_rc_tag is_rc ${tag}
# Note that the script does not prevent tags x.y.z-0 (reserved for rc),
# even if the policy for the new workflow is to use x.y.z-1 and above.
# Extra checks on Ubuntu
......@@ -514,6 +518,9 @@ elif [ "$step" == "release" ]; then
# Is this a legacy RedHat release using rpmbuild in the CI instead of koji?
is_legacy_tag use_rpmbuild ${tag} ${brn}
# Is this a (pre)-release-candidate?
is_rc_tag is_rc ${tag}
# Check that the tags expected from previous steps exist
# Check that the tags to be created do not already exist
ts_ok=
......@@ -572,16 +579,25 @@ elif [ "$step" == "release" ]; then
echo Retrieving ${url}
curl -s -L --cookie $CKF --cookie-jar $CKF ${url} > downloads/${deb64}
\rm -f $CKF # cleanup: remove the cookie
# Commit and push downloads to git if they differ from the previous ones
if [ "${mkdow}" == "1" ]; then git add downloads; fi
dows=$(git status --porcelain | grep '^?? downloads/' | cut -d' ' -f2)
for dow in ${dows}; do git add ${dow}; done
if [ "${mkdow}" == "1" ] || [ "${dows}" != "" ] || git status --porcelain | egrep "^ M downloads/"; then
printf "=== Committing downloads ===\n"
git commit -m "Downloads for release ${tag}" downloads
git push
# Is this a (pre)-release-candidate?
if ${is_rc}; then
# (Pre)-release-candidate: do not distribute for installation
echo "WARNING: tag ${tag} is a (pre-)release-candidate"
echo "WARNING: skip committing to downloads area of git repo"
\rm -f downloads/${deb64}
else
printf "[No need to commit: no change in downloads/]\n"
# Production release
# Commit and push downloads to git if they differ from the previous ones
if [ "${mkdow}" == "1" ]; then git add downloads; fi
dows=$(git status --porcelain | grep '^?? downloads/' | cut -d' ' -f2)
for dow in ${dows}; do git add ${dow}; done
if [ "${mkdow}" == "1" ] || [ "${dows}" != "" ] || git status --porcelain | egrep "^ M downloads/"; then
printf "=== Committing downloads ===\n"
git commit -m "Downloads for release ${tag}" downloads
git push
else
printf "[No need to commit: no change in downloads/]\n"
fi
fi
printf "\n"
else
......@@ -606,10 +622,8 @@ elif [ "$step" == "release" ]; then
wget -q https://koji.cern.ch/kojifiles/packages/${nam}/${vrs}/${dis}/${arc64}/${rpm64} -O ${tmpd}/${rpm64}
wget -q https://koji.cern.ch/kojifiles/packages/${nam}/${vrs}/${dis}/src/${rpmsr} -O ${tmpd}/${rpmsr}
# Is this a (pre)-release-candidate?
is_rc_tag is_rc ${tag}
# (Pre)-release-candidate: do nothing
# [or, at most, test signing rpm's with the WLCG signature]
if ${is_rc}; then
# (Pre)-release-candidate: do not distribute for installation
# Sign (without installing) for the WLCG repository
pushd ${tmpd} > /dev/null
printf "=== Sign rpm's with the WLCG signature ===\n"
......@@ -621,6 +635,7 @@ elif [ "$step" == "release" ]; then
echo "WARNING: tag ${tag} is a (pre-)release-candidate"
echo "WARNING: skip installation in WLCG and SLC6/CC7 repos"
else
# Production release
# Sign and install in the WLCG repository
pushd ${tmpd} > /dev/null
printf "=== Sign and install rpm's in the the WLCG repo ===\n"
......
<!-- Copyright 2017 CERN. Licensed under LGPLv3+. -->
# Workflow for preparing new HEP_OSlibs releases
The workflow for preparing new releases of the HEP_OSlibs meta-package
is now based on the gitlab repository of the project
(and for RedHat on the koji build system) at CERN.
The following inputs are taken from the git repository:
- All spec files needed to build the RedHat meta-rpm
and all ctl and other relevant files needed to build the Ubuntu meta-deb.
- The template documentation files used to prepare
the README and release notes for the new release.
- All scripts needed to build, test, deploy and document
the meta-package (outside or inside the gitlab CI).
The following artefacts are produced
whenever a new release is tagged and prepared:
- The unsigned src.rpm and x86_64.rpm (*) on RedHat,
as created by the build tool (koji).
- The amd64.deb on Ubuntu, as created by the build tool (equivs-build).
- The signed src.rpm and x86_64.rpm on RedHat,
for distribution via the WLCG repository.
- The signed src.rpm and x86_64.rpm on RedHat,
for distribution via the linuxsoft repository.
- The updated README and release notes for the new release on gitlab.
- The lists of direct and indirect dependencies for the meta-package.
(*) _Notes about RedHat:_
- _the x86_64.rpm contains both 32-bit and 64-bit dependencies on el6,
but only 64-bit dependencies on el7_
- _the i686.rpm, which would contain only 32-bit dependencies on el7,
is no longer built in this new workflow_
## 1. Workflow for building, testing, deploying and documenting the meta-rpm on Redhat
The x86_64.rpm meta-rpm and the corresponding source rpm on RedHat
are created using the koji infrastructure of CERN IT
(see [koji.cern.ch](http://koji.cern.ch/koji/index)).
The new workflow based on koji has been used
for el7 since the 7.2.0 and for el6 since the 1.1.0 release.
- All legacy releases in the 7.0 and 7.1 release series on el7
and in the 1.0 release series on el6
had been built using a simpler workflow
using `rpmbuild -bb` and `rpmbuild -bs`.
- The previous workflow is described in the README's of the last legacy releases
[7.1.10-0.el7](https://gitlab.cern.ch/linuxsupport/rpms/HEP_OSlibs/blob/7.1.10-0.el7/README-el7.md) and
[1.0.20-0.el6](https://gitlab.cern.ch/linuxsupport/rpms/HEP_OSlibs/blob/1.0.20-0.el6/README-el6.md).
The new workflow includes the following steps.
Note that the pkg.sh script determines the new package version
for the meta-rpm from the spec file in the local directory.
### Preliminary steps
Preliminary steps (repeat as often as necessary):
- Get a **Kerberos ticket** for CERN SSO authentication and authorization.
- **Checkout** the relevant branch.
* Different git branches exist for el6 and el7.
- **Modify and commit** the spec file for the new release.
* Also modify all relevant scripts and documentation templates.
- **"Scratch" build** step (**./pkg.sh build --scratch**).
* This triggers *koji build --scratch &lt;target&gt;*
for the last commit hash in the local git branch
(where the koji *&lt;target&gt;* is aux6 on el6 and aux7 on el7).
<br/>&nbsp; -
This checks that an rpm may be built, but discards this rpm at the end.
* While not strictly necessary, this step is useful to test koji builds
without changing version numbers (remember that
[rpm version numbers are unique in koji](https://twiki.cern.ch/twiki/bin/view/LinuxSupport/BuildingRPMswithKoji#RPM_101)).
### Mandatory steps
Mandatory steps (each step can be executed only once for each package version):
1. **Build** step (**./pkg.sh build**).
* This triggers *koji build &lt;target&gt;*
for the last commit hash in the local git branch.
<br/>&nbsp; -
This in turn *builds the meta-rpm and src rpm* for the new package version
and maintains them within koji with the *&lt;target&gt;-testing* koji tag.
* If the build is successful,
the script *creates a new &lt;version&gt;-testing git tag*.
<br/>&nbsp; -
This in turn triggers a git CI job
that *tests the installation* of the rpm on the relevant O/S
and *produces dependency lists* (which are kept forever in the gitlab CI).
2. **QA** step (**./pkg.sh qa**).
* This triggers *koji tag-build &lt;target&gt;-qa*
for the new package version,
which *promotes the meta-rpm and src rpm to qa*,
i.e. to the *&lt;target&gt;-qa* koji tag.
* The script then *commits the dependency lists* to git
and *creates a new &lt;version&gt;-qa git tag*.
3. **Release** step (**./pkg.sh release**).
* This *signs and installs the rpm's in the WLCG repository*.
* It also triggers *koji tag tag-build &lt;target&gt;-stable*
for the new package version,
which *promotes the meta-rpm and src rpm to stable*,
i.e. to the *&lt;target&gt;-stable* koji tag.
<br/>&nbsp; -
This in turn eventually *signs and installs the rpm's
in the linuxsoft repository*.
* The script then *updates and commits the README* file for the new release,
it *creates a new &lt;version&gt; git tag*
and *annotates it with release notes*.
* Finally, the script
*deletes the &lt;version&gt;-testing and &lt;version&gt;-qa git tags*.
Note that the following differences with respect to the recommended
[CERN IT koji workflow](https://twiki.cern.ch/twiki/bin/view/LinuxSupport/BuildingRPMswithKoji)
were agreed with the koji support team:
* It is not necessary to open a JIRA ticket when packages enter QA.
* It is not necessary to wait for one week after packages enter QA:
new versions of HEP_OSlibs can be built and promoted to stable
during the same day.
### Recommended post-release steps
To prevent accidental deletions or modifications of release tags,
it is useful to add them to the list of Protected Tags
from the gitlab [settings/repository](/../settings/repository) page.
### Useful koji links
The following direct links may be useful for debugging issues:
* **el7**: koji [information for HEP_OSlibs](http://koji.cern.ch/koji/packageinfo?packageID=5738)
- koji [output directory for HEP_OSlibs](https://koji.cern.ch/kojifiles/packages/HEP_OSlibs/)
* **el6**: koji [information for HEP_OSlibs_SL6](http://koji.cern.ch/koji/packageinfo?packageID=3057)
- koji [output directory for HEP_OSlibs_SL6](https://koji.cern.ch/kojifiles/packages/HEP_OSlibs_SL6/)
## 2. Workflow for building, testing, deploying and documenting the meta-deb on Ubuntu
The amd64.deb meta-deb on Ubuntu is created
using the command-line tool equivs-build.
This is executed within the gitlab CI, when a new tag is created.
This workflow has been used for all releases on Ubuntu.
It has been defined at the same time as the switch to koji and gitlab
for releases on RedHat el6 and el7.
The workflow includes the following steps.
Note that the pkg.sh script determines the new package version
for the meta-deb from the ctl file in the local directory.
### Preliminary steps
Preliminary steps (repeat as often as necessary):
- **Checkout** the relevant branch.
* Different git branches exist for ubuntu1604 and ubuntu1704.
- **Modify and commit** the ctl and changelog files for the new release.
* Also modify all relevant scripts and documentation templates.
### Mandatory steps
Mandatory steps (each step can be executed only once for each package version,
but the --rebuild option makes it possible
to start over again for the same version):
1. **Build** step (**./pkg.sh build [--rebuild]**).
* This *creates a new &lt;version&gt;-testing git tag*.
<br/>&nbsp; -
This in turn triggers a git CI job
that *builds the meta-deb* on the relevant O/S.
<br/>&nbsp; -
The CI job also *tests the installation* of the meta-deb
and *produces dependency lists* (which are kept forever in the gitlab CI).
* The --rebuild option makes it possible to delete
existing &lt;version&gt;-testing or &lt;version&gt;-qa git tags
and restart the workflow from the ctl file.
<br/>&nbsp; -
(Note that this is not possible on RedHat, where the version number
must be increased because rpm version numbers are unique in koji.)
2. **QA** step (**./pkg.sh qa**).
* This *commits the dependency lists* to git
and *creates a new &lt;version&gt;-qa git tag*.
3. **Release** step (**./pkg.sh release**).
* This *commits the meta-deb* to git.
* The script then *updates and commits the README* file for the new release,
it *creates a new &lt;version&gt; git tag*
and *annotates it with release notes*.
* Finally, the script
*deletes the &lt;version&gt;-testing and &lt;version&gt;-qa git tags*.
### Recommended post-release steps
To prevent accidental deletions or modifications of release tags,
it is useful to add them to the list of Protected Tags
from the gitlab [settings/repository](/../settings/repository) page.
# Notes about the workflow goals and implementation constraints
The current design and implementation of the workflow
were chosen to address a few specific goals,
under the limitations imposed by the available gitlab and koji infrastructures.
The workflow goals include:
- Automating the workflow as much as possible,
to require as few manual operations (commits and script calls) as possible.
- Keeping the spec and ctl files in the gitlab repository,
allowing easy diffs between different releases.
- Keeping the dependency lists in the gitlab repository,
allowing easy diffs between different releases.
- Separating the "build" and "release" steps on RedHat,
to leave time for reviewing the CI logs and dependency lists
before pushing rpm's to the WLCG and linuxsoft repositories.
- Keeping the meta-deb for Ubuntu in the gitlab repository,
rather than only in the CI artifacts area.
- Automating the update of the README for each O/S,
pointing to the most recent version released on that O/S.
- Having an easy access to the relevant changelog and README
for each release from the gitlab tag release notes.
At the time of switching to the new workflow in October 2017,
the current implementation constraints include the following:
- It is impossible to call koji from the gitlab CI
due to missing Kerberos credentials
(this may eventually be addressed by creating a dedicated service account, see
[RQF0768395](https://cern.service-now.com/service-portal/view-request.do?n=RQF0768395)).
* As a consequence, koji builds on RedHat are triggered directly
from the command line script (pkg.sh),
using the Kerberos credentials of a HEP_OSlibs developer.
* The gitlab CI however is used to create the meta-deb package on Ubuntu
and to create dependency lists on both RedHat and Ubuntu.
- It is impossible to trigger gitlab CI jobs
when a specific file in the gitlab repository changes
(this may become possible in future gitlab releases, see gitlab issues
[#19813](https://gitlab.com/gitlab-org/gitlab-ce/issues/19813)
and [#19232](https://gitlab.com/gitlab-org/gitlab-ce/issues/19232)).
* As a consequence, the CI job that creates dependency lists
(and the Ubuntu meta-deb) is triggered during the "build" step
using a temporary git tag *"&lt;version&gt;-testing"*.
* This temporary tag is then deleted and replaced
by the final *"&lt;version&gt;"* tag during the final "release" step.
- It is impossible to commit files to the gitlab repository
from the gitlab CI due to missing credentials
(this may become possible in future gitlab releases, see in gitlab issue
[#18106](https://gitlab.com/gitlab-org/gitlab-ce/issues/18106)).
* As a consequence, the meta-deb and dependency lists
created by a CI job during the "build" step must be committed
to the gitlab repository during a separate "qa" step.
<br/>&nbsp; -
This is the main reason why the workflow includes
three mandatory steps on RedHat, even after agreeing
with the koji team that the qa step could be skipped in koji.
* For consistency with RedHat, the workflow also includes
three mandatory steps on Ubuntu,
even if this could easily be reduced to only two.
* A temporary git tag *"&lt;version&gt;-qa"* is created during the "qa" step
to keep track of the progress of the release process.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment