Commit a80aa895 authored by Dainius Simelevicius's avatar Dainius Simelevicius
Browse files

references #153: 'yum' job with two upload methods

parent 55ecee24
......@@ -2,7 +2,6 @@ variables:
EXTERN_PACKAGES: "extern/oracle extern/dim extern/smi extern/yui extern/caen/a2818 extern/caen/a3818 extern/caen/CAENUSBdrvB extern/caen/caenvmelib extern/caen/caencomm extern/caen/caenbridgeupgrade extern/caen/caenupgrader"
WORKSUITE_PACKAGES: "xdaq2rc interface/evb interface/shared evb jobcontrol xpci/drv xpci hal/generic hal/utilities hal/busAdapter/dummy hal/busAdapter/caen hal/busAdapter/pci hal/PyHAL pheaps/drv/cmem_rcc pheaps jal/jtagChain jal/jtagController jal/jtagSVFSequencer ttc/utils ttc/monitoring ttc/ltc ttc/ttcci tts/ttsbase tts/atts tts/cpcibase tts/fmmtd tts/ipcutils tts/fmm tts/fmmcontroller tts/fmmdbi tts/fmmtester d2s/utils d2s/fedemulator d2s/gtpe d2s/gtpecontroller d2s/firmwareloader fedstreamer fedkit ferol ferol40 psx/sapi psx/mapi psx psx/watchdog sentinel sentinel/utils sentinel/probe sentinel/sentineld sentinel/tester sentinel/bridge2g sentinel/spotlight2g sentinel/spotlightocci sentinel/arc/utils sentinel/arc tstore/utils tstore/client tstore tstore/api xmas/admin xmas/heartbeat xmas/heartbeat/probe xmas/heartbeat/heartbeatd xmas/utils xmas/sensord xmas/probe xmas/tester xmas/bridge2g xmas/slash2g xmas/store2g xmas/smarthub ibvla pt/udapl pt/ibv gevb2g es/api es/xtreme es/xbeat elastic/api elastic/timestream amc13controller dipbridge"
RPM_REPO_DIR: "/tmp/$CI_PROJECT_NAME/$CI_JOB_NAME/repo"
YUM_DIR: "/tmp/$CI_PROJECT_NAME/$CI_COMMIT_REF_NAME/cc7/x86_64/RPMS"
DOCKER_IMAGE: "gitlab-registry.cern.ch/cmsos/docker/docker-image-cmsos-15-cc7-x86_64-worksuite:2.1.0.0"
ARTIFACT_PREREQUISITE_DIR: "$CI_PROJECT_DIR/prerequisites/"
ARTIFACT_DISTRIBUTED_DIR: "$CI_PROJECT_DIR/distributed/rpm/"
......@@ -849,23 +848,19 @@ yum:
- amc13controller
- dipbridge
script:
- find $ARTIFACT_DISTRIBUTED_DIR -name "*.rpm" -exec touch -r $ARTIFACT_PREREQUISITE_DIR/marktime {} \; # touch artifacts
- export MARK_TIME=$(date -r $ARTIFACT_PREREQUISITE_DIR/marktime '+%Y-%m-%d %H:%M:%S')
- echo "Remove all RPMs newer than "$MARK_TIME
- export YUM_DIR="/tmp/RPMS/" #trailing slash is obligatory (needed by rsync)
- mkdir -p $YUM_DIR
- find $YUM_DIR -name "*.rpm" -newermt "$MARK_TIME" -print #replace -print with -delete when tested
- find $YUM_DIR -name "*.rpm" -newermt "$MARK_TIME" -delete
- find $ARTIFACT_DISTRIBUTED_DIR -name "*.rpm" -exec touch -r $ARTIFACT_PREREQUISITE_DIR/marktime {} \; # touch artifacts
- rsync --archive --verbose --times $ARTIFACT_DISTRIBUTED_DIR $YUM_DIR
- rm -rf $YUM_DIR/.olddata/
- createrepo -g $CI_PROJECT_DIR/yumgroups.xml $YUM_DIR
- cp $CI_PROJECT_DIR/yumgroups.xml $YUM_DIR
# ***************************** IMPORTANT ***********************************************************
# Repository is always managed locally on /tmp and then uploaded.
# At this point a lftp command using SFTP to xdaq.web.cern.ch/x/xdaq/repo/ can be used to upload
# RPMS to xdaq web site. No need for mount anymore. A secret variable for credentials XDAQ_LOGIN
# can be used for secure access of the the website.
# The remote repository could be used, when available, to initialize local repository, otherwise empty
# ***************************** IMPORTANT ***********************************************************
# ***************************** IMPORTANT ***********************************************************
# Repository is always managed locally in YUM_DIR and then uploaded.
# RPMS are uploaded to xdaq.web.cern.ch/x/xdaq/repo/ either using ftps protocol using lftp tool
# or via samba mount. A secret variable for credentials XDAQ_WEBSITE_CREDENTIALS is used for secure
# access to the website. Upload method should be defined by Gitlab variable UPLOAD_METHOD.
# The following options are available: "ftps", "samba"
# ***************************** IMPORTANT ***********************************************************
- export YUM_USER=$(echo $XDAQ_WEBSITE_CREDENTIALS | awk '{split($0,a,","); print a[1]}')
- export YUM_PASS=$(echo $XDAQ_WEBSITE_CREDENTIALS | awk '{split($0,a,","); print a[2]}')
- echo $YUM_USER
......@@ -874,14 +869,23 @@ yum:
- echo "YUM_DIR="$YUM_DIR
- export DISTRIBUTE_PATH=$XDAQ_WEBSITE_PATH/development/$CI_PROJECT_NAME/$CI_COMMIT_REF_NAME/cc7/x86_64/RPMS
- echo $DISTRIBUTE_PATH
# - lftp -u $XDAQ_WEBSITE_CREDENTIALS $XDAQ_WEBSITE_HOST -e "set ftp:ssl-protect-data true; set ftp:ssl-force true; set ssl:verify-certificate no; set ftp:use-hftp no; set ftp:proxy $XDAQ_WEBSITE_PROXY; rm -rf $DISTRIBUTE_PATH; mirror -R $YUM_DIR $DISTRIBUTE_PATH"
- export MOUNTED_DIR=/dfs/cern.ch/x
- mkdir -p $MOUNTED_DIR
- mount -t cifs -o user=$YUM_USER,password=$YUM_PASS,iocharset=utf8,file_mode=0777,dir_mode=0777,nolock,vers=2.0 //cerndataw01.cern.ch/Websites-x$ $MOUNTED_DIR
- rm -rf /dfs/cern.ch/$DISTRIBUTE_PATH
- mkdir -p /dfs/cern.ch/$DISTRIBUTE_PATH
- rsync --archive --verbose --times $YUM_DIR /dfs/cern.ch/$DISTRIBUTE_PATH
- umount $MOUNTED_DIR
- >
if [ $UPLOAD_METHOD == "ftps" ]; then
echo "Uploading using ftps protocol"
lftp -u $XDAQ_WEBSITE_CREDENTIALS $XDAQ_WEBSITE_HOST -e "set ftp:ssl-protect-data true; set ftp:ssl-force true; set ssl:verify-certificate no; set ftp:use-hftp no; set ftp:proxy $XDAQ_WEBSITE_PROXY; rm -rf $DISTRIBUTE_PATH; mirror -R $YUM_DIR $DISTRIBUTE_PATH"
elif [ $UPLOAD_METHOD == "samba" ]; then
echo "Uploading using samba mount"
export MOUNTED_DIR=/dfs/cern.ch/x
mkdir -p $MOUNTED_DIR
mount -t cifs -o user=$YUM_USER,password=$YUM_PASS,iocharset=utf8,file_mode=0777,dir_mode=0777,nolock,vers=2.0 //cerndataw01.cern.ch/Websites-x$ $MOUNTED_DIR
rm -rf /dfs/cern.ch/$DISTRIBUTE_PATH
mkdir -p /dfs/cern.ch/$DISTRIBUTE_PATH
rsync --archive --verbose --times $YUM_DIR /dfs/cern.ch/$DISTRIBUTE_PATH
umount $MOUNTED_DIR
else
echo "Unknown upload method: "$UPLOAD_METHOD
exit 1
fi
environment:
name: development
url: http://$XDAQ_WEBSITE_HOST/xdaq/repo/development/$CI_PROJECT_NAME/$CI_COMMIT_REF_NAME/cc7/x86_64/RPMS/
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment