From d204bd7bde26e4098a48764686bcaf9890d78dda Mon Sep 17 00:00:00 2001 From: Pere Mato Date: Fri, 17 Nov 2017 14:38:35 +0100 Subject: [PATCH 1/9] Changes needed to support the use of binary tarfiles. This is interesting when access to CVMFS is not avaibable --- CMakeLists.txt | 5 +-- cmake/modules/lcgsoft-macros.cmake | 52 ++++++++++++++++++++++++++++-- cmake/scripts/DownloadURL.cmake | 35 +++++++++----------- 3 files changed, 67 insertions(+), 25 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f7d231f6..fa79a9aee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,9 @@ message(STATUS "Preparing .post-install.sh scripts : ${POST_INSTALL}") message(STATUS "Stripping RPATH from binaries : ${STRIP_RPATH}") message(STATUS "Ignored packages from LCG installation : ${LCG_IGNORE}") +set(GenURL http://lcgpackages.web.cern.ch/lcgpackages/tarFiles/sources) +set(BinURL http://lcgpackages.web.cern.ch/lcgpackages/tarFiles/releases) + include(${CMAKE_SOURCE_DIR}/toolchain.cmake) include(lcgsoft-macros) @@ -45,8 +48,6 @@ mark_as_advanced(env_cmd) set(EXEC ${env_cmd}) set(MAKE ${CMAKE_SOURCE_DIR}/cmake/scripts/make_verbose $(MAKE)) set(LOCKFILE ${CMAKE_SOURCE_DIR}/cmake/scripts/lockfile.py) -#set(GenURL http://service-spi.web.cern.ch/service-spi/external/tarFiles) -set(GenURL http://lcgpackages.web.cern.ch/lcgpackages/tarFiles/sources) set(PYTHON python${Python_version_major}) #--- Select the vector instruction set -------------------------------------------------------------------------------- diff --git a/cmake/modules/lcgsoft-macros.cmake b/cmake/modules/lcgsoft-macros.cmake index 0199ffd09..57a96a757 100644 --- a/cmake/modules/lcgsoft-macros.cmake +++ b/cmake/modules/lcgsoft-macros.cmake @@ -1,6 +1,16 @@ include(ExternalProject) include(CMakeParseArguments) +cmake_policy(SET CMP0057 NEW) # Support new IN_LIST if() operator +set (dependency_split_pattern "([^-]+)-(.+)") +find_package(Git) + +#---Cache the list of binary tarfiles +if(NOT EXISTS ${CMAKE_BINARY_DIR}/bintarlisting.txt) + file(DOWNLOAD ${BinURL}/summary-${LCG_system}.txt ${CMAKE_BINARY_DIR}/bintarlisting.txt) +endif() +file(STRINGS ${CMAKE_BINARY_DIR}/bintarlisting.txt bintars) + #---------------------------------------------------------------------------------------------------- #---LCGPackage_Add macro --------------------------------------------------------------------------- # @@ -19,9 +29,6 @@ include(CMakeParseArguments) # o It uses ${name}_native_version which has to be set externally # #---------------------------------------------------------------------------------------------------- -set (dependency_split_pattern "([^-]+)-(.+)") -find_package(Git) - macro(LCGPackage_Add name) #---If version is not defined (package not mentioned in toolchain) skip the whole macro------------ @@ -156,6 +163,13 @@ macro(LCGPackage_Add name) endif() endif() + #---Check whether the package is already available in binary form + set(tarname ${name}-${version}_${targethash}-${LCG_system}.tgz) + set(tarurl ${BinURL}/${tarname}) + if(${tarname} IN_LIST bintars) + set(${targetname}_binary_exists 1) + endif() + #---Package to be ignord from LCG_INSTALL_PREFIX list(FIND LCG_IGNORE ${name} lcg_ignore) set(${name}_lcg_exists) @@ -202,6 +216,38 @@ macro(LCGPackage_Add name) add_custom_target(cleanmore-${targetname} COMMENT "${targetname} nothing to be done") endif() + #---Get the buildinfo from the already installed package + if(EXISTS ${LCG_INSTALL_PREFIX}/${lcg_install_path}/.buildinfo_${name}.txt) + file(STRINGS ${LCG_INSTALL_PREFIX}/${lcg_install_path}/.buildinfo_${name}.txt ${targetname}_buildinfo) + # Cope with obsolete formats of .buildinfo file in release area + string(REPLACE "SVNREVISION:" "DIRECTORY: ${${dest_name}_directory_name}, SVNREVISION:" ${targetname}_buildinfo ${${targetname}_buildinfo}) + string(REPLACE "GITHASH: \"format:" "DIRECTORY: ${${dest_name}_directory_name}, GITHASH: '" ${targetname}_buildinfo ${${targetname}_buildinfo} ) + string(REPLACE "\"" "'" ${targetname}_buildinfo ${${targetname}_buildinfo}) + if(NOT ${${targetname}_buildinfo} MATCHES "DIRECTORY:") + string(REPLACE "VERSION:" "DIRECTORY: ${${dest_name}_directory_name}, VERSION:" ${targetname}_buildinfo ${${targetname}_buildinfo} ) + endif() + endif() + elseif(${targetname}_binary_exists) + set(${name}_home ${CMAKE_INSTALL_PREFIX}/${install_path}) + set(${targetname}_home ${${name}_home}) + set(${targetname}_dest_name ${dest_name}) + add_custom_target(${targetname} ALL DEPENDS ${${targetname}-dependencies} ${CMAKE_INSTALL_PREFIX}/${install_path}) + add_custom_command(OUTPUT ${CMAKE_INSTALL_PREFIX}/${install_path} + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_PREFIX}/${${dest_name}_directory_name}/${dest_version} + COMMAND ${CMAKE_COMMAND} -Durl=${tarurl} -Dsource_dir=. -Dnodecompress=1 -P ${CMAKE_SOURCE_DIR}/cmake/scripts/DownloadURL.cmake + COMMAND ${CMAKE_TAR} -xzf ${CMAKE_CURRENT_BINARY_DIR}/${tarname} -C ${CMAKE_INSTALL_PREFIX} + COMMAND ${CMAKE_INSTALL_PREFIX}/${install_path}/.post-install + COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_BINARY_DIR}/timestamps/${targetname}-start.timestamp + COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_BINARY_DIR}/timestamps/${targetname}-stop.timestamp + COMMENT "${targetname} binary already existing in ${tarurl}. Downloading and installing it.") + + add_custom_target(clean-${targetname} COMMAND ${CMAKE_COMMAND} -E remove ${CMAKE_INSTALL_PREFIX}/${install_path} + COMMENT "Deleting binaries for package ${targetname}") + if(LCG_SAFE_INSTALL) + #### ---- this has to be replaced by actual content if it is really needed + add_custom_target(cleanmore-${targetname} COMMENT "${targetname} nothing to be done") + endif() + #---Get the buildinfo from the already installed package if(EXISTS ${LCG_INSTALL_PREFIX}/${lcg_install_path}/.buildinfo_${name}.txt) file(STRINGS ${LCG_INSTALL_PREFIX}/${lcg_install_path}/.buildinfo_${name}.txt ${targetname}_buildinfo) diff --git a/cmake/scripts/DownloadURL.cmake b/cmake/scripts/DownloadURL.cmake index 7e71f1c1d..1153714f5 100644 --- a/cmake/scripts/DownloadURL.cmake +++ b/cmake/scripts/DownloadURL.cmake @@ -4,6 +4,7 @@ # source_dir - source directory # timeout - timeout in seconds # md5 - expected md5 +# nodecompress - flag to skip decompression # if(timeout) @@ -31,7 +32,7 @@ if("${url}" MATCHES "^[a-z]+://") timeout='${timeout_msg}' md5='${md5_msg}'") - file(DOWNLOAD "${url}" "${file}" SHOW_PROGRESS ${md5_args} ${timeout_args} STATUS status LOG log) + file(DOWNLOAD "${url}" "${file}" ${md5_args} ${timeout_args} STATUS status LOG log) list(GET status 0 status_code) list(GET status 1 status_string) @@ -45,25 +46,19 @@ if("${url}" MATCHES "^[a-z]+://") message(STATUS "downloading... done") #---extract file------------------------------------------------------------------------------------ - if(file MATCHES "(\\.|=)(gz|tgz|zip)$") - - get_filename_component(filename "${file}" ABSOLUTE) - if(NOT EXISTS "${filename}") - message(FATAL_ERROR "error: file to decompress does not exist: '${filename}'") - endif() - - message(STATUS "decompressing... src='${filename}'") - - execute_process(COMMAND gzip -df ${filename} RESULT_VARIABLE rv) - - if(NOT rv EQUAL 0) - message(FATAL_ERROR "error: decompressing of '${filename}' failed") + if(NOT nodecompress) + if(file MATCHES "(\\.|=)(gz|tgz|zip)$") + get_filename_component(filename "${file}" ABSOLUTE) + if(NOT EXISTS "${filename}") + message(FATAL_ERROR "error: file to decompress does not exist: '${filename}'") + endif() + message(STATUS "decompressing... src='${filename}'") + execute_process(COMMAND gzip -df ${filename} RESULT_VARIABLE rv) + if(NOT rv EQUAL 0) + message(FATAL_ERROR "error: decompressing of '${filename}' failed") + endif() + else() + message(STATUS "file is already de-compressed") endif() - - else() - - message(STATUS "file is already de-compressed src='${filename}'") - endif() - endif() -- GitLab From eba4a8f0a988e525f0f5af60ca0033c995d27aff Mon Sep 17 00:00:00 2001 From: Pere Mato Date: Fri, 17 Nov 2017 15:23:17 +0100 Subject: [PATCH 2/9] Add INSTALLDIR environment to post-install --- cmake/modules/lcgsoft-macros.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/modules/lcgsoft-macros.cmake b/cmake/modules/lcgsoft-macros.cmake index 57a96a757..c0bfe3501 100644 --- a/cmake/modules/lcgsoft-macros.cmake +++ b/cmake/modules/lcgsoft-macros.cmake @@ -236,7 +236,7 @@ macro(LCGPackage_Add name) COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_PREFIX}/${${dest_name}_directory_name}/${dest_version} COMMAND ${CMAKE_COMMAND} -Durl=${tarurl} -Dsource_dir=. -Dnodecompress=1 -P ${CMAKE_SOURCE_DIR}/cmake/scripts/DownloadURL.cmake COMMAND ${CMAKE_TAR} -xzf ${CMAKE_CURRENT_BINARY_DIR}/${tarname} -C ${CMAKE_INSTALL_PREFIX} - COMMAND ${CMAKE_INSTALL_PREFIX}/${install_path}/.post-install + COMMAND INSTALLDIR=${CMAKE_INSTALL_PREFIX} ${CMAKE_INSTALL_PREFIX}/${install_path}/.post-install.sh COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_BINARY_DIR}/timestamps/${targetname}-start.timestamp COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_BINARY_DIR}/timestamps/${targetname}-stop.timestamp COMMENT "${targetname} binary already existing in ${tarurl}. Downloading and installing it.") -- GitLab From 3863edfeb20789ecae8c590dd37f8c0e6ea6906c Mon Sep 17 00:00:00 2001 From: Pere Mato Date: Wed, 20 Dec 2017 17:37:14 +0100 Subject: [PATCH 3/9] fix clean command --- cmake/modules/lcgsoft-macros.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/modules/lcgsoft-macros.cmake b/cmake/modules/lcgsoft-macros.cmake index c0bfe3501..a8ae256e4 100644 --- a/cmake/modules/lcgsoft-macros.cmake +++ b/cmake/modules/lcgsoft-macros.cmake @@ -241,7 +241,7 @@ macro(LCGPackage_Add name) COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_BINARY_DIR}/timestamps/${targetname}-stop.timestamp COMMENT "${targetname} binary already existing in ${tarurl}. Downloading and installing it.") - add_custom_target(clean-${targetname} COMMAND ${CMAKE_COMMAND} -E remove ${CMAKE_INSTALL_PREFIX}/${install_path} + add_custom_target(clean-${targetname} COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_INSTALL_PREFIX}/${install_path} COMMENT "Deleting binaries for package ${targetname}") if(LCG_SAFE_INSTALL) #### ---- this has to be replaced by actual content if it is really needed -- GitLab From c29fa108c964a0013982c7a6fbb0105f95e09ecc Mon Sep 17 00:00:00 2001 From: Pere Mato Date: Fri, 15 Jun 2018 14:57:03 +0200 Subject: [PATCH 4/9] Add a new flag USE_BINARIES and option for quiet downloads --- CMakeLists.txt | 2 ++ cmake/modules/lcgsoft-macros.cmake | 5 +++-- cmake/scripts/DownloadURL.cmake | 19 ++++++++++++------- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fa79a9aee..c9cf27500 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,7 @@ set(LCG_SOURCE_INSTALL ON CACHE BOOL "Turn ON/OFF installation of sources set(VALIDATION OFF CACHE BOOL "Enable validation settings.") set(POST_INSTALL ON CACHE BOOL "Enable validation settings.") set(STRIP_RPATH ON CACHE BOOL "Strip RPATH from binaries.") +set(USE_BINARIES ON CACHE BOOL "Download binaries if they exists in repository.") string(REPLACE " " ";" LCG_IGNORE "${LCG_IGNORE}") #---Report the values of the options----------------------------------------------------------------------------------- @@ -34,6 +35,7 @@ message(STATUS "Source installation in /share : ${LCG_SOURCE_INSTALL}") message(STATUS "Validation mode : ${VALIDATION}") message(STATUS "Preparing .post-install.sh scripts : ${POST_INSTALL}") message(STATUS "Stripping RPATH from binaries : ${STRIP_RPATH}") +message(STATUS "Download binaries if they exists : ${USE_BINARIES}") message(STATUS "Ignored packages from LCG installation : ${LCG_IGNORE}") set(GenURL http://lcgpackages.web.cern.ch/lcgpackages/tarFiles/sources) diff --git a/cmake/modules/lcgsoft-macros.cmake b/cmake/modules/lcgsoft-macros.cmake index a8ae256e4..00cdd2961 100644 --- a/cmake/modules/lcgsoft-macros.cmake +++ b/cmake/modules/lcgsoft-macros.cmake @@ -6,7 +6,8 @@ set (dependency_split_pattern "([^-]+)-(.+)") find_package(Git) #---Cache the list of binary tarfiles -if(NOT EXISTS ${CMAKE_BINARY_DIR}/bintarlisting.txt) +if(USE_BINARIES AND NOT EXISTS ${CMAKE_BINARY_DIR}/bintarlisting.txt) + message("Download ${BinURL}/summary-${LCG_system}.txt") file(DOWNLOAD ${BinURL}/summary-${LCG_system}.txt ${CMAKE_BINARY_DIR}/bintarlisting.txt) endif() file(STRINGS ${CMAKE_BINARY_DIR}/bintarlisting.txt bintars) @@ -234,7 +235,7 @@ macro(LCGPackage_Add name) add_custom_target(${targetname} ALL DEPENDS ${${targetname}-dependencies} ${CMAKE_INSTALL_PREFIX}/${install_path}) add_custom_command(OUTPUT ${CMAKE_INSTALL_PREFIX}/${install_path} COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_PREFIX}/${${dest_name}_directory_name}/${dest_version} - COMMAND ${CMAKE_COMMAND} -Durl=${tarurl} -Dsource_dir=. -Dnodecompress=1 -P ${CMAKE_SOURCE_DIR}/cmake/scripts/DownloadURL.cmake + COMMAND ${CMAKE_COMMAND} -Durl=${tarurl} -Dsource_dir=. -Dnodecompress=1 -Dquiet=1 -P ${CMAKE_SOURCE_DIR}/cmake/scripts/DownloadURL.cmake COMMAND ${CMAKE_TAR} -xzf ${CMAKE_CURRENT_BINARY_DIR}/${tarname} -C ${CMAKE_INSTALL_PREFIX} COMMAND INSTALLDIR=${CMAKE_INSTALL_PREFIX} ${CMAKE_INSTALL_PREFIX}/${install_path}/.post-install.sh COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_BINARY_DIR}/timestamps/${targetname}-start.timestamp diff --git a/cmake/scripts/DownloadURL.cmake b/cmake/scripts/DownloadURL.cmake index 1153714f5..6db8bbdfa 100644 --- a/cmake/scripts/DownloadURL.cmake +++ b/cmake/scripts/DownloadURL.cmake @@ -5,6 +5,7 @@ # timeout - timeout in seconds # md5 - expected md5 # nodecompress - flag to skip decompression +# quiet - flag to avoid verbosity # if(timeout) @@ -26,11 +27,13 @@ if("${url}" MATCHES "^[a-z]+://") set(file ${source_dir}/${fname}) #---Downloading------------------------------------------------------------------------------------- - message(STATUS "downloading... - src='${url}' - dst='${file}' - timeout='${timeout_msg}' - md5='${md5_msg}'") + if(NOT quiet) + message(STATUS "downloading... + src='${url}' + dst='${file}' + timeout='${timeout_msg}' + md5='${md5_msg}'") + endif() file(DOWNLOAD "${url}" "${file}" ${md5_args} ${timeout_args} STATUS status LOG log) @@ -42,8 +45,10 @@ if("${url}" MATCHES "^[a-z]+://") status_string: ${status_string} log: ${log}") endif() - - message(STATUS "downloading... done") + + if(NOT quiet) + message(STATUS "downloading... done") + endif() #---extract file------------------------------------------------------------------------------------ if(NOT nodecompress) -- GitLab From 94deb1eeb80f132d4ad17f6ffd684a5bff814b6f Mon Sep 17 00:00:00 2001 From: Pere Mato Date: Fri, 15 Jun 2018 18:06:55 +0200 Subject: [PATCH 5/9] cosmetics --- .gitignore | 2 ++ compiler_wrapper.in | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) delete mode 100755 compiler_wrapper.in diff --git a/.gitignore b/.gitignore index e3463c544..ec4fd7de3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ .vscode/* .browse.VC.* +.DS_Store + diff --git a/compiler_wrapper.in b/compiler_wrapper.in deleted file mode 100755 index bc1c33255..000000000 --- a/compiler_wrapper.in +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -@COMPILER_PATH@ @WRAPPER_FLAGS@ "$@" -- GitLab From 84bdc6bc094234d1913b63b2054888664c7862e4 Mon Sep 17 00:00:00 2001 From: Pere Mato Date: Tue, 19 Jun 2018 17:55:31 +0200 Subject: [PATCH 6/9] Changed the installation locatikon for the binaries --- CMakeLists.txt | 2 +- bin/lcgcmake | 189 +++++++++++++++++++++++++++++ cmake/modules/lcgsoft-macros.cmake | 15 ++- toolchain.cmake | 5 + 4 files changed, 205 insertions(+), 6 deletions(-) create mode 100755 bin/lcgcmake diff --git a/CMakeLists.txt b/CMakeLists.txt index c9cf27500..3a4283fd0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,7 @@ set(LCG_SOURCE_INSTALL ON CACHE BOOL "Turn ON/OFF installation of sources set(VALIDATION OFF CACHE BOOL "Enable validation settings.") set(POST_INSTALL ON CACHE BOOL "Enable validation settings.") set(STRIP_RPATH ON CACHE BOOL "Strip RPATH from binaries.") -set(USE_BINARIES ON CACHE BOOL "Download binaries if they exists in repository.") +set(USE_BINARIES OFF CACHE BOOL "Download binaries if they exists in repository (changes installation layout).") string(REPLACE " " ";" LCG_IGNORE "${LCG_IGNORE}") #---Report the values of the options----------------------------------------------------------------------------------- diff --git a/bin/lcgcmake b/bin/lcgcmake new file mode 100755 index 000000000..429c73e43 --- /dev/null +++ b/bin/lcgcmake @@ -0,0 +1,189 @@ +#!/usr/bin/env python + +# Author: Pere Mato +# Usage: lcgcmake [command] [options] +# This command drives the full process of building a LCG release using the 'lcgcmake' tool +# commands +# bootstrap +# configure +# build +# install +# optional arguments: +# -h, --help show this help message and exit +# + +#---------------------------------------------------------------------------------------------------- +from __future__ import print_function +import argparse, sys, os, json, platform, re, tarfile, urllib, subprocess + +#---Globals------------------------------------------------------------------------------------------ + +srcURL = 'http://lcgpackages.web.cern.ch/lcgpackages/tarFiles/sources' +binURL = 'http://lcgpackages.web.cern.ch/lcgpackages/tarFiles/releases' +rcPATH = os.path.join(os.environ['HOME'],'.lcgcmake') + +depends = {'gcc62binutils':['binutils-2.28'], + 'gcc62':[], + 'gcc7binutils':['binutils-2.28'], + 'gcc8binutils':['binutils-2.28'], + } + +config = {} +helpstring = '{0} [sub-command] [options]\n\nThis command drives the full process of building a LCG release using the lcgcmake tool.\n' + +#---Bootstrap----------------------------------------------------------------------------------------- +def lcgcmake_bootstrap(args) : + print('Boostrapping...') + + compiler = args.compiler + prefix = args.prefix + arch = args.arch + osvers = args.osvers + environment = {} + sourcedir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + cmake = subprocess.check_output(['/usr/bin/bash','-i', '-c','which cmake']).strip().split('\n')[-1].strip() + + #---Check compiler + if compiler != 'native' : + c, v = extract_compilerversion(compiler) + compilerpath = os.path.join(prefix, c, v, "-".join([arch,osvers])) + if not os.path.exists(compilerpath) : + print('Requested compiler %s not found. Installing it.' % compiler) + install_tarfile(c, v, arch, osvers, prefix) + if compiler in depends: + for dep in depends[compiler]: + p, v = dep.split('-') + packagepath = os.path.join(prefix, p, v, "-".join([arch,osvers])) + if not os.path.exists(packagepath) : + print('Dependent package %s-%s not found. Installing it.' % (p,v)) + install_tarfile(p, v, arch, osvers, prefix) + else: + print('compiler already installed') + #---define environment for compiler + if compiler in depends: + for dep in depends[compiler]: + p, v = dep.split('-') + packagepath = os.path.join(prefix, p, v, "-".join([arch,osvers])) + set_env_variable(environment, 'PATH', os.path.join(packagepath,'bin')) + set_env_variable(environment, 'LD_LIBRARY_PATH', os.path.join(packagepath,'lib')) + set_env_variable(environment, 'PATH', os.path.join(compilerpath,'bin')) + set_env_variable(environment, 'LD_LIBRARY_PATH', os.path.join(compilerpath,'lib64')) + set_env_variable(environment, 'CC', os.path.join(compilerpath,'bin','gcc')) + set_env_variable(environment, 'CXX', os.path.join(compilerpath,'bin','g++')) + set_env_variable(environment, 'FC', os.path.join(compilerpath,'bin','gfortran')) + else: + print('using native compiler') + + #---Save configuration for next commands + config['COMPILER'] = args.compiler + config['PREFIX'] = args.prefix + config['ARCH'] = args.arch + config['OSVERS'] = args.osvers + config['ENVIRONMENT'] = environment + config['SOURCEDIR'] = sourcedir + config['CMAKE'] = cmake + + if not os.path.exists(os.path.join(rcPATH,'config.json')) : os.mkdir(rcPATH) + json.dump(config, open(os.path.join(rcPATH,'config.json'),'w'), sort_keys=True, indent=2, separators=(',', ': ')) + +#---Configure----------------------------------------------------------------------------------------- +def lcgcmake_configure(args) : + if not os.path.exists(os.path.join(rcPATH,'config.json')): + print("lcgcmake session has not been bootstrapped. Use the command 'lcgcmake bootstrap' first.") + return + + version = args.version + #---read configuration + config = json.load(open(os.path.join(rcPATH,'config.json'),'r')) + options = [ + '-DLCG_VERSION=%s' % version, + '-DCMAKE_INSTALL_PREFIX=%s' % config['PREFIX'] + ] + setup_environment(config['ENVIRONMENT']) + subprocess.call([config['CMAKE'], config['SOURCEDIR']] + options) + +#---Info------------------------------------------------------------------------------------------------ +def lcgcmake_info(args) : + print('Current configuration') + print(open(os.path.join(rcPATH,'config.json'),'r').read()) + +#---Utility functions----------------------------------------------------------------------------------- +def extract_compilerversion(compvers): + patt = re.compile('([a-z]+)([0-9]+)([a-z]*)') + mobj = patt.match(compvers) + compiler = mobj.group(1) + if compiler == 'clang' : compiler = 'llvm' + version = '.'.join(list(mobj.group(2))) + mobj.group(3) + return compiler, version + +def install_tarfile(package, version, arch, osvers, prefix): + urltarfile = os.path.join(binURL, "-".join([package, version, arch, osvers])+'.tgz') + tmptarfile = os.path.join('/tmp', "-".join([package, version, arch, osvers])+'.tgz') + print('Downloading %s' % urltarfile) + filename, headers = urllib.urlretrieve(urltarfile, tmptarfile) + print('Installing %s-%s in %s' % (package, version, prefix)) + tar = tarfile.open(tmptarfile) + tar.extractall(path=prefix) + tar.close() + +def set_env_variable(env, var, val): + if('PATH' in var): + if var in env: env[var] += [val] + else: env[var] = [val] + else: + env[var] = val + +def setup_environment(env): + for var in env: + if('PATH' in var): + if var in os.environ: os.environ[var] = ':'.join(env[var]+[os.environ[var]]) + else: os.environ[var] = ':'.join(env[var]) + else: + os.environ[var] = env[var] + +if __name__ == '__main__': + + #---default values + arch = platform.machine() + system = platform.system() + if system == 'Darwin' : + osvers = 'mac' + string.join(platform.mac_ver()[0].split('.')[:2],'') + elif system == 'Linux' : + dist = platform.linux_distribution() + if re.search('SLC', dist[0]): + osvers = 'slc' + dist[1].split('.')[0] + elif re.search('CentOS', dist[0]): + osvers = 'centos' + dist[1].split('.')[0] + elif re.search('Ubuntu', dist[0]): + osvers = 'ubuntu' + dist[1].split('.')[0] + dist[1].split('.')[1] + elif re.search('Fedora', dist[0]): + osvers = 'fedora' + dist[1].split('.')[0] + else: + osvers = 'linux' + string.join(platform.linux_distribution()[1].split('.')[:2],'') + elif system == 'Windows': + osvers = win + platform.win32_ver()[0] + else: + osvers = 'unk-os' + + parser = argparse.ArgumentParser(usage=helpstring.format(sys.argv[0])) + parser.add_argument('--verbose', action='count', dest='verbose_level', help='Increase logging verbosity', default=0) + parser.add_argument('-q', '--quiet', action='count', dest='quiet_level', help='Decrease logging verbosity', default=0) + parser.add_argument('-a', '--arch', dest='arch', help='processor architecture', default=arch) + parser.add_argument('-o', '--os', dest='osvers', help='operating system keyword (e.g. slc6, mac1013, etc.) ', default=osvers) + subparsers = parser.add_subparsers( dest='command', help='available sub-commands') + p_v = subparsers.add_parser('version', help='print the version of lcgcmake') + p_i = subparsers.add_parser('info', help='dump the current configuration') + p_b = subparsers.add_parser('bootstrap', help='bootstrap the lcgcmake session by selecting the compiler, prefix, etc.') + p_b.add_argument('-c', '--compiler', dest='compiler', help='compiler keyword to be used (e.g. gcc62, gcc7binutils, etc. )', default='native') + p_b.add_argument('-p', '--prefix', dest='prefix', help='prefix to the final installations', default='/opt/lcg') + p_c = subparsers.add_parser('configure', help='configure the lcgcmake session by selecting the software stack version, etc.') + p_c.add_argument('-v', '--version', dest='version', help='LCG software stack version', default='dev3') + args = parser.parse_args() + + if args.command == 'bootstrap' : + lcgcmake_bootstrap(args) + elif args.command == 'configure' : + lcgcmake_configure(args) + elif args.command == 'info' : + lcgcmake_info(args) + diff --git a/cmake/modules/lcgsoft-macros.cmake b/cmake/modules/lcgsoft-macros.cmake index 00cdd2961..ca339363a 100644 --- a/cmake/modules/lcgsoft-macros.cmake +++ b/cmake/modules/lcgsoft-macros.cmake @@ -145,9 +145,10 @@ macro(LCGPackage_Add name) endif() #---Install path---------------------------------------------------------------------------------- # we support both with the hash and without - set(install_path ${${dest_name}_directory_name}/${dest_version}/${LCG_system}) - if(EXISTS ${LCG_INSTALL_PREFIX}/${${dest_name}_directory_name}/${dest_version}-${${dest_name}-${dest_version}_hash}/${LCG_system}) - set(lcg_install_path ${${dest_name}_directory_name}/${dest_version}-${${dest_name}-${dest_version}_hash}/${LCG_system}) + set(install_path ${${dest_name}_directory_name}/${dest_version}/${LCG_system}) + set(install_hash_path ${${dest_name}_directory_name}/${dest_version}-${${dest_name}-${dest_version}_hash}/${LCG_system}) + if(EXISTS ${LCG_INSTALL_PREFIX}/${install_hash_path}) + set(lcg_install_path ${install_hash_path}) else() set(lcg_install_path ${install_path}) endif() @@ -230,14 +231,18 @@ macro(LCGPackage_Add name) endif() elseif(${targetname}_binary_exists) set(${name}_home ${CMAKE_INSTALL_PREFIX}/${install_path}) + get_filename_component(preprefix ${CMAKE_INSTALL_PREFIX}/.. REALPATH) set(${targetname}_home ${${name}_home}) set(${targetname}_dest_name ${dest_name}) add_custom_target(${targetname} ALL DEPENDS ${${targetname}-dependencies} ${CMAKE_INSTALL_PREFIX}/${install_path}) add_custom_command(OUTPUT ${CMAKE_INSTALL_PREFIX}/${install_path} COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_PREFIX}/${${dest_name}_directory_name}/${dest_version} COMMAND ${CMAKE_COMMAND} -Durl=${tarurl} -Dsource_dir=. -Dnodecompress=1 -Dquiet=1 -P ${CMAKE_SOURCE_DIR}/cmake/scripts/DownloadURL.cmake - COMMAND ${CMAKE_TAR} -xzf ${CMAKE_CURRENT_BINARY_DIR}/${tarname} -C ${CMAKE_INSTALL_PREFIX} - COMMAND INSTALLDIR=${CMAKE_INSTALL_PREFIX} ${CMAKE_INSTALL_PREFIX}/${install_path}/.post-install.sh + COMMAND ${CMAKE_TAR} -xzf ${CMAKE_CURRENT_BINARY_DIR}/${tarname} -C ${preprefix} + COMMAND INSTALLDIR=${preprefix} ${preprefix}/${install_path}/.post-install.sh + COMMAND ${CMAKE_COMMAND} -E rename ${preprefix}/${${dest_name}_directory_name}/${dest_version} + ${preprefix}/${${dest_name}_directory_name}/${dest_version}-${${dest_name}-${dest_version}_hash} + COMMAND ${CMAKE_COMMAND} -E create_symlink ${preprefix}/${install_hash_path} ${CMAKE_INSTALL_PREFIX}/${install_path} COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_BINARY_DIR}/timestamps/${targetname}-start.timestamp COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_BINARY_DIR}/timestamps/${targetname}-stop.timestamp COMMENT "${targetname} binary already existing in ${tarurl}. Downloading and installing it.") diff --git a/toolchain.cmake b/toolchain.cmake index 0fa7aeca8..fe88a9969 100644 --- a/toolchain.cmake +++ b/toolchain.cmake @@ -20,4 +20,9 @@ set(CMAKE_TOOLCHAIN_FILE ${CMAKE_TOOLCHAIN_FILE} CACHE FILEPATH "The CMake toolchain file" FORCE) message(STATUS "Using toolchain file : ${CMAKE_TOOLCHAIN_FILE}") +# Suffix the effective install prefix with the LCG version +if(USE_BINARIES) + set(CMAKE_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}/${LCG_VERSION}) +endif() + include(${CMAKE_TOOLCHAIN_FILE}) -- GitLab From 929ffa609947603f82b4b6f69d9cb1530d71b5c8 Mon Sep 17 00:00:00 2001 From: Pere Mato Date: Mon, 2 Jul 2018 15:33:26 +0200 Subject: [PATCH 7/9] Finlizing the 'lcgcmake' command, README and adding latest heptools --- README.md | 25 +- bin/lcgcmake | 233 ++++++++--- cmake/modules/lcgsoft-macros.cmake | 24 +- cmake/scripts/create_lcg_view.py | 2 +- .../heptools-geantvext20180306.cmake | 2 + cmake/toolchain/heptools-latest.cmake | 369 ++++++++++++++++++ 6 files changed, 587 insertions(+), 68 deletions(-) create mode 100644 cmake/toolchain/heptools-latest.cmake diff --git a/README.md b/README.md index 0a134bcc4..4bdcbbaa2 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,29 @@ Check the [pre-requisites](#pre-requisites) section for the complete list of req On CVMFS (CERN) set PATH to use one of latest CMake versions (system default is 2.6 and even 3.6.X are too old) ``` - ARCH=$(uname -m) - export PATH=/cvmfs/sft.cern.ch/lcg/contrib/CMake/3.8.1/Linux-${ARCH}/bin:${PATH} +ARCH=$(uname -m) +export PATH=/cvmfs/sft.cern.ch/lcg/contrib/CMake/3.8.1/Linux-${ARCH}/bin:${PATH} ``` -# A Quick Start: +# Installation and very Quick Start +1. **Install lcgcmake** by just cloning the lcgcmake package from CERN GitLab and ensure that the PATH environment variable contains the the bin directory from the just cloned repository. + ``` + git clone https://gitlab.cern.ch/sft/lcgcmake.git + export PATH $PWD/lcgcmake/bin:$PATH + ``` +2. **Configure the software stack** by selecting the compiler and version of the stack to be use +``` +lcgcmake configure --compiler=gcc73binutils --version=latest [--prefix=...] +``` +3. **Show the current configuration** by using the show sub-command +``` +lcgcmake show configuration +``` +4. Installation of required packages +``` +lcgcmake install ROOT Boost +``` + +# Low level interface 1. Checkout the lcgcmake package from lcgsoft GIT repository: ``` git clone https://gitlab.cern.ch/sft/lcgcmake.git diff --git a/bin/lcgcmake b/bin/lcgcmake index 429c73e43..cd9392585 100755 --- a/bin/lcgcmake +++ b/bin/lcgcmake @@ -14,7 +14,7 @@ #---------------------------------------------------------------------------------------------------- from __future__ import print_function -import argparse, sys, os, json, platform, re, tarfile, urllib, subprocess +import argparse, sys, os, json, platform, re, tarfile, urllib, subprocess, shutil #---Globals------------------------------------------------------------------------------------------ @@ -22,27 +22,19 @@ srcURL = 'http://lcgpackages.web.cern.ch/lcgpackages/tarFiles/sources' binURL = 'http://lcgpackages.web.cern.ch/lcgpackages/tarFiles/releases' rcPATH = os.path.join(os.environ['HOME'],'.lcgcmake') -depends = {'gcc62binutils':['binutils-2.28'], - 'gcc62':[], - 'gcc7binutils':['binutils-2.28'], - 'gcc8binutils':['binutils-2.28'], - } +c_depends = {'native':[], + 'gcc62binutils':['binutils-2.28'], + 'gcc62':[], + 'gcc73binutils':['binutils-2.28'], + 'gcc81binutils':['binutils-2.28'], + } config = {} helpstring = '{0} [sub-command] [options]\n\nThis command drives the full process of building a LCG release using the lcgcmake tool.\n' #---Bootstrap----------------------------------------------------------------------------------------- -def lcgcmake_bootstrap(args) : - print('Boostrapping...') - - compiler = args.compiler - prefix = args.prefix - arch = args.arch - osvers = args.osvers +def lcgcmake_bootstrap(compiler, arch, osvers, prefix) : environment = {} - sourcedir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - cmake = subprocess.check_output(['/usr/bin/bash','-i', '-c','which cmake']).strip().split('\n')[-1].strip() - #---Check compiler if compiler != 'native' : c, v = extract_compilerversion(compiler) @@ -50,18 +42,18 @@ def lcgcmake_bootstrap(args) : if not os.path.exists(compilerpath) : print('Requested compiler %s not found. Installing it.' % compiler) install_tarfile(c, v, arch, osvers, prefix) - if compiler in depends: - for dep in depends[compiler]: + if compiler in c_depends: + for dep in c_depends[compiler]: p, v = dep.split('-') packagepath = os.path.join(prefix, p, v, "-".join([arch,osvers])) if not os.path.exists(packagepath) : print('Dependent package %s-%s not found. Installing it.' % (p,v)) install_tarfile(p, v, arch, osvers, prefix) else: - print('compiler already installed') + print('Compiler %s already installed' % compiler) #---define environment for compiler - if compiler in depends: - for dep in depends[compiler]: + if compiler in c_depends: + for dep in c_depends[compiler]: p, v = dep.split('-') packagepath = os.path.join(prefix, p, v, "-".join([arch,osvers])) set_env_variable(environment, 'PATH', os.path.join(packagepath,'bin')) @@ -73,39 +65,121 @@ def lcgcmake_bootstrap(args) : set_env_variable(environment, 'FC', os.path.join(compilerpath,'bin','gfortran')) else: print('using native compiler') + set_env_variable(environment, 'COMPILER',compiler) + return environment + + +#---Configure----------------------------------------------------------------------------------------- +def lcgcmake_configure(args) : + #---setup some variables with arguments and defaults + environment = {} + config = {} + sourcedir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + cmake = subprocess.check_output(['/usr/bin/bash','-i', '-c','which cmake']).strip().split('\n')[-1].strip() + + #---read the old configuration if exists + if os.path.exists(os.path.join(rcPATH,'config.json')): + config = json.load(open(os.path.join(rcPATH,'config.json'),'r')) + + compiler = args.compiler or config.get('COMPILER', 'native') + version = args.version or config.get('VERSION','latest') + prefix = args.prefix or config.get('PREFIX', '/opt/lcg') + arch = args.arch + osvers = args.osvers + + #---deal with new compiler + if 'COMPILER' not in config or config['COMPILER'] != compiler: + # need to setup a new compiler + environment = lcgcmake_bootstrap(compiler, arch, osvers, prefix) + # need to delete caches + if os.path.exists('CMakeCache.txt') : os.remove('CMakeCache.txt') + if os.path.exists('CMakeFiles') : shutil.rmtree('CMakeFiles') + if os.path.exists('bintarlisting.txt') : os.remove('bintarlisting.txt') + else: + # recover the environment from previous run + environment = config['ENVIRONMENT'] + + #---execute cmake to configure + options = [ + '-DLCG_VERSION=%s' % version, + '-DCMAKE_INSTALL_PREFIX=%s' % prefix, + '-DLCG_SOURCE_INSTALL=OFF', + '-DUSE_BINARIES=ON' + ] + setup_environment(environment) + rc = subprocess.call([cmake, sourcedir] + options) + if rc != 0 : + print('Configuration has failed. Please correct the problem guided by the error messages and try again') + return - #---Save configuration for next commands - config['COMPILER'] = args.compiler - config['PREFIX'] = args.prefix - config['ARCH'] = args.arch - config['OSVERS'] = args.osvers + #---save configuration + config['COMPILER'] = compiler + config['PREFIX'] = prefix + config['ARCH'] = arch + config['OSVERS'] = osvers config['ENVIRONMENT'] = environment config['SOURCEDIR'] = sourcedir config['CMAKE'] = cmake - - if not os.path.exists(os.path.join(rcPATH,'config.json')) : os.mkdir(rcPATH) + config['VERSION'] = version + config['PLATFORM'] = subprocess.check_output(['grep','LCG_platform:STRING=','CMakeCache.txt']).strip()[20:] + if not os.path.exists(rcPATH) : os.mkdir(rcPATH) json.dump(config, open(os.path.join(rcPATH,'config.json'),'w'), sort_keys=True, indent=2, separators=(',', ': ')) -#---Configure----------------------------------------------------------------------------------------- -def lcgcmake_configure(args) : + +#---Install----------------------------------------------------------------------------------------- +def lcgcmake_install(args) : if not os.path.exists(os.path.join(rcPATH,'config.json')): - print("lcgcmake session has not been bootstrapped. Use the command 'lcgcmake bootstrap' first.") + print("lcgcmake session has not been configured. Use the command 'lcgcmake configure' first.") return + targets = args.targets - version = args.version #---read configuration config = json.load(open(os.path.join(rcPATH,'config.json'),'r')) - options = [ - '-DLCG_VERSION=%s' % version, - '-DCMAKE_INSTALL_PREFIX=%s' % config['PREFIX'] - ] setup_environment(config['ENVIRONMENT']) - subprocess.call([config['CMAKE'], config['SOURCEDIR']] + options) -#---Info------------------------------------------------------------------------------------------------ -def lcgcmake_info(args) : - print('Current configuration') - print(open(os.path.join(rcPATH,'config.json'),'r').read()) + rc = subprocess.call(['make', '-j8'] + targets) + if rc == 0 : + rc = subprocess.call(['make','view']) + if rc == 0 : + print("Installation of '%s' has been successful." % (' '.join(targets),)) + print(" To set the running environment do 'source %s/%s/%s/setup.sh" % (config['PREFIX'],config['VERSION'],config['PLATFORM'])) + print(" Or run the command 'lcgcmake run [command]'") + +#---Show------------------------------------------------------------------------------------------------ +def lcgcmake_show(args) : + if args.what == 'config': + if not os.path.exists(os.path.join(rcPATH,'config.json')): + print("lcgcmake session has not been configured. Use the command 'lcgcmake configure' first.") + else: + print('Current configuration:') + print(open(os.path.join(rcPATH,'config.json'),'r').read()) + elif args.what == 'targets': + config = json.load(open(os.path.join(rcPATH,'config.json'),'r')) + targets = subprocess.check_output([config['CMAKE'], '--build', '.', '--target', 'help']) + # start from '... clean' + i = targets.find('... clean') + targets = [x for x in targets[i:].split() if x not in ('...') and 'clean-' not in x and '-dependencies' not in x] + targets.sort() + stargets=[] + last=None + for t in targets: + if '-' in t : stargets[-1].append(t) + else: stargets.append([t]) + for t in stargets: + print(','.join(t)) + elif args.what == 'compilers': + get_available_compilers(args.arch, args.osvers) + +#---Run------------------------------------------------------------------------------------------------- +def lcgcmake_run(args): + if not os.path.exists(os.path.join(rcPATH,'config.json')): + print("lcgcmake session has not been configured. Use the command 'lcgcmake configure' first.") + return + #---read configuration + config = json.load(open(os.path.join(rcPATH,'config.json'),'r')) + + setup = '%s/%s/%s/setup.sh' % (config['PREFIX'],config['VERSION'],config['PLATFORM']) + rc = subprocess.call(['/usr/bin/bash', '-i', '-c','export PS1=[lcgcmake\ \\\W]\$\ && source '+ setup + ' &&' + ' '.join(args.runcmd)]) #---Utility functions----------------------------------------------------------------------------------- def extract_compilerversion(compvers): @@ -120,11 +194,26 @@ def install_tarfile(package, version, arch, osvers, prefix): urltarfile = os.path.join(binURL, "-".join([package, version, arch, osvers])+'.tgz') tmptarfile = os.path.join('/tmp', "-".join([package, version, arch, osvers])+'.tgz') print('Downloading %s' % urltarfile) - filename, headers = urllib.urlretrieve(urltarfile, tmptarfile) + urllib.urlretrieve(urltarfile, tmptarfile) print('Installing %s-%s in %s' % (package, version, prefix)) - tar = tarfile.open(tmptarfile) - tar.extractall(path=prefix) - tar.close() + try: + tar = tarfile.open(tmptarfile, errorlevel=1) + tar.extractall(path=prefix) + except tarfile.ReadError as detail: + print ('Error untaring %s : %s' %(tmptarfile, detail)) + except: + print ('Unexpected error:', sys.exc_info()[0]) + else: + tar.close() + +def get_available_compilers(arch, osvers): + urlsummary = os.path.join(binURL, "-".join(['summary', arch, osvers])+'.txt') + tmpsummary = os.path.join('/tmp', "-".join(['summary', arch, osvers])+'.txt') + filename, headers = urllib.urlretrieve(urlsummary, tmpsummary) + for l in open(tmpsummary).readlines(): + p,v = l.split('-')[0:2] + if p == 'gcc' : + print(p + v.replace('.','')) def set_env_variable(env, var, val): if('PATH' in var): @@ -141,6 +230,29 @@ def setup_environment(env): else: os.environ[var] = env[var] +# Code taken from https://gist.github.com/sampsyo/471779 +class AliasedSubParsersAction(argparse._SubParsersAction): + class _AliasedPseudoAction(argparse.Action): + def __init__(self, name, aliases, help): + dest = name + if aliases: + dest += ' (%s)' % ','.join(aliases) + sup = super(AliasedSubParsersAction._AliasedPseudoAction, self) + sup.__init__(option_strings=[], dest=dest, help=help) + def add_parser(self, name, **kwargs): + aliases = kwargs.pop('aliases', []) + parser = super(AliasedSubParsersAction, self).add_parser(name, **kwargs) + # Make the aliases work. + for alias in aliases: + self._name_parser_map[alias] = parser + # Make the help text reflect them, first removing old help entry. + if 'help' in kwargs: + help = kwargs.pop('help') + self._choices_actions.pop() + pseudo_action = self._AliasedPseudoAction(name, aliases, help) + self._choices_actions.append(pseudo_action) + return parser + if __name__ == '__main__': #---default values @@ -166,24 +278,31 @@ if __name__ == '__main__': osvers = 'unk-os' parser = argparse.ArgumentParser(usage=helpstring.format(sys.argv[0])) + parser.register('action', 'parsers', AliasedSubParsersAction) parser.add_argument('--verbose', action='count', dest='verbose_level', help='Increase logging verbosity', default=0) parser.add_argument('-q', '--quiet', action='count', dest='quiet_level', help='Decrease logging verbosity', default=0) parser.add_argument('-a', '--arch', dest='arch', help='processor architecture', default=arch) parser.add_argument('-o', '--os', dest='osvers', help='operating system keyword (e.g. slc6, mac1013, etc.) ', default=osvers) subparsers = parser.add_subparsers( dest='command', help='available sub-commands') p_v = subparsers.add_parser('version', help='print the version of lcgcmake') - p_i = subparsers.add_parser('info', help='dump the current configuration') - p_b = subparsers.add_parser('bootstrap', help='bootstrap the lcgcmake session by selecting the compiler, prefix, etc.') - p_b.add_argument('-c', '--compiler', dest='compiler', help='compiler keyword to be used (e.g. gcc62, gcc7binutils, etc. )', default='native') - p_b.add_argument('-p', '--prefix', dest='prefix', help='prefix to the final installations', default='/opt/lcg') - p_c = subparsers.add_parser('configure', help='configure the lcgcmake session by selecting the software stack version, etc.') - p_c.add_argument('-v', '--version', dest='version', help='LCG software stack version', default='dev3') + p_c = subparsers.add_parser('configure', help='configure the lcgcmake session by selecting the software stack version, etc.', aliases=['config', 'conf']) + p_c.add_argument('-c', '--compiler', dest='compiler', help='compiler keyword to be used (e.g. gcc62, gcc7binutils, etc. )', default=None) + p_c.add_argument('-p', '--prefix', dest='prefix', help='prefix to the final installations', default=None) + p_c.add_argument('-v', '--version', dest='version', help='LCG software stack version', default=None) + p_m = subparsers.add_parser('install', help='Install or build a set of given targets with all their dependencies') + p_m.add_argument('targets', nargs='+', help='Select the target[s]', default=None) + p_s = subparsers.add_parser('show', help='get information from the lcgcmake session', aliases=['sh']) + p_s.add_argument('what', help='What to show', choices=['config','targets','compilers']) + p_r = subparsers.add_parser('run', help='run a command with the just installed software stack') + p_r.add_argument('runcmd', nargs='*', help='Command to run', default=['/usr/bin/bash']) + args = parser.parse_args() - if args.command == 'bootstrap' : - lcgcmake_bootstrap(args) - elif args.command == 'configure' : + if args.command in ['configure','config','conf'] : lcgcmake_configure(args) - elif args.command == 'info' : - lcgcmake_info(args) - + elif args.command in ['install'] : + lcgcmake_install(args) + elif args.command in ['show', 'sh'] : + lcgcmake_show(args) + elif args.command in ['run'] : + lcgcmake_run(args) diff --git a/cmake/modules/lcgsoft-macros.cmake b/cmake/modules/lcgsoft-macros.cmake index ca339363a..8238663af 100644 --- a/cmake/modules/lcgsoft-macros.cmake +++ b/cmake/modules/lcgsoft-macros.cmake @@ -235,19 +235,29 @@ macro(LCGPackage_Add name) set(${targetname}_home ${${name}_home}) set(${targetname}_dest_name ${dest_name}) add_custom_target(${targetname} ALL DEPENDS ${${targetname}-dependencies} ${CMAKE_INSTALL_PREFIX}/${install_path}) - add_custom_command(OUTPUT ${CMAKE_INSTALL_PREFIX}/${install_path} + if(EXISTS ${preprefix}/${install_hash_path}) + add_custom_command(OUTPUT ${CMAKE_INSTALL_PREFIX}/${install_path} + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_PREFIX}/${${dest_name}_directory_name}/${dest_version} + COMMAND ${CMAKE_COMMAND} -E create_symlink ${preprefix}/${install_hash_path} ${CMAKE_INSTALL_PREFIX}/${install_path} + COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_BINARY_DIR}/timestamps/${targetname}-start.timestamp + COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_BINARY_DIR}/timestamps/${targetname}-stop.timestamp + COMMENT "${targetname} binary already installed. Linking to it.") + else() + set(tar_options "--transform='s,/${dest_version}/${LCG_system},/${dest_version}-${${dest_name}-${dest_version}_hash}/${LCG_system},g'") + add_custom_command(OUTPUT ${CMAKE_INSTALL_PREFIX}/${install_path} COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_PREFIX}/${${dest_name}_directory_name}/${dest_version} COMMAND ${CMAKE_COMMAND} -Durl=${tarurl} -Dsource_dir=. -Dnodecompress=1 -Dquiet=1 -P ${CMAKE_SOURCE_DIR}/cmake/scripts/DownloadURL.cmake - COMMAND ${CMAKE_TAR} -xzf ${CMAKE_CURRENT_BINARY_DIR}/${tarname} -C ${preprefix} - COMMAND INSTALLDIR=${preprefix} ${preprefix}/${install_path}/.post-install.sh - COMMAND ${CMAKE_COMMAND} -E rename ${preprefix}/${${dest_name}_directory_name}/${dest_version} - ${preprefix}/${${dest_name}_directory_name}/${dest_version}-${${dest_name}-${dest_version}_hash} + COMMAND ${CMAKE_TAR} ${tar_options} -xzf ${CMAKE_CURRENT_BINARY_DIR}/${tarname} -C ${preprefix} + COMMAND ${CMAKE_COMMAND} -E remove_directory ${preprefix}/${${dest_name}_directory_name}/${dest_version} + COMMAND INSTALLDIR=${preprefix} ${preprefix}/${install_hash_path}/.post-install.sh + #COMMAND ${CMAKE_COMMAND} -E rename ${preprefix}/${${dest_name}_directory_name}/${dest_version} + # ${preprefix}/${${dest_name}_directory_name}/${dest_version}-${${dest_name}-${dest_version}_hash} COMMAND ${CMAKE_COMMAND} -E create_symlink ${preprefix}/${install_hash_path} ${CMAKE_INSTALL_PREFIX}/${install_path} COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_BINARY_DIR}/timestamps/${targetname}-start.timestamp COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_BINARY_DIR}/timestamps/${targetname}-stop.timestamp COMMENT "${targetname} binary already existing in ${tarurl}. Downloading and installing it.") - - add_custom_target(clean-${targetname} COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_INSTALL_PREFIX}/${install_path} + endif() + add_custom_target(clean-${targetname} COMMAND ${CMAKE_COMMAND} -E remove ${CMAKE_INSTALL_PREFIX}/${install_path} COMMENT "Deleting binaries for package ${targetname}") if(LCG_SAFE_INSTALL) #### ---- this has to be replaced by actual content if it is really needed diff --git a/cmake/scripts/create_lcg_view.py b/cmake/scripts/create_lcg_view.py index 5cd5f1f08..950d58c12 100755 --- a/cmake/scripts/create_lcg_view.py +++ b/cmake/scripts/create_lcg_view.py @@ -143,7 +143,7 @@ class LCGViewMaker(object): compiler = mobj.group(1) if compiler == 'clang' : compiler = 'llvm' version = '.'.join(list(mobj.group(2))) + mobj.group(3) - for p in (os.path.join(i,j) for i in ('..', '../../../..', '../../..', '../..') for j in ('contrib','external')): + for p in (os.path.join(i,j) for i in ('..', '../../../..', '../../..', '../..') for j in ('contrib','external','.')): if os.path.exists(os.path.normpath(os.path.join(self.lcg_root, p, compiler))): compiler = os.path.normpath(os.path.join(self.lcg_root, p, compiler, version, '-'.join((arch, osvers)))) break diff --git a/cmake/toolchain/heptools-geantvext20180306.cmake b/cmake/toolchain/heptools-geantvext20180306.cmake index 04331babc..1cc4942a6 100644 --- a/cmake/toolchain/heptools-geantvext20180306.cmake +++ b/cmake/toolchain/heptools-geantvext20180306.cmake @@ -44,6 +44,7 @@ LCG_external_package(decorator 4.0.9 LCG_external_package(delphes 3.4.0 ) LCG_external_package(dill 0.2.5 ) LCG_external_package(distribute 0.6.49 ) +LCG_external_package(enum34 1.1.6 ) LCG_external_package(joblib 0.10.0 ) LCG_external_package(jsonc 0.12 ) LCG_external_package(jsoncpp 1.7.2 ) @@ -198,6 +199,7 @@ LCG_external_package(pkg_config 0.28 LCG_external_package(pox 0.2.2 ) LCG_external_package(png 1.6.17 ) LCG_external_package(ppft 1.6.4.6 ) +LCG_external_package(prctl 1.7 ) LCG_external_package(prettytable 0.7.2 ) LCG_external_package(processing 0.52 ) LCG_external_package(prompt_toolkit 1.0.3 ) diff --git a/cmake/toolchain/heptools-latest.cmake b/cmake/toolchain/heptools-latest.cmake new file mode 100644 index 000000000..b77682012 --- /dev/null +++ b/cmake/toolchain/heptools-latest.cmake @@ -0,0 +1,369 @@ +cmake_minimum_required(VERSION 2.8.5) + +# Declare the version of HEP Tools we use +# (must be done before including heptools-common to allow evolution of the +# structure) +set(heptools_version latest) + +include(${CMAKE_CURRENT_LIST_DIR}/heptools-common.cmake) + +# please keep alphabetic order and the structure (tabbing). +# it makes it much easier to edit/read this file! + + +# Application Area Projects +LCG_AA_project(COOL 3_2_0) +LCG_AA_project(CORAL 3_2_0) +LCG_AA_project(RELAX root6) +LCG_AA_project(ROOT 6.12.06) +LCG_AA_project(HepMC 2.06.09) +LCG_AA_project(Geant4 10.04.p01) +LCG_AA_project(DD4hep 01-05) + +# Externals +LCG_external_package(lcgenv 1.3.5 ) +LCG_external_package(hepmc3 3.0.0 ) +LCG_external_package(4suite 1.0.2p1 ) +LCG_external_package(absl_py 0.2.0 ) +LCG_external_package(AIDA 3.2.1 ) +LCG_external_package(arrow 0.8.0 ) +LCG_external_package(pyarrow 0.8.0 ) +LCG_external_package(astroid 1.6.4 ) +LCG_external_package(autoconf 2.69 ) +LCG_external_package(automake 1.14 ) +LCG_external_package(backports 1.0.0 ) +LCG_external_package(backports.ssl_match_hostname 3.4.0.2 ) +LCG_external_package(backports.shutil_get_terminal_size 1.0.0 ) +LCG_external_package(backports.lzma 0.0.6 ) +LCG_external_package(backports.functools_lru_cache 1.4 ) +LCG_external_package(benchmark 1.4.0 ) +LCG_external_package(blas 3.8.0 ) +LCG_external_package(bleach 2.0.0 ) +LCG_external_package(Boost 1.66.0 ) +LCG_external_package(C50 2.07 ) +LCG_external_package(cairo 1.15.8 ) +if(NOT LCG_COMP STREQUAL "clang") +LCG_external_package(catboost 0.5 ) +endif() +LCG_external_package(certifi 14.05.14 ) +LCG_external_package(chardet 3.0.4 ) +LCG_external_package(cmmnbuild 2.1.3 ) +LCG_external_package(configparser 3.5.0 ) +LCG_external_package(cppgsl b07383ea ) +LCG_external_package(cycler 0.10.0 ) +LCG_external_package(decorator 4.0.9 ) +LCG_external_package(delphes 3.4.2pre12 ) +LCG_external_package(dill 0.2.5 ) +LCG_external_package(joblib 0.10.0 ) +LCG_external_package(jsonc 0.12 ) +LCG_external_package(jsoncpp 1.7.2 ) +LCG_external_package(ccache 3.3.4 ) +LCG_external_package(CLHEP 2.4.0.1 clhep ) +LCG_external_package(clang 3.9.0 ) +LCG_external_package(CouchDB 1.0.1 ) +LCG_external_package(cmaketools 1.6 ) +LCG_external_package(CMake 3.11.1 ) +LCG_external_package(cmt v1r20p20090520 ) +LCG_external_package(coin3d 3.1.3p2 ) +LCG_external_package(coverage 4.4.1 ) +LCG_external_package(CppUnit 1.12.1_p1 author=1.12.1 ) +LCG_external_package(cx_oracle 5.1.1 ) +if(${LCG_OS}${LCG_OSVERS} STREQUAL slc6) + LCG_external_package(curl 7.19.7 ) +else() + LCG_external_package(curl 7.59.0 ) +endif() +LCG_external_package(cython 0.27.3 ) +if(NOT ${LCG_OS} STREQUAL mac) + LCG_external_package(Davix 0.6.7 ) +endif() +LCG_external_package(Django 1.9.12 ) +LCG_external_package(doxygen 1.8.11 ) +LCG_external_package(expat 2.2.5 ) +LCG_external_package(eigen 3.2.9 ) +LCG_external_package(elasticsearch 6.2.0 ) +LCG_external_package(entrypoints 0.2.2 ) +LCG_external_package(enum34 1.1.6 ) +LCG_external_package(fastjet 3.3.0 ) +LCG_external_package(flatbuffers 1.8.0 ) +LCG_external_package(fjcontrib 1.030 ) +LCG_external_package(fftw 3.3.4 fftw3 ) +LCG_external_package(fontconfig 2.12.6 ) +LCG_external_package(freetype 2.6.3 ) +LCG_external_package(fplll 5.0.3 ) +LCG_external_package(Frontier_Client 2.8.19 frontier_client) +LCG_external_package(ftjam 2.5.2 ) +LCG_external_package(funcsigs 1.0.2 ) +LCG_external_package(future 0.16.0 ) +LCG_external_package(futures 3.0.5 ) +#LCG_external_package(GCCXML 0.9.0_20131026 gccxml ) +LCG_external_package(gast 0.2.0 ) +if(NOT ${LCG_OS} MATCHES mac) + LCG_external_package(gdb 7.12.1 ) +endif() +LCG_external_package(genshi 0.7 ) +LCG_external_package(gettext 0.19.8.1 ) +LCG_external_package(gmp 6.0.0 ) +LCG_external_package(gnuplot 5.0.6 ) +if(${LCG_OS} STREQUAL slc OR ${LCG_OS} STREQUAL centos OR ${LCG_OS} STREQUAL cc) + LCG_external_package(go 1.9.2 ) + LCG_external_package(gophernotes 1.0.0-rc1 ) +endif() +LCG_external_package(glib 2.52.2 ) +LCG_external_package(gperf 3.1 ) +if(NOT LCG_COMP STREQUAL "clang") + LCG_external_package(gperftools 2.5 ) +endif() +LCG_external_package(gtest master20160308 ) +LCG_external_package(graphviz 2.28.0 ) +LCG_external_package(GSL 2.1 ) +LCG_external_package(hadoop 2.7.5.1 ) +LCG_external_package(harfbuzz 1.6.3 ) +LCG_external_package(hbase 1.2.0 ) +LCG_external_package(hepdata_converter 0.1.23 ) +LCG_external_package(hepdata_validator 0.1.8 ) +LCG_external_package(HepPDT 2.06.01 ) +LCG_external_package(hdf5 1.8.18 ) +LCG_external_package(hive 1.1.1 ) +LCG_external_package(hspy 1.6.4 ) +LCG_external_package(h5py 2.6.0 ) +LCG_external_package(html5lib 0.999999999 ) +LCG_external_package(idna 2.6 ) +LCG_external_package(ipython 5.4.1 ) +LCG_external_package(ipython_genutils 0.1.0 ) +LCG_external_package(ipykernel 4.6.1 ) +LCG_external_package(ipywidgets 5.2.2 ) +LCG_external_package(isort 4.3.3 ) +LCG_external_package(java 8u91 ) +LCG_external_package(jemalloc 4.1.0 ) +LCG_external_package(Jinja2 2.10 ) +LCG_external_package(jpype 0.6.2 ) +LCG_external_package(jsonschema 2.4.0 ) +LCG_external_package(jupyter 1.0.0 ) +LCG_external_package(jupyter_client 5.1.0 ) +LCG_external_package(jupyter_core 4.4.0 ) +LCG_external_package(jupyter_console 5.0.0 ) +LCG_external_package(jupyter_contrib_nbextensions 0.2.1 ) +LCG_external_package(jupyter_contrib_core 0.3.0 ) +LCG_external_package(jupyter_nbextensions_configurator 0.2.2 ) +LCG_external_package(keras 2.1.1 ) +LCG_external_package(lapack 3.5.0 ) +LCG_external_package(lazy_object_proxy 1.3.1 ) +LCG_external_package(lcov 1.9 ) +LCG_external_package(libaio 0.3.110-1 ) +LCG_external_package(libffi 3.2.1 ) +LCG_external_package(libgit2 0.27.0 ) +LCG_external_package(libsvm 2.86 ) +LCG_external_package(libtool 2.4.2 ) +LCG_external_package(libxml2 2.9.7 ) +LCG_external_package(libxslt 1.1.28 ) +LCG_external_package(logilabcommon 1.0.1 ) +LCG_external_package(lxml 4.1.1 ) +if (NOT (${LCG_OS} STREQUAL ubuntu )) +LCG_external_package(libxkbcommon 0.7.1 ) +endif() +LCG_external_package(m4 1.4.17 ) +LCG_external_package(m2crypto 0.25.1 ) +LCG_external_package(MarkupSafe 0.23 ) +LCG_external_package(maven 3.3.9 ) +LCG_external_package(metakernel 0.20.12 ) +LCG_external_package(markdown 2.6.11 ) +LCG_external_package(matplotlib 2.1.0 ) +LCG_external_package(mccabe 0.6.1 ) +LCG_external_package(messaging 1.0 ) +LCG_external_package(mistune 0.5.1 ) +LCG_external_package(mock 2.0.0 ) +LCG_external_package(mpfr 3.1.5 ) +LCG_external_package(mpfi 1.5.1 ) +LCG_external_package(mpich2 1.5 ) +LCG_external_package(mpmath 1.0.0 ) +LCG_external_package(multiprocess 0.70.4 ) +LCG_external_package(multiprocessing 2.6.2.1 ) +LCG_external_package(mysql 5.7.20 ) +LCG_external_package(mysql_python 1.2.3 ) +LCG_external_package(nanomsg 1.1.3 ) +LCG_external_package(nbconvert 5.1.1 ) +LCG_external_package(nbformat 4.0.1 ) +LCG_external_package(networkx 1.11 ) +LCG_external_package(ninja 1.7.2.gcc0ea.kitware.dyndep ) +LCG_external_package(nose 1.1.2 ) +LCG_external_package(notebook 4.2.1 ) +LCG_external_package(numexpr 2.6.4 ) +LCG_external_package(numpy 1.14.2 ) +LCG_external_package(octave 4.2.1 ) +LCG_external_package(octavekernel 0.28.3 ) +LCG_external_package(omniorb 4.2.2 ) +if(${LCG_OS} MATCHES mac) + LCG_external_package(openssl 1.0.2d ) +endif() +LCG_external_package(oracle 11.2.0.3.0 ) +LCG_external_package(pacparser 1.3.5 ) +LCG_external_package(pandas 0.21.0 ) +LCG_external_package(pandocfilters 1.4.1 ) +LCG_external_package(pango 1.40.13 ) +LCG_external_package(pathlib2 2.1.0 ) +LCG_external_package(pathos 0.2.0 ) +LCG_external_package(patsy 0.4.1 ) +LCG_external_package(pbr 2.0.0 ) +LCG_external_package(pcre 8.38 ) +LCG_external_package(pexpect 4.2.0 ) +LCG_external_package(pickleshare 0.7.2 ) +LCG_external_package(pip 9.0.1 ) +LCG_external_package(pixman 0.34.0 ) +LCG_external_package(pjlsa 0.0.12 ) +LCG_external_package(pkg_config 0.28 ) +LCG_external_package(pox 0.2.2 ) +LCG_external_package(png 1.6.17 ) +LCG_external_package(ppft 1.6.4.6 ) +LCG_external_package(prctl 1.7 ) +LCG_external_package(prettytable 0.7.2 ) +LCG_external_package(processing 0.52 ) +LCG_external_package(prompt_toolkit 1.0.3 ) +LCG_external_package(protobuf 2.5.0 ) +LCG_external_package(ptyprocess 0.5.1 ) +LCG_external_package(psutil 4.3.1 ) +LCG_external_package(py 1.4.33 ) +LCG_external_package(py2neo 1.4.6 ) +LCG_external_package(py4j 0.10.3 ) +LCG_external_package(pyanalysis 2.0 ) +LCG_external_package(pydot 1.2.3 ) +LCG_external_package(pydot_ng 1.0.0 ) +LCG_external_package(pygments 2.0.2 ) +LCG_external_package(pygraphics 2.0 ) +if(NOT ${LCG_OS} MATCHES mac) + LCG_external_package(pygsi 0.6.3 ) +endif() +LCG_external_package(pyjapc 1.4.1 ) +LCG_external_package(pylint 1.9.1 ) +LCG_external_package(pyparsing 2.1.8 ) +LCG_external_package(pyqt5 5.9.1 ) +LCG_external_package(pystan 2.16.0.0 ) +LCG_external_package(pytest 3.2.5 ) +LCG_external_package(pytimber 2.6.2 ) +LCG_external_package(pytz 2017.3 ) +LCG_external_package(Python 2.7.15 ) +LCG_external_package(PythonFWK 2.7.15 Python ) +LCG_external_package(python_dateutil 2.4.0 ) +LCG_external_package(python_gitlab 0.20 ) +if(${LCG_OS} STREQUAL slc OR ${LCG_OS} STREQUAL centos OR ${LCG_OS} STREQUAL cc) +LCG_external_package(pythonsollya 0.1 ) +endif() +LCG_external_package(pytools 2.0 ) +LCG_external_package(PyHEADTAIL 1.12.1 ) +LCG_external_package(PyYAML 3.12 pyyaml ) +LCG_external_package(pyxml 0.8.4p1 ) +LCG_external_package(pyzmq 17.0.0b3 ) +LCG_external_package(QMtest 2.4.1 ) +##LCG_external_package(Qt 4.8.7 qt ) +LCG_external_package(Qt5 5.9.2 qt5 ) +LCG_external_package(qtconsole 4.2.1 ) +##LCG_external_package(qwt 6.0.1 ) +LCG_external_package(R 3.2.5 ) +LCG_external_package(rangev3 0.3.0 ) +LCG_external_package(requests 2.18.4 ) +LCG_external_package(rise 5.3.0 ) +LCG_external_package(rootpy 0.8.1 ) +LCG_external_package(root_numpy 4.6.0 ) +LCG_external_package(rpy2 2.8.0 ) +LCG_external_package(singledispatch 3.4.0.3 ) +LCG_external_package(scikitlearn 0.18.1 ) +LCG_external_package(scipy 1.0.0 ) +LCG_external_package(seaborn 0.7.1 ) +LCG_external_package(setuptools 36.0.1 ) +LCG_external_package(simplegeneric 0.8.1 ) +LCG_external_package(simplejson 3.8.2 ) +LCG_external_package(sip 4.19.5 ) +LCG_external_package(six 1.10.0 ) +LCG_external_package(sollya 6.0 ) +###LCG_external_package(soqt 1.5.0 ) +if(${LCG_OS} STREQUAL slc OR ${LCG_OS} STREQUAL centos) + LCG_external_package(spark 2.3.0 ) +endif() +LCG_external_package(sqlalchemy 1.1.15 ) +LCG_external_package(sqlite 3210000 ) +LCG_external_package(statsmodels 0.8.0 ) +LCG_external_package(stomppy 3.1.3 ) +LCG_external_package(storm 0.19 ) +LCG_external_package(subprocess32 3.2.6 ) +LCG_external_package(swig 3.0.12 author=3.0.12 ) +LCG_external_package(sympy 1.1.1 ) +LCG_external_package(tbb 2018_U1 ) +#LCG_external_package(tcmalloc 2.4 ) +LCG_external_package(tensorboard 1.8.0 ) +LCG_external_package(tensorflow 1.8.0 ) +LCG_external_package(terminado 0.5 ) +LCG_external_package(testpath 0.3.1 ) +LCG_external_package(texinfo 6.3 ) +LCG_external_package(theano 0.8.2 ) +LCG_external_package(tornado 4.0.2 ) +LCG_external_package(traitlets 4.2.2 ) +LCG_external_package(tricktrack 1.0.2 ) +LCG_external_package(typing 3.5.2.2 ) +LCG_external_package(urllib3 1.22 ) +LCG_external_package(uuid 1.42 ) +if(NOT ${LCG_OS} STREQUAL mac) + LCG_external_package(valgrind 3.11.0 ) +endif() +LCG_external_package(umesimd 0.8.1 ) +LCG_external_package(veccore 0.4.2 ) +LCG_external_package(vectorclass 1.30 ) +LCG_external_package(VecGeom v00.05.01 ) +LCG_external_package(vdt 0.3.9 ) +LCG_external_package(wcwidth 0.1.7 ) +LCG_external_package(Vc 1.3.2 ) +LCG_external_package(wheel 0.29.0 ) +LCG_external_package(webencodings 0.5.1 ) +LCG_external_package(werkzeug 0.14.1 ) +LCG_external_package(widgetsnbextension 1.2.6 ) +LCG_external_package(wrapt 1.10.11 ) +LCG_external_package(xapian 1.2.21 ) +LCG_external_package(XercesC 3.1.3 author=3.1.3 ) +LCG_external_package(xgboost 0.71 ) +LCG_external_package(xqilla 2.3.3 ) +LCG_external_package(xrootd 4.8.2 ) +LCG_external_package(xz 5.0.4 ) +if(NOT ${LCG_OS} STREQUAL mac) + LCG_external_package(xrootd_python 0.3.0 ) +endif() +LCG_external_package(yamlcpp 0.5.1 ) +LCG_external_package(zeromq 4.1.6 ) +LCG_external_package(zlib 1.2.11 ) +#---EMI-2 grid externals and other binary packages--------------------- +if(NOT ${LCG_OS} MATCHES mac) + LCG_external_package(libunwind 5c2cade ) + LCG_external_package(igprof 5.9.16 ) +endif() +if(${LCG_OS} STREQUAL slc OR ${LCG_OS} STREQUAL centos) + if (NOT ${LCG_HOST_ARCH} STREQUAL i686) + LCG_external_package(CASTOR 2.1.13-6 castor ) + LCG_external_package(cream 1.14.0-4 Grid/cream ) + LCG_external_package(dcap 2.47.7-1 Grid/dcap ) + LCG_external_package(dm-util 1.15.0-0 Grid/dm-util ) + LCG_external_package(dpm 1.8.5-1 Grid/DPM ) + LCG_external_package(epel 20130408 Grid/epel ) + LCG_external_package(FTS 2.2.8emi2 Grid/FTS ) + if(${LCG_OS}${LCG_OSVERS} STREQUAL slc6) + LCG_external_package(FTS3 0.0.1-88 Grid/FTS3 ) + endif() + LCG_external_package(gfal 1.13.0-0 Grid/gfal ) +# LCG_external_package(gfal2 2.2.0-1 Grid/gfal2 ) + LCG_external_package(gridftp_ifce 2.3.1-0 Grid/gridftp-ifce ) + LCG_external_package(gridsite 1.7.25-1.emi2 Grid/gridsite ) + LCG_external_package(is_ifce 1.15.0-0 Grid/is-ifce ) + LCG_external_package(lb 3.2.9 Grid/lb ) + LCG_external_package(lcgdmcommon 1.8.5-1 Grid/lcg-dm-common) + LCG_external_package(lcginfosites 3.1.0-3 Grid/lcg-infosites) + LCG_external_package(lfc 1.8.5-1 Grid/LFC ) + LCG_external_package(srm_ifce 1.13.0-0 Grid/srm-ifce ) + LCG_external_package(voms 2.0.9-1 Grid/voms ) + LCG_external_package(WMS 3.4.0 Grid/WMS ) + LCG_external_package(neurobayes 3.7.0 ) + LCG_external_package(neurobayes_expert 3.7.0 ) + endif() +endif() + +#---Additional External packages------(Generators)----------------- +include(${CMAKE_CURRENT_LIST_DIR}/heptools-dev-generators.cmake) + +# Prepare the search paths according to the versions above +LCG_prepare_paths() -- GitLab From 6e31cd5b09ece4fe839df87b9f8c766707226ce3 Mon Sep 17 00:00:00 2001 From: Pere Mato Date: Mon, 2 Jul 2018 16:49:40 +0200 Subject: [PATCH 8/9] Ensure compatibility with CMake 2.8 --- README.md | 2 +- bin/lcgcmake | 17 ++++++++++--- cmake/modules/lcgsoft-macros.cmake | 41 +++++++++++++++--------------- 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 4bdcbbaa2..af7aa9aa1 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ export PATH=/cvmfs/sft.cern.ch/lcg/contrib/CMake/3.8.1/Linux-${ARCH}/bin:${PATH} ``` 2. **Configure the software stack** by selecting the compiler and version of the stack to be use ``` -lcgcmake configure --compiler=gcc73binutils --version=latest [--prefix=...] +lcgcmake configure --compiler=gcc62binutils --version=latest [--prefix=...] ``` 3. **Show the current configuration** by using the show sub-command ``` diff --git a/bin/lcgcmake b/bin/lcgcmake index cd9392585..ecfeceabb 100755 --- a/bin/lcgcmake +++ b/bin/lcgcmake @@ -75,7 +75,7 @@ def lcgcmake_configure(args) : environment = {} config = {} sourcedir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - cmake = subprocess.check_output(['/usr/bin/bash','-i', '-c','which cmake']).strip().split('\n')[-1].strip() + cmake = subprocess.check_output(['/bin/bash','-i', '-c','which cmake']).strip().split('\n')[-1].strip() #---read the old configuration if exists if os.path.exists(os.path.join(rcPATH,'config.json')): @@ -179,7 +179,11 @@ def lcgcmake_run(args): config = json.load(open(os.path.join(rcPATH,'config.json'),'r')) setup = '%s/%s/%s/setup.sh' % (config['PREFIX'],config['VERSION'],config['PLATFORM']) - rc = subprocess.call(['/usr/bin/bash', '-i', '-c','export PS1=[lcgcmake\ \\\W]\$\ && source '+ setup + ' &&' + ' '.join(args.runcmd)]) + rc = subprocess.call(['/bin/bash', '-i', '-c','export PS1=[lcgcmake\ \\\W]\$\ && source '+ setup + ' &&' + ' '.join(args.runcmd)]) + +#---Version------------------------------------------------------------------------------------------------- +def lcgcmake_version(args): + print('0.1') #---Utility functions----------------------------------------------------------------------------------- def extract_compilerversion(compvers): @@ -253,6 +257,11 @@ class AliasedSubParsersAction(argparse._SubParsersAction): self._choices_actions.append(pseudo_action) return parser +if sys.version_info < (2,7) : + def subprocess_check_output(args): + return subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0] + subprocess.check_output = subprocess_check_output + if __name__ == '__main__': #---default values @@ -294,7 +303,7 @@ if __name__ == '__main__': p_s = subparsers.add_parser('show', help='get information from the lcgcmake session', aliases=['sh']) p_s.add_argument('what', help='What to show', choices=['config','targets','compilers']) p_r = subparsers.add_parser('run', help='run a command with the just installed software stack') - p_r.add_argument('runcmd', nargs='*', help='Command to run', default=['/usr/bin/bash']) + p_r.add_argument('runcmd', nargs='*', help='Command to run', default=['/bin/bash']) args = parser.parse_args() @@ -306,3 +315,5 @@ if __name__ == '__main__': lcgcmake_show(args) elif args.command in ['run'] : lcgcmake_run(args) + elif args.command in ['version'] : + lcgcmake_version(args) diff --git a/cmake/modules/lcgsoft-macros.cmake b/cmake/modules/lcgsoft-macros.cmake index 8238663af..3e650a0f4 100644 --- a/cmake/modules/lcgsoft-macros.cmake +++ b/cmake/modules/lcgsoft-macros.cmake @@ -93,24 +93,22 @@ macro(LCGPackage_Add name) "Add a call to 'LCG_external_package(${dep} )' in ${CMAKE_TOOLCHAIN_FILE}.") endif() endif() - if(${${dep}_home} STREQUAL TakenFromSystem) - continue() + if(NOT ${${dep}_home} STREQUAL TakenFromSystem) + list(APPEND ${targetname}_dependencies ${dep}) endif() - list(APPEND ${targetname}_dependencies ${dep}) endforeach() endif() #---Get list of dependencies as package-version if(ARG_DEPENDS) set (${targetname}-dependencies) foreach(dep ${ARG_DEPENDS}) - if(${${dep}_home} STREQUAL TakenFromSystem) - continue() - endif() - if(dep MATCHES "${dependency_split_pattern}") - list (APPEND ${targetname}-dependencies "${CMAKE_MATCH_1}-${CMAKE_MATCH_2}") - else() - list(GET ${dep}_native_version -1 dep_vers) - list(APPEND ${targetname}-dependencies "${dep}-${dep_vers}") + if(NOT ${${dep}_home} STREQUAL TakenFromSystem) + if(dep MATCHES "${dependency_split_pattern}") + list (APPEND ${targetname}-dependencies "${CMAKE_MATCH_1}-${CMAKE_MATCH_2}") + else() + list(GET ${dep}_native_version -1 dep_vers) + list(APPEND ${targetname}-dependencies "${dep}-${dep_vers}") + endif() endif() endforeach() endif() @@ -168,7 +166,9 @@ macro(LCGPackage_Add name) #---Check whether the package is already available in binary form set(tarname ${name}-${version}_${targethash}-${LCG_system}.tgz) set(tarurl ${BinURL}/${tarname}) - if(${tarname} IN_LIST bintars) + #if(${tarname} IN_LIST bintars) + list(FIND bintars ${tarname} _index) + if(_index NOT EQUAL -1) set(${targetname}_binary_exists 1) endif() @@ -416,15 +416,14 @@ macro(LCGPackage_Add name) set(buildinfostring "${buildinfostring} DEPENDS: ") foreach(dep ${ARG_DEPENDS}) # ignore packages taken from system - if(${${dep}_home} STREQUAL TakenFromSystem) - continue() - endif() - # dependent package may have the form name-version - if(dep MATCHES "${dependency_split_pattern}") - set(buildinfostring "${buildinfostring}${dep}-${${dep}_hash},") - else() - list(GET ${dep}_native_version -1 vers) - set(buildinfostring "${buildinfostring}${dep}-${vers}-${${dep}-${vers}_hash},") + if(NOT ${${dep}_home} STREQUAL TakenFromSystem) + # dependent package may have the form name-version + if(dep MATCHES "${dependency_split_pattern}") + set(buildinfostring "${buildinfostring}${dep}-${${dep}_hash},") + else() + list(GET ${dep}_native_version -1 vers) + set(buildinfostring "${buildinfostring}${dep}-${vers}-${${dep}-${vers}_hash},") + endif() endif() endforeach() set(${targetname}_buildinfo "${buildinfostring}") -- GitLab From 0ef5180789a2845279f923a429441e0fa275f833 Mon Sep 17 00:00:00 2001 From: Pere Mato Date: Mon, 2 Jul 2018 18:18:13 +0200 Subject: [PATCH 9/9] Improvem ents from testing in native SLC6 --- CMakeLists.txt | 2 +- bin/lcgcmake | 4 ++-- cmake/modules/lcgsoft-macros.cmake | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a4283fd0..ecfa76f7d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -162,6 +162,6 @@ file(WRITE ${CMAKE_BINARY_DIR}/fail-logs.txt "") #---Target to create a LCG view in the install prefix area add_custom_target(view - COMMAND ${CMAKE_COMMAND} -E env LCG_VERSION=${LCG_VERSION} + COMMAND /bin/env LCG_VERSION=${LCG_VERSION} ${CMAKE_SOURCE_DIR}/cmake/scripts/create_lcg_view.py -l ${CMAKE_INSTALL_PREFIX} -p ${LCG_system} -d -B ${CMAKE_INSTALL_PREFIX}/${LCG_system}) diff --git a/bin/lcgcmake b/bin/lcgcmake index ecfeceabb..1f39547d9 100755 --- a/bin/lcgcmake +++ b/bin/lcgcmake @@ -14,7 +14,7 @@ #---------------------------------------------------------------------------------------------------- from __future__ import print_function -import argparse, sys, os, json, platform, re, tarfile, urllib, subprocess, shutil +import argparse, sys, os, json, platform, re, tarfile, urllib, subprocess, shutil, multiprocessing #---Globals------------------------------------------------------------------------------------------ @@ -137,7 +137,7 @@ def lcgcmake_install(args) : config = json.load(open(os.path.join(rcPATH,'config.json'),'r')) setup_environment(config['ENVIRONMENT']) - rc = subprocess.call(['make', '-j8'] + targets) + rc = subprocess.call(['make', '-j%d' % (multiprocessing.cpu_count())] + targets) if rc == 0 : rc = subprocess.call(['make','view']) if rc == 0 : diff --git a/cmake/modules/lcgsoft-macros.cmake b/cmake/modules/lcgsoft-macros.cmake index 3e650a0f4..b72e1d5cd 100644 --- a/cmake/modules/lcgsoft-macros.cmake +++ b/cmake/modules/lcgsoft-macros.cmake @@ -1,7 +1,9 @@ include(ExternalProject) include(CMakeParseArguments) -cmake_policy(SET CMP0057 NEW) # Support new IN_LIST if() operator +if(POLICY CMP0057) + cmake_policy(SET CMP0057 NEW) # Support new IN_LIST if() operator +endif() set (dependency_split_pattern "([^-]+)-(.+)") find_package(Git) @@ -168,7 +170,7 @@ macro(LCGPackage_Add name) set(tarurl ${BinURL}/${tarname}) #if(${tarname} IN_LIST bintars) list(FIND bintars ${tarname} _index) - if(_index NOT EQUAL -1) + if(NOT _index EQUAL -1) set(${targetname}_binary_exists 1) endif() -- GitLab