From 4048ee0f8a4549141e1360ab3739954a90a0da0b Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Mon, 13 May 2024 11:10:38 +0200 Subject: [PATCH 01/11] pytensor: this is the successor for theano pytensor 2.19 is the last one with python 3.9 support --- cmake/toolchain/heptools-dev-base.cmake | 2 +- pyexternals/CMakeLists.txt | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/cmake/toolchain/heptools-dev-base.cmake b/cmake/toolchain/heptools-dev-base.cmake index 6a572d0c8d..50c4a766cb 100644 --- a/cmake/toolchain/heptools-dev-base.cmake +++ b/cmake/toolchain/heptools-dev-base.cmake @@ -636,6 +636,7 @@ LCG_external_package(pystan 3.9.0 LCG_external_package(pyshp 2.1.0 ) LCG_external_package(pyshtools 4.12.2 ) LCG_external_package(pyslha 3.2.6 ) +LCG_external_package(pytensor 2.19.0 ) LCG_external_package(pytables 3.6.1 ) LCG_external_package(pytest 8.0.0 ) LCG_external_package(pytest_cov 4.1.0 ) @@ -786,7 +787,6 @@ LCG_external_package(texinfo 6.3 LCG_external_package(texttable 1.7.0 ) LCG_external_package(textual 0.1.18 ) LCG_external_package(tfs_pandas 3.5.1 ) -LCG_external_package(theano 1.0.5 ) LCG_external_package(threadpoolctl 2.2.0 ) LCG_external_package(thrift 0.16.0 ) LCG_external_package(tiff 4.2.0 ) diff --git a/pyexternals/CMakeLists.txt b/pyexternals/CMakeLists.txt index e811a53bae..7b3fa0c723 100644 --- a/pyexternals/CMakeLists.txt +++ b/pyexternals/CMakeLists.txt @@ -4268,6 +4268,20 @@ LCGPackage_Add( DEPENDS numpy scipy nose pygments graphviz pydot six pydot_ng ) +#---pytensor-------------------------------------------------------------------------------------------------------- +LCGPackage_Add( + pytensor + URL ${GenURL}/pytensor-${pytensor_native_version}.tar.gz + ENVIRONMENT HDF5_DIR=${hdf5_home} + CONFIGURE_COMMAND <VOID> + BUILD_COMMAND <VOID> + INSTALL_COMMAND ${MakeSitePackagesDir} + COMMAND ${PYTHON} -m pip install ${PySetupOptions} --no-deps . + COMMAND ${CMAKE_SOURCE_DIR}/pyexternals/Python_postinstall.sh <INSTALL_DIR> + BUILD_IN_SOURCE 1 + DEPENDS Python pip +) + #---keras------------------------------------------------------------------------------------------------------ LCGPackage_Add( keras -- GitLab From d6885e0d14ec31d7229621a0e671b8a7dceea537 Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Mon, 13 May 2024 11:11:03 +0200 Subject: [PATCH 02/11] Keras: remove theano dependency (no longer required), simplify recipe drop recipe for older version --- pyexternals/CMakeLists.txt | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/pyexternals/CMakeLists.txt b/pyexternals/CMakeLists.txt index 7b3fa0c723..0fdf10400c 100644 --- a/pyexternals/CMakeLists.txt +++ b/pyexternals/CMakeLists.txt @@ -4285,25 +4285,13 @@ LCGPackage_Add( #---keras------------------------------------------------------------------------------------------------------ LCGPackage_Add( keras - IF <VERSION> VERSION_GREATER_EQUAL 2.8.0 THEN - DOWNLOAD_COMMAND <VOID> - CONFIGURE_COMMAND <VOID> - BUILD_COMMAND <VOID> - INSTALL_COMMAND ${MakeSitePackagesDir} - COMMAND ${PYTHON} ${pip_home}/bin/pip install --no-deps --prefix=<INSTALL_DIR> -i ${lcg_pip_mirror} - keras==<VERSION> - BUILD_IN_SOURCE 1 - DEPENDS Python setuptools pip theano PyYAML six h5py scipy numpy - ELSE - URL ${GenURL}/Keras-${keras_native_version}.tar.gz - CONFIGURE_COMMAND <VOID> - BUILD_COMMAND <VOID> - INSTALL_COMMAND ${MakeSitePackagesDir} - COMMAND ${PYTHON} setup.py install ${PySetupOptions} - COMMAND ${CMAKE_SOURCE_DIR}/pyexternals/Python_postinstall.sh <INSTALL_DIR> - BUILD_IN_SOURCE 1 - DEPENDS Python setuptools theano PyYAML six h5py scipy numpy - ENDIF + DOWNLOAD_COMMAND <VOID> + CONFIGURE_COMMAND <VOID> + BUILD_COMMAND <VOID> + INSTALL_COMMAND ${MakeSitePackagesDir} + COMMAND ${PYTHON} ${pip_home}/bin/pip install --no-deps ${PySetupOptions} -i ${lcg_pip_mirror} keras==<VERSION> + BUILD_IN_SOURCE 1 + DEPENDS Python setuptools pip PyYAML six h5py scipy numpy ) #---keras_preprocessing------------------------------------------------------------------------------------------- -- GitLab From 5ce1a26ebc8a1b888bb7ae75ab36b43df4ff2d91 Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Mon, 13 May 2024 12:49:04 +0200 Subject: [PATCH 03/11] pysimdjson: update for pystan --- cmake/toolchain/heptools-dev-base.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/toolchain/heptools-dev-base.cmake b/cmake/toolchain/heptools-dev-base.cmake index 50c4a766cb..048cda3b02 100644 --- a/cmake/toolchain/heptools-dev-base.cmake +++ b/cmake/toolchain/heptools-dev-base.cmake @@ -631,7 +631,7 @@ LCG_external_package(pyroscope 1.2.0 LCG_external_package(pyroscope_io 0.8.5 ) LCG_external_package(pyrsistent 0.18.0 ) LCG_external_package(pyserial 3.4 ) -LCG_external_package(pysimdjson 3.2.0 ) +LCG_external_package(pysimdjson 5.0.2 ) LCG_external_package(pystan 3.9.0 ) LCG_external_package(pyshp 2.1.0 ) LCG_external_package(pyshtools 4.12.2 ) -- GitLab From f6fcb0a7b4d85f4236a03ce3f1441de95eeaf744 Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Mon, 13 May 2024 12:49:24 +0200 Subject: [PATCH 04/11] pytables: update for newer typing_extension compatibility --- cmake/toolchain/heptools-dev-base.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/toolchain/heptools-dev-base.cmake b/cmake/toolchain/heptools-dev-base.cmake index 048cda3b02..e453513aa1 100644 --- a/cmake/toolchain/heptools-dev-base.cmake +++ b/cmake/toolchain/heptools-dev-base.cmake @@ -637,7 +637,7 @@ LCG_external_package(pyshp 2.1.0 LCG_external_package(pyshtools 4.12.2 ) LCG_external_package(pyslha 3.2.6 ) LCG_external_package(pytensor 2.19.0 ) -LCG_external_package(pytables 3.6.1 ) +LCG_external_package(pytables 3.9.2 ) LCG_external_package(pytest 8.0.0 ) LCG_external_package(pytest_cov 4.1.0 ) LCG_external_package(pytest_runner 5.1 ) -- GitLab From b70d1314a53b8d34f13e382e9a8573b01713d19e Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Mon, 13 May 2024 12:49:53 +0200 Subject: [PATCH 05/11] genshi: add missing six dependency --- pyexternals/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyexternals/CMakeLists.txt b/pyexternals/CMakeLists.txt index 0fdf10400c..f44a307355 100644 --- a/pyexternals/CMakeLists.txt +++ b/pyexternals/CMakeLists.txt @@ -1114,7 +1114,7 @@ LCGPackage_Add( COMMAND ${CMAKE_SOURCE_DIR}/pyexternals/Python_postinstall.sh <INSTALL_DIR> BUILD_IN_SOURCE 1 DEPENDS Python setuptools - IF ${genshi_native_version} VERSION_GREATER_EQUAL 0.7.7 THEN pip ENDIF + IF ${genshi_native_version} VERSION_GREATER_EQUAL 0.7.7 THEN pip six ENDIF ) #---prctl------------------------------------------------------------------------------------------------------ -- GitLab From 8630e87da36673f38f2e2bca5223ece200051131 Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Mon, 13 May 2024 12:50:15 +0200 Subject: [PATCH 06/11] blosc2: add missing msgpack dependency --- pyexternals/CMakeLists.txt | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/pyexternals/CMakeLists.txt b/pyexternals/CMakeLists.txt index f44a307355..8e86a45525 100644 --- a/pyexternals/CMakeLists.txt +++ b/pyexternals/CMakeLists.txt @@ -3640,6 +3640,18 @@ LCGPackage_Add( DEPENDS Python pip ) +#---msgpack----------------------------------------------------------------------------------------------- +LCGPackage_Add( + msgpack + URL ${GenURL}/msgpack-${msgpack_native_version}.tar.gz + CONFIGURE_COMMAND <VOID> + BUILD_COMMAND <VOID> + INSTALL_COMMAND ${MakeSitePackagesDir} + COMMAND ${PYTHON} -m pip install ${PySetupOptions} --no-deps . + COMMAND ${CMAKE_SOURCE_DIR}/pyexternals/Python_postinstall.sh <INSTALL_DIR> + BUILD_IN_SOURCE 1 + DEPENDS Python pip +) #---blosc2--------------------------------------------------------------------------- LCGPackage_Add( @@ -3653,7 +3665,7 @@ LCGPackage_Add( COMMAND ${CMAKE_SOURCE_DIR}/pyexternals/Python_postinstall.sh <INSTALL_DIR> BUILD_IN_SOURCE 1 DEPENDS Python setuptools wheel scikitbuild CMake ninja numpy pycpuinfo - IF <VERSION> VERSION_GREATER_EQUAL 2.5.1 THEN ndindex ENDIF + IF <VERSION> VERSION_GREATER_EQUAL 2.5.1 THEN ndindex msgpack ENDIF ) #---blosc--------------------------------------------------------------------------- @@ -4424,19 +4436,6 @@ LCGPackage_Add( DEPENDS Python setuptools ) -#---msgpack----------------------------------------------------------------------------------------------- -LCGPackage_Add( - msgpack - URL ${GenURL}/msgpack-${msgpack_native_version}.tar.gz - CONFIGURE_COMMAND <VOID> - BUILD_COMMAND <VOID> - INSTALL_COMMAND ${MakeSitePackagesDir} - COMMAND ${PYTHON} -m pip install ${PySetupOptions} --no-deps . - COMMAND ${CMAKE_SOURCE_DIR}/pyexternals/Python_postinstall.sh <INSTALL_DIR> - BUILD_IN_SOURCE 1 - DEPENDS Python setuptools pip -) - #---grpcio------------------------------------------------------------------------------------------------------------ LCGPackage_Add( -- GitLab From db5f981da588441d113c2a450cc16ba9df6c0150 Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Mon, 13 May 2024 12:50:37 +0200 Subject: [PATCH 07/11] dask_awkward: add missing cachetools dependency --- pyexternals/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyexternals/CMakeLists.txt b/pyexternals/CMakeLists.txt index 8e86a45525..53b037a64e 100644 --- a/pyexternals/CMakeLists.txt +++ b/pyexternals/CMakeLists.txt @@ -5931,7 +5931,7 @@ LCGPackage_Add( COMMAND ${PYTHON} -m pip install --no-deps ${PySetupOptions} . COMMAND ${CMAKE_SOURCE_DIR}/pyexternals/Python_postinstall.sh <INSTALL_DIR> BUILD_IN_SOURCE 1 - DEPENDS Python pip dask awkward typing_extensions + DEPENDS Python pip dask awkward typing_extensions cachetools ) -- GitLab From 1572c771f100e0fda37f955c19c73ebde8f2dc76 Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Mon, 13 May 2024 12:50:51 +0200 Subject: [PATCH 08/11] coffea: add missing aiohttp dependency --- pyexternals/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyexternals/CMakeLists.txt b/pyexternals/CMakeLists.txt index 53b037a64e..395c08cfdb 100644 --- a/pyexternals/CMakeLists.txt +++ b/pyexternals/CMakeLists.txt @@ -7467,7 +7467,7 @@ LCGPackage_Add( ENDIF COMMAND ${CMAKE_SOURCE_DIR}/pyexternals/Python_postinstall.sh <INSTALL_DIR> BUILD_IN_SOURCE 1 - DEPENDS blosc bokeh dask distributed ipywidgets parsl tcut_to_qastle torch + DEPENDS blosc bokeh dask distributed ipywidgets parsl tcut_to_qastle torch aiohttp awkward cachetools cloudpickle correctionlib fsspec fsspec_xrootd hist py_lz4 lz4 matplotlib mplhep numba numpy packaging pandas pyarrow scipy toml tqdm uproot aiostream dask_awkward dask_histogram ) -- GitLab From 119e0f60a08185358548bd10f84362ec03df70bf Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Mon, 13 May 2024 12:51:15 +0200 Subject: [PATCH 09/11] faiss: add missing packaging dependency --- pyexternals/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyexternals/CMakeLists.txt b/pyexternals/CMakeLists.txt index 395c08cfdb..7c35190e9f 100644 --- a/pyexternals/CMakeLists.txt +++ b/pyexternals/CMakeLists.txt @@ -8193,7 +8193,7 @@ LCGPackage_Add( COMMAND ${CMAKE_COMMAND} -E chdir faiss/python ${PYTHON} setup.py install ${PySetupOptions} COMMAND ${MAKE} install COMMAND ${CMAKE_COMMAND} -E copy faiss/python/libfaiss_python_callbacks.so <INSTALL_DIR>/lib/ - DEPENDS blas Python numpy swig + DEPENDS blas Python numpy swig packaging REVISION 2 ) -- GitLab From d1bba968289c4cebb2b754725fecb4481201676f Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Mon, 13 May 2024 12:52:23 +0200 Subject: [PATCH 10/11] uproot: update for coffea requirement --- cmake/toolchain/heptools-dev-base.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/toolchain/heptools-dev-base.cmake b/cmake/toolchain/heptools-dev-base.cmake index e453513aa1..709e9a79d7 100644 --- a/cmake/toolchain/heptools-dev-base.cmake +++ b/cmake/toolchain/heptools-dev-base.cmake @@ -826,7 +826,7 @@ LCG_external_package(umesimd 0.8.1 LCG_external_package(uncertainties 3.1.7 ) LCG_external_package(unigen 2.2 ) LCG_External_package(upandas 0.2.0 ) -LCG_external_package(uproot 5.3.4 ) +LCG_external_package(uproot 5.3.7 ) LCG_external_package(uproot_browser 0.4.0 ) LCG_external_package(urllib3 1.26.16 ) LCG_external_package(utf8cpp 3.2.3 ) -- GitLab From 0d564c42afd56c407a0fd9468a78f3afa5a0d67f Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Mon, 13 May 2024 12:59:04 +0200 Subject: [PATCH 11/11] openqasm3: add antlr4_python3_runtime this is part of the openqasm3[parser] optionals --- cmake/toolchain/heptools-dev-base.cmake | 1 + pyexternals/CMakeLists.txt | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/cmake/toolchain/heptools-dev-base.cmake b/cmake/toolchain/heptools-dev-base.cmake index 709e9a79d7..c1bab57079 100644 --- a/cmake/toolchain/heptools-dev-base.cmake +++ b/cmake/toolchain/heptools-dev-base.cmake @@ -33,6 +33,7 @@ LCG_external_package(alabaster 0.7.12 LCG_external_package(alpaka 0.9.0 ) LCG_external_package(altair 5.2.0 ) LCG_external_package(ant 1.10.7 ) +LCG_external_package(antlr4_python3_runtime 4.13.1 ) LCG_external_package(anyio 3.7.1 ) LCG_external_package(appdirs 1.4.4 ) LCG_external_package(argon2_cffi 20.1.0 ) diff --git a/pyexternals/CMakeLists.txt b/pyexternals/CMakeLists.txt index 7c35190e9f..f38c5b19ee 100644 --- a/pyexternals/CMakeLists.txt +++ b/pyexternals/CMakeLists.txt @@ -8610,6 +8610,19 @@ LCGPackage_Add( DEPENDS Python pip cloudpickle future networkx numpy py4j scipy six tqdm ) +#---antlr4_python3_runtime----------------------------------------------------------------------------------------------------- +LCGPackage_Add( + antlr4_python3_runtime + URL ${GenURL}/antlr4-python3-runtime-<VERSION>.tar.gz + CONFIGURE_COMMAND <VOID> + BUILD_COMMAND <VOID> + INSTALL_COMMAND ${MakeSitePackagesDir} + COMMAND ${PYTHON} -m pip install ${PySetupOptions} --no-deps . + COMMAND ${CMAKE_SOURCE_DIR}/pyexternals/Python_postinstall.sh <INSTALL_DIR> + BUILD_IN_SOURCE 1 + DEPENDS Python pip +) + #---openqasm3----------------------------------------------------------------------------------------------------- LCGPackage_Add( openqasm3 @@ -8620,7 +8633,7 @@ LCGPackage_Add( COMMAND ${PYTHON} -m pip install ${PySetupOptions} --no-deps . COMMAND ${CMAKE_SOURCE_DIR}/pyexternals/Python_postinstall.sh <INSTALL_DIR> BUILD_IN_SOURCE 1 - DEPENDS Python pip + DEPENDS Python pip antlr4_python3_runtime ) #---qibo----------------------------------------------------------------------------------------------------- -- GitLab