diff --git a/cmake/toolchain/heptools-dev-base.cmake b/cmake/toolchain/heptools-dev-base.cmake index e95371934a850926371a22d7c1fee230baa793a0..48feac79c613769c5b87f76eb9225ab6e4a51418 100644 --- a/cmake/toolchain/heptools-dev-base.cmake +++ b/cmake/toolchain/heptools-dev-base.cmake @@ -87,7 +87,7 @@ endif() LCG_external_package(lcgenv 1.3.8 ) LCG_external_package(absl_py 0.7.1 ) LCG_external_package(AIDA 3.2.1 ) -LCG_external_package(arrow 0.13.0 ) +LCG_external_package(arrow 0.14.1 ) LCG_external_package(asn1crypto 0.24.0 ) LCG_external_package(astor 0.8.0 ) LCG_external_package(atomicwrites 1.3.0 ) @@ -339,6 +339,7 @@ LCG_external_package(pandocfilters 1.4.2 if(NOT ${LCG_OS} MATCHES mac) LCG_external_package(pango 1.40.13 ) endif() +LCG_external_package(parquet 0.11.0 ) LCG_external_package(parso 0.4.0 ) LCG_external_package(pathlib2 2.3.3 ) LCG_external_package(pathos 0.2.3 ) @@ -373,6 +374,7 @@ LCG_external_package(py 1.5.4 LCG_external_package(py2neo 4.3.0 ) LCG_external_package(py4j 0.10.7 ) LCG_external_package(pyanalysis 2.0 ) +# PSA: arrow and pyarrow use the same tarball LCG_external_package(pyarrow ${arrow_native_version} ) LCG_external_package(pyasn1 0.4.5 ) LCG_external_package(pyasn1_modules 0.2.5 ) @@ -457,6 +459,7 @@ LCG_external_package(terminado 0.8.2 LCG_external_package(testpath 0.4.2 ) LCG_external_package(texinfo 6.3 ) LCG_external_package(theano 1.0.4 ) +LCG_external_package(thrift 0.12.0 ) LCG_external_package(tiff 4.0.10 ) LCG_external_package(toolz 0.9.0 ) if(NOT ${LCG_OS}${LCG_OSVERS} STREQUAL slc6) diff --git a/documentation/packages.json b/documentation/packages.json index 92461eb52679ddcd5e0fac0f5f563d5bd2e3e953..8f44c7cb97af15a94c07797b5550e736ccdc9afd 100644 --- a/documentation/packages.json +++ b/documentation/packages.json @@ -2709,12 +2709,25 @@ }, { "category": "Other", - "contacts": [], + "contacts": [ + { + "email": "dev@arrow.apache.org", + "name": "Apache arrow developer mailing list" + }, + { + "email": "https://arrow.apache.org/", + "name": "Official Apache Arrow homepage" + }, + { + "email": "https://issues.apache.org/jira/projects/ARROW", + "name": "Apache Arrow Jira issue tracker" + } + ], "description": "Apache Arrow is a cross-language development platform for in-memory data. It specifies a standardized language-independent columnar memory format for flat and hierarchical data, organized for efficient analytic operations on modern hardware. It also provides computational libraries and zero-copy streaming messaging and interprocess communication. Languages currently supported include C, C++, Java, JavaScript, Python, and Ruby.", - "fullname": "arrow", + "fullname": "Arrow", "homepage": "https://arrow.apache.org/", "language": "C++", - "license": "Apache", + "license": "Apache-2.0", "name": "arrow" }, { @@ -5706,12 +5719,25 @@ }, { "category": "Other", - "contacts": [], - "description": "Python library for Apache Arrow", + "contacts": [ + { + "email": "dev@arrow.apache.org", + "name": "Apache arrow developer mailing list" + }, + { + "email": "https://arrow.apache.org/", + "name": "Official Apache Arrow homepage" + }, + { + "email": "https://issues.apache.org/jira/projects/ARROW", + "name": "Apache Arrow Jira issue tracker" + } + ], + "description": "This library provides a Python API for functionality provided by the Arrow C++ libraries", "fullname": "pyarrow", "homepage": "https://pypi.org/project/pyarrow/", "language": "Python", - "license": "Apache", + "license": "Apache-2.0", "name": "pyarrow" }, { @@ -7197,5 +7223,43 @@ "language": "C++", "license": "http://zlib.net/zlib_license.html", "name": "zlib" + }, + { + "category": "Other", + "contacts": [ + { + "email": "https://thrift.apache.org/mailing", + "name": "Thrift mailing list" + }, + { + "email": "http://issues.apache.org/jira/browse/THRIFT", + "name": "Thrift Jira issue tracking" + } + ], + "description": "Thrift is a lightweight, language-independent software stack for point-to-point RPC implementation", + "fullname": "Thrift", + "homepage": "https://thrift.apache.org", + "language": "C++", + "license": "Apache-2.0", + "name": "thrift" + }, + { + "category": "Other", + "contacts": [ + { + "email": "https://parquet.apache.org/community/", + "name": "Apache Parquet community page (mailing lists)" + }, + { + "email": "http://issues.apache.org/jira/browse/PARQUET", + "name": "Parquet Jira issue tracking" + } + ], + "description": "Parquet-MR contains the java implementation of the Parquet format. Parquet is a columnar storage format for Hadoop; it provides efficient storage and encoding of data.", + "fullname": "Parquet MR", + "homepage": "https://parquet.apache.org/", + "language": "Java", + "license": "Apache-2.0", + "name": "parquet_mr" } -] \ No newline at end of file +] diff --git a/externals/CMakeLists.txt b/externals/CMakeLists.txt index 37f31fe40a36b3f7eb51daaa6870d6837879d842..9b33b52b0f2103d264058079c0c40a070d67803c 100644 --- a/externals/CMakeLists.txt +++ b/externals/CMakeLists.txt @@ -2519,3 +2519,38 @@ LCGPackage_Add( -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -DJSON_MultipleHeaders=ON ) + +#---thrift------------------------------------------------------------------- +LCGPackage_Add( + thift + URL ${GenURL}/thrift-${thrift_native_version}.tar.gz + CONFIGURE_COMMAND ./boostrap.sh + COMMAND ./configure + --with-boost=${Boost_home} + --prefix=<INSTALL_DIR> + --disable-gen-erl + --disable-gen-hs + --without-ruby + --without-haskell + --without-erlang + --without-php + --without-nodejs + BUILD_COMMAND ${MAKE} + INSTALL_COMMAND ${MAKE} install + BUILD_IN_SOURCE 1 + DEPENDS Boost flex bison +) + +#---parquet-mr------------------------------------------------------------------- +# Renamed tarball name to match github package name +LCGPackage_Add( + parquet_mr + ENVIRONMENT JAVA_HOME=${JAVA_HOME} LC_ALL=C + URL ${GenURL}/parquet-mr-${parquet_mr_native_version}.tar.gz + BUILD_COMMAND mvn clean install + #INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory hadoop-dist/target/hadoop-<VERSION> <INSTALL_DIR> + # COMMAND ${CMAKE_COMMAND} -E copy_directory hadoop-client/target/hadoop-client-<VERSION> <INSTALL_DIR> + # COMMAND chmod -R go+r <INSTALL_DIR>/share + BUILD_IN_SOURCE 1 + DEPENDS maven protobuf thrift +) diff --git a/projects/CMakeLists.txt b/projects/CMakeLists.txt index 28c52bffdd21f72f3237c237590c0a7b536b46ab..d69f99401dd762a70dc47807407ee102f01564d1 100644 --- a/projects/CMakeLists.txt +++ b/projects/CMakeLists.txt @@ -36,6 +36,7 @@ LCGPackage_Add( ) #---arrow--------------------------------------------------------------------------------------------- +# Remember to rename the tarball to match url LCGPackage_Add( arrow URL ${GenURL}/apache-arrow-${arrow_native_version}.tar.gz @@ -46,6 +47,7 @@ LCGPackage_Add( -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR> -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD} + -DARROW_PARQUET=ON ${Boost_extra_configuration} BUILD_COMMAND ${MAKE} INSTALL_COMMAND ${MAKE} install diff --git a/pyexternals/CMakeLists.txt b/pyexternals/CMakeLists.txt index 5f70dc1196c3e9244a3dd616391763d474005ba2..8744cb3da627ef9300fe1bf6cf0e8ae6a7838ce7 100644 --- a/pyexternals/CMakeLists.txt +++ b/pyexternals/CMakeLists.txt @@ -3326,7 +3326,7 @@ LCGPackage_Add( --build-type=${ARROW_BUILD_TYPE} --inplace INSTALL_COMMAND ${MakeSitePackagesDir} COMMAND ${CMAKE_COMMAND} -E chdir <SOURCE_DIR>/python env ARROW_HOME=${arrow_home} ${PYTHON} setup.py install ${PySetupOptions} - DEPENDS numpy six cython arrow setuptools_scm IF NOT LCG_PYTHON_VERSION EQUAL 3 THEN futures ENDIF + DEPENDS numpy six cython arrow setuptools_scm IF NOT LCG_PYTHON_VERSION EQUAL 3 THEN futures enum34 ENDIF ) #----minrpc---------------------------------------------------------------------