Merge branch 'cleanup' into 'devel'

Cleanup

See merge request YARR/YARR!101

.gitlab-ci.yml

+image: gitlab-registry.cern.ch/yarr/yarr_container:yarr_build
+
 job_build_and_test:
-    image: gitlab-registry.cern.ch/ci-tools/ci-worker:cc7
     stage: build
     script:
-      - yum -y install centos-release-scl-rh devtoolset-7 make gnuplot
       - scl enable devtoolset-7 bash
       - source /opt/rh/devtoolset-7/enable
-      - cd src
-      - which g++
-      - make
+      - mkdir  build
+      - cd build
+      - cmake3 .. -DCMAKE_TOOLCHAIN_FILE=../cmake/linux-gcc
+      - make install -j4
       - export USER=test
       - echo $USER
       - /bin/pwd
       - mkdir /tmp/test
       - ls /tmp/
+      - cd ../
       - bin/scanConsole -r configs/controller/emuCfg.json -c configs/connectivity/example_fei4b_setup.json -s configs/scans/fei4/std_digitalscan.json -p
       - ls
       - ls ./data/
@@ -22,95 +24,20 @@ job_build_and_test:
       - if [ "$NUM100" != "26880" ]; then exit 2; fi
       - bin/scanConsole -k
 
-job_build_cmake:
-    image: gitlab-registry.cern.ch/ci-tools/ci-worker:cc7
+job_build_clang:
     stage: build
     script:
-      - yum -y install centos-release-scl-rh devtoolset-7 make cmake gnuplot
-      - scl enable devtoolset-7 bash
-      - source /opt/rh/devtoolset-7/enable
-      - mkdir -p src/cmake_build
-      - cd src/cmake_build
-      - cmake .. -DCMAKE_TOOLCHAIN_FILE=../cmake/linux-gcc
-      - make -j4
-      - bin/scanConsole -k
-
-job_build_cmake_minimum_gcc:
-    # Default is gcc 4.8.5
-    image: gitlab-registry.cern.ch/ci-tools/ci-worker:cc7
-    stage: build
-    script:
-      - yum -y install make cmake gnuplot gcc-c++
-      - mkdir -p src/cmake_build
-      - cd src/cmake_build
-      - cmake .. -DCMAKE_TOOLCHAIN_FILE=../cmake/linux-gcc
-      - make -j4
+      - scl enable   llvm-toolset-7  bash
+      - source /opt/rh/llvm-toolset-7/enable
+      - mkdir -p build
+      - cd build 
+      - cmake3 .. -DCMAKE_TOOLCHAIN_FILE=../cmake/linux-clang
+      - make install -j4
+      - cd ..
       - bin/scanConsole -k
 
 job_test_json:
-    image: gitlab-registry.cern.ch/ci-tools/ci-worker:cc7
     stage: build
     script:
-      - yum -y install npm
       - npm install jsonlint -g
-      - for file in $(find src/configs/ -iname "*.json"); do echo ${file}; jsonlint -q ${file} || break -1; done
-
-job_build_with_felix:
-    # Image of pre-built felix software
-    image: gitlab-registry.cern.ch/atlas-tdaq-felix/felix-image:cc7
-    variables:
-     # This is what is in the docker image!
-     FELIX_DIR: /opt/felix/felix-nightly
-     # LCG style config identifier
-     CONFIG: x86_64-centos7-gcc62-opt
-     GCC_SCRIPT: /cvmfs/sft.cern.ch/lcg/releases/gcc/6.2.0/x86_64-centos7/setup.sh
-    tags:
-     - cvmfs
-    script:
-     # Which versions are present
-     - ls ${FELIX_DIR}/..
-     # Which configurations are present
-     - ls ${FELIX_DIR}
-     # Which include files exist
-     - find ${FELIX_DIR}/${CONFIG}/include -type f
-     # Temporary fix for unexpected change in container (Apr '19)
-     - "[[ -d ${FELIX_DIR}/${CONFIG}/include/netio ]] || mkdir ${FELIX_DIR}/${CONFIG}/include/netio"
-     - "[[ -f ${FELIX_DIR}/${CONFIG}/include/netio/netio.hpp ]] || mv ${FELIX_DIR}/${CONFIG}/include/{,netio/}netio.hpp"
-     - yum -y install make cmake gnuplot glibc-devel which
-     # TBB lib is in FELIX image, but we need the includes to build
-     - yum -y install tbb tbb-devel
-     # Run setup for YARR
-     - source ${FELIX_DIR}/${CONFIG}/setup.sh
-     - source ${GCC_SCRIPT}
-     - gcc --version
-     - mkdir -p src/cmake_build
-     - cd src/cmake_build
-     - cmake .. -DCMAKE_TOOLCHAIN_FILE=../cmake/linux-gcc -DENABLE_NETIO:BOOL=ON -DNETIO_DIR:PATH=${FELIX_DIR}/${CONFIG}
-     - make -j4
-     - bin/scanConsole -k
-
-job_build_make_with_felix:
-    # Image of pre-built felix software
-    image: gitlab-registry.cern.ch/atlas-tdaq-felix/felix-image:cc7
-    variables:
-     # This is what is in the docker image!
-     FELIX_DIR: /opt/felix/felix-nightly
-     # LCG style config identifier
-     CONFIG: x86_64-centos7-gcc8-opt
-     GCC_SCRIPT: /cvmfs/sft.cern.ch/lcg/releases/gcc/8.2.0/x86_64-centos7/setup.sh
-    tags:
-     - cvmfs
-    script:
-     # Temporary fix for unexpected change in container (Apr '19)
-     - "[[ -d ${FELIX_DIR}/${CONFIG}/include/netio ]] || mkdir ${FELIX_DIR}/${CONFIG}/include/netio"
-     - "[[ -f ${FELIX_DIR}/${CONFIG}/include/netio/netio.hpp ]] || mv ${FELIX_DIR}/${CONFIG}/include/{,netio/}netio.hpp"
-     - yum -y install make gnuplot glibc-devel which
-     # TBB lib is in FELIX image, but we need the includes to build
-     - yum -y install tbb tbb-devel
-     # Run setup for YARR
-     - source ${FELIX_DIR}/${CONFIG}/setup.sh
-     - source ${GCC_SCRIPT}
-     - gcc --version
-     - cd src
-     - make NETIO_DIR=${FELIX_DIR}/${CONFIG}
-     - bin/scanConsole -k
+      - for file in $(find ../configs/ -iname "*.json"); do echo ${file}; jsonlint -q ${file} || break -1; done

src/.syntastic_cpp_config → .syntastic_cpp_config

 -D__STDC_CONSTANT_MACROS
 -D__STDC_LIMIT_MACROS
 -std=c++17
--IlibUtil/include
--IlibYarr/include
--IlibFei4/include
--IlibFe65p2/include
--IlibRd53a/include
--IlibSpec/include
+-Isrc/libUtil/include
+-Isrc/libYarr/include
+-Isrc/libFei4/include
+-Isrc/libFe65p2/include
+-Isrc/libRd53a/include
+-Isrc/libSpec/include

CMakeLists.txt

+project (YARR)
+set (YARR_VERSION_MAJOR 1)
+set (YARR_VERSION_MINOR 0)
+
+# Build dirs
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+
+# cmake req
 cmake_minimum_required (VERSION 3.6)
 
+# Compiler req
+if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.3)
+        message(FATAL_ERROR "GCC version must be at least 7.3!")
+    endif()
+elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)
+        message(FATAL_ERROR "Clang version must be at least 5.0!")
+    endif()
+else()
+    message(WARNING "You are using an unsupported compiler! Compilation has only been tested with Clang and GCC.")
+endif()
+
+# External Toolchains
+include(ExternalProject)
+if(CMAKE_TOOLCHAIN_FILE)
+    set(TOOLCHAIN "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}")
+else()
+    set(TOOLCHAIN "")
+endif()
+
+set(TARGET_INSTALL_AREA ${PROJECT_SOURCE_DIR})
+set(SHARED_INSTALL_AREA ${PROJECT_SOURCE_DIR}/installed/)
+add_compile_options(-std=c++17)
+
+# Architecture
+if ("${TARGET_ARCH}" STREQUAL "arm")
+    set(TBB_ARCH "arch=arm")
+else()
+    if ("${TARGET_ARCH}" STREQUAL "arm64")
+        set(TBB_ARCH "arch=arm64")
+    else()
+        set(TBB_ARCH "")
+    endif()
+endif()
+if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+    set(TBB_COMP compiler=clang)
+else()
+    set(TBB_COMP CXX=${CMAKE_CXX_COMPILER})
+endif()
+
+# TBB
+ExternalProject_Add (
+    tbb_2019
+    GIT_REPOSITORY https://github.com/intel/tbb.git
+    GIT_TAG tbb_2019
+    UPDATE_COMMAND ""
+    INSTALL_COMMAND ${PROJECT_SOURCE_DIR}/scripts/install_tbb.sh ${CMAKE_CURRENT_BINARY_DIR}  ${CMAKE_CURRENT_BINARY_DIR}
+    BUILD_COMMAND cd ${CMAKE_CURRENT_BINARY_DIR}/external/src/tbb_2019 &&  make -j4 ${TBB_COMP} ${TBB_ARCH} extra_inc=big_iron.inc
+    CONFIGURE_COMMAND ""
+    PREFIX "${CMAKE_CURRENT_BINARY_DIR}/external"
+    )
+
+# NetIO
+ExternalProject_Add (
+    netio4
+    GIT_REPOSITORY https://:@gitlab.cern.ch:8443/wittgen/netio.git
+    GIT_TAG felix-4.0.x
+    UPDATE_COMMAND ""
+    PATCH_COMMAND cp ${PROJECT_SOURCE_DIR}/cmake/CMakeLists.txt.netio ${CMAKE_CURRENT_BINARY_DIR}/external/src/netio4/CMakeLists.txt
+    INSTALL_COMMAND ""
+    PREFIX "${CMAKE_CURRENT_BINARY_DIR}/external"
+    CMAKE_ARGS "${TOOLCHAIN}"
+    DEPENDS tbb_2019
+    )
+
+# FELIX
+ExternalProject_Add (
+    felixbase4
+    GIT_REPOSITORY https://:@gitlab.cern.ch:8443/wittgen/felixbase.git
+    GIT_TAG felix-4.0.x
+    UPDATE_COMMAND ""
+    PATCH_COMMAND ""
+    INSTALL_COMMAND ""
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+    PREFIX "${CMAKE_CURRENT_BINARY_DIR}/external"
+    DEPENDS netio4
+    )
+
+# Rogue
+ExternalProject_Add (
+    rogue
+    GIT_REPOSITORY https://github.com/slaclab/rogue.git
+    GIT_TAG atlas-fixes
+    UPDATE_COMMAND ""
+    INSTALL_COMMAND make install
+    PREFIX "${CMAKE_CURRENT_BINARY_DIR}/external"
+    CMAKE_ARGS "-DNO_PYTHON=1" "-DNO_ZMQ=1" "-DNO_BZIP=1" "-DSTATIC_LIB=1" "${TOOLCHAIN}"
+    )
+
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/external/src/felixbase4)
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/external/src/netio4)
+link_directories(${CMAKE_CURRENT_BINARY_DIR}/external/src/netio4-build)
+
 add_subdirectory(src)
+

README.md

 ![docs](http://readthedocs.org/projects/yarr/badge/?version=latest)
 
+![https://gitlab.cern.ch/YARR/YARR/pipelines](https://gitlab.cern.ch/YARR/YARR/badges/master/pipeline.svg)
+
+![logo](docs/images/logo_blue_inv.png)
+
 # YARR: Yet Another Rapid Readout
 
 ## Documentation
@@ -12,7 +16,7 @@ Users should subscribe to the CERN mailing list to receive announcements for imp
 
 Developers and potential developers please refer to [Contribution](CONTRIBUTING.md) guide.
 
-## Quick Install Guide
+## Requirements
 
 ### Hardware:
 
@@ -20,36 +24,39 @@ Developers and potential developers please refer to [Contribution](CONTRIBUTING.
 
 ### Software:
 
-Requirements:
-
-- CC7 (CERN CentOs 7) 
-- GCC version 7.0 or higher
-    - for example from devtoolset-7, instruction can be found [here](https://yarr.readthedocs.io/en/latest/install/#hardware-setup-and-software-installation)
-
-Quick Default Install Guide:
-
-- Using make:
-    - ``cd YARR/src``
-    - ``make``
-- Using cmake:
-    - ``cd YARR``
-    - ``mkdir build``
-    - ``cd build``
-    - or select one of the supported toolchains
-        - make ..  -DCMAKE_TOOLCHAIN_FILE=../cmake/linux-clang # requires clang installed on Linux
-        - make ..  -DCMAKE_TOOLCHAIN_FILE=../cmake/linux-gcc # gcc 4.8 or higher
-        - make ..  -DCMAKE_TOOLCHAIN_FILE=../cmake/rce-gcc # ARM/Archlinux on RCE
-        - make ..  -DCMAKE_TOOLCHAIN_FILE=../cmake/macos-clang # MacOS build
-    - ``make``
-
-
-Quick RCE Install Guide:
-
-- source /opt/AtkasRceSDK/V0.11.0/setup.sh # for RCEs
-- Atlas RCE SDK for cross compilation and running on RCEs (HSIO2 or COB)
-- to install in /opt/AtlasRceSDK
-    - sudo mkdir -p /opt/AtlasRceSDK
-    - cd /opt/AtkasRceSDK/ ; curl -s  http://rceprojectportal.web.cern.ch/RceProjectPortal/software/SDK/V0.11.1.tar.gz | sudo tar xvfz - 
-        
-
-Quick NetIO Install Guide:
+- CentOS 7
+- cmake 3.6
+- GCC version 7 or higher
+    - for example from devtoolset-7, instruction can be found here https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7
+- Some misc packages (can be installed via yum):
+    - gnuplot
+    - texlive-epstopdf
+    - zeromq, zeromq-devel
+
+## Quick Install Guide:
+- Clone from git 
+	- ``$ git clone https://gitlab.cern.ch/YARR/YARR.git Yarr``
+- Compilation:
+    - ``$ cd Yarr``
+    - ``$ mkdir build``
+    - ``$ cd build``
+    - ``$ cmake3 ..``
+    - ``$ make install -j4``
+    - ``$ cd ..``
+- Running
+    - execute programs from the repository top folder
+
+### RCE Guide
+- for ARM target cross compilers are provided by the RCE_SDK
+    - installtion instructions: https://twiki.cern.ch/twiki/bin/viewauth/Atlas/RCEGen3SDK
+- using CMake:
+    - ``$ source /opt/rce/setup.sh (for RCE cross compilation setup)``
+    - ``$ export CENTOS7_ARM32_ROOT=/opt/rce/rootfs/centos7 #(points to cross installed CentOS7)``
+    - ``$ export CENTOS7_ARM64_ROOT=/opt/rce/rootfs/centos7_64 #for ZCU102 ``
+    - ``$ cd build``
+    - select one of the supported toolchains
+        - ``$ cmake3 ..  -DCMAKE_TOOLCHAIN_FILE=../cmake/linux-clang # requires clang installed on Linux ``
+        - ``$ cmake3 ..  -DCMAKE_TOOLCHAIN_FILE=../cmake/linux-gcc # gcc 4.8 or higher ``
+        - ``$ cmake3 ..  -DCMAKE_TOOLCHAIN_FILE=../cmake/rce-arm32 # ARM32/Centos7 on RCE ``
+        - ``$ cmake3 ..  -DCMAKE_TOOLCHAIN_FILE=../cmake/rce-arm64 # ARM64/Centos7 on zcu102 ``
+    - ``$ make -j4 install ``

cmake/CMakeLists.txt.netio

+cmake_minimum_required(VERSION 3.4.3)
+
+set(PACKAGE netio)
+set(PACKAGE_VERSION 0.8.0)
+
+
+
+set(VERSION "0.8.0")
+
+## BINARY DISTRIBUTION USING CPACK ##
+add_compile_options(-std=c++17)
+set(CPACK_RPM_PACKAGE_AUTOREQPROV " no")
+set(CPACK_PACKAGE_VERSION ${VERSION})
+set(CPACK_GENERATOR "RPM" "TGZ")
+set(CPACK_PACKAGE_NAME "netio")
+set(CPACK_PACKAGE_RELEASE 1)
+set(CPACK_PACKAGE_CONTACT "Jörn Schumacher")
+set(CPACK_PACKAGE_VENDOR "CERN")
+#set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
+
+## Options ##
+
+set(VERSION_STRING ${VERSION})
+configure_file(src/config.h.in config.h NEWLINE_STYLE UNIX)
+
+
+## SOURCES AND TARGETS ##
+include_directories(${PROJECT_BINARY_DIR}/../netio4)
+include_directories(${PROJECT_BINARY_DIR}/../../../include)
+link_directories(${PROJECT_BINARY_DIR}/../../../lib)
+include_directories(${CMAKE_BINARY_DIR})
+
+set(NETIO_SOURCES src/netio.cpp src/serialization.cpp src/posix.cpp  src/sockcfg.cpp src/endpoint.cpp
+	src/message.cpp src/eventloop.cpp src/backend.cpp src/sockets.cpp src/context.cpp src/buffer.cpp)
+
+add_library(netio STATIC ${NETIO_SOURCES})
+target_link_libraries(netio rt tbb)
+
+set(NETIO_TEST_SOURCES test/test_main.cpp test/test_backend.cpp  test/test_buffer.cpp
+    test/test_event_loop.cpp test/test_message.cpp  test/test_serialization.cpp test/test_sockets.cpp )
+add_executable(netio_test ${NETIO_TEST_SOURCES})
+target_link_libraries(netio_test  	netio rt pthread)
+
+add_executable(netio_cat src/main_cat.cpp)
+target_link_libraries(netio_cat netio rt pthread)
+
+add_executable(netio_throughput src/main_throughput.cpp)
+target_link_libraries(netio_throughput netio rt pthread)
+
+add_executable(netio_bw src/main_bw.cpp)
+target_link_libraries(netio_bw netio rt pthread)
+
+add_executable(netio_pingpong src/main_pingpong.cpp)
+target_link_libraries(netio_pingpong netio rt pthread)
+
+add_executable(netio_data_pub src/main_data_pub.cpp)
+target_link_libraries(netio_data_pub netio rt pthread)
+
+add_executable(netio_recv src/main_recv.cpp)
+target_link_libraries(netio_recv netio rt pthread)
+
+  add_executable(netio_zeromq src/main_zeromq.cpp)
+  target_link_libraries(netio_zeromq rt pthread zmq)
+
+  add_executable(netio_zeromq_pingpong src/main_zeromq_pingpong.cpp)
+  target_link_libraries(netio_zeromq_pingpong rt pthread zmq)
+
+  add_executable(netio_zeromq_recv src/main_zeromq_recv.cpp)
+  target_link_libraries(netio_zeromq_recv rt pthread zmq)
+

src/cmake/linux-clang → cmake/linux-clang

 SET(CMAKE_C_COMPILER clang )
 SET(CMAKE_CXX_COMPILER clang++)
+SET(TARGET_ARCH x86_64)
\ No newline at end of file

src/cmake/linux-gcc → cmake/linux-gcc

 SET(CMAKE_SYSTEM_NAME Linux)
 SET(CMAKE_C_COMPILER gcc )
 SET(CMAKE_CXX_COMPILER g++)
+SET(TARGET_ARCH x86_64)
 
 

src/cmake/rce-gcc → cmake/rce-arm32

-SET(CMAKE_SYSTEM_NAME Linux)
 SET(CMAKE_C_COMPILER   "arm-unknown-linux-gnueabihf-gcc")
 SET(CMAKE_CXX_COMPILER "arm-unknown-linux-gnueabihf-g++")
-SET(CMAKE_FIND_ROOT_PATH  $ENV{SDK_ROOT}/ArchLinux)
+SET(CMAKE_FIND_ROOT_PATH  $ENV{CENTOS7_ARM32_ROOT})
 # search for programs in the build host directories
 SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
 # for libraries and headers in the target directories
 SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
 SET(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
+add_compile_options("-B$ENV{CENTOS7_ARM32_ROOT}/opt/rh/devtoolset-7/root/lib/gcc/armv7hl-redhat-linux-gnueabi/7 -isystem $ENV{CENTOS7_ARM32_ROOT}/usr/include")
+SET(CMAKE_EXE_LINKER_FLAGS "-B$ENV{CENTOS7_ARM32_ROOT}/opt/rh/devtoolset-7/root/lib/gcc/armv7hl-redhat-linux-gnueabi/7  --sysroot=$ENV{CENTOS7_ARM32_ROOT}")
+if(CMAKE_BUILD_TYPE MATCHES Debug)
+add_compile_options(-g -O4 -pg)
+else()
+add_compile_options( -O4)
+endif()
+add_compile_options(-Wfatal-errors) 
+add_compile_options(-Wno-deprecated) 
+include_directories(AFTER SYSTEM $ENV{CENTOS7_ARM32_ROOT}/usr/include)
+SET(TARGET_ARCH arm)
\ No newline at end of file

cmake/rce-arm64

+SET(CMAKE_C_COMPILER   "aarch64-unknown-linux-gnu-gcc")
+SET(CMAKE_CXX_COMPILER "aarch64-unknown-linux-gnu-g++")
+SET(CMAKE_FIND_ROOT_PATH  $ENV{CENTOS7_ARM64_ROOT})
+# search for programs in the build host directories
+SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+# for libraries and headers in the target directories
+SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+SET(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
+add_compile_options("-B$ENV{CENTOS7_ARM64_ROOT}/opt/rh/devtoolset-7/root/lib/gcc/aarch64-redhat-linux/7 -isystem $ENV{CENTOS7_ARM64_ROOT}/usr/include")
+SET(CMAKE_EXE_LINKER_FLAGS "-B$ENV{CENTOS7_ARM64_ROOT}/opt/rh/devtoolset-7/root/lib/gcc/aarch64-redhat-linux/7 --sysroot=$ENV{CENTOS7_ARM64_ROOT} -pg")
+if(CMAKE_BUILD_TYPE MATCHES Debug)
+add_compile_options(-g -O4 -pg)
+else()
+add_compile_options( -O4)
+endif()
+add_compile_options(-Wfatal-errors) 
+add_compile_options(-Wno-deprecated) 
+include_directories(AFTER SYSTEM $ENV{CENTOS7_ARM64_ROOT}/usr/include)
+SET(TARGET_ARCH arm64)