From bf12e1614f77b4d399bd6103138cc3808dfb69e4 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 12:47:57 +0100
Subject: [PATCH 01/72] REG: fix string related python 3 regressions

---
 bdaq53/firmware_downloader.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bdaq53/firmware_downloader.py b/bdaq53/firmware_downloader.py
index f8b766c4a..07aac0ec6 100644
--- a/bdaq53/firmware_downloader.py
+++ b/bdaq53/firmware_downloader.py
@@ -32,7 +32,7 @@ def get_web_page(url):
     ''' Read web page.
     '''
     response = urllib.request.urlopen(url)  # Add md for markdown
-    return response.read()
+    return response.read().decode('utf-8')
 
 
 def download_firmware(name, url):
@@ -96,13 +96,13 @@ def flash_firmware(name):
 
             More complexity needed here since Xilinx does multi line returns
         '''
-        flushed = ''
+        flushed = bytearray()
         try:
             while not vivado.expect(r'.+', timeout=timeout):
                 flushed += vivado.match.group(0)
         except pexpect.exceptions.TIMEOUT:
             pass
-        return flushed
+        return flushed.decode('utf-8')
 
     logger.info('Flash firmware %s', name)
 
-- 
GitLab


From 00b22a87f21117e35d606b760aaee53c016a343e Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 12:51:23 +0100
Subject: [PATCH 02/72] MAINT: renaming of EUDAQ repo

---
 bdaq53/tests/setup_eudaq.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bdaq53/tests/setup_eudaq.sh b/bdaq53/tests/setup_eudaq.sh
index 5125d1811..14c9e318d 100644
--- a/bdaq53/tests/setup_eudaq.sh
+++ b/bdaq53/tests/setup_eudaq.sh
@@ -1,4 +1,4 @@
-git clone -b v1.7-dev https://github.com/eudaq/eudaq
+git clone -b v1.x-dev https://github.com/eudaq/eudaq
 cd eudaq/build
 cmake -DBUILD_python=ON -DBUILD_gui=OFF -DBUILD_onlinemon=OFF -DBUILD_runsplitter=OFF ..
 make -j 4
-- 
GitLab


From 5671c0ebfed8d7db6167abab53acc16cca6cfc91 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 12:57:51 +0100
Subject: [PATCH 03/72] MAINT

---
 bdaq53/firmware_downloader.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bdaq53/firmware_downloader.py b/bdaq53/firmware_downloader.py
index 07aac0ec6..7fb31d9d0 100644
--- a/bdaq53/firmware_downloader.py
+++ b/bdaq53/firmware_downloader.py
@@ -19,14 +19,14 @@ import pexpect
 
 import bdaq53
 
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
 repository = r'https://gitlab.cern.ch/silab/bdaq53/'
 
 firmware_dev_url = repository + r'wikis/Hardware/Firmware-(development-versions)'
 firmware_url = repository + r'tags'
 
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-
 
 def get_web_page(url):
     ''' Read web page.
@@ -94,7 +94,7 @@ def flash_firmware(name):
     def get_return_string(timeout=1):
         ''' Helper function to get full return string.
 
-            More complexity needed here since Xilinx does multi line returns
+            This complexity needed here since Xilinx does multi line returns
         '''
         flushed = bytearray()
         try:
-- 
GitLab


From cf8517311681a9911676c0940f63f83eb9bce119 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 12:58:13 +0100
Subject: [PATCH 04/72] REG: python 3 incompatibility

---
 firmware/vivado/run.tcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/firmware/vivado/run.tcl b/firmware/vivado/run.tcl
index c83e771d2..c29192651 100644
--- a/firmware/vivado/run.tcl
+++ b/firmware/vivado/run.tcl
@@ -10,7 +10,7 @@
 #
 
 
-set basil_dir [exec python -c "import basil, os; print(str(os.path.dirname(basil.__file__)))"]
+set basil_dir [exec python -c "import basil, os; print(str(os.path.dirname(os.path.dirname(basil.__file__))))"]
 set include_dirs [list $basil_dir/firmware/modules $basil_dir/firmware/modules/utils]
 
 file mkdir output reports
-- 
GitLab


From cccd9f1a272cc5e4507bcf5caa5b58bd180295b9 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 13:22:07 +0100
Subject: [PATCH 05/72] PRJ: use new hardware runner with shell executor

---
 .gitlab-ci.yml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 21b3a3c4b..c5864bc3c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -53,18 +53,19 @@ test:software:
       - pytest -v test_software --ignore=test_software/test_monitor.py
 
 # Tests that need bdaq53 readout hardware and RD53 FE
+# Needs Ubuntu system with xvfb installed and
+# Gitlab-runner with shell executor: https://docs.gitlab.com/runner/executors/
 test:hardware:
     allow_failure: true
     tags:  # tags to differentiate runners able to run the job
       - hardware  # Use Silab hardware runner
-    image: continuumio/miniconda3:latest  # Ubuntu based miniconda image
     before_script:
-      # Update miniconda python and install required binary packages
-      - conda update --yes conda
+      # Install miniconda python and required binary python packages
+      - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+      - bash miniconda.sh -b -p $HOME/miniconda
+      - export PATH=$HOME/miniconda/bin:$PATH
       - conda install --yes numpy bitarray pytest pyyaml scipy numba pytables pyqt matplotlib tqdm pyzmq blosc psutil pexpect coloredlogs
       # Install virtual x server for matplotlib Qt error
-      - apt-get update
-      - apt-get install -y xvfb
       - pip install xvfbwrapper
       # Install basil from github
       - git clone -b development --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..;
-- 
GitLab


From 9226e79c9f8e6bc571708caf8610e3dc46d8a68b Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 14:12:23 +0100
Subject: [PATCH 06/72] PRJ: rename since it does more now

---
 bdaq53/{firmware_downloader.py => firmware_manager.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename bdaq53/{firmware_downloader.py => firmware_manager.py} (100%)

diff --git a/bdaq53/firmware_downloader.py b/bdaq53/firmware_manager.py
similarity index 100%
rename from bdaq53/firmware_downloader.py
rename to bdaq53/firmware_manager.py
-- 
GitLab


From 23bf0565b855ebcc794dca136e5ca06673ea968d Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 14:13:13 +0100
Subject: [PATCH 07/72] ENH: add sitcp install option

---
 bdaq53/firmware_manager.py | 120 +++++++++++++++++++++++--------------
 1 file changed, 76 insertions(+), 44 deletions(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index 7fb31d9d0..f704cea18 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -5,6 +5,8 @@
 # ------------------------------------------------------------
 #
 
+''' Module to manage firmware download, compilation and flashing using vivado '''
+
 import urllib
 import re
 import logging
@@ -12,10 +14,13 @@ import requests
 import math
 import pkg_resources
 import tarfile
+import shutil
 import os.path
 from sys import platform
-from tqdm import tqdm
+
 import pexpect
+from tqdm import tqdm
+from git import Repo
 
 import bdaq53
 
@@ -23,7 +28,6 @@ logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
 repository = r'https://gitlab.cern.ch/silab/bdaq53/'
-
 firmware_dev_url = repository + r'wikis/Hardware/Firmware-(development-versions)'
 firmware_url = repository + r'tags'
 
@@ -185,11 +189,15 @@ def where(name, path, flags=os.F_OK):
     return result
 
 
-def main(name, path=None):
-    ''' All steps to upload matching firmware to FPGA
+def main(name, path=None, create=False):
+    ''' Steps to download/compile/flash matching firmware to FPGA
 
-        If name has .bit suffix try to flash from local file
-        If not available find suitable firmware online, download, extract, and  flash
+        name: str
+            Firmware name:
+                If name has .bit suffix try to flash from local file
+                If not available find suitable firmware online, download, extract, and  flash
+        compile: boolean
+            Compile firmware
     '''
 
     vivado_path = find_vivado(path)
@@ -200,46 +208,54 @@ def main(name, path=None):
         if path:
             logger.error('Cannot find vivado installation in %s', path)
         else:
-            logger.error('Cannot find vivado installation! Install vivado lab from here:\nhttps://www.xilinx.com/support/download.html')
+            logger.error('Cannot find vivado installation!')
+            if not create:
+                logger.error('Install vivado lab from here:\nhttps://www.xilinx.com/support/download.html')
+            else:
+                logger.error('Install vivado paid version to be able to compile firmware')
         return
 
-    if os.path.isfile(name):
-        logger.info('Found existing local bit file')
-        bit_file = name
-    else:
-        if not name.endswith('.tar.gz'):
-            name += '.tar.gz'
-        stable_firmware = True  # std. setting: use stable (tag) firmware
-        version = pkg_resources.get_distribution("bdaq53").version
-        try:
-            import git
+    if not create:
+        if os.path.isfile(name):
+            logger.info('Found existing local bit file')
+            bit_file = name
+        else:
+            if not name.endswith('.tar.gz'):
+                name += '.tar.gz'
+            stable_firmware = True  # std. setting: use stable (tag) firmware
+            version = pkg_resources.get_distribution("bdaq53").version
             try:
-                bdaq53_path = os.path.dirname(bdaq53.__file__)
-                repo = git.Repo(search_parent_directories=True, path=bdaq53_path)
-                active_branch = repo.active_branch
-                if active_branch != 'master':
-                    stable_firmware = False  # use development firmware
-            except git.InvalidGitRepositoryError:  # no github repo --> use stable firmware
+                import git
+                try:
+                    bdaq53_path = os.path.dirname(bdaq53.__file__)
+                    repo = git.Repo(search_parent_directories=True, path=bdaq53_path)
+                    active_branch = repo.active_branch
+                    if active_branch != 'master':
+                        stable_firmware = False  # use development firmware
+                except git.InvalidGitRepositoryError:  # no github repo --> use stable firmware
+                    pass
+            except ImportError:  # git not available
+                logger.warning(
+                    'Git not properly installed, assume software release %s', version)
                 pass
-        except ImportError:  # git not available
-            logger.warning(
-                'Git not properly installed, assume software release %s', version)
-            pass
-        if stable_firmware:
-            tag_list = get_tag_list(firmware_url)
-            matches = [i for i in range(len(tag_list)) if version in tag_list[i]]
-            if not matches:
-                raise RuntimeError('Cannot find tag version %s at %s', version, firmware_url)
-            tag_url = firmware_url + '/' + tag_list[matches[0]]
-            logger.info('Download stable firmware version %s', version)
-            archiv_name = download_firmware(name, tag_url)
-        else:
-            logger.info('Download development firmware')
-            archiv_name = download_firmware(name, firmware_dev_url + '.md')
-        if not archiv_name:
-            return
-        bit_file = unpack_bit_file(archiv_name)
-    flash_firmware(bit_file)
+            if stable_firmware:
+                tag_list = get_tag_list(firmware_url)
+                matches = [i for i in range(len(tag_list)) if version in tag_list[i]]
+                if not matches:
+                    raise RuntimeError('Cannot find tag version %s at %s', version, firmware_url)
+                tag_url = firmware_url + '/' + tag_list[matches[0]]
+                logger.info('Download stable firmware version %s', version)
+                archiv_name = download_firmware(name, tag_url)
+            else:
+                logger.info('Download development firmware')
+                archiv_name = download_firmware(name, firmware_dev_url + '.md')
+            if not archiv_name:
+                return
+            bit_file = unpack_bit_file(archiv_name)
+        flash_firmware(bit_file)
+    else:
+        logger.info('Compiling firmware')
+        get_si_tcp()
 
 
 def get_tag_list(url):
@@ -252,6 +268,22 @@ def get_tag_list(url):
     return re.findall(r'href="/silab/bdaq53/tags/(.*?)"', response)
 
 
+def get_si_tcp():
+    ''' Download SiTCP sources from official github repo '''
+    bdaq53_path = os.path.dirname(bdaq53.__file__)
+    sitcp_folder = os.path.join(bdaq53_path, '..', 'firmware/SiTCP/')
+    # Has to be moved to be allowed to use existing folder for git checkout
+    shutil.move(sitcp_folder + '.gitkeep', os.path.join(sitcp_folder, '..'))
+    Repo.clone_from(url=r'https://github.com/BeeBeansTechnologies/SiTCP_Netlist_for_Kintex7',
+                    to_path=r'../firmware/SiTCP/', branch='master')
+    shutil.move(os.path.join(sitcp_folder, '..', '.gitkeep'), sitcp_folder)
+
+
 if __name__ == '__main__':
-    logging.basicConfig(level=logging.DEBUG)
-    main('BDAQ53_RX640')
+#     logging.basicConfig(level=logging.DEBUG)
+#     main('BDAQ53_RX640')
+
+    get_si_tcp()
+
+    # get_si_tcp()
+
-- 
GitLab


From ff4165ce036c6e846af685c32632a6e7bba6e77a Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 14:13:29 +0100
Subject: [PATCH 08/72] ENH: add firmware compilation option

---
 bdaq53/bdaq53.py     | 505 +++++++------------------------------------
 bdaq53/bdaq53_cli.py |   2 +-
 2 files changed, 85 insertions(+), 422 deletions(-)

diff --git a/bdaq53/bdaq53.py b/bdaq53/bdaq53.py
index 438ece2b6..ddcb605fb 100644
--- a/bdaq53/bdaq53.py
+++ b/bdaq53/bdaq53.py
@@ -5,438 +5,101 @@
 # ------------------------------------------------------------
 #
 
-import yaml
-import logging
+import argparse
 import os
-import time
-import struct
-import coloredlogs
-import numpy as np
-import tables as tb
-
-from tables.exceptions import NoSuchNodeError
-
-from basil.dut import Dut
-from basil.utils.BitLogic import BitLogic
-
-import pkg_resources
-VERSION = pkg_resources.get_distribution("bdaq53").version
-
-loglevel = logging.INFO
-
-''' Set up main logger '''
-for handler in logging.root.handlers[:]:
-    logging.root.removeHandler(handler)
-logging.getLogger('basil.HL.RegisterHardwareLayer').setLevel(logging.WARNING)
-
-logging.SUCCESS = 25  # WARNING(30) > SUCCESS(25) > INFO(20)
-logging.addLevelName(logging.SUCCESS, 'SUCCESS')
+import logging
+import yaml
 
-coloredlogs.DEFAULT_FIELD_STYLES = {'asctime': {},
-                                    'hostname': {},
-                                    'levelname': {'bold': True},
-                                    'name': {},
-                                    'programname': {}}
-coloredlogs.DEFAULT_LEVEL_STYLES = {'critical': {'color': 'red', 'bold': True},
-                                    'debug': {'color': 'magenta'},
-                                    'error': {'color': 'red', 'bold': True},
-                                    'info': {},
-                                    'success': {'color': 'green'},
-                                    'warning': {'color': 'yellow'}}
+from importlib import import_module
+from inspect import getmembers
+from bdaq53 import firmware_manager
 
-coloredlogs.install(fmt="%(asctime)s - [%(name)-15s] - %(levelname)-7s %(message)s", milliseconds=True)
 
 logger = logging.getLogger('BDAQ53')
-logger.setLevel(loglevel)
-logger.success = lambda msg, *args, **kwargs: logger.log(logging.SUCCESS, msg, *args, **kwargs)
-
-
-class BDAQ53(Dut):
-    '''
-    Main class for BDAQ53 readout system
-    '''
-
-    ''' Map hardware IDs for board identification '''
-    hw_map = {
-        0: 'SIMULATION',
-        1: 'BDAQ53',
-        2: 'USBPix3',
-        3: 'KC705',
-        4: 'GENESYS 2'
-    }
-
-    hw_con_map = {
-        0: 'SMA',
-        1: 'FMC_LPC',
-        2: 'FMC_HPC',
-        3: 'Displayport'
-
-    }
-
-    ''' Options concerning the readout hardware '''
-    board_options_map = {
-        '640Mbps': 0x01
-    }
-
-    def __init__(self, conf=None, bench_config=None):
-        self.proj_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-        self.configuration = {}
-
-        try:
-            if bench_config is None:
-                bench_config = os.path.join(self.proj_dir, 'bdaq53' + os.sep + 'testbench.yaml')
-            with open(bench_config) as f:
-                self.configuration['bench'] = yaml.load(f)
-        except TypeError:
-            self.configuration['bench'] = bench_config
-
-        if not conf:
-            conf = os.path.join(self.proj_dir, 'bdaq53' + os.sep + 'bdaq53.yaml')
-        logger.debug("Loading configuration file from %s" % conf)
-
-        super(BDAQ53, self).__init__(conf)
-
-    def init(self, **kwargs):
-        super(BDAQ53, self).init()
-
-        self.fw_version, self.board_version, self.board_options, self.connector_version = self.get_daq_version()
-        logger.success('Found board %s with %s running firmware version %s' % (self.board_version, self.connector_version, self.fw_version))
-
-        if self.fw_version != VERSION.split('.')[0] + '.' + VERSION.split('.')[1]:  # Compare only the first two blocks
-            raise Exception("Firmware version (%s) is different than software version (%s)! Please update." % (self.fw_version, VERSION))
-
-        if self.board_version == 'BDAQ53' or self.board_version == 'USBPix3':
-            if self['rx'].get_Si570_is_configured() is False:
-                from bdaq53 import si570
-                si570_conf = {'name': 'si570', 'type': 'bdaq53.si570', 'interface': 'intf', 'base_addr': 0xba, 'init': {'frequency': 160.0}}
-                bdaq53a_clk_gen = si570.si570(self['i2c'], si570_conf)
-                self['cmd'].set_output_en(False)
-                self['rx'].reset()
-                time.sleep(0.1)
-                bdaq53a_clk_gen.init()
-                time.sleep(0.1)
-                self['cmd'].set_output_en(True)
-                self['rx'].set_Si570_is_configured(True)
-            else:
-                logger.info('Si570 oscillator is already configured')
-        elif self.board_version == 'KC705':
-            self._kc705_setup_si5324(**kwargs)
-        elif self.board_version == 'SIMULATION':
-            pass
-
-        # Configure cmd encoder
-        self['cmd'].reset()
-        time.sleep(0.1)
-        # Wait for PLL lock
-        self.wait_for_pll_lock()
-
-        self.set_aurora()
-
-        self.print_powered_dp_connectors()
-
-    def get_daq_version(self):
-        ret = self['intf'].read(0x0000, 2)
-        fw_version = str('%s.%s' % (ret[1], ret[0]))
-
-        ret = self['intf'].read(0x0002, 2)
-        board_version = self.hw_map[ret[0] + (ret[1] << 8)]
-
-        ret = self['intf'].read(0x0004, 1)
-        board_options = ret[0]
-
-        ret = self['intf'].read(0x0005, 2)
-        connector_version = self.hw_con_map[ret[0] + (ret[1] << 8)]
-
-        return fw_version, board_version, board_options, connector_version
-
-    def _kc705_set_i2c_mux(self, value):
-        ''' Configure the I2C MUX and returns the base address of the selected device '''
-        _i2c_mux_map = {
-            'Si570': (0x01, 0x5d),
-            'FMC_HPC': (0x02, 0x00),
-            'FMC_LPC': (0x04, 0x00),
-            'I2C_EEPROM': (0x08, 0x54),
-            'SFP_MODULE': (0x10, 0x50),
-            'ADV7511': (0x20, 0x39),
-            'DDR3_SODIMM': (0x40, (0x50, 0x18)),
-            'Si5324': (0x80, 0x68)
-        }
-        if value in _i2c_mux_map:
-            self['i2c'].write(0xe8, [_i2c_mux_map[value][0]])
-            logger.debug('I2C mux set to: %s' % value)
+logging.basicConfig()
+logger.setLevel(logging.INFO)
+
+
+def main():
+    scans = {'test_registers': 'RegisterTest',
+             'test_dac_linearity': 'DACLinearTest',
+             'scan_digital': 'DigitalScan',
+             'scan_analog': 'AnalogScan',
+             'scan_threshold': 'ThresholdScan',
+             'scan_noise_occupancy': 'NoiseOccScan',
+             'tune_global_threshold': 'ThresholdTuning',
+             'tune_local_threshold': 'TDACTuning',
+             'meta_mask_noisy_pixels': 'MetaNOccScan',
+             'meta_tune_threshold': 'MetaThrTuning',
+             'meta_tune_threshold_simple': 'MetaTDACTuning'
+             }
+
+    scan_names = [key for key in scans.keys()]
+
+    parser = argparse.ArgumentParser(
+        description='Bonn DAQ system for RD53A prototype\nexample: bdaq53 scan_digital -p start_column=10 stop_column=20', formatter_class=argparse.RawTextHelpFormatter)
+
+    parser.add_argument('scan',
+                        type=str,
+                        nargs='?',
+                        choices=scan_names,
+                        help='Scan name. Allowed values are:\n' + ', '.join(scan_names),
+                        metavar='scan_name')
+
+    parser.add_argument('-f', '--parameter_file',
+                        type=str,
+                        nargs='?',
+                        help='Path to scan parameter file. If not given the default configuration is used.',
+                        metavar='parameter_file')
+
+    parser.add_argument('-p', '--parameters',
+                        type=str,
+                        nargs='+',
+                        help='or list of parameters. E.g. paramA=9 paramB=1',
+                        metavar='',
+                        default=[])
+
+    parser.add_argument('--firmware',
+                        nargs=1,
+                        help='Firmware file name (e.g. auto, BDAQ53, KC705, ...)\n See https://gitlab.cern.ch/silab/bdaq53/tags',)
+
+    parser.add_argument('--vivado_path',
+                        default=[None],
+                        nargs=1,
+                        help='Path of vivado installation',)
+
+    args = parser.parse_args()
+
+    if args.firmware is not None:
+        firmware_downloader.main(args.firmware[0], path=args.vivado_path[0])
+
+    if args.scan:
+        mod = import_module('bdaq53.scans.' + args.scan)
+
+        if args.parameter_file:
+            parameter_file = args.parameter_file
         else:
-            logger.error('I2C mux setting invalid: %s' % value)
-
-        base_addr = _i2c_mux_map[value][1]
-        logger.debug('I2C base address: %s' % hex(base_addr))
-
-        return base_addr
-
-    def _kc705_setup_si5324(self, **kwargs):
-        ''' Calculated register values for Si5324 have to be modified in order to work!
-            N1_HS, NC1_LS, N2_HS, N2_LS, N32, BWSEL '''
-        _si5324_f_map = {
-            200: (7, 4, 10, 112000, 22857, 2),
-            180: (7, 4, 10, 100800, 22857, 2),
-            170: (5, 6, 10, 102000, 22857, 2),
-            160: (8, 4, 10, 102400, 22857, 2),
-            150: (5, 3, 6, 33249, 7036, 2),
-            140: (9, 4, 10, 100800, 22857, 2),
-            120: (11, 4, 11, 32000, 7619, 2),
-            100: (9, 4, 11, 32000, 15238, 2)
-        }
-
-        frequency = kwargs.get('aurora_ref', 160)
+            parameter_file = os.path.dirname(os.path.abspath(mod.__file__)) + '/default_chip.yaml'
 
-        _si5324_base_address = self._kc705_set_i2c_mux('Si5324')
+        logger.info('Using parameter file: ' + parameter_file + '\n')
 
-        def si5324_read(addr):
-            self['i2c'].write(0xd0, [addr])
-            return self['i2c'].read(0xd0, 1)[0]
+        with open(parameter_file, 'r') as f:
+            config = yaml.load(f)
 
-        def si5324_write(addr, data):
-            self['i2c'].write(0xd0, [addr, data & 0xff])
+        config.update(mod.local_configuration)
 
-        # Based on: https://github.com/m-labs/si5324_test/blob/master/firmware/runtime/si5324.c
-        self['i2c'].write(0xd0, [134])
-        ident = struct.unpack(">H", bytearray(self['i2c'].read(0xd0, 2)))[0]
-        if ident != 0x0182:
-            raise ValueError("It is not Si5324 chip.")
-
-        # Select XA/XB input
-        si5324_write(0, si5324_read(0) | 0x40)  # Free running mode=1, CKOUT_ALWAYS_ON = 0
-        si5324_write(11, 0x41)  # Disable CLKIN1
-        si5324_write(6, 0x0F)  # Disable CKOUT2 (SFOUT2_REG=001), set CKOUT1 to LVDS (SFOUT1_REG=111)
-        si5324_write(21, si5324_read(21) & 0xfe)  # CKSEL_PIN = 0
-        si5324_write(3, 0x55)  # CKIN2 selected, SQ_ICAL=1
-
-        if frequency in _si5324_f_map:
-            register_set = _si5324_f_map[frequency]
-
-            N1_HS = register_set[0] - 4
-            NC1_LS = register_set[1] - 1
-            N2_HS = register_set[2] - 4
-            N2_LS = register_set[3] - 1
-            N32 = register_set[4] - 1
-            BWSEL = register_set[5]
-
-            logging.debug('Si5324: Setting registers to %s' % str(register_set))
-        else:
-            logger.error('Si5324: No valid frequency specified: %u' % frequency)
+        for param in args.parameters:
+            key, value = param.split('=')
+            config[key] = eval(value)
 
-        si5324_write(2, (si5324_read(2) & 0x0f) | (BWSEL << 4))
-        si5324_write(25, N1_HS << 5)
-        si5324_write(31, NC1_LS >> 16)
-        si5324_write(32, NC1_LS >> 8)
-        si5324_write(33, NC1_LS)
-        si5324_write(40, (N2_HS << 5) | (N2_LS >> 16))
-        si5324_write(41, N2_LS >> 8)
-        si5324_write(42, N2_LS)
-        si5324_write(46, N32 >> 16)
-        si5324_write(47, N32 >> 8)
-        si5324_write(48, N32)
-        si5324_write(137, si5324_read(137) | 0x01)  # FASTLOCK=1
-        si5324_write(136, 0x40)  # ICAL=1
-
-        time.sleep(0.1)
-
-        LOS1_INT = (si5324_read(129) & 0x02) == 0
-        LOSX_INT = (si5324_read(129) & 0x01) == 0
-        LOL_INT = (si5324_read(130) & 0x01) == 0
-
-        logger.debug('Si5324: Has input: %d' % (LOS1_INT))
-        logger.debug('Si5324: Has xtal %d:' % (LOSX_INT))
-        logger.debug('Si5324: Locked: %d' % (LOL_INT))
-
-        logger.info('Si5324: Clock set to %u MHz.' % frequency)
-
-        if LOL_INT is False:
-            logger.warning('Si5324: Not locked.')
-
-    def _kc705_get_temperature_NTC_CERNFMC(self):
-        ''' Measure the temperature of the SCC NTC using the CERN FMC card connected to KC705 board'''
-
-        if self.board_version != 'KC705':
-            raise RuntimeError('_kc705_get_temperature_NTC_CERNFMC() is only available with KC705 and CERN FMC card')
-
-        # -----constants-------------
-        # FIXME: ove to calibration object?
-        ntc_adc_vdd = 2.5  # ntc_adc_vdd of the ADC
-        ntc_R1 = 39000  # Resistance 1 of the voltage divider, in series with NTC in FMC-card
-        ntc_R25C = 10e3  # NTC constant
-        ntc_T25 = 298.15
-        ntc_beta = 3435  # Beta NTC constant
-        ntc_adc_lsb = 0.001  # ntc_adc_lsb value of the ADC for normal configuration. Can be changed by changing the configuration register.
-        # ---------------------------
-
-        ntc_adc_base_address = 0x90
-
-        # Read the values of the 1 bit ADC in CERN-FMC card:
-        if self.connector_version == 'FMC_HPC':
-            self._kc705_set_i2c_mux('FMC_HPC')
-            logger.debug('I2C mux: FMC HPC selected')
-        elif self.connector_version == 'FMC_LPC':
-            self._kc705_set_i2c_mux('FMC_LPC')
-            logger.debug('I2C mux: FMC LPC selected')
-        else:
-            raise RuntimeError('_kc705_get_temperature_NTC_CERNFMC() is only available with KC705 and CERN FMC card')
-
-        self['i2c'].write(ntc_adc_base_address, [0b00000001])  # address of ADC and write the addresss pointer register to point the configuration register(default 0x8583)
-        self['i2c'].write(ntc_adc_base_address, [0b00000001, 0x85, 0x83])  # Reset the ADC to start adc_raw single conversion. with this config (which is the default) in the conversion register we will read the voltage drop between the terminals of the NTC resistor.
-
-        self['i2c'].write(ntc_adc_base_address, [0b00000000])
-        adc_raw = self['i2c'].read(ntc_adc_base_address, 2)  # read two bytes of the conversion register of adc
-
-        logger.debug('NTC ADC raw data: %s' % (hex(adc_raw[0]) + ' ' + hex(adc_raw[1])))
-        adc = (((adc_raw[0] << 8) | adc_raw[1]) >> 4)
-        V_adc = ntc_adc_lsb * adc
-        R_ntc = (ntc_R1 * V_adc) / (ntc_adc_vdd - V_adc)
-        T_ntc = (1.0 / ((1.0 / ntc_T25) + ((1.0 / ntc_beta) * (np.log(R_ntc / ntc_R25C))))) - (ntc_T25 - 25)
-
-        return round(T_ntc, 3)
-
-    def get_temperature_NTC(self):
-        if self.board_version == 'KC705':
-            return self._kc705_get_temperature_NTC_CERNFMC()
-        elif self.board_version == 'BDAQ53':
-            raise NotImplementedError('NTC readout is not yet implemented with BDAQ board.')
-        else:
-            raise NotImplementedError('NTC readout is not not supported on this hardware platform.')
-
-    def get_DP_SENSE(self, DP_ID):
-        ''' Read back the vddd_sense lines to identify powered chips '''
-        if self.board_version == 'BDAQ53':
-            sense = self['DP_CONTROL'].get_data()
-            logging.debug('Slow_control sense: %s' % bin(sense[0]))
-            if 0 <= DP_ID < 4:
-                return ((sense[0] & (1 << DP_ID)) is False)
-            else:
-                logger.error('Invalid DP_ID (0..3)')
-        else:
-            logger.warning('RD53A slow control is only available for BDAQ53 hardware')
-            return False
-
-    def set_DP_RESET(self, DP_ID, value):
-        ''' Controls the POR lines. if set to 1, the POR is pulled to local chip ground via opto-coupler '''
-        if self.board_version == 'BDAQ53':
-            if 0 <= DP_ID < 4:
-                self['DP_CONTROL'].set_data([value << (4 + DP_ID)])
-            else:
-                logger.error('Invalid DP_ID (0..3)')
-        else:
-            logger.warning('RD53A slow control is only available for BDAQ53 hardware')
-            return False
-
-    def print_powered_dp_connectors(self):
-        ''' Report the VDDD_SENSE signal status for the given Displayport connectors'''
-        if self.board_version == 'BDAQ53':
-            for i in range(4):
-                if self.get_DP_SENSE(i):
-                    logger.info('VDDD_SENSE detected at Displayport ID %i' % i)
-
-    def dp_power_on_reset(self, DP_ID):
-        ''' Short reset pulse for given DP_ID '''
-        self.set_DP_RESET(DP_ID, True)
-        time.sleep(0.1)
-        self.set_DP_RESET(DP_ID, False)
-
-    def set_monitor_filter(self, state=True):
-        self['rx'].set_USER_K_FILTER_MASK_1(0x01)  # only allow frames containing register data
-        self['rx'].set_USER_K_FILTER_MASK_2(0x02)  # only allow frames containing register data
-        self['rx'].set_USER_K_FILTER_MASK_3(0x04)  # only allow frames containing register data
-        self['rx'].set_USER_K_FILTER_EN(state)  # set the filter
-        logger.debug('USER_K filter set to %s' % state)
-
-    def write_digilent_dac(self, value):
-        ''' Writes integer upto 16 bits to the externally via PMOD connected Digilent DAC '''
-        if self.board_version == 'BDAQ53':
-            byts = []
-            value = int(value)
-            for i in range(0, 2):
-                byts.append(value >> (i * 8) & 0xff)
-            byts.reverse()
-            self['spi_dac'].set_data(byts, addr=0)
-            self['spi_dac'].start()
-        else:
-            logger.warning('Digilent DAC is only available for BDAQ53 hardware')
-            return False
-
-    def set_aurora(self):
-        if self.configuration['bench']['bypass_mode'] is True:
-            if self.board_version == 'KC705'and self.connector_version == 'FMC_LPC' and self.board_options & self.board_options_map['640Mbps']:
-                logger.info("Switching FMC card to BYPASS MODE @ 640Mb/s")
-                self['cmd'].set_bypass_mode(True)
-            else:
-                raise NotImplementedError('Bypass mode is only supported for the KC705+FMC_LPC readout hardware @ 640Mb/s')
-        elif self.board_options & self.board_options_map['640Mbps']:
-            logger.info("Aurora receiver running at 640Mb/s")
-            self['cmd'].set_bypass_mode(False)
-        else:
-            logger.info("Aurora receiver running at 1.28Gb/s")
-            self['cmd'].set_bypass_mode(False)
-
-    def wait_for_aurora_sync(self, timeout=1000):
-        logger.debug("Waiting for Aurora sync...")
-        times = 0
-
-        while times < timeout and self['rx'].get_rx_ready() == 0:
-            times += 1
-
-        if self['rx'].get_rx_ready() == 1:
-            logger.debug("Aurora link synchronized")
-            return True
-        else:
-            self['cmd'].reset()
-            raise RuntimeError('Timeout while waiting for Aurora Sync.')
-
-    def wait_for_pll_lock(self, timeout=1000):
-        logger.debug("Waiting for PLL lock...")
-        times = 0
-
-        while times < timeout and self['rx'].get_pll_locked() == 0:
-            times += 1
-
-        if self['rx'].get_pll_locked() == 1:
-            logger.debug("PLL locked")
-            return True
-        else:
-            raise RuntimeError('Timeout while waiting for PLL to lock.')
-
-    def get_trigger_counter(self):
-        return self['tlu']['TRIGGER_COUNTER']
-
-    def set_tlu_module(self, trigger_enable):
-        self['tlu']['TRIGGER_ENABLE'] = trigger_enable
-
-    def set_trigger_data_delay(self, trigger_data_delay):
-        self['tlu']['TRIGGER_DATA_DELAY'] = trigger_data_delay
-
-    def configure_tlu_module(self, **kwargs):
-        # Reset first TLU module
-        self['tlu']['RESET'] = 1
-        # Set specified registers
-        for key, value in kwargs['TRIGGER'].items():
-            self['tlu'][key] = value
-        # Set maximum number of triggers
-        if kwargs['max_triggers']:
-            self['tlu']['MAX_TRIGGERS'] = kwargs['max_triggers']
-        else:
-            # unlimited number of triggers
-            self['tlu']['MAX_TRIGGERS'] = 0
+        for name, cls in getmembers(mod):
+            if name == scans[args.scan]:
+                break
 
-    def get_tlu_erros(self):
-        return (self['tlu']['TRIGGER_LOW_TIMEOUT_ERROR_COUNTER'], self['tlu']['TLU_TRIGGER_ACCEPT_ERROR_COUNTER'])
+        scan = cls()
+        scan.start(**config)
+        scan.analyze()
 
-    def configure_trigger_cmd_pulse(self, **kwargs):
-        # configures pulse which is sent to CMD for incoming triggers; factor 4 is needed for conversion from 160 MHz to 40 MHz (BC)
-        self['pulser_trig'].set_en(True)
-        self['pulser_trig'].set_width(kwargs['trigger_length'] * 4)
-        self['pulser_trig'].set_delay(kwargs['trigger_delay'] * 4)
-        self['pulser_trig'].set_repeat(1)
 
-    def configure_tlu_veto_pulse(self, **kwargs):
-        # configures pulse for veto of new triggers; factor 4 is needed for conversion from 160 MHz to 40 MHz (BC)
-        self['pulser_veto'].set_en(True)
-        self['pulser_veto'].set_width(4)
-        self['pulser_veto'].set_delay(kwargs['veto_length'] * 4)
-        self['pulser_veto'].set_repeat(1)
+if __name__ == '__main__':
+    main()
diff --git a/bdaq53/bdaq53_cli.py b/bdaq53/bdaq53_cli.py
index 7879fe330..ddcb605fb 100644
--- a/bdaq53/bdaq53_cli.py
+++ b/bdaq53/bdaq53_cli.py
@@ -12,7 +12,7 @@ import yaml
 
 from importlib import import_module
 from inspect import getmembers
-from bdaq53 import firmware_downloader
+from bdaq53 import firmware_manager
 
 
 logger = logging.getLogger('BDAQ53')
-- 
GitLab


From 5c088b444874a671f6680aa5b3551a4f99ba6a5a Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 15:17:52 +0100
Subject: [PATCH 09/72] BUG: sitcp folder path

---
 bdaq53/firmware_manager.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index f704cea18..84bad3964 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -270,20 +270,17 @@ def get_tag_list(url):
 
 def get_si_tcp():
     ''' Download SiTCP sources from official github repo '''
+    logger.info('Downloading SiTCP')
     bdaq53_path = os.path.dirname(bdaq53.__file__)
     sitcp_folder = os.path.join(bdaq53_path, '..', 'firmware/SiTCP/')
     # Has to be moved to be allowed to use existing folder for git checkout
     shutil.move(sitcp_folder + '.gitkeep', os.path.join(sitcp_folder, '..'))
     Repo.clone_from(url=r'https://github.com/BeeBeansTechnologies/SiTCP_Netlist_for_Kintex7',
-                    to_path=r'../firmware/SiTCP/', branch='master')
+                    to_path=sitcp_folder, branch='master')
     shutil.move(os.path.join(sitcp_folder, '..', '.gitkeep'), sitcp_folder)
 
 
 if __name__ == '__main__':
-#     logging.basicConfig(level=logging.DEBUG)
-#     main('BDAQ53_RX640')
-
-    get_si_tcp()
-
-    # get_si_tcp()
+    logging.basicConfig(level=logging.DEBUG)
+    main('BDAQ53_RX640')
 
-- 
GitLab


From cd9337815e72f06d0cedf0271a0b5cb6e69e3ffe Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 16:08:00 +0100
Subject: [PATCH 10/72] PRJ: Add conda git package

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c5864bc3c..2e75292e4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -9,7 +9,7 @@ test:rd53:
       - bash miniconda.sh -b -p $HOME/miniconda
       - export PATH=$HOME/miniconda/bin:$PATH
       - conda update --yes conda
-      - conda install --yes numpy bitarray pytest pyyaml scipy numba pytables pyqt matplotlib tqdm pyzmq blosc psutil pexpect coloredlogs
+      - conda install --yes numpy bitarray pytest pyyaml scipy numba pytables pyqt matplotlib tqdm pyzmq blosc psutil pexpect coloredlogs ptyprocess
       # Setup co-simulation
       - pip install cocotb==1.0.dev3
       - git clone -b development --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..;
@@ -64,7 +64,7 @@ test:hardware:
       - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
       - bash miniconda.sh -b -p $HOME/miniconda
       - export PATH=$HOME/miniconda/bin:$PATH
-      - conda install --yes numpy bitarray pytest pyyaml scipy numba pytables pyqt matplotlib tqdm pyzmq blosc psutil pexpect coloredlogs
+      - conda install --yes numpy bitarray pytest pyyaml scipy numba pytables pyqt matplotlib tqdm pyzmq blosc psutil pexpect coloredlogs ptyprocess
       # Install virtual x server for matplotlib Qt error
       - pip install xvfbwrapper
       # Install basil from github
-- 
GitLab


From bb7e2f51f19a18f3cbc09d0bc557afa7b1dd28e3 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 16:08:18 +0100
Subject: [PATCH 11/72] ENH: patch SiTCP sources

---
 bdaq53/firmware_manager.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index 84bad3964..f03790265 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -12,6 +12,7 @@ import re
 import logging
 import requests
 import math
+import fileinput
 import pkg_resources
 import tarfile
 import shutil
@@ -269,7 +270,16 @@ def get_tag_list(url):
 
 
 def get_si_tcp():
-    ''' Download SiTCP sources from official github repo '''
+    ''' Download SiTCP sources from official github repo 
+        and apply patches
+    '''
+
+    def line_prepender(filename, line):
+        with open(filename, 'r+') as f:
+            content = f.read()
+            f.seek(0, 0)
+            f.write(line.rstrip('\r\n') + '\n' + content)
+
     logger.info('Downloading SiTCP')
     bdaq53_path = os.path.dirname(bdaq53.__file__)
     sitcp_folder = os.path.join(bdaq53_path, '..', 'firmware/SiTCP/')
@@ -278,6 +288,12 @@ def get_si_tcp():
     Repo.clone_from(url=r'https://github.com/BeeBeansTechnologies/SiTCP_Netlist_for_Kintex7',
                     to_path=sitcp_folder, branch='master')
     shutil.move(os.path.join(sitcp_folder, '..', '.gitkeep'), sitcp_folder)
+    # Patch sources, see README of bdaq53
+    line_prepender(filename=sitcp_folder + 'TIMER.v', line=r'`default_nettype wire')
+    line_prepender(filename=sitcp_folder + 'WRAP_SiTCP_GMII_XC7K_32K.V', line=r'`default_nettype wire')
+    for line in fileinput.input([sitcp_folder + 'WRAP_SiTCP_GMII_XC7K_32K.V'], inplace=True):
+        print(line.replace("assign\tMY_IP_ADDR[31:0]\t= (~FORCE_DEFAULTn | (EXT_IP_ADDR[31:0]==32'd0) \t? DEFAULT_IP_ADDR[31:0]\t\t: EXT_IP_ADDR[31:0]\t\t);",
+                           'assign\tMY_IP_ADDR[31:0]\t= EXT_IP_ADDR[31:0];'), end='')
 
 
 if __name__ == '__main__':
-- 
GitLab


From e40f0276a7b1e269636c4227b577fadd810ec037 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 16:18:18 +0100
Subject: [PATCH 12/72] MAINT: new basil firmware path MAINT: only create one
 firmware as std setting

---
 firmware/vivado/run.tcl | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/firmware/vivado/run.tcl b/firmware/vivado/run.tcl
index c29192651..2fbd84244 100644
--- a/firmware/vivado/run.tcl
+++ b/firmware/vivado/run.tcl
@@ -11,7 +11,7 @@
 
 
 set basil_dir [exec python -c "import basil, os; print(str(os.path.dirname(os.path.dirname(basil.__file__))))"]
-set include_dirs [list $basil_dir/firmware/modules $basil_dir/firmware/modules/utils]
+set include_dirs [list $basil_dir/basil/firmware/modules $basil_dir/basil/firmware/modules/utils]
 
 file mkdir output reports
 
@@ -98,16 +98,16 @@ proc run_bit { part board connector xdc_file size option} {
 
 # Bitfiles for the 1.28 Gb/s Aurora ip core configuration
 #       FPGA type           board name	connector  	constraints file     flash size  option
-run_bit xc7k160tffg676-2    BDAQ53      ""          ../src/bdaq53.xdc       64        ""
-run_bit xc7k160tfbg676-1    USBPIX3     ""          ../src/usbpix3.xdc      64        ""
-run_bit xc7k325tffg900-2    KC705       _SMA        ../src/kc705_gmii.xdc   16        ""
-run_bit xc7k325tffg900-2    KC705       _FMC_LPC    ../src/kc705_gmii.xdc   16        ""
+#run_bit xc7k160tffg676-2    BDAQ53      ""          ../src/bdaq53.xdc       64        ""
+#run_bit xc7k160tfbg676-1    USBPIX3     ""          ../src/usbpix3.xdc      64        ""
+#run_bit xc7k325tffg900-2    KC705       _SMA        ../src/kc705_gmii.xdc   16        ""
+#run_bit xc7k325tffg900-2    KC705       _FMC_LPC    ../src/kc705_gmii.xdc   16        ""
 
 # Bitfiles for the 640 Mb/s Aurora ip core configuration
 #       FPGA type           board name  connector	constraints file     flash size  option
 run_bit xc7k160tffg676-2    BDAQ53      ""          ../src/bdaq53.xdc       64      _RX640
-run_bit xc7k160tfbg676-1    USBPIX3     ""          ../src/usbpix3.xdc      64      _RX640
-run_bit xc7k325tffg900-2    KC705       _SMA        ../src/kc705_gmii.xdc   16      _RX640
-run_bit xc7k325tffg900-2    KC705       _FMC_LPC    ../src/kc705_gmii.xdc   16      _RX640
+#run_bit xc7k160tfbg676-1    USBPIX3     ""          ../src/usbpix3.xdc      64      _RX640
+#run_bit xc7k325tffg900-2    KC705       _SMA        ../src/kc705_gmii.xdc   16      _RX640
+#run_bit xc7k325tffg900-2    KC705       _FMC_LPC    ../src/kc705_gmii.xdc   16      _RX640
 
 exit
-- 
GitLab


From 46af353e1475fb4cebd3cb15326e354a105778a0 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 16:45:53 +0100
Subject: [PATCH 13/72] MAINT: enable all FW again

---
 firmware/vivado/run.tcl | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/firmware/vivado/run.tcl b/firmware/vivado/run.tcl
index 2fbd84244..2a0bcd05b 100644
--- a/firmware/vivado/run.tcl
+++ b/firmware/vivado/run.tcl
@@ -98,16 +98,16 @@ proc run_bit { part board connector xdc_file size option} {
 
 # Bitfiles for the 1.28 Gb/s Aurora ip core configuration
 #       FPGA type           board name	connector  	constraints file     flash size  option
-#run_bit xc7k160tffg676-2    BDAQ53      ""          ../src/bdaq53.xdc       64        ""
-#run_bit xc7k160tfbg676-1    USBPIX3     ""          ../src/usbpix3.xdc      64        ""
-#run_bit xc7k325tffg900-2    KC705       _SMA        ../src/kc705_gmii.xdc   16        ""
-#run_bit xc7k325tffg900-2    KC705       _FMC_LPC    ../src/kc705_gmii.xdc   16        ""
+run_bit xc7k160tffg676-2    BDAQ53      ""          ../src/bdaq53.xdc       64        ""
+run_bit xc7k160tfbg676-1    USBPIX3     ""          ../src/usbpix3.xdc      64        ""
+run_bit xc7k325tffg900-2    KC705       _SMA        ../src/kc705_gmii.xdc   16        ""
+run_bit xc7k325tffg900-2    KC705       _FMC_LPC    ../src/kc705_gmii.xdc   16        ""
 
 # Bitfiles for the 640 Mb/s Aurora ip core configuration
 #       FPGA type           board name  connector	constraints file     flash size  option
 run_bit xc7k160tffg676-2    BDAQ53      ""          ../src/bdaq53.xdc       64      _RX640
-#run_bit xc7k160tfbg676-1    USBPIX3     ""          ../src/usbpix3.xdc      64      _RX640
-#run_bit xc7k325tffg900-2    KC705       _SMA        ../src/kc705_gmii.xdc   16      _RX640
-#run_bit xc7k325tffg900-2    KC705       _FMC_LPC    ../src/kc705_gmii.xdc   16      _RX640
+run_bit xc7k160tfbg676-1    USBPIX3     ""          ../src/usbpix3.xdc      64      _RX640
+run_bit xc7k325tffg900-2    KC705       _SMA        ../src/kc705_gmii.xdc   16      _RX640
+run_bit xc7k325tffg900-2    KC705       _FMC_LPC    ../src/kc705_gmii.xdc   16      _RX640
 
 exit
-- 
GitLab


From 1ec1f3c237d45315354b71e8afc02da5ab41b317 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 16:46:40 +0100
Subject: [PATCH 14/72] PRJ: for CI FW builder

---
 firmware/vivado/ci_run.tcl | 91 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 91 insertions(+)
 create mode 100644 firmware/vivado/ci_run.tcl

diff --git a/firmware/vivado/ci_run.tcl b/firmware/vivado/ci_run.tcl
new file mode 100644
index 000000000..3bb99a08c
--- /dev/null
+++ b/firmware/vivado/ci_run.tcl
@@ -0,0 +1,91 @@
+
+# ---------------------------------------------------------------
+# Copyright (c) SILAB ,  Institute of Physics, University of Bonn
+# ---------------------------------------------------------------
+#
+#   This script creates Vivado projects and bitfiles for the supported hardware platforms
+#
+#   Start vivado in tcl mode by typing:
+#       vivado -mode tcl -source run.tcl
+#
+
+
+set basil_dir [exec python -c "import basil, os; print(str(os.path.dirname(os.path.dirname(basil.__file__))))"]
+set include_dirs [list $basil_dir/basil/firmware/modules $basil_dir/basil/firmware/modules/utils]
+
+file mkdir output reports
+
+
+proc read_design_files {} {
+    read_verilog ../src/bdaq53.v
+    read_verilog ../src/bdaq53_core.v
+
+    read_edif ../SiTCP/SiTCP_XC7K_32K_BBT_V110.ngc
+    read_verilog ../SiTCP/TIMER.v
+    read_verilog ../SiTCP/SiTCP_XC7K_32K_BBT_V110.V
+    read_verilog ../SiTCP/WRAP_SiTCP_GMII_XC7K_32K.V
+
+
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/rx_aurora_64b66b.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/rx_aurora_64b66b_core.v
+
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/exdes/aurora_64b66b_1lane_exdes.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/exdes/aurora_64b66b_1lane_cdc_sync_exdes.v
+
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane_core.v
+
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_64b66b_descrambler.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_block_sync_sm.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_cbcc_gtx_6466.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_cdc_sync.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_common_logic_cbcc.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_common_reset_cbcc.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_ll_to_axi.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_reset_logic.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_aurora_lane_simplex.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_channel_err_detect_simplex.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_channel_init_sm_simplex.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_err_detect_simplex.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_global_logic_simplex.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_lane_init_sm_simplex.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_ll_datapath_simplex.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_ll_simplex.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_ll_user_k_datapath_simplex.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_startup_fsm.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_sym_dec.v
+
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/support/aurora_64b66b_1lane_clock_module.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/support/aurora_64b66b_1lane_gt_common_wrapper.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/support/aurora_64b66b_1lane_support.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/support/aurora_64b66b_1lane_support_reset_logic.v
+
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/gt/aurora_64b66b_1lane_multi_wrapper.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/gt/aurora_64b66b_1lane_wrapper.v
+    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/gt/aurora_64b66b_1lane_gtx.v
+}
+
+
+proc run_bit { part board connector xdc_file size option} {
+    create_project -force -part $part $board$option$connector designs
+
+    read_design_files
+    read_xdc $xdc_file
+
+    #read_ip ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane.xci
+    global include_dirs
+
+    synth_design -top bdaq53 -include_dirs $include_dirs -verilog_define "$board=1" -verilog_define "$connector=1" -verilog_define "SYNTHESIS=1" -verilog_define "$option=1"
+    opt_design
+    place_design
+    phys_opt_design
+    route_design
+    report_utilization
+    report_timing -file "reports/report_timing.$board$option$connector.log"
+    write_bitstream -force -file output/$board$option$connector
+    write_cfgmem -format mcs -size $size -interface SPIx4 -loadbit "up 0x0 output/$board$option$connector.bit" -force -file output/$board$option$connector
+    close_project
+
+    exec tar -C ./output -cvzf output/$board$option$connector.tar.gz $board$option$connector.bit $board$option$connector.mcs
+}
+
-- 
GitLab


From 0e5fb88567a0fd99aa8f3070537f2aeaaeb8c881 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 16:57:34 +0100
Subject: [PATCH 15/72] ENH: only download SiTCP if needed

---
 bdaq53/firmware_manager.py | 65 ++++++++++++++++++++------------------
 1 file changed, 35 insertions(+), 30 deletions(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index f03790265..5bd9fb7a5 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -32,6 +32,8 @@ repository = r'https://gitlab.cern.ch/silab/bdaq53/'
 firmware_dev_url = repository + r'wikis/Hardware/Firmware-(development-versions)'
 firmware_url = repository + r'tags'
 
+sitcp_repo = r'https://github.com/BeeBeansTechnologies/SiTCP_Netlist_for_Kintex7'
+
 
 def get_web_page(url):
     ''' Read web page.
@@ -201,20 +203,20 @@ def main(name, path=None, create=False):
             Compile firmware
     '''
 
-    vivado_path = find_vivado(path)
-    if vivado_path:
-        logger.debug('Found vivado binary at %s', vivado_path)
-        os.environ["PATH"] += os.pathsep + vivado_path
-    else:
-        if path:
-            logger.error('Cannot find vivado installation in %s', path)
-        else:
-            logger.error('Cannot find vivado installation!')
-            if not create:
-                logger.error('Install vivado lab from here:\nhttps://www.xilinx.com/support/download.html')
-            else:
-                logger.error('Install vivado paid version to be able to compile firmware')
-        return
+#     vivado_path = find_vivado(path)
+#     if vivado_path:
+#         logger.debug('Found vivado binary at %s', vivado_path)
+#         os.environ["PATH"] += os.pathsep + vivado_path
+#     else:
+#         if path:
+#             logger.error('Cannot find vivado installation in %s', path)
+#         else:
+#             logger.error('Cannot find vivado installation!')
+#             if not create:
+#                 logger.error('Install vivado lab from here:\nhttps://www.xilinx.com/support/download.html')
+#             else:
+#                 logger.error('Install vivado paid version to be able to compile firmware')
+#         return
 
     if not create:
         if os.path.isfile(name):
@@ -270,30 +272,33 @@ def get_tag_list(url):
 
 
 def get_si_tcp():
-    ''' Download SiTCP sources from official github repo 
-        and apply patches
+    ''' Download SiTCP sources from official github repo and apply patches
     '''
 
     def line_prepender(filename, line):
-        with open(filename, 'r+') as f:
+        with open(filename, 'rb+') as f:
             content = f.read()
             f.seek(0, 0)
-            f.write(line.rstrip('\r\n') + '\n' + content)
+            f.write(line + '\n' + content)
 
-    logger.info('Downloading SiTCP')
     bdaq53_path = os.path.dirname(bdaq53.__file__)
     sitcp_folder = os.path.join(bdaq53_path, '..', 'firmware/SiTCP/')
-    # Has to be moved to be allowed to use existing folder for git checkout
-    shutil.move(sitcp_folder + '.gitkeep', os.path.join(sitcp_folder, '..'))
-    Repo.clone_from(url=r'https://github.com/BeeBeansTechnologies/SiTCP_Netlist_for_Kintex7',
-                    to_path=sitcp_folder, branch='master')
-    shutil.move(os.path.join(sitcp_folder, '..', '.gitkeep'), sitcp_folder)
-    # Patch sources, see README of bdaq53
-    line_prepender(filename=sitcp_folder + 'TIMER.v', line=r'`default_nettype wire')
-    line_prepender(filename=sitcp_folder + 'WRAP_SiTCP_GMII_XC7K_32K.V', line=r'`default_nettype wire')
-    for line in fileinput.input([sitcp_folder + 'WRAP_SiTCP_GMII_XC7K_32K.V'], inplace=True):
-        print(line.replace("assign\tMY_IP_ADDR[31:0]\t= (~FORCE_DEFAULTn | (EXT_IP_ADDR[31:0]==32'd0) \t? DEFAULT_IP_ADDR[31:0]\t\t: EXT_IP_ADDR[31:0]\t\t);",
-                           'assign\tMY_IP_ADDR[31:0]\t= EXT_IP_ADDR[31:0];'), end='')
+
+    # Only download if not already existing
+    if not os.path.isdir(os.path.join(sitcp_folder, '.git')):
+        logger.info('Downloading SiTCP')
+
+        # Has to be moved to be allowed to use existing folder for git checkout
+        shutil.move(sitcp_folder + '.gitkeep', os.path.join(sitcp_folder, '..'))
+        Repo.clone_from(url=sitcp_repo,
+                        to_path=sitcp_folder, branch='master')
+        shutil.move(os.path.join(sitcp_folder, '..', '.gitkeep'), sitcp_folder)
+        # Patch sources, see README of bdaq53
+        line_prepender(filename=sitcp_folder + 'TIMER.v', line=r'`default_nettype wire')
+        line_prepender(filename=sitcp_folder + 'WRAP_SiTCP_GMII_XC7K_32K.V', line=r'`default_nettype wire')
+        for line in fileinput.input([sitcp_folder + 'WRAP_SiTCP_GMII_XC7K_32K.V'], inplace=True):
+            print(line.replace("assign\tMY_IP_ADDR[31:0]\t= (~FORCE_DEFAULTn | (EXT_IP_ADDR[31:0]==32'd0) \t? DEFAULT_IP_ADDR[31:0]\t\t: EXT_IP_ADDR[31:0]\t\t);",
+                               'assign\tMY_IP_ADDR[31:0]\t= EXT_IP_ADDR[31:0];'), end='')
 
 
 if __name__ == '__main__':
-- 
GitLab


From 294a4ad423d5f51fa550bee9b13624a358e3f9e3 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 18:07:38 +0100
Subject: [PATCH 16/72] ENH: compilation working

---
 bdaq53/firmware_manager.py | 129 +++++++++++++++++++++++++++++++------
 1 file changed, 111 insertions(+), 18 deletions(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index 5bd9fb7a5..96edb4968 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -5,7 +5,9 @@
 # ------------------------------------------------------------
 #
 
-''' Module to manage firmware download, compilation and flashing using vivado '''
+''' Module to manage firmware download, compilation and flashing using vivado.
+    Mainly for CI runner but also useful during headless session (e.g. test beams)
+'''
 
 import urllib
 import re
@@ -34,6 +36,8 @@ firmware_url = repository + r'tags'
 
 sitcp_repo = r'https://github.com/BeeBeansTechnologies/SiTCP_Netlist_for_Kintex7'
 
+bdaq53_path = os.path.dirname(bdaq53.__file__)
+
 
 def get_web_page(url):
     ''' Read web page.
@@ -159,6 +163,96 @@ def flash_firmware(name):
     vivado.expect('Exiting')
 
     logger.info('SUCCESS!')
+    
+    
+def compile_firmware(name):
+    ''' Compile firmware using vivado in tcl mode
+    '''
+
+    def get_return_string(timeout=1):
+        ''' Helper function to get full return string.
+
+            This complexity needed here since Xilinx does multi line returns
+        '''
+        flushed = bytearray()
+        try:
+            while not vivado.expect(r'.+', timeout=timeout):
+                flushed += vivado.match.group(0)
+        except pexpect.exceptions.TIMEOUT:
+            pass
+        return flushed.decode('utf-8')
+
+    supported_firmwares = ['BDAQ53', 'USBPIX3', 'KC705',
+                           'BDAQ53_RX640', 'USBPIX3_RX640', 'KC705_RX640'] 
+    if not name in supported_firmwares:
+        logger.error('Can only compile firmwares: %s', ','.join(supported_firmwares))
+        return
+
+    logger.info('Compile firmware %s', name)
+    
+    vivado_tcl = os.path.join(bdaq53_path, '..', 'firmware/vivado')
+
+    try:
+        vivado = pexpect.spawn('vivado -mode tcl -source ci_run.tcl', cwd=vivado_tcl, timeout=10)  # try full version
+        vivado.expect('Vivado', timeout=5)
+    except pexpect.exceptions.ExceptionPexpect:
+        logger.error('Cannot execute vivado command. Maybe paid version is missing, that is needed for compilation?')
+        return
+    vivado.expect('Vivado%')  # Booted up when showing prompt
+    
+    # Use mappings from run.tcl
+    fpga_types = {'BDAQ53': 'xc7k160tffg676-2',
+                 'USBPIX3': 'xc7k160tfbg676-1',
+                 'KC705':  'xc7k325tffg900-2'}
+    constrains_files = {'BDAQ53': '../src/bdaq53.xdc',
+                 'USBPIX3': '../src/usbpix3.xdc',
+                 'KC705':  '../src/kc705_gmii.xdc'}
+    flash_sizes = {'BDAQ53': '64',
+                 'USBPIX3': '64',
+                 'KC705':  '16'}
+    
+    for k, v in fpga_types.items():
+        if k in name:
+            fpga_type = v
+            constrain_files = constrains_files[k]
+            flash_size = flash_sizes[k]
+            
+    if '_SMA' in name:
+        connector = '_SMA'
+    elif '_FMC_LPC' in name:
+        connector = '_FMC_LPC'
+    else:
+        connector = '""'
+            
+    if 'RX640' in name:
+        option = '_RX640'
+    else:
+        option = '""'
+    
+    compile_command = 'run_bit ' + fpga_type + " " + name + " " + connector + " " + constrain_files + " " + flash_size + " " + option
+    
+#     compile_command = 'run_bit xc7k160tffg676-2    BDAQ53      ""          ../src/bdaq53.xdc       64        ""'
+    
+    logger.info('Compiling firmware. Takes about 10 minutes!')
+    vivado.sendline(compile_command)
+    
+    import time
+    timeout = 30  # 30 seconds with no new print to screen
+    t = 0
+    while t < timeout:
+        r = get_return_string()
+        if r:
+            if 'write_cfgmem completed successfully' in r:
+                break
+            print(r)
+            t = 0
+        else:
+            time.sleep(1)
+            t += 1
+    else:
+        raise RuntimeError('Timeout during compilation, check log!')
+
+    logger.info('SUCCESS!')
 
 
 def find_vivado(path):
@@ -203,20 +297,20 @@ def main(name, path=None, create=False):
             Compile firmware
     '''
 
-#     vivado_path = find_vivado(path)
-#     if vivado_path:
-#         logger.debug('Found vivado binary at %s', vivado_path)
-#         os.environ["PATH"] += os.pathsep + vivado_path
-#     else:
-#         if path:
-#             logger.error('Cannot find vivado installation in %s', path)
-#         else:
-#             logger.error('Cannot find vivado installation!')
-#             if not create:
-#                 logger.error('Install vivado lab from here:\nhttps://www.xilinx.com/support/download.html')
-#             else:
-#                 logger.error('Install vivado paid version to be able to compile firmware')
-#         return
+    vivado_path = find_vivado(path)
+    if vivado_path:
+        logger.debug('Found vivado binary at %s', vivado_path)
+        os.environ["PATH"] += os.pathsep + vivado_path
+    else:
+        if path:
+            logger.error('Cannot find vivado installation in %s', path)
+        else:
+            logger.error('Cannot find vivado installation!')
+            if not create:
+                logger.error('Install vivado lab from here:\nhttps://www.xilinx.com/support/download.html')
+            else:
+                logger.error('Install vivado paid version to be able to compile firmware')
+        return
 
     if not create:
         if os.path.isfile(name):
@@ -230,7 +324,6 @@ def main(name, path=None, create=False):
             try:
                 import git
                 try:
-                    bdaq53_path = os.path.dirname(bdaq53.__file__)
                     repo = git.Repo(search_parent_directories=True, path=bdaq53_path)
                     active_branch = repo.active_branch
                     if active_branch != 'master':
@@ -258,7 +351,8 @@ def main(name, path=None, create=False):
         flash_firmware(bit_file)
     else:
         logger.info('Compiling firmware')
-        get_si_tcp()
+        get_si_tcp()  # get missing SiTCP sources
+        compile_firmware(name)
 
 
 def get_tag_list(url):
@@ -281,7 +375,6 @@ def get_si_tcp():
             f.seek(0, 0)
             f.write(line + '\n' + content)
 
-    bdaq53_path = os.path.dirname(bdaq53.__file__)
     sitcp_folder = os.path.join(bdaq53_path, '..', 'firmware/SiTCP/')
 
     # Only download if not already existing
-- 
GitLab


From 5223525b6d06f677ec4bb46bea1d597a9ac8800f Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 18:13:06 +0100
Subject: [PATCH 17/72] ENH: better logging

---
 bdaq53/firmware_manager.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index 96edb4968..850369105 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -237,17 +237,17 @@ def compile_firmware(name):
     vivado.sendline(compile_command)
     
     import time
-    timeout = 30  # 30 seconds with no new print to screen
+    timeout = 5  # 30 seconds with no new print to screen
     t = 0
     while t < timeout:
         r = get_return_string()
         if r:
             if 'write_cfgmem completed successfully' in r:
                 break
-            print(r)
+            print('.', end='', flush=True)
             t = 0
         else:
-            time.sleep(1)
+            time.sleep(5)
             t += 1
     else:
         raise RuntimeError('Timeout during compilation, check log!')
@@ -350,7 +350,6 @@ def main(name, path=None, create=False):
             bit_file = unpack_bit_file(archiv_name)
         flash_firmware(bit_file)
     else:
-        logger.info('Compiling firmware')
         get_si_tcp()  # get missing SiTCP sources
         compile_firmware(name)
 
-- 
GitLab


From 3f622a2f1020b9d1a3243c7c07c9c76e8b04a08d Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 18:28:26 +0100
Subject: [PATCH 18/72] BUG: fix line ending in SiTCP file

---
 bdaq53/firmware_manager.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index 850369105..3e9656107 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -372,7 +372,11 @@ def get_si_tcp():
         with open(filename, 'rb+') as f:
             content = f.read()
             f.seek(0, 0)
-            f.write(line + '\n' + content)
+            # Python 3, wtf?
+            add = bytearray()
+            add.extend(map(ord, line))
+            add.extend(map(ord, '\n'))
+            f.write(add + content)
 
     sitcp_folder = os.path.join(bdaq53_path, '..', 'firmware/SiTCP/')
 
-- 
GitLab


From 9c2b7448a9433a6e349f1fa7ff95d0dc3d7eb33a Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 18:32:05 +0100
Subject: [PATCH 19/72] PRJ: first compilation try

---
 .gitlab-ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2e75292e4..188f87f68 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -53,7 +53,7 @@ test:software:
       - pytest -v test_software --ignore=test_software/test_monitor.py
 
 # Tests that need bdaq53 readout hardware and RD53 FE
-# Needs Ubuntu system with xvfb installed and
+# Needs Ubuntu system with xvfb and full Xilinx installed and
 # Gitlab-runner with shell executor: https://docs.gitlab.com/runner/executors/
 test:hardware:
     allow_failure: true
@@ -72,8 +72,8 @@ test:hardware:
       # - if [ -z "$CI_COMMIT_TAG"]; then git clone -b v3.0.0 --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; else git clone -b development --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; fi
     script:
       - python setup.py develop
-      # FIXME: This should work
-      # - bdaq53 --firmware BDAQ53  # Flash newest development firmware
+      - bdaq53 --firmware BDAQ53 -c  # Compile new firmware
+      - bdaq53 --firmware BDAQ53  # Flash firmware
       - cd bdaq53/tests
       - pytest -v test_hardware
     artifacts:
-- 
GitLab


From 9d74163912be1ed85bd1b51bfa2badf369f3259b Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 17 Jan 2019 18:35:02 +0100
Subject: [PATCH 20/72] PRJ: conda does not have to be installed everytime

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 188f87f68..956ac38b8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -61,8 +61,8 @@ test:hardware:
       - hardware  # Use Silab hardware runner
     before_script:
       # Install miniconda python and required binary python packages
-      - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
-      - bash miniconda.sh -b -p $HOME/miniconda
+      # - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+      # - bash miniconda.sh -b -p $HOME/miniconda
       - export PATH=$HOME/miniconda/bin:$PATH
       - conda install --yes numpy bitarray pytest pyyaml scipy numba pytables pyqt matplotlib tqdm pyzmq blosc psutil pexpect coloredlogs ptyprocess
       # Install virtual x server for matplotlib Qt error
-- 
GitLab


From 0ec6f27703e2d0c781f0a35d06f615a017b0d448 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 18 Jan 2019 10:00:14 +0100
Subject: [PATCH 21/72] MAINT: always dev firmware for firmware ENH: copy
 firmware to current folder

---
 bdaq53/firmware_manager.py | 48 +++++++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 21 deletions(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index 3e9656107..050c3896b 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -321,29 +321,31 @@ def main(name, path=None, create=False):
                 name += '.tar.gz'
             stable_firmware = True  # std. setting: use stable (tag) firmware
             version = pkg_resources.get_distribution("bdaq53").version
-            try:
-                import git
+            if not os.getenv('CI'):
                 try:
-                    repo = git.Repo(search_parent_directories=True, path=bdaq53_path)
-                    active_branch = repo.active_branch
-                    if active_branch != 'master':
-                        stable_firmware = False  # use development firmware
-                except git.InvalidGitRepositoryError:  # no github repo --> use stable firmware
+                    import git
+                    try:
+                        repo = git.Repo(search_parent_directories=True, path=bdaq53_path)
+                        active_branch = repo.active_branch
+                        if active_branch != 'master':
+                            stable_firmware = False  # use development firmware
+                    except git.InvalidGitRepositoryError:  # no github repo --> use stable firmware
+                        pass
+                except ImportError:  # git not available
+                    logger.warning('Git not properly installed, assume software release %s', version)
                     pass
-            except ImportError:  # git not available
-                logger.warning(
-                    'Git not properly installed, assume software release %s', version)
-                pass
-            if stable_firmware:
-                tag_list = get_tag_list(firmware_url)
-                matches = [i for i in range(len(tag_list)) if version in tag_list[i]]
-                if not matches:
-                    raise RuntimeError('Cannot find tag version %s at %s', version, firmware_url)
-                tag_url = firmware_url + '/' + tag_list[matches[0]]
-                logger.info('Download stable firmware version %s', version)
-                archiv_name = download_firmware(name, tag_url)
-            else:
-                logger.info('Download development firmware')
+                if stable_firmware:
+                    tag_list = get_tag_list(firmware_url)
+                    matches = [i for i in range(len(tag_list)) if version in tag_list[i]]
+                    if not matches:
+                        raise RuntimeError('Cannot find tag version %s at %s', version, firmware_url)
+                    tag_url = firmware_url + '/' + tag_list[matches[0]]
+                    logger.info('Download stable firmware version %s', version)
+                    archiv_name = download_firmware(name, tag_url)
+                else:
+                    logger.info('Download development firmware')
+                    archiv_name = download_firmware(name, firmware_dev_url + '.md')
+            else:  # always use development version for CI runner
                 archiv_name = download_firmware(name, firmware_dev_url + '.md')
             if not archiv_name:
                 return
@@ -352,6 +354,10 @@ def main(name, path=None, create=False):
     else:
         get_si_tcp()  # get missing SiTCP sources
         compile_firmware(name)
+        # Move firmware to current folder
+        cwd = os.getcwd()
+        vivado_tcl = os.path.join(bdaq53_path, '..', 'firmware/vivado')
+        shutil.move(os.path.join(vivado_tcl, r'output/%s' % name + '.bit'), cwd)
 
 
 def get_tag_list(url):
-- 
GitLab


From 9d75bf87843b3423edd38458617081db97713722 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 18 Jan 2019 10:00:40 +0100
Subject: [PATCH 22/72] PRJ: only compile for firmware changes

---
 .gitlab-ci.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 956ac38b8..77f0bf50e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -53,16 +53,14 @@ test:software:
       - pytest -v test_software --ignore=test_software/test_monitor.py
 
 # Tests that need bdaq53 readout hardware and RD53 FE
-# Needs Ubuntu system with xvfb and full Xilinx installed and
+# Needs Ubuntu system with xvfb, Miniconda3 and full Xilinx installed and
 # Gitlab-runner with shell executor: https://docs.gitlab.com/runner/executors/
 test:hardware:
     allow_failure: true
     tags:  # tags to differentiate runners able to run the job
       - hardware  # Use Silab hardware runner
     before_script:
-      # Install miniconda python and required binary python packages
-      # - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
-      # - bash miniconda.sh -b -p $HOME/miniconda
+      # Install required binary python packages
       - export PATH=$HOME/miniconda/bin:$PATH
       - conda install --yes numpy bitarray pytest pyyaml scipy numba pytables pyqt matplotlib tqdm pyzmq blosc psutil pexpect coloredlogs ptyprocess
       # Install virtual x server for matplotlib Qt error
@@ -72,13 +70,15 @@ test:hardware:
       # - if [ -z "$CI_COMMIT_TAG"]; then git clone -b v3.0.0 --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; else git clone -b development --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; fi
     script:
       - python setup.py develop
-      - bdaq53 --firmware BDAQ53 -c  # Compile new firmware
-      - bdaq53 --firmware BDAQ53  # Flash firmware
+      # Compile firmware if changes detected and flash
+      - if [[ `git status --porcelain firmware/` ]]; then bdaq53 --firmware BDAQ53 -c; bdaq53 --firmware BDAQ53.bit; else bdaq53 --firmware BDAQ53; fi
       - cd bdaq53/tests
       - pytest -v test_hardware
     artifacts:
       paths:
-        - "output_data/*.pdf"
+        - "bdaq53/tests/output_data/*.pdf"
+        - "BDAQ53.bit"
+        - "firmware/vivado/vivado.log"
       expire_in: 1 month
 
 # Tests for code style violations in new code lines
-- 
GitLab


From 28f3a263c31e3387be30e707cee80a9bf7a158a5 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 18 Jan 2019 10:08:34 +0100
Subject: [PATCH 23/72] PRJ: use slow firmware

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 77f0bf50e..c99883ba9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -71,7 +71,7 @@ test:hardware:
     script:
       - python setup.py develop
       # Compile firmware if changes detected and flash
-      - if [[ `git status --porcelain firmware/` ]]; then bdaq53 --firmware BDAQ53 -c; bdaq53 --firmware BDAQ53.bit; else bdaq53 --firmware BDAQ53; fi
+      - if [[ `git status --porcelain firmware/` ]]; then bdaq53 --firmware BDAQ53_RX640 -c; bdaq53 --firmware BDAQ53_RX640.bit; else bdaq53 --firmware BDAQ53_RX640; fi
       - cd bdaq53/tests
       - pytest -v test_hardware
     artifacts:
-- 
GitLab


From a91eea2376d05a60d604c5525abb1d92d1736031 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 18 Jan 2019 10:08:53 +0100
Subject: [PATCH 24/72] MAINT: formatting

---
 bdaq53/firmware_manager.py | 39 ++++++++++++++++++--------------------
 1 file changed, 18 insertions(+), 21 deletions(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index 050c3896b..8d8316229 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -163,8 +163,8 @@ def flash_firmware(name):
     vivado.expect('Exiting')
 
     logger.info('SUCCESS!')
-    
-    
+
+
 def compile_firmware(name):
     ''' Compile firmware using vivado in tcl mode
     '''
@@ -183,13 +183,13 @@ def compile_firmware(name):
         return flushed.decode('utf-8')
 
     supported_firmwares = ['BDAQ53', 'USBPIX3', 'KC705',
-                           'BDAQ53_RX640', 'USBPIX3_RX640', 'KC705_RX640'] 
-    if not name in supported_firmwares:
+                           'BDAQ53_RX640', 'USBPIX3_RX640', 'KC705_RX640']
+    if name not in supported_firmwares:
         logger.error('Can only compile firmwares: %s', ','.join(supported_firmwares))
         return
 
     logger.info('Compile firmware %s', name)
-    
+
     vivado_tcl = os.path.join(bdaq53_path, '..', 'firmware/vivado')
 
     try:
@@ -199,43 +199,41 @@ def compile_firmware(name):
         logger.error('Cannot execute vivado command. Maybe paid version is missing, that is needed for compilation?')
         return
     vivado.expect('Vivado%')  # Booted up when showing prompt
-    
+
     # Use mappings from run.tcl
     fpga_types = {'BDAQ53': 'xc7k160tffg676-2',
-                 'USBPIX3': 'xc7k160tfbg676-1',
-                 'KC705':  'xc7k325tffg900-2'}
+                  'USBPIX3': 'xc7k160tfbg676-1',
+                  'KC705': 'xc7k325tffg900-2'}
     constrains_files = {'BDAQ53': '../src/bdaq53.xdc',
-                 'USBPIX3': '../src/usbpix3.xdc',
-                 'KC705':  '../src/kc705_gmii.xdc'}
+                        'USBPIX3': '../src/usbpix3.xdc',
+                        'KC705': '../src/kc705_gmii.xdc'}
     flash_sizes = {'BDAQ53': '64',
-                 'USBPIX3': '64',
-                 'KC705':  '16'}
-    
+                   'USBPIX3': '64',
+                   'KC705': '16'}
+
     for k, v in fpga_types.items():
         if k in name:
             fpga_type = v
             constrain_files = constrains_files[k]
             flash_size = flash_sizes[k]
-            
+
     if '_SMA' in name:
         connector = '_SMA'
     elif '_FMC_LPC' in name:
         connector = '_FMC_LPC'
     else:
         connector = '""'
-            
+
     if 'RX640' in name:
         option = '_RX640'
     else:
         option = '""'
-    
+
     compile_command = 'run_bit ' + fpga_type + " " + name + " " + connector + " " + constrain_files + " " + flash_size + " " + option
-    
-#     compile_command = 'run_bit xc7k160tffg676-2    BDAQ53      ""          ../src/bdaq53.xdc       64        ""'
-    
+
     logger.info('Compiling firmware. Takes about 10 minutes!')
     vivado.sendline(compile_command)
-    
+
     import time
     timeout = 5  # 30 seconds with no new print to screen
     t = 0
@@ -406,4 +404,3 @@ def get_si_tcp():
 if __name__ == '__main__':
     logging.basicConfig(level=logging.DEBUG)
     main('BDAQ53_RX640')
-
-- 
GitLab


From 565d77840625a73eaf2e6704ae88b0315325c349 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 18 Jan 2019 10:50:56 +0100
Subject: [PATCH 25/72] MAINT: check string fix

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c99883ba9..9d55847e8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -66,7 +66,7 @@ test:hardware:
       # Install virtual x server for matplotlib Qt error
       - pip install xvfbwrapper
       # Install basil from github
-      - git clone -b development --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..;
+      # - git clone -b development --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..;
       # - if [ -z "$CI_COMMIT_TAG"]; then git clone -b v3.0.0 --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; else git clone -b development --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; fi
     script:
       - python setup.py develop
-- 
GitLab


From dc1e4dbc4f8f174a8d748fee4f205e292ec9c092 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 18 Jan 2019 10:52:08 +0100
Subject: [PATCH 26/72] MAINT: try bytes fix

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9d55847e8..ff26db867 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -66,7 +66,7 @@ test:hardware:
       # Install virtual x server for matplotlib Qt error
       - pip install xvfbwrapper
       # Install basil from github
-      # - git clone -b development --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..;
+      - git clone -b fix_array_tobytes --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..;
       # - if [ -z "$CI_COMMIT_TAG"]; then git clone -b v3.0.0 --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; else git clone -b development --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; fi
     script:
       - python setup.py develop
-- 
GitLab


From 52da5cc27559de95fd5160687f1bb4d03332d8dc Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 18 Jan 2019 11:24:38 +0100
Subject: [PATCH 27/72] PRJ: try fix basil install

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ff26db867..317883712 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -66,6 +66,7 @@ test:hardware:
       # Install virtual x server for matplotlib Qt error
       - pip install xvfbwrapper
       # Install basil from github
+      - pip uninstall basil-daq
       - git clone -b fix_array_tobytes --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..;
       # - if [ -z "$CI_COMMIT_TAG"]; then git clone -b v3.0.0 --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; else git clone -b development --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; fi
     script:
-- 
GitLab


From 848bbdca5b4de7784e4fe9fa7e9e10e590a571c7 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 18 Jan 2019 12:26:56 +0100
Subject: [PATCH 28/72] DBG: try fix shell runner

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 317883712..dda4be900 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -66,8 +66,8 @@ test:hardware:
       # Install virtual x server for matplotlib Qt error
       - pip install xvfbwrapper
       # Install basil from github
-      - pip uninstall basil-daq
       - git clone -b fix_array_tobytes --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..;
+      - python -c "from pkg_resources import get_distribution; print(get_distribution('basil-daq').version)"
       # - if [ -z "$CI_COMMIT_TAG"]; then git clone -b v3.0.0 --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; else git clone -b development --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; fi
     script:
       - python setup.py develop
-- 
GitLab


From 04cbe3e87aa193f84047bc2abcd6ff0051d97aa3 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 18 Jan 2019 12:30:37 +0100
Subject: [PATCH 29/72] MAINT

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index dda4be900..b666748db 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -66,7 +66,7 @@ test:hardware:
       # Install virtual x server for matplotlib Qt error
       - pip install xvfbwrapper
       # Install basil from github
-      - git clone -b fix_array_tobytes --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..;
+      - git clone -b development --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..;
       - python -c "from pkg_resources import get_distribution; print(get_distribution('basil-daq').version)"
       # - if [ -z "$CI_COMMIT_TAG"]; then git clone -b v3.0.0 --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; else git clone -b development --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; fi
     script:
-- 
GitLab


From 390ff1abdd078d3d38f893e0310ecbb897cc44d6 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 18 Jan 2019 12:58:39 +0100
Subject: [PATCH 30/72] PRJ: try fix weird runner behavior

---
 .gitlab-ci.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b666748db..c2a67d165 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -65,10 +65,8 @@ test:hardware:
       - conda install --yes numpy bitarray pytest pyyaml scipy numba pytables pyqt matplotlib tqdm pyzmq blosc psutil pexpect coloredlogs ptyprocess
       # Install virtual x server for matplotlib Qt error
       - pip install xvfbwrapper
-      # Install basil from github
-      - git clone -b development --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..;
-      - python -c "from pkg_resources import get_distribution; print(get_distribution('basil-daq').version)"
-      # - if [ -z "$CI_COMMIT_TAG"]; then git clone -b v3.0.0 --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; else git clone -b development --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; fi
+      # Update basil development from github
+      - git --git-dir=$HOME/git/basil/.git pull
     script:
       - python setup.py develop
       # Compile firmware if changes detected and flash
-- 
GitLab


From ac690abea2dffe486f0ee5bc0ab204e071b28fc7 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 18 Jan 2019 13:58:09 +0100
Subject: [PATCH 31/72] TST: try firmware build

---
 firmware/trigger | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 firmware/trigger

diff --git a/firmware/trigger b/firmware/trigger
new file mode 100644
index 000000000..e69de29bb
-- 
GitLab


From da700b52c0724c5b9d3acde429346a66337c2e74 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 18 Jan 2019 14:00:48 +0100
Subject: [PATCH 32/72] TST

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c2a67d165..c6a2ab0ef 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -71,6 +71,7 @@ test:hardware:
       - python setup.py develop
       # Compile firmware if changes detected and flash
       - if [[ `git status --porcelain firmware/` ]]; then bdaq53 --firmware BDAQ53_RX640 -c; bdaq53 --firmware BDAQ53_RX640.bit; else bdaq53 --firmware BDAQ53_RX640; fi
+      - git status --porcelain firmware/
       - cd bdaq53/tests
       - pytest -v test_hardware
     artifacts:
-- 
GitLab


From 2e1e1e2fc3e6d56819fad585ca33668556a0fa5f Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 18 Jan 2019 14:09:21 +0100
Subject: [PATCH 33/72] MAINT: dedicated runner does not have to be polite

---
 .../tests/test_hardware/test_scan_scripts.py  | 26 +------------------
 1 file changed, 1 insertion(+), 25 deletions(-)

diff --git a/bdaq53/tests/test_hardware/test_scan_scripts.py b/bdaq53/tests/test_hardware/test_scan_scripts.py
index b8ba28136..9813c14a9 100644
--- a/bdaq53/tests/test_hardware/test_scan_scripts.py
+++ b/bdaq53/tests/test_hardware/test_scan_scripts.py
@@ -5,39 +5,16 @@
 # ------------------------------------------------------------
 #
 
-import os
-import time
 import logging
 import unittest
 
 import tables as tb
 import numpy as np
-from slackclient import SlackClient
 
 logger = logging.getLogger(__file__)
 
 
 class TestScanScripts(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        if os.getenv('CI', False):
-            job_id = os.environ.get('CI_JOB_ID', 'unknown')
-            text = 'Use setup for hardware based test job #%s in 5 minutes! Visit https://gitlab.cern.ch/silab/bdaq53/-/jobs/%s to cancel.' % (job_id, job_id)
-            logger.info(text)
-            if os.getenv('SLACK_TOKEN', False):
-                cls.slack = SlackClient(os.environ.get('SLACK_TOKEN'))
-                if not cls.slack.api_call('chat.postMessage', channel='rd53a_module_setup', text=text, username='BDAQ53 Bot', icon_emoji=':robot_face:')['ok']:
-                    raise unittest.SkipTest("Cannot use Slack to inform users about hardware usage. Skip hardware based test!")
-            else:
-                raise unittest.SkipTest("Cannot get Slack credentials to inform users about hardware usage. Skip hardware based test!")
-            time.sleep(60 * 5)
-            cls.slack.api_call('chat.postMessage', channel='rd53a_module_setup', text='Starting...', username='BDAQ53 Bot', icon_emoji=':robot_face:')
-
-    @classmethod
-    def tearDownClass(cls):
-        if os.getenv('CI', False):
-            cls.slack.api_call('chat.postMessage', channel='rd53a_module_setup', text='Done!', username='BDAQ53 Bot', icon_emoji=':robot_face:')
-
     def test_digital_scan(self):
         ''' Test digital scan '''
         from bdaq53.scans import scan_digital
@@ -64,9 +41,8 @@ class TestScanScripts(unittest.TestCase):
         scan.close()
 
         with tb.open_file(scan.output_filename + '_interpreted.h5') as in_file:
-            # 60 % of the pixels see all injections, since FE issues
             logger.error(np.count_nonzero((in_file.root.HistOcc[:].sum(axis=2) != 100)))
-            self.assertTrue(np.count_nonzero(in_file.root.HistOcc[:].sum(axis=2) == 100) > 0.6 * 400 * 192)
+            self.assertTrue(np.count_nonzero(in_file.root.HistOcc[:].sum(axis=2) == 100) > 0.99 * 400 * 192)
             self.assertTrue(np.any(in_file.root.HistRelBCID[:]))
             self.assertTrue(np.any(in_file.root.HistTot[:]))
             # We expect BCID errors from SYNC
-- 
GitLab


From e7467ed6164afad9800749302b330da476f57928 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 18 Jan 2019 16:44:36 +0100
Subject: [PATCH 34/72] BUG: fix check

---
 .gitlab-ci.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c6a2ab0ef..b91ff236e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -70,8 +70,7 @@ test:hardware:
     script:
       - python setup.py develop
       # Compile firmware if changes detected and flash
-      - if [[ `git status --porcelain firmware/` ]]; then bdaq53 --firmware BDAQ53_RX640 -c; bdaq53 --firmware BDAQ53_RX640.bit; else bdaq53 --firmware BDAQ53_RX640; fi
-      - git status --porcelain firmware/
+      - if [[ `git diff ..development -- firmware` ]]; then bdaq53 --firmware BDAQ53_RX640 -c; bdaq53 --firmware BDAQ53_RX640.bit; else bdaq53 --firmware BDAQ53_RX640; fi
       - cd bdaq53/tests
       - pytest -v test_hardware
     artifacts:
-- 
GitLab


From c94d007758ff0471236adfc8011ded2e658f5b36 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 18 Jan 2019 16:50:49 +0100
Subject: [PATCH 35/72] MAINT

---
 firmware/trigger | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 firmware/trigger

diff --git a/firmware/trigger b/firmware/trigger
deleted file mode 100644
index e69de29bb..000000000
-- 
GitLab


From 8f4bd4b1878f105b4231476dbceb999355f37b84 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 18 Jan 2019 17:08:54 +0100
Subject: [PATCH 36/72] BUG: correct board name for vivado

---
 bdaq53/firmware_manager.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index 8d8316229..a71d89b46 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -216,6 +216,7 @@ def compile_firmware(name):
             fpga_type = v
             constrain_files = constrains_files[k]
             flash_size = flash_sizes[k]
+            board_name = k
 
     if '_SMA' in name:
         connector = '_SMA'
@@ -229,7 +230,7 @@ def compile_firmware(name):
     else:
         option = '""'
 
-    compile_command = 'run_bit ' + fpga_type + " " + name + " " + connector + " " + constrain_files + " " + flash_size + " " + option
+    compile_command = 'run_bit ' + fpga_type + " " + board_name + " " + connector + " " + constrain_files + " " + flash_size + " " + option
 
     logger.info('Compiling firmware. Takes about 10 minutes!')
     vivado.sendline(compile_command)
-- 
GitLab


From 751dfe5fcaea0fa052a02d6e7f64a1a574e455ab Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Mon, 21 Jan 2019 09:57:58 +0100
Subject: [PATCH 37/72] BUG: only move compiled firmware

---
 bdaq53/firmware_manager.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index a71d89b46..5cfe2241f 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -251,6 +251,10 @@ def compile_firmware(name):
     else:
         raise RuntimeError('Timeout during compilation, check log!')
 
+    # Move firmware to current folder
+    cwd = os.getcwd()
+    vivado_tcl = os.path.join(bdaq53_path, '..', 'firmware/vivado')
+    shutil.move(os.path.join(vivado_tcl, r'output/%s' % name + '.bit'), cwd)
     logger.info('SUCCESS!')
 
 
@@ -353,10 +357,6 @@ def main(name, path=None, create=False):
     else:
         get_si_tcp()  # get missing SiTCP sources
         compile_firmware(name)
-        # Move firmware to current folder
-        cwd = os.getcwd()
-        vivado_tcl = os.path.join(bdaq53_path, '..', 'firmware/vivado')
-        shutil.move(os.path.join(vivado_tcl, r'output/%s' % name + '.bit'), cwd)
 
 
 def get_tag_list(url):
-- 
GitLab


From 9ded13094bf5b5703d0c4f86fcda8caa5e98f506 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Mon, 21 Jan 2019 10:05:11 +0100
Subject: [PATCH 38/72] ENH: upload verilog log also on failure

---
 .gitlab-ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b91ff236e..a9a96c7ef 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -74,10 +74,11 @@ test:hardware:
       - cd bdaq53/tests
       - pytest -v test_hardware
     artifacts:
+      when: always  # also upload verilog log on failure
       paths:
+        - "firmware/vivado/vivado.log"
         - "bdaq53/tests/output_data/*.pdf"
         - "BDAQ53.bit"
-        - "firmware/vivado/vivado.log"
       expire_in: 1 month
 
 # Tests for code style violations in new code lines
-- 
GitLab


From 51fcc827babf7292933c7b7fd3ab8b068c77d632 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Mon, 21 Jan 2019 10:11:10 +0100
Subject: [PATCH 39/72] BUG: check properly if SiTCP sources are available

---
 bdaq53/firmware_manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index 5cfe2241f..c459bf8fb 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -386,7 +386,7 @@ def get_si_tcp():
     sitcp_folder = os.path.join(bdaq53_path, '..', 'firmware/SiTCP/')
 
     # Only download if not already existing
-    if not os.path.isdir(os.path.join(sitcp_folder, '.git')):
+    if not os.path.isfile(os.path.join(sitcp_folder, 'SiTCP_XC7K_32K_BBT_V110.ngc')):
         logger.info('Downloading SiTCP')
 
         # Has to be moved to be allowed to use existing folder for git checkout
-- 
GitLab


From aac08a97d7363d4f8d5dd78fd9a3f6f846589dae Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Mon, 21 Jan 2019 10:17:54 +0100
Subject: [PATCH 40/72] BUG: SiTCP download

---
 bdaq53/firmware_manager.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index c459bf8fb..1b00108aa 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -23,7 +23,7 @@ from sys import platform
 
 import pexpect
 from tqdm import tqdm
-from git import Repo
+import git
 
 import bdaq53
 
@@ -385,14 +385,14 @@ def get_si_tcp():
 
     sitcp_folder = os.path.join(bdaq53_path, '..', 'firmware/SiTCP/')
 
-    # Only download if not already existing
-    if not os.path.isfile(os.path.join(sitcp_folder, 'SiTCP_XC7K_32K_BBT_V110.ngc')):
+    # Only download if not already existing SiTCP git repository
+    if not os.path.isdir(os.path.join(sitcp_folder, '.git')):
         logger.info('Downloading SiTCP')
 
         # Has to be moved to be allowed to use existing folder for git checkout
         shutil.move(sitcp_folder + '.gitkeep', os.path.join(sitcp_folder, '..'))
-        Repo.clone_from(url=sitcp_repo,
-                        to_path=sitcp_folder, branch='master')
+        git.Repo.clone_from(url=sitcp_repo,
+                            to_path=sitcp_folder, branch='master')
         shutil.move(os.path.join(sitcp_folder, '..', '.gitkeep'), sitcp_folder)
         # Patch sources, see README of bdaq53
         line_prepender(filename=sitcp_folder + 'TIMER.v', line=r'`default_nettype wire')
@@ -400,6 +400,9 @@ def get_si_tcp():
         for line in fileinput.input([sitcp_folder + 'WRAP_SiTCP_GMII_XC7K_32K.V'], inplace=True):
             print(line.replace("assign\tMY_IP_ADDR[31:0]\t= (~FORCE_DEFAULTn | (EXT_IP_ADDR[31:0]==32'd0) \t? DEFAULT_IP_ADDR[31:0]\t\t: EXT_IP_ADDR[31:0]\t\t);",
                                'assign\tMY_IP_ADDR[31:0]\t= EXT_IP_ADDR[31:0];'), end='')
+    else:  # update if existing
+        g = git.cmd.Git(sitcp_folder)
+        g.pull()
 
 
 if __name__ == '__main__':
-- 
GitLab


From 0dd8352cb66b83154982a990cf5a1c4628f78b42 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Mon, 21 Jan 2019 10:46:25 +0100
Subject: [PATCH 41/72] BUG: upload slow firmware

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a9a96c7ef..dfebbfa49 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -78,7 +78,7 @@ test:hardware:
       paths:
         - "firmware/vivado/vivado.log"
         - "bdaq53/tests/output_data/*.pdf"
-        - "BDAQ53.bit"
+        - "BDAQ53_RX640.bit"
       expire_in: 1 month
 
 # Tests for code style violations in new code lines
-- 
GitLab


From 9e5ef9641a8acf2e9e567c21fec20bc52691a3b6 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Mon, 21 Jan 2019 10:46:36 +0100
Subject: [PATCH 42/72] MAINT

---
 bdaq53/firmware_manager.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index 1b00108aa..29286894d 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -249,7 +249,7 @@ def compile_firmware(name):
             time.sleep(5)
             t += 1
     else:
-        raise RuntimeError('Timeout during compilation, check log!')
+        raise RuntimeError('Timeout during compilation, check vivado.log')
 
     # Move firmware to current folder
     cwd = os.getcwd()
@@ -275,7 +275,7 @@ def find_vivado(path):
             if 'bin' in path:
                 return os.path.dirname(os.path.realpath(path))
     else:
-        raise NotImplementedError('Only Linux supported thus far')
+        raise NotImplementedError('Only Linux supported')
 
 
 def where(name, path, flags=os.F_OK):
-- 
GitLab


From aef356924009321cc39213dd0a0d79e06cf44bc4 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Mon, 21 Jan 2019 10:47:20 +0100
Subject: [PATCH 43/72] ENH: add more scans to test

---
 .../tests/test_hardware/test_scan_scripts.py  | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/bdaq53/tests/test_hardware/test_scan_scripts.py b/bdaq53/tests/test_hardware/test_scan_scripts.py
index 9813c14a9..0096ce4d3 100644
--- a/bdaq53/tests/test_hardware/test_scan_scripts.py
+++ b/bdaq53/tests/test_hardware/test_scan_scripts.py
@@ -49,6 +49,41 @@ class TestScanScripts(unittest.TestCase):
             # self.assertTrue(np.all(in_file.root.HistBCIDError[:] == 0))
             # self.assertTrue(np.all(in_file.root.HistEventStatus[:] == 0))
 
+    def test_threshold_scan(self):
+        ''' Test threshold scan and results '''
+        from bdaq53.scans import scan_threshold
+        scan = scan_threshold.ThresholdScan()
+        scan.start(**scan_threshold.local_configuration)
+        scan.close()
+
+#         with tb.open_file(scan.output_filename + '_interpreted.h5') as in_file:
+#             logger.error(np.count_nonzero((in_file.root.HistOcc[:].sum(axis=2) != 100)))
+#             self.assertTrue(np.count_nonzero(in_file.root.HistOcc[:].sum(axis=2) == 100) > 0.99 * 400 * 192)
+#             self.assertTrue(np.any(in_file.root.HistRelBCID[:]))
+#             self.assertTrue(np.any(in_file.root.HistTot[:]))
+#             # We expect BCID errors from SYNC
+#             # self.assertTrue(np.all(in_file.root.HistBCIDError[:] == 0))
+#             # self.assertTrue(np.all(in_file.root.HistEventStatus[:] == 0))
+
+    def test_fast_threshold_scan(self):
+        ''' Test fast threshold scan and results '''
+        from bdaq53.scans import scan_threshold_fast
+        scan = scan_threshold_fast.FastThresholdScan()
+        scan.start(**scan_threshold_fast.local_configuration)
+        scan.close()
+
+    def test_register_test(self):
+        from bdaq53.scans import test_registers
+        scan = test_registers.RegisterTest()
+        scan.start(**test_registers.local_configuration)
+        scan.close()
+
+    def test_threshold_noise_tuning(self):
+        from bdaq53.scans import tune_local_threshold_noise
+        scan = tune_local_threshold_noise.NoiseTuning()
+        scan.start(**tune_local_threshold_noise.local_configuration)
+        scan.close()
+
 
 if __name__ == '__main__':
     unittest.main()
-- 
GitLab


From b371125b68e20dd84f6463f44be4d01e9f652f88 Mon Sep 17 00:00:00 2001
From: Marco Vogt <marco.vogt@cern.ch>
Date: Thu, 24 Jan 2019 13:06:41 +0000
Subject: [PATCH 44/72] ENH: firmware build script

---
 bdaq53/tests/.gitignore                       |  6 +-
 firmware/src/bdaq53_core.v                    |  3 +-
 .../ip/exdes/aurora_64b66b_1lane_exdes.v      | 81 +++++++++----------
 .../rx_aurora_64b66b_core.v                   |  3 -
 firmware/vivado/run.tcl                       | 73 ++++++-----------
 5 files changed, 71 insertions(+), 95 deletions(-)

diff --git a/bdaq53/tests/.gitignore b/bdaq53/tests/.gitignore
index 92b911c56..0b8273d32 100644
--- a/bdaq53/tests/.gitignore
+++ b/bdaq53/tests/.gitignore
@@ -6,7 +6,11 @@ unisims
 secureip
 /cds.lib
 
-/sim_build
+/test_rd53/sim_build
+/test_rd53/output_data
+/test_rd53/*.log
+/test_rd53/Makefile
+
 /results.xml
 /*.gtkw
 /simvision#
diff --git a/firmware/src/bdaq53_core.v b/firmware/src/bdaq53_core.v
index c12af14f6..dd2d88ecf 100755
--- a/firmware/src/bdaq53_core.v
+++ b/firmware/src/bdaq53_core.v
@@ -37,7 +37,6 @@
 `include "utils/rgmii_io.v"
 `include "utils/rbcp_to_bus.v"
 `include "utils/fifo_32_to_8.v"
-`include "utils/clock_divider.v"
 `include "utils/bus_to_ip.v"
 `include "utils/ddr_des.v"
 `include "rrp_arbiter/rrp_arbiter.v"
@@ -169,7 +168,7 @@ localparam BDAQ53 = 16'd1;
 localparam USBPIX3 = 16'd2;
 localparam KC705 = 16'd3;
 
-`ifdef COCOTB_SIM
+`ifdef RTL_SIM
     localparam BOARD = SIM;
     localparam CONNECTOR = CON_SMA;
 `elsif BDAQ53
diff --git a/firmware/src/rx_aurora/rx_aurora_64b66b_1lane/ip/exdes/aurora_64b66b_1lane_exdes.v b/firmware/src/rx_aurora/rx_aurora_64b66b_1lane/ip/exdes/aurora_64b66b_1lane_exdes.v
index 094dae930..ba9cf0e26 100644
--- a/firmware/src/rx_aurora/rx_aurora_64b66b_1lane/ip/exdes/aurora_64b66b_1lane_exdes.v
+++ b/firmware/src/rx_aurora/rx_aurora_64b66b_1lane/ip/exdes/aurora_64b66b_1lane_exdes.v
@@ -68,47 +68,46 @@
 // This is sample simplex exdes file
 `timescale 1 ns / 10 ps
 
-`define SIM_ENCRYPTED
-
-`ifndef SYNTHESIS
-    `include "rx_aurora_64b66b_1lane/ip/exdes/aurora_64b66b_1lane_cdc_sync_exdes.v"
-    `ifdef SIM_ENCRYPTED    // use the encrypted ip core for the GitLab CI
-        `include "rx_aurora_64b66b_1lane/ip/sim/core.vp"
-    `else   // otherwise we use the non-encrypted version
-
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane_core.v"
-
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_64b66b_descrambler.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_block_sync_sm.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_cbcc_gtx_6466.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_cdc_sync.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_common_logic_cbcc.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_common_reset_cbcc.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_ll_to_axi.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_reset_logic.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_aurora_lane_simplex.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_channel_err_detect_simplex.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_channel_init_sm_simplex.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_err_detect_simplex.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_global_logic_simplex.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_lane_init_sm_simplex.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_ll_datapath_simplex.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_ll_simplex.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_ll_user_k_datapath_simplex.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_startup_fsm.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_sym_dec.v"
-
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/support/aurora_64b66b_1lane_clock_module.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/support/aurora_64b66b_1lane_gt_common_wrapper.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/support/aurora_64b66b_1lane_support.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/support/aurora_64b66b_1lane_support_reset_logic.v"
-
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/gt/aurora_64b66b_1lane_multi_wrapper.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/gt/aurora_64b66b_1lane_wrapper.v"
-        `include "rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/gt/aurora_64b66b_1lane_gtx.v"
-
-    `endif
+`ifdef RTL_SIM
+    `define SIM_ENCRYPTED
+`endif
+
+`include "rx_aurora/rx_aurora_64b66b_1lane/ip/exdes/aurora_64b66b_1lane_cdc_sync_exdes.v"
+
+`ifdef SIM_ENCRYPTED    // use the encrypted ip core for the GitLab CI
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/sim/core.vp"
+`else   // otherwise we use the non-encrypted version
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane_core.v"
+
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_64b66b_descrambler.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_block_sync_sm.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_cbcc_gtx_6466.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_cdc_sync.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_common_logic_cbcc.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_common_reset_cbcc.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_ll_to_axi.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_reset_logic.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_aurora_lane_simplex.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_channel_err_detect_simplex.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_channel_init_sm_simplex.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_err_detect_simplex.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_global_logic_simplex.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_lane_init_sm_simplex.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_ll_datapath_simplex.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_ll_simplex.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_ll_user_k_datapath_simplex.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_startup_fsm.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_sym_dec.v"
+
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/support/aurora_64b66b_1lane_clock_module.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/support/aurora_64b66b_1lane_gt_common_wrapper.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/support/aurora_64b66b_1lane_support.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/support/aurora_64b66b_1lane_support_reset_logic.v"
+
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/gt/aurora_64b66b_1lane_multi_wrapper.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/gt/aurora_64b66b_1lane_wrapper.v"
+    `include "rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/gt/aurora_64b66b_1lane_gtx.v"
 `endif
 
 
diff --git a/firmware/src/rx_aurora/rx_aurora_64b66b_1lane/rx_aurora_64b66b_core.v b/firmware/src/rx_aurora/rx_aurora_64b66b_1lane/rx_aurora_64b66b_core.v
index 81c26d90e..b3521f9a0 100755
--- a/firmware/src/rx_aurora/rx_aurora_64b66b_1lane/rx_aurora_64b66b_core.v
+++ b/firmware/src/rx_aurora/rx_aurora_64b66b_1lane/rx_aurora_64b66b_core.v
@@ -8,10 +8,7 @@
 `timescale 1ns/1ps
 `default_nettype none
 
-`ifndef SYNTHESIS
     `include "rx_aurora/rx_aurora_64b66b_1lane/ip/exdes/aurora_64b66b_1lane_exdes.v"
-`endif
-
 
 module rx_aurora_64b66b_core
 #(
diff --git a/firmware/vivado/run.tcl b/firmware/vivado/run.tcl
index 2a0bcd05b..f201e6372 100644
--- a/firmware/vivado/run.tcl
+++ b/firmware/vivado/run.tcl
@@ -3,11 +3,16 @@
 # Copyright (c) SILAB ,  Institute of Physics, University of Bonn
 # ---------------------------------------------------------------
 #
-#   This script creates Vivado projects and bitfiles for the supported hardware platforms
+#   This script creates Vivado projects and bitfiles for the supported hardware platforms.
 #
-#   Start vivado in tcl mode by typing:
+#   Start vivado in tcl mode by executing:
 #       vivado -mode tcl -source run.tcl
 #
+#   NOTE: This will build firmware versions for every supported hardware. See the section "Create projects and bitfiles" below.
+#   Alternatively, a list of 6 arguments can be used to build only the specified firmware.
+#   The arguments have to be in the correct order. Just copy&paste from the "Create projects and bitfiles" section and remove all but one space in between the args.
+#       vivado -mode tcl -source run.tcl -tclargs xc7k160tffg676-2 BDAQ53 "" ../src/bdaq53.xdc 64 _RX640
+#
 
 
 set basil_dir [exec python -c "import basil, os; print(str(os.path.dirname(os.path.dirname(basil.__file__))))"]
@@ -24,45 +29,6 @@ proc read_design_files {} {
     read_verilog ../SiTCP/TIMER.v
     read_verilog ../SiTCP/SiTCP_XC7K_32K_BBT_V110.V
     read_verilog ../SiTCP/WRAP_SiTCP_GMII_XC7K_32K.V
-
-
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/rx_aurora_64b66b.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/rx_aurora_64b66b_core.v
-
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/exdes/aurora_64b66b_1lane_exdes.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/exdes/aurora_64b66b_1lane_cdc_sync_exdes.v
-
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane_core.v
-
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_64b66b_descrambler.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_block_sync_sm.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_cbcc_gtx_6466.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_cdc_sync.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_common_logic_cbcc.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_common_reset_cbcc.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_ll_to_axi.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_reset_logic.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_aurora_lane_simplex.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_channel_err_detect_simplex.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_channel_init_sm_simplex.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_err_detect_simplex.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_global_logic_simplex.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_lane_init_sm_simplex.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_ll_datapath_simplex.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_ll_simplex.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_ll_user_k_datapath_simplex.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_rx_startup_fsm.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/src/aurora_64b66b_1lane_sym_dec.v
-
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/support/aurora_64b66b_1lane_clock_module.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/support/aurora_64b66b_1lane_gt_common_wrapper.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/support/aurora_64b66b_1lane_support.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/support/aurora_64b66b_1lane_support_reset_logic.v
-
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/gt/aurora_64b66b_1lane_multi_wrapper.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/gt/aurora_64b66b_1lane_wrapper.v
-    read_verilog ../src/rx_aurora/rx_aurora_64b66b_1lane/ip/aurora_64b66b_1lane/aurora_64b66b_1lane/example_design/gt/aurora_64b66b_1lane_gtx.v
 }
 
 
@@ -96,18 +62,29 @@ proc run_bit { part board connector xdc_file size option} {
 # Create projects and bitfiles
 #
 
-# Bitfiles for the 1.28 Gb/s Aurora ip core configuration
-#       FPGA type           board name	connector  	constraints file     flash size  option
-run_bit xc7k160tffg676-2    BDAQ53      ""          ../src/bdaq53.xdc       64        ""
-run_bit xc7k160tfbg676-1    USBPIX3     ""          ../src/usbpix3.xdc      64        ""
-run_bit xc7k325tffg900-2    KC705       _SMA        ../src/kc705_gmii.xdc   16        ""
-run_bit xc7k325tffg900-2    KC705       _FMC_LPC    ../src/kc705_gmii.xdc   16        ""
+if {$argc == 0} {
+# By default, all firmware versions are generated. You can comment the ones you don't need.
 
 # Bitfiles for the 640 Mb/s Aurora ip core configuration
-#       FPGA type           board name  connector	constraints file     flash size  option
+#       FPGA type           board name  connector   constraints file     flash size  option
 run_bit xc7k160tffg676-2    BDAQ53      ""          ../src/bdaq53.xdc       64      _RX640
 run_bit xc7k160tfbg676-1    USBPIX3     ""          ../src/usbpix3.xdc      64      _RX640
 run_bit xc7k325tffg900-2    KC705       _SMA        ../src/kc705_gmii.xdc   16      _RX640
 run_bit xc7k325tffg900-2    KC705       _FMC_LPC    ../src/kc705_gmii.xdc   16      _RX640
 
+# Bitfiles for the 1.28 Gb/s Aurora ip core configuration
+#       FPGA type           board name	connector  	constraints file     flash size  option
+run_bit xc7k160tffg676-2    BDAQ53      ""          ../src/bdaq53.xdc       64        ""
+run_bit xc7k160tfbg676-1    USBPIX3     ""          ../src/usbpix3.xdc      64        ""
+run_bit xc7k325tffg900-2    KC705       _SMA        ../src/kc705_gmii.xdc   16        ""
+run_bit xc7k325tffg900-2    KC705       _FMC_LPC    ../src/kc705_gmii.xdc   16        ""
+
+# In order to build only one specific firmware version, the tun.tcl can be executed with arguments
+} else {
+    if {$argc == 6} {
+        run_bit {*}$argv
+    } else {
+        puts "ERROR: Invalid args"
+    }
+}
 exit
-- 
GitLab


From 1b0813e795597d98a7b7c8b26036b88658b76e1f Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 24 Jan 2019 14:14:31 +0100
Subject: [PATCH 45/72] TST: try add needed cleanup after CI shell executor run

---
 .gitlab-ci.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index dfebbfa49..e39ea8689 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -71,8 +71,11 @@ test:hardware:
       - python setup.py develop
       # Compile firmware if changes detected and flash
       - if [[ `git diff ..development -- firmware` ]]; then bdaq53 --firmware BDAQ53_RX640 -c; bdaq53 --firmware BDAQ53_RX640.bit; else bdaq53 --firmware BDAQ53_RX640; fi
-      - cd bdaq53/tests
-      - pytest -v test_hardware
+      - pytest -v bdaq53/tests/test_hardware
+    after_script:
+      - pwd
+      - ls
+      - rm -r -f bdaq53  # manual cleanup needed to trigger new checkout for each commit
     artifacts:
       when: always  # also upload verilog log on failure
       paths:
-- 
GitLab


From bb2b0fc02aaabe51b573262e866874a0845eafd2 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 24 Jan 2019 14:50:00 +0100
Subject: [PATCH 46/72] MAINT: use new build script that supports arguments

---
 bdaq53/firmware_manager.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index 29286894d..fb29beded 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -192,14 +192,6 @@ def compile_firmware(name):
 
     vivado_tcl = os.path.join(bdaq53_path, '..', 'firmware/vivado')
 
-    try:
-        vivado = pexpect.spawn('vivado -mode tcl -source ci_run.tcl', cwd=vivado_tcl, timeout=10)  # try full version
-        vivado.expect('Vivado', timeout=5)
-    except pexpect.exceptions.ExceptionPexpect:
-        logger.error('Cannot execute vivado command. Maybe paid version is missing, that is needed for compilation?')
-        return
-    vivado.expect('Vivado%')  # Booted up when showing prompt
-
     # Use mappings from run.tcl
     fpga_types = {'BDAQ53': 'xc7k160tffg676-2',
                   'USBPIX3': 'xc7k160tfbg676-1',
@@ -230,10 +222,15 @@ def compile_firmware(name):
     else:
         option = '""'
 
-    compile_command = 'run_bit ' + fpga_type + " " + board_name + " " + connector + " " + constrain_files + " " + flash_size + " " + option
-
+    command_args = fpga_type + ' ' + board_name + ' ' + connector + ' ' + constrain_files + ' ' + flash_size + ' ' + option
+    command = 'vivado -mode tcl -source run.tcl -tclargs %s' % command_args
     logger.info('Compiling firmware. Takes about 10 minutes!')
-    vivado.sendline(compile_command)
+    try:
+        vivado = pexpect.spawn(command, cwd=vivado_tcl, timeout=10)
+        vivado.expect('****** Vivado', timeout=5)
+    except pexpect.exceptions.ExceptionPexpect:
+        logger.error('Cannot execute vivado command %d.\nMaybe paid version is missing, that is needed for compilation?', command)
+        return
 
     import time
     timeout = 5  # 30 seconds with no new print to screen
-- 
GitLab


From 224f7d3e0d7b55d82b7582ec6b14cdbf211b940e Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 24 Jan 2019 14:55:15 +0100
Subject: [PATCH 47/72] BUG: python 3 str f**k up

---
 bdaq53/firmware_manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index fb29beded..cc1534b54 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -227,7 +227,7 @@ def compile_firmware(name):
     logger.info('Compiling firmware. Takes about 10 minutes!')
     try:
         vivado = pexpect.spawn(command, cwd=vivado_tcl, timeout=10)
-        vivado.expect('****** Vivado', timeout=5)
+        vivado.expect('Vivado', timeout=5)
     except pexpect.exceptions.ExceptionPexpect:
         logger.error('Cannot execute vivado command %d.\nMaybe paid version is missing, that is needed for compilation?', command)
         return
-- 
GitLab


From 42ddf720ada2d31bc37042053638a37bbf5d9bdd Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 24 Jan 2019 15:16:35 +0100
Subject: [PATCH 48/72] PRJ: debug runner

---
 bdaq53/firmware_manager.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index cc1534b54..c4ff222d7 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -240,7 +240,8 @@ def compile_firmware(name):
         if r:
             if 'write_cfgmem completed successfully' in r:
                 break
-            print('.', end='', flush=True)
+            print(r)
+            #print('.', end='', flush=True)
             t = 0
         else:
             time.sleep(5)
-- 
GitLab


From 7e431a726ab90b4e4ee36556139269e53a83844d Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 24 Jan 2019 15:17:18 +0100
Subject: [PATCH 49/72] TST: try save artifacts

---
 .gitlab-ci.yml | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e39ea8689..484e7f13b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -48,9 +48,8 @@ test:software:
       # - if [ -z "$CI_COMMIT_TAG"]; then git clone -b v3.0.0 --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; else git clone -b development --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; fi
     script:
       - python setup.py develop
-      - cd bdaq53/tests
       # Do not run virtual x server tests (monitor) in this runner due to segfault
-      - pytest -v test_software --ignore=test_software/test_monitor.py
+      - pytest -v bdaq53/tests/test_software --ignore=bdaq53/tests/test_software/test_monitor.py
 
 # Tests that need bdaq53 readout hardware and RD53 FE
 # Needs Ubuntu system with xvfb, Miniconda3 and full Xilinx installed and
@@ -72,15 +71,17 @@ test:hardware:
       # Compile firmware if changes detected and flash
       - if [[ `git diff ..development -- firmware` ]]; then bdaq53 --firmware BDAQ53_RX640 -c; bdaq53 --firmware BDAQ53_RX640.bit; else bdaq53 --firmware BDAQ53_RX640; fi
       - pytest -v bdaq53/tests/test_hardware
-    after_script:
-      - pwd
-      - ls
-      - rm -r -f bdaq53  # manual cleanup needed to trigger new checkout for each commit
+    after_script:  # manual cleanup needed to trigger new checkout for each commit
+      - mv firmware/vivado/vivado.log ../
+      - mv BDAQ53_RX640.bit ../
+      - mv output_data ../
+      - cd ..
+      - rm -r -f bdaq53  
     artifacts:
       when: always  # also upload verilog log on failure
       paths:
-        - "firmware/vivado/vivado.log"
-        - "bdaq53/tests/output_data/*.pdf"
+        - "vivado.log"
+        - "output_data/*.pdf"
         - "BDAQ53_RX640.bit"
       expire_in: 1 month
 
-- 
GitLab


From c5550e5ba3a92ee87060203942afa960d908b5b9 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 24 Jan 2019 15:45:38 +0100
Subject: [PATCH 50/72] PRJ: try save artifacts

---
 .gitlab-ci.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 484e7f13b..d45e77df1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -72,6 +72,7 @@ test:hardware:
       - if [[ `git diff ..development -- firmware` ]]; then bdaq53 --firmware BDAQ53_RX640 -c; bdaq53 --firmware BDAQ53_RX640.bit; else bdaq53 --firmware BDAQ53_RX640; fi
       - pytest -v bdaq53/tests/test_hardware
     after_script:  # manual cleanup needed to trigger new checkout for each commit
+      # Save artifacts first
       - mv firmware/vivado/vivado.log ../
       - mv BDAQ53_RX640.bit ../
       - mv output_data ../
@@ -80,9 +81,9 @@ test:hardware:
     artifacts:
       when: always  # also upload verilog log on failure
       paths:
-        - "vivado.log"
-        - "output_data/*.pdf"
-        - "BDAQ53_RX640.bit"
+        - "../vivado.log"
+        - "../output_data/*.pdf"
+        - "../BDAQ53_RX640.bit"
       expire_in: 1 month
 
 # Tests for code style violations in new code lines
-- 
GitLab


From 6916d5f825fe765e6c7c8fb39dc9987867817855 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 24 Jan 2019 15:46:14 +0100
Subject: [PATCH 51/72] BUG: treat EOF

---
 bdaq53/firmware_manager.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index c4ff222d7..8089928e7 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -178,7 +178,7 @@ def compile_firmware(name):
         try:
             while not vivado.expect(r'.+', timeout=timeout):
                 flushed += vivado.match.group(0)
-        except pexpect.exceptions.TIMEOUT:
+        except (pexpect.exceptions.TIMEOUT, pexpect.exceptions.EOF):
             pass
         return flushed.decode('utf-8')
 
@@ -240,8 +240,7 @@ def compile_firmware(name):
         if r:
             if 'write_cfgmem completed successfully' in r:
                 break
-            print(r)
-            #print('.', end='', flush=True)
+            print('.', end='', flush=True)
             t = 0
         else:
             time.sleep(5)
-- 
GitLab


From 7dc8c9820a81771dbf87c0cf4d312a7c6840e1f3 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 24 Jan 2019 17:05:10 +0100
Subject: [PATCH 52/72] PRJ: keep artifacts in build directory since it is
 needed

---
 .gitlab-ci.yml | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d45e77df1..05732aed7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -71,19 +71,16 @@ test:hardware:
       # Compile firmware if changes detected and flash
       - if [[ `git diff ..development -- firmware` ]]; then bdaq53 --firmware BDAQ53_RX640 -c; bdaq53 --firmware BDAQ53_RX640.bit; else bdaq53 --firmware BDAQ53_RX640; fi
       - pytest -v bdaq53/tests/test_hardware
-    after_script:  # manual cleanup needed to trigger new checkout for each commit
-      # Save artifacts first
-      - mv firmware/vivado/vivado.log ../
-      - mv BDAQ53_RX640.bit ../
-      - mv output_data ../
-      - cd ..
-      - rm -r -f bdaq53  
+    after_script:  # manual cleanup needed to force new checkout for each commit
+      - mv firmware/vivado/vivado.log .
+      - rm -r -f bdaq53
+      - rm -r -f firmware/SiTCP
     artifacts:
       when: always  # also upload verilog log on failure
       paths:
-        - "../vivado.log"
-        - "../output_data/*.pdf"
-        - "../BDAQ53_RX640.bit"
+        - "vivado.log"
+        - "output_data/*.pdf"
+        - "BDAQ53_RX640.bit"
       expire_in: 1 month
 
 # Tests for code style violations in new code lines
-- 
GitLab


From 95c1590056fa6be639a8e38dea0f684d2c3f4da5 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 24 Jan 2019 17:07:25 +0100
Subject: [PATCH 53/72] TST: CI

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 05732aed7..d0e7be384 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -74,7 +74,7 @@ test:hardware:
     after_script:  # manual cleanup needed to force new checkout for each commit
       - mv firmware/vivado/vivado.log .
       - rm -r -f bdaq53
-      - rm -r -f firmware/SiTCP
+      - rm -r -f firmware/SiTCP/*.*
     artifacts:
       when: always  # also upload verilog log on failure
       paths:
-- 
GitLab


From 2b8a6799b6756797f9b7f898031aef80b308c065 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 25 Jan 2019 15:15:54 +0100
Subject: [PATCH 54/72] PRJ: cleanup in before script

---
 .gitlab-ci.yml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d0e7be384..ab5e6dccb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -66,19 +66,17 @@ test:hardware:
       - pip install xvfbwrapper
       # Update basil development from github
       - git --git-dir=$HOME/git/basil/.git pull
+      # Clean potential preceding runner run SiTCP git checkout
+      - rm -r -f firmware/SiTCP/*.*
     script:
       - python setup.py develop
       # Compile firmware if changes detected and flash
       - if [[ `git diff ..development -- firmware` ]]; then bdaq53 --firmware BDAQ53_RX640 -c; bdaq53 --firmware BDAQ53_RX640.bit; else bdaq53 --firmware BDAQ53_RX640; fi
       - pytest -v bdaq53/tests/test_hardware
-    after_script:  # manual cleanup needed to force new checkout for each commit
-      - mv firmware/vivado/vivado.log .
-      - rm -r -f bdaq53
-      - rm -r -f firmware/SiTCP/*.*
     artifacts:
       when: always  # also upload verilog log on failure
       paths:
-        - "vivado.log"
+        - "firmware/vivado/vivado.log"
         - "output_data/*.pdf"
         - "BDAQ53_RX640.bit"
       expire_in: 1 month
-- 
GitLab


From c9416c44ea038f35561c84f0f65af1262440edc4 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Fri, 25 Jan 2019 15:28:48 +0100
Subject: [PATCH 55/72] BUG

---
 .gitlab-ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ab5e6dccb..e353b9984 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -67,7 +67,8 @@ test:hardware:
       # Update basil development from github
       - git --git-dir=$HOME/git/basil/.git pull
       # Clean potential preceding runner run SiTCP git checkout
-      - rm -r -f firmware/SiTCP/*.*
+      - rm -rf firmware/SiTCP/{*,.*}
+      
     script:
       - python setup.py develop
       # Compile firmware if changes detected and flash
-- 
GitLab


From cbb2edfc6afa7d72f9b185ea5a78233f783b5999 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Tue, 5 Feb 2019 11:09:01 +0100
Subject: [PATCH 56/72] DEP: rm bug
 https://unix.stackexchange.com/questions/369530

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e353b9984..9f25d68d1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -67,7 +67,7 @@ test:hardware:
       # Update basil development from github
       - git --git-dir=$HOME/git/basil/.git pull
       # Clean potential preceding runner run SiTCP git checkout
-      - rm -rf firmware/SiTCP/{*,.*}
+      - rm -rf firmware/SiTCP/{*,.git}
       
     script:
       - python setup.py develop
-- 
GitLab


From 5d98c64b0c1c8a3a2149cdb3f94f376e3b5d5ac5 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Wed, 13 Feb 2019 15:52:30 +0100
Subject: [PATCH 57/72] GIT: rebase

---
 bdaq53/bdaq53_cli.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/bdaq53/bdaq53_cli.py b/bdaq53/bdaq53_cli.py
index ddcb605fb..2628f230c 100644
--- a/bdaq53/bdaq53_cli.py
+++ b/bdaq53/bdaq53_cli.py
@@ -68,10 +68,14 @@ def main():
                         nargs=1,
                         help='Path of vivado installation',)
 
+    parser.add_argument('-c', '--compile',
+                    action='store_true',
+                    help='Compile firmware',)
+
     args = parser.parse_args()
 
     if args.firmware is not None:
-        firmware_downloader.main(args.firmware[0], path=args.vivado_path[0])
+        firmware_manager.main(args.firmware[0], path=args.vivado_path[0], create=args.compile)
 
     if args.scan:
         mod = import_module('bdaq53.scans.' + args.scan)
-- 
GitLab


From 46d63b334800925d08596cf1a717bba2b5e88d13 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Wed, 13 Feb 2019 17:07:57 +0100
Subject: [PATCH 58/72] MAINT: increase time out, since log was hanging for >
 30 s

---
 bdaq53/firmware_manager.py | 2 +-
 firmware/SiTCP/.gitkeep    | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 delete mode 100644 firmware/SiTCP/.gitkeep

diff --git a/bdaq53/firmware_manager.py b/bdaq53/firmware_manager.py
index 8089928e7..3279664e4 100644
--- a/bdaq53/firmware_manager.py
+++ b/bdaq53/firmware_manager.py
@@ -233,7 +233,7 @@ def compile_firmware(name):
         return
 
     import time
-    timeout = 5  # 30 seconds with no new print to screen
+    timeout = 10  # 50 seconds with no new print to screen
     t = 0
     while t < timeout:
         r = get_return_string()
diff --git a/firmware/SiTCP/.gitkeep b/firmware/SiTCP/.gitkeep
deleted file mode 100644
index e69de29bb..000000000
-- 
GitLab


From 327f31f0030756e13fb89704e56baf900f3b3c15 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Wed, 13 Feb 2019 17:14:25 +0100
Subject: [PATCH 59/72] MAINT: style

---
 bdaq53/bdaq53_cli.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bdaq53/bdaq53_cli.py b/bdaq53/bdaq53_cli.py
index 2628f230c..35a0eec9c 100644
--- a/bdaq53/bdaq53_cli.py
+++ b/bdaq53/bdaq53_cli.py
@@ -69,8 +69,8 @@ def main():
                         help='Path of vivado installation',)
 
     parser.add_argument('-c', '--compile',
-                    action='store_true',
-                    help='Compile firmware',)
+                        action='store_true',
+                        help='Compile firmware',)
 
     args = parser.parse_args()
 
-- 
GitLab


From 4d85dc3bf7fea88dbfebca621642a68321527f8a Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Wed, 13 Feb 2019 17:25:04 +0100
Subject: [PATCH 60/72] GIT: rebase regression

---
 bdaq53/bdaq53.py | 501 +++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 417 insertions(+), 84 deletions(-)

diff --git a/bdaq53/bdaq53.py b/bdaq53/bdaq53.py
index ddcb605fb..b55cd8622 100644
--- a/bdaq53/bdaq53.py
+++ b/bdaq53/bdaq53.py
@@ -5,101 +5,434 @@
 # ------------------------------------------------------------
 #
 
-import argparse
-import os
-import logging
 import yaml
+import logging
+import os
+import time
+import struct
+import coloredlogs
+import numpy as np
+
+from basil.dut import Dut
+
+import pkg_resources
+VERSION = pkg_resources.get_distribution("bdaq53").version
 
-from importlib import import_module
-from inspect import getmembers
-from bdaq53 import firmware_manager
+loglevel = logging.INFO
 
+''' Set up main logger '''
+for handler in logging.root.handlers[:]:
+    logging.root.removeHandler(handler)
+logging.getLogger('basil.HL.RegisterHardwareLayer').setLevel(logging.WARNING)
+
+logging.SUCCESS = 25  # WARNING(30) > SUCCESS(25) > INFO(20)
+logging.addLevelName(logging.SUCCESS, 'SUCCESS')
+
+coloredlogs.DEFAULT_FIELD_STYLES = {'asctime': {},
+                                    'hostname': {},
+                                    'levelname': {'bold': True},
+                                    'name': {},
+                                    'programname': {}}
+coloredlogs.DEFAULT_LEVEL_STYLES = {'critical': {'color': 'red', 'bold': True},
+                                    'debug': {'color': 'magenta'},
+                                    'error': {'color': 'red', 'bold': True},
+                                    'info': {},
+                                    'success': {'color': 'green'},
+                                    'warning': {'color': 'yellow'}}
+
+coloredlogs.install(fmt="%(asctime)s - [%(name)-15s] - %(levelname)-7s %(message)s", milliseconds=True)
 
 logger = logging.getLogger('BDAQ53')
-logging.basicConfig()
-logger.setLevel(logging.INFO)
-
-
-def main():
-    scans = {'test_registers': 'RegisterTest',
-             'test_dac_linearity': 'DACLinearTest',
-             'scan_digital': 'DigitalScan',
-             'scan_analog': 'AnalogScan',
-             'scan_threshold': 'ThresholdScan',
-             'scan_noise_occupancy': 'NoiseOccScan',
-             'tune_global_threshold': 'ThresholdTuning',
-             'tune_local_threshold': 'TDACTuning',
-             'meta_mask_noisy_pixels': 'MetaNOccScan',
-             'meta_tune_threshold': 'MetaThrTuning',
-             'meta_tune_threshold_simple': 'MetaTDACTuning'
-             }
-
-    scan_names = [key for key in scans.keys()]
-
-    parser = argparse.ArgumentParser(
-        description='Bonn DAQ system for RD53A prototype\nexample: bdaq53 scan_digital -p start_column=10 stop_column=20', formatter_class=argparse.RawTextHelpFormatter)
-
-    parser.add_argument('scan',
-                        type=str,
-                        nargs='?',
-                        choices=scan_names,
-                        help='Scan name. Allowed values are:\n' + ', '.join(scan_names),
-                        metavar='scan_name')
-
-    parser.add_argument('-f', '--parameter_file',
-                        type=str,
-                        nargs='?',
-                        help='Path to scan parameter file. If not given the default configuration is used.',
-                        metavar='parameter_file')
-
-    parser.add_argument('-p', '--parameters',
-                        type=str,
-                        nargs='+',
-                        help='or list of parameters. E.g. paramA=9 paramB=1',
-                        metavar='',
-                        default=[])
-
-    parser.add_argument('--firmware',
-                        nargs=1,
-                        help='Firmware file name (e.g. auto, BDAQ53, KC705, ...)\n See https://gitlab.cern.ch/silab/bdaq53/tags',)
-
-    parser.add_argument('--vivado_path',
-                        default=[None],
-                        nargs=1,
-                        help='Path of vivado installation',)
-
-    args = parser.parse_args()
-
-    if args.firmware is not None:
-        firmware_downloader.main(args.firmware[0], path=args.vivado_path[0])
-
-    if args.scan:
-        mod = import_module('bdaq53.scans.' + args.scan)
-
-        if args.parameter_file:
-            parameter_file = args.parameter_file
+logger.setLevel(loglevel)
+logger.success = lambda msg, *args, **kwargs: logger.log(logging.SUCCESS, msg, *args, **kwargs)
+
+
+class BDAQ53(Dut):
+    '''
+    Main class for BDAQ53 readout system
+    '''
+
+    ''' Map hardware IDs for board identification '''
+    hw_map = {
+        0: 'SIMULATION',
+        1: 'BDAQ53',
+        2: 'USBPix3',
+        3: 'KC705',
+        4: 'GENESYS 2'
+    }
+
+    hw_con_map = {
+        0: 'SMA',
+        1: 'FMC_LPC',
+        2: 'FMC_HPC',
+        3: 'Displayport'
+
+    }
+
+    ''' Options concerning the readout hardware '''
+    board_options_map = {
+        '640Mbps': 0x01
+    }
+
+    def __init__(self, conf=None, bench_config=None):
+        self.proj_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        self.configuration = {}
+
+        try:
+            if bench_config is None:
+                bench_config = os.path.join(self.proj_dir, 'bdaq53' + os.sep + 'testbench.yaml')
+            with open(bench_config) as f:
+                self.configuration['bench'] = yaml.load(f)
+        except TypeError:
+            self.configuration['bench'] = bench_config
+
+        if not conf:
+            conf = os.path.join(self.proj_dir, 'bdaq53' + os.sep + 'bdaq53.yaml')
+        logger.debug("Loading configuration file from %s" % conf)
+
+        super(BDAQ53, self).__init__(conf)
+
+    def init(self, **kwargs):
+        super(BDAQ53, self).init()
+
+        self.fw_version, self.board_version, self.board_options, self.connector_version = self.get_daq_version()
+        logger.success('Found board %s with %s running firmware version %s' % (self.board_version, self.connector_version, self.fw_version))
+
+        if self.fw_version != VERSION.split('.')[0] + '.' + VERSION.split('.')[1]:  # Compare only the first two blocks
+            raise Exception("Firmware version (%s) is different than software version (%s)! Please update." % (self.fw_version, VERSION))
+
+        if self.board_version == 'BDAQ53' or self.board_version == 'USBPix3':
+            if self['rx'].get_Si570_is_configured() is False:
+                from bdaq53 import si570
+                si570_conf = {'name': 'si570', 'type': 'bdaq53.si570', 'interface': 'intf', 'base_addr': 0xba, 'init': {'frequency': 160.0}}
+                bdaq53a_clk_gen = si570.si570(self['i2c'], si570_conf)
+                self['cmd'].set_output_en(False)
+                self['rx'].reset()
+                time.sleep(0.1)
+                bdaq53a_clk_gen.init()
+                time.sleep(0.1)
+                self['cmd'].set_output_en(True)
+                self['rx'].set_Si570_is_configured(True)
+            else:
+                logger.info('Si570 oscillator is already configured')
+        elif self.board_version == 'KC705':
+            self._kc705_setup_si5324(**kwargs)
+        elif self.board_version == 'SIMULATION':
+            pass
+
+        # Configure cmd encoder
+        self['cmd'].reset()
+        time.sleep(0.1)
+        # Wait for PLL lock
+        self.wait_for_pll_lock()
+
+        self.set_aurora()
+
+        self.print_powered_dp_connectors()
+
+    def get_daq_version(self):
+        ret = self['intf'].read(0x0000, 2)
+        fw_version = str('%s.%s' % (ret[1], ret[0]))
+
+        ret = self['intf'].read(0x0002, 2)
+        board_version = self.hw_map[ret[0] + (ret[1] << 8)]
+
+        ret = self['intf'].read(0x0004, 1)
+        board_options = ret[0]
+
+        ret = self['intf'].read(0x0005, 2)
+        connector_version = self.hw_con_map[ret[0] + (ret[1] << 8)]
+
+        return fw_version, board_version, board_options, connector_version
+
+    def _kc705_set_i2c_mux(self, value):
+        ''' Configure the I2C MUX and returns the base address of the selected device '''
+        _i2c_mux_map = {
+            'Si570': (0x01, 0x5d),
+            'FMC_HPC': (0x02, 0x00),
+            'FMC_LPC': (0x04, 0x00),
+            'I2C_EEPROM': (0x08, 0x54),
+            'SFP_MODULE': (0x10, 0x50),
+            'ADV7511': (0x20, 0x39),
+            'DDR3_SODIMM': (0x40, (0x50, 0x18)),
+            'Si5324': (0x80, 0x68)
+        }
+        if value in _i2c_mux_map:
+            self['i2c'].write(0xe8, [_i2c_mux_map[value][0]])
+            logger.debug('I2C mux set to: %s' % value)
+        else:
+            logger.error('I2C mux setting invalid: %s' % value)
+
+        base_addr = _i2c_mux_map[value][1]
+        logger.debug('I2C base address: %s' % hex(base_addr))
+
+        return base_addr
+
+    def _kc705_setup_si5324(self, **kwargs):
+        ''' Calculated register values for Si5324 have to be modified in order to work!
+            N1_HS, NC1_LS, N2_HS, N2_LS, N32, BWSEL '''
+        _si5324_f_map = {
+            200: (7, 4, 10, 112000, 22857, 2),
+            180: (7, 4, 10, 100800, 22857, 2),
+            170: (5, 6, 10, 102000, 22857, 2),
+            160: (8, 4, 10, 102400, 22857, 2),
+            150: (5, 3, 6, 33249, 7036, 2),
+            140: (9, 4, 10, 100800, 22857, 2),
+            120: (11, 4, 11, 32000, 7619, 2),
+            100: (9, 4, 11, 32000, 15238, 2)
+        }
+
+        frequency = kwargs.get('aurora_ref', 160)
+
+        _si5324_base_address = self._kc705_set_i2c_mux('Si5324')
+
+        def si5324_read(addr):
+            self['i2c'].write(0xd0, [addr])
+            return self['i2c'].read(0xd0, 1)[0]
+
+        def si5324_write(addr, data):
+            self['i2c'].write(0xd0, [addr, data & 0xff])
+
+        # Based on: https://github.com/m-labs/si5324_test/blob/master/firmware/runtime/si5324.c
+        self['i2c'].write(0xd0, [134])
+        ident = struct.unpack(">H", bytearray(self['i2c'].read(0xd0, 2)))[0]
+        if ident != 0x0182:
+            raise ValueError("It is not Si5324 chip.")
+
+        # Select XA/XB input
+        si5324_write(0, si5324_read(0) | 0x40)  # Free running mode=1, CKOUT_ALWAYS_ON = 0
+        si5324_write(11, 0x41)  # Disable CLKIN1
+        si5324_write(6, 0x0F)  # Disable CKOUT2 (SFOUT2_REG=001), set CKOUT1 to LVDS (SFOUT1_REG=111)
+        si5324_write(21, si5324_read(21) & 0xfe)  # CKSEL_PIN = 0
+        si5324_write(3, 0x55)  # CKIN2 selected, SQ_ICAL=1
+
+        if frequency in _si5324_f_map:
+            register_set = _si5324_f_map[frequency]
+
+            N1_HS = register_set[0] - 4
+            NC1_LS = register_set[1] - 1
+            N2_HS = register_set[2] - 4
+            N2_LS = register_set[3] - 1
+            N32 = register_set[4] - 1
+            BWSEL = register_set[5]
+
+            logging.debug('Si5324: Setting registers to %s' % str(register_set))
         else:
-            parameter_file = os.path.dirname(os.path.abspath(mod.__file__)) + '/default_chip.yaml'
+            logger.error('Si5324: No valid frequency specified: %u' % frequency)
+
+        si5324_write(2, (si5324_read(2) & 0x0f) | (BWSEL << 4))
+        si5324_write(25, N1_HS << 5)
+        si5324_write(31, NC1_LS >> 16)
+        si5324_write(32, NC1_LS >> 8)
+        si5324_write(33, NC1_LS)
+        si5324_write(40, (N2_HS << 5) | (N2_LS >> 16))
+        si5324_write(41, N2_LS >> 8)
+        si5324_write(42, N2_LS)
+        si5324_write(46, N32 >> 16)
+        si5324_write(47, N32 >> 8)
+        si5324_write(48, N32)
+        si5324_write(137, si5324_read(137) | 0x01)  # FASTLOCK=1
+        si5324_write(136, 0x40)  # ICAL=1
+
+        time.sleep(0.1)
 
-        logger.info('Using parameter file: ' + parameter_file + '\n')
+        LOS1_INT = (si5324_read(129) & 0x02) == 0
+        LOSX_INT = (si5324_read(129) & 0x01) == 0
+        LOL_INT = (si5324_read(130) & 0x01) == 0
 
-        with open(parameter_file, 'r') as f:
-            config = yaml.load(f)
+        logger.debug('Si5324: Has input: %d' % (LOS1_INT))
+        logger.debug('Si5324: Has xtal %d:' % (LOSX_INT))
+        logger.debug('Si5324: Locked: %d' % (LOL_INT))
 
-        config.update(mod.local_configuration)
+        logger.info('Si5324: Clock set to %u MHz.' % frequency)
 
-        for param in args.parameters:
-            key, value = param.split('=')
-            config[key] = eval(value)
+        if LOL_INT is False:
+            logger.warning('Si5324: Not locked.')
 
-        for name, cls in getmembers(mod):
-            if name == scans[args.scan]:
-                break
+    def _kc705_get_temperature_NTC_CERNFMC(self):
+        ''' Measure the temperature of the SCC NTC using the CERN FMC card connected to KC705 board'''
+
+        if self.board_version != 'KC705':
+            raise RuntimeError('_kc705_get_temperature_NTC_CERNFMC() is only available with KC705 and CERN FMC card')
+
+        # -----constants-------------
+        # FIXME: ove to calibration object?
+        ntc_adc_vdd = 2.5  # ntc_adc_vdd of the ADC
+        ntc_R1 = 39000  # Resistance 1 of the voltage divider, in series with NTC in FMC-card
+        ntc_R25C = 10e3  # NTC constant
+        ntc_T25 = 298.15
+        ntc_beta = 3435  # Beta NTC constant
+        ntc_adc_lsb = 0.001  # ntc_adc_lsb value of the ADC for normal configuration. Can be changed by changing the configuration register.
+        # ---------------------------
+
+        ntc_adc_base_address = 0x90
+
+        # Read the values of the 1 bit ADC in CERN-FMC card:
+        if self.connector_version == 'FMC_HPC':
+            self._kc705_set_i2c_mux('FMC_HPC')
+            logger.debug('I2C mux: FMC HPC selected')
+        elif self.connector_version == 'FMC_LPC':
+            self._kc705_set_i2c_mux('FMC_LPC')
+            logger.debug('I2C mux: FMC LPC selected')
+        else:
+            raise RuntimeError('_kc705_get_temperature_NTC_CERNFMC() is only available with KC705 and CERN FMC card')
+
+        self['i2c'].write(ntc_adc_base_address, [0b00000001])  # address of ADC and write the addresss pointer register to point the configuration register(default 0x8583)
+        self['i2c'].write(ntc_adc_base_address, [0b00000001, 0x85, 0x83])  # Reset the ADC to start adc_raw single conversion. with this config (which is the default) in the conversion register we will read the voltage drop between the terminals of the NTC resistor.
+
+        self['i2c'].write(ntc_adc_base_address, [0b00000000])
+        adc_raw = self['i2c'].read(ntc_adc_base_address, 2)  # read two bytes of the conversion register of adc
+
+        logger.debug('NTC ADC raw data: %s' % (hex(adc_raw[0]) + ' ' + hex(adc_raw[1])))
+        adc = (((adc_raw[0] << 8) | adc_raw[1]) >> 4)
+        V_adc = ntc_adc_lsb * adc
+        R_ntc = (ntc_R1 * V_adc) / (ntc_adc_vdd - V_adc)
+        T_ntc = (1.0 / ((1.0 / ntc_T25) + ((1.0 / ntc_beta) * (np.log(R_ntc / ntc_R25C))))) - (ntc_T25 - 25)
+
+        return round(T_ntc, 3)
+
+    def get_temperature_NTC(self):
+        if self.board_version == 'KC705':
+            return self._kc705_get_temperature_NTC_CERNFMC()
+        elif self.board_version == 'BDAQ53':
+            raise NotImplementedError('NTC readout is not yet implemented with BDAQ board.')
+        else:
+            raise NotImplementedError('NTC readout is not not supported on this hardware platform.')
+
+    def get_DP_SENSE(self, DP_ID):
+        ''' Read back the vddd_sense lines to identify powered chips '''
+        if self.board_version == 'BDAQ53':
+            sense = self['DP_CONTROL'].get_data()
+            logging.debug('Slow_control sense: %s' % bin(sense[0]))
+            if 0 <= DP_ID < 4:
+                return ((sense[0] & (1 << DP_ID)) is False)
+            else:
+                logger.error('Invalid DP_ID (0..3)')
+        else:
+            logger.warning('RD53A slow control is only available for BDAQ53 hardware')
+            return False
+
+    def set_DP_RESET(self, DP_ID, value):
+        ''' Controls the POR lines. if set to 1, the POR is pulled to local chip ground via opto-coupler '''
+        if self.board_version == 'BDAQ53':
+            if 0 <= DP_ID < 4:
+                self['DP_CONTROL'].set_data([value << (4 + DP_ID)])
+            else:
+                logger.error('Invalid DP_ID (0..3)')
+        else:
+            logger.warning('RD53A slow control is only available for BDAQ53 hardware')
+            return False
+
+    def print_powered_dp_connectors(self):
+        ''' Report the VDDD_SENSE signal status for the given Displayport connectors'''
+        if self.board_version == 'BDAQ53':
+            for i in range(4):
+                if self.get_DP_SENSE(i):
+                    logger.info('VDDD_SENSE detected at Displayport ID %i' % i)
+
+    def dp_power_on_reset(self, DP_ID):
+        ''' Short reset pulse for given DP_ID '''
+        self.set_DP_RESET(DP_ID, True)
+        time.sleep(0.1)
+        self.set_DP_RESET(DP_ID, False)
+
+    def set_monitor_filter(self, state=True):
+        self['rx'].set_USER_K_FILTER_MASK_1(0x01)  # only allow frames containing register data
+        self['rx'].set_USER_K_FILTER_MASK_2(0x02)  # only allow frames containing register data
+        self['rx'].set_USER_K_FILTER_MASK_3(0x04)  # only allow frames containing register data
+        self['rx'].set_USER_K_FILTER_EN(state)  # set the filter
+        logger.debug('USER_K filter set to %s' % state)
+
+    def write_digilent_dac(self, value):
+        ''' Writes integer upto 16 bits to the externally via PMOD connected Digilent DAC '''
+        if self.board_version == 'BDAQ53':
+            byts = []
+            value = int(value)
+            for i in range(0, 2):
+                byts.append(value >> (i * 8) & 0xff)
+            byts.reverse()
+            self['spi_dac'].set_data(byts, addr=0)
+            self['spi_dac'].start()
+        else:
+            logger.warning('Digilent DAC is only available for BDAQ53 hardware')
+            return False
+
+    def set_aurora(self):
+        if self.configuration['bench']['bypass_mode'] is True:
+            if self.board_version == 'KC705'and self.connector_version == 'FMC_LPC' and self.board_options & self.board_options_map['640Mbps']:
+                logger.info("Switching FMC card to BYPASS MODE @ 640Mb/s")
+                self['cmd'].set_bypass_mode(True)
+            else:
+                raise NotImplementedError('Bypass mode is only supported for the KC705+FMC_LPC readout hardware @ 640Mb/s')
+        elif self.board_options & self.board_options_map['640Mbps']:
+            logger.info("Aurora receiver running at 640Mb/s")
+            self['cmd'].set_bypass_mode(False)
+        else:
+            logger.info("Aurora receiver running at 1.28Gb/s")
+            self['cmd'].set_bypass_mode(False)
+
+    def wait_for_aurora_sync(self, timeout=1000):
+        logger.debug("Waiting for Aurora sync...")
+        times = 0
+
+        while times < timeout and self['rx'].get_rx_ready() == 0:
+            times += 1
+
+        if self['rx'].get_rx_ready() == 1:
+            logger.debug("Aurora link synchronized")
+            return True
+        else:
+            self['cmd'].reset()
+            raise RuntimeError('Timeout while waiting for Aurora Sync.')
+
+    def wait_for_pll_lock(self, timeout=1000):
+        logger.debug("Waiting for PLL lock...")
+        times = 0
+
+        while times < timeout and self['rx'].get_pll_locked() == 0:
+            times += 1
+
+        if self['rx'].get_pll_locked() == 1:
+            logger.debug("PLL locked")
+            return True
+        else:
+            raise RuntimeError('Timeout while waiting for PLL to lock.')
+
+    def get_trigger_counter(self):
+        return self['tlu']['TRIGGER_COUNTER']
+
+    def set_tlu_module(self, trigger_enable):
+        self['tlu']['TRIGGER_ENABLE'] = trigger_enable
+
+    def set_trigger_data_delay(self, trigger_data_delay):
+        self['tlu']['TRIGGER_DATA_DELAY'] = trigger_data_delay
+
+    def configure_tlu_module(self, **kwargs):
+        # Reset first TLU module
+        self['tlu']['RESET'] = 1
+        # Set specified registers
+        for key, value in kwargs['TRIGGER'].items():
+            self['tlu'][key] = value
+        # Set maximum number of triggers
+        if kwargs['max_triggers']:
+            self['tlu']['MAX_TRIGGERS'] = kwargs['max_triggers']
+        else:
+            # unlimited number of triggers
+            self['tlu']['MAX_TRIGGERS'] = 0
 
-        scan = cls()
-        scan.start(**config)
-        scan.analyze()
+    def get_tlu_erros(self):
+        return (self['tlu']['TRIGGER_LOW_TIMEOUT_ERROR_COUNTER'], self['tlu']['TLU_TRIGGER_ACCEPT_ERROR_COUNTER'])
 
+    def configure_trigger_cmd_pulse(self, **kwargs):
+        # configures pulse which is sent to CMD for incoming triggers; factor 4 is needed for conversion from 160 MHz to 40 MHz (BC)
+        self['pulser_trig'].set_en(True)
+        self['pulser_trig'].set_width(kwargs['trigger_length'] * 4)
+        self['pulser_trig'].set_delay(kwargs['trigger_delay'] * 4)
+        self['pulser_trig'].set_repeat(1)
 
-if __name__ == '__main__':
-    main()
+    def configure_tlu_veto_pulse(self, **kwargs):
+        # configures pulse for veto of new triggers; factor 4 is needed for conversion from 160 MHz to 40 MHz (BC)
+        self['pulser_veto'].set_en(True)
+        self['pulser_veto'].set_width(4)
+        self['pulser_veto'].set_delay(kwargs['veto_length'] * 4)
+        self['pulser_veto'].set_repeat(1)
-- 
GitLab


From a0b7a3df8422a375e23004f21a7bf8d49676a7f3 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Wed, 13 Feb 2019 17:25:26 +0100
Subject: [PATCH 61/72] CI: try conda fix

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9f25d68d1..a7a405a9b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -49,6 +49,7 @@ test:software:
     script:
       - python setup.py develop
       # Do not run virtual x server tests (monitor) in this runner due to segfault
+      - conda activate
       - pytest -v bdaq53/tests/test_software --ignore=bdaq53/tests/test_software/test_monitor.py
 
 # Tests that need bdaq53 readout hardware and RD53 FE
-- 
GitLab


From 9c9de47b9741a9479fcaef5053cb7df8729cce21 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Wed, 13 Feb 2019 17:32:16 +0100
Subject: [PATCH 62/72] CI: try fix

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a7a405a9b..6a1ea9ca5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -69,11 +69,11 @@ test:hardware:
       - git --git-dir=$HOME/git/basil/.git pull
       # Clean potential preceding runner run SiTCP git checkout
       - rm -rf firmware/SiTCP/{*,.git}
-      
+      - touch firmware/SiTCP/.gitkeep  # recreate gitkeep file
     script:
       - python setup.py develop
       # Compile firmware if changes detected and flash
-      - if [[ `git diff ..development -- firmware` ]]; then bdaq53 --firmware BDAQ53_RX640 -c; bdaq53 --firmware BDAQ53_RX640.bit; else bdaq53 --firmware BDAQ53_RX640; fi
+      - if [[ `git diff ..development -- firmware` ]]; then ; bdaq53 --firmware BDAQ53_RX640.bit; else bdaq53 --firmware BDAQ53_RX640; fi
       - pytest -v bdaq53/tests/test_hardware
     artifacts:
       when: always  # also upload verilog log on failure
-- 
GitLab


From 87f4cccb066f0d28dcdb741334ff16896dca8ad7 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Wed, 13 Feb 2019 17:56:37 +0100
Subject: [PATCH 63/72] CI: anaconda issues

---
 .gitlab-ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6a1ea9ca5..5f03bd2f0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -49,7 +49,8 @@ test:software:
     script:
       - python setup.py develop
       # Do not run virtual x server tests (monitor) in this runner due to segfault
-      - conda activate
+      - conda init bash
+      - conda activate  # https://github.com/ContinuumIO/docker-images/issues/89
       - pytest -v bdaq53/tests/test_software --ignore=bdaq53/tests/test_software/test_monitor.py
 
 # Tests that need bdaq53 readout hardware and RD53 FE
-- 
GitLab


From 187f7d1bfe8520bb4f719c61080bd4f113916a14 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Wed, 13 Feb 2019 18:01:11 +0100
Subject: [PATCH 64/72] TST debug

---
 .gitlab-ci.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5f03bd2f0..ea2db034b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -49,8 +49,8 @@ test:software:
     script:
       - python setup.py develop
       # Do not run virtual x server tests (monitor) in this runner due to segfault
-      - conda init bash
-      - conda activate  # https://github.com/ContinuumIO/docker-images/issues/89
+      - conda init bash  # https://github.com/ContinuumIO/docker-images/issues/89
+      - conda activate
       - pytest -v bdaq53/tests/test_software --ignore=bdaq53/tests/test_software/test_monitor.py
 
 # Tests that need bdaq53 readout hardware and RD53 FE
@@ -70,6 +70,7 @@ test:hardware:
       - git --git-dir=$HOME/git/basil/.git pull
       # Clean potential preceding runner run SiTCP git checkout
       - rm -rf firmware/SiTCP/{*,.git}
+      - pwd
       - touch firmware/SiTCP/.gitkeep  # recreate gitkeep file
     script:
       - python setup.py develop
-- 
GitLab


From 02a7146698e8fbd777412e689fa453eb035eac7c Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Wed, 13 Feb 2019 18:03:46 +0100
Subject: [PATCH 65/72] TST fix

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ea2db034b..a2d6cad61 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -68,9 +68,9 @@ test:hardware:
       - pip install xvfbwrapper
       # Update basil development from github
       - git --git-dir=$HOME/git/basil/.git pull
-      # Clean potential preceding runner run SiTCP git checkout
+      # Clean SiTCP git checkout from potential preceding runner run  
       - rm -rf firmware/SiTCP/{*,.git}
-      - pwd
+      - mkdir firmware/SiTCP
       - touch firmware/SiTCP/.gitkeep  # recreate gitkeep file
     script:
       - python setup.py develop
-- 
GitLab


From 7df2c96dd5ff77c766fb227c7a60c2993671545b Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Wed, 13 Feb 2019 18:08:28 +0100
Subject: [PATCH 66/72] BUG

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a2d6cad61..c41a297f3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -75,7 +75,7 @@ test:hardware:
     script:
       - python setup.py develop
       # Compile firmware if changes detected and flash
-      - if [[ `git diff ..development -- firmware` ]]; then ; bdaq53 --firmware BDAQ53_RX640.bit; else bdaq53 --firmware BDAQ53_RX640; fi
+      - if [[ `git diff ..development -- firmware` ]]; then bdaq53 --firmware BDAQ53_RX640 -c; bdaq53 --firmware BDAQ53_RX640.bit; else bdaq53 --firmware BDAQ53_RX640; fi
       - pytest -v bdaq53/tests/test_hardware
     artifacts:
       when: always  # also upload verilog log on failure
-- 
GitLab


From 98637b57e69553f7918aff52b143fe706b81ed82 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Wed, 13 Feb 2019 18:14:41 +0100
Subject: [PATCH 67/72] TST: try fix broken miniconda docker

---
 .gitlab-ci.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c41a297f3..c2fa9abe3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -48,9 +48,11 @@ test:software:
       # - if [ -z "$CI_COMMIT_TAG"]; then git clone -b v3.0.0 --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; else git clone -b development --depth 1 https://github.com/SiLab-Bonn/basil.git; cd basil; python setup.py develop; cd ..; fi
     script:
       - python setup.py develop
-      # Do not run virtual x server tests (monitor) in this runner due to segfault
+      # Try to activate conda evironment in the newest broken miniconda docker
       - conda init bash  # https://github.com/ContinuumIO/docker-images/issues/89
-      - conda activate
+      - source ~/.bashrc  # since ==> For changes to take effect, close and re-open your current shell. <==
+      - conda activate  # to link properly to pytest
+      # Do not run virtual x server tests (monitor) in this runner due to segfault
       - pytest -v bdaq53/tests/test_software --ignore=bdaq53/tests/test_software/test_monitor.py
 
 # Tests that need bdaq53 readout hardware and RD53 FE
-- 
GitLab


From 3931661152c7654604345ce3d2dbdb6743124fef Mon Sep 17 00:00:00 2001
From: Michael Daas <daas@physik.uni-bonn.de>
Date: Thu, 14 Feb 2019 13:05:30 +0100
Subject: [PATCH 68/72] HOTFIX: Readd init_communication()

---
 bdaq53/rd53a.py | 36 ++++++++++++++++++++++++++++++++++--
 1 file changed, 34 insertions(+), 2 deletions(-)

diff --git a/bdaq53/rd53a.py b/bdaq53/rd53a.py
index 6208641ee..5f92282a3 100644
--- a/bdaq53/rd53a.py
+++ b/bdaq53/rd53a.py
@@ -310,8 +310,7 @@ class RD53A(object):
         self.reset_masks()
 
     def init(self, **kwargs):
-        self.set_aurora(**kwargs)
-        self.bdaq.wait_for_aurora_sync()
+        self.init_communication(**kwargs)
         self.write_ecr()
 
     def set_aurora(self, tx_lanes=1, CB_Wait=255, CB_Send=1, chip_id=8, only_cb=False, write=True, **_):
@@ -379,6 +378,39 @@ class RD53A(object):
             self.write_command(indata)
         return indata
 
+    def init_communication(self, **kwargs):
+        logger.info('Initializing communication...')
+
+        # Configure cmd encoder
+        self.bdaq['cmd'].reset()
+        time.sleep(0.1)
+        self.write_command(self.write_sync(write=False) * 32)
+        # Wait for PLL lock
+        self.bdaq.wait_for_pll_lock()
+
+        aurora_data = self.set_aurora(**kwargs)
+
+        # Workaround for problems locking
+        for _ in range(30):
+            self.write_command(self.write_sync(write=False) * 32)
+            self.write_ecr()
+
+            self.write_command(aurora_data)
+            time.sleep(0.01)
+            try:
+                self.bdaq.wait_for_aurora_sync()
+            except RuntimeError:
+                pass
+            else:
+                break
+
+            self.write_command([0x00] * 1000, repetitions=1)
+            time.sleep(0.01)
+        else:
+            self.bdaq.wait_for_aurora_sync()
+
+        logger.success('Communication established')
+
     def write_global_pulse(self, width, chip_id=8, write=True):
         # 0101_1100    ChipId<3:0>,0    Width<3:0>,0
         indata = [self.CMD_GLOBAL_PULSE] * 2  # [0b01011100]
-- 
GitLab


From e5e75c784615ff5b9eb23063481a4696f162c7a9 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 14 Feb 2019 15:36:15 +0100
Subject: [PATCH 69/72] CI: set to std ubuntu 18 settings to ease runner
 operation

---
 bdaq53/periphery.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bdaq53/periphery.yaml b/bdaq53/periphery.yaml
index a75382c68..8001eec00 100644
--- a/bdaq53/periphery.yaml
+++ b/bdaq53/periphery.yaml
@@ -8,7 +8,7 @@ transfer_layer:
   - name     : Serial
     type     : Serial
     init     :
-        port     : /dev/ttyACM0
+        port     : /dev/ttyUSB0
         read_termination : "\r\n"
         write_termination : "\n"
         baudrate : 19200
-- 
GitLab


From 9332eb858d2b1295f4205efbdb3b1799d88f3d38 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 14 Feb 2019 15:37:02 +0100
Subject: [PATCH 70/72] BUG: fix #250

---
 bdaq53/periphery.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bdaq53/periphery.py b/bdaq53/periphery.py
index 4b1b8d110..6c847347f 100644
--- a/bdaq53/periphery.py
+++ b/bdaq53/periphery.py
@@ -61,7 +61,7 @@ class BDAQ53Periphery(object):
             return
         try:
             self.devices.init()
-            self.logger.debug('Initialized SCC powersupply %s' % (self.devices['SCC_Powersupply'].get_info()))
+            self.logger.debug('Initialized SCC powersupply %s' % (self.devices['SCC_Powersupply'].get_name()))
             try:
                 self.logger.debug('Initialized sensor bias powersupply %s' % (self.devices['SensorBias'].get_name()))
             except KeyError:
-- 
GitLab


From c3b2819d56a147d27ad85cfa658f53669d1a8350 Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 14 Feb 2019 17:04:31 +0100
Subject: [PATCH 71/72] TST: debug

---
 .../tests/test_hardware/test_scan_scripts.py  | 36 +++++++++----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/bdaq53/tests/test_hardware/test_scan_scripts.py b/bdaq53/tests/test_hardware/test_scan_scripts.py
index 0096ce4d3..14473a299 100644
--- a/bdaq53/tests/test_hardware/test_scan_scripts.py
+++ b/bdaq53/tests/test_hardware/test_scan_scripts.py
@@ -65,24 +65,24 @@ class TestScanScripts(unittest.TestCase):
 #             # self.assertTrue(np.all(in_file.root.HistBCIDError[:] == 0))
 #             # self.assertTrue(np.all(in_file.root.HistEventStatus[:] == 0))
 
-    def test_fast_threshold_scan(self):
-        ''' Test fast threshold scan and results '''
-        from bdaq53.scans import scan_threshold_fast
-        scan = scan_threshold_fast.FastThresholdScan()
-        scan.start(**scan_threshold_fast.local_configuration)
-        scan.close()
-
-    def test_register_test(self):
-        from bdaq53.scans import test_registers
-        scan = test_registers.RegisterTest()
-        scan.start(**test_registers.local_configuration)
-        scan.close()
-
-    def test_threshold_noise_tuning(self):
-        from bdaq53.scans import tune_local_threshold_noise
-        scan = tune_local_threshold_noise.NoiseTuning()
-        scan.start(**tune_local_threshold_noise.local_configuration)
-        scan.close()
+#     def test_fast_threshold_scan(self):
+#         ''' Test fast threshold scan and results '''
+#         from bdaq53.scans import scan_threshold_fast
+#         scan = scan_threshold_fast.FastThresholdScan()
+#         scan.start(**scan_threshold_fast.local_configuration)
+#         scan.close()
+# 
+#     def test_register_test(self):
+#         from bdaq53.scans import test_registers
+#         scan = test_registers.RegisterTest()
+#         scan.start(**test_registers.local_configuration)
+#         scan.close()
+# 
+#     def test_threshold_noise_tuning(self):
+#         from bdaq53.scans import tune_local_threshold_noise
+#         scan = tune_local_threshold_noise.NoiseTuning()
+#         scan.start(**tune_local_threshold_noise.local_configuration)
+#         scan.close()
 
 
 if __name__ == '__main__':
-- 
GitLab


From f123c06c3616cc23029b0bae11389c88076148fc Mon Sep 17 00:00:00 2001
From: DavidLP <pohl@physik.uni-bonn.de>
Date: Thu, 14 Feb 2019 19:42:31 +0100
Subject: [PATCH 72/72] MAINT: style

---
 bdaq53/tests/test_hardware/test_scan_scripts.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bdaq53/tests/test_hardware/test_scan_scripts.py b/bdaq53/tests/test_hardware/test_scan_scripts.py
index 14473a299..c0119b955 100644
--- a/bdaq53/tests/test_hardware/test_scan_scripts.py
+++ b/bdaq53/tests/test_hardware/test_scan_scripts.py
@@ -71,13 +71,13 @@ class TestScanScripts(unittest.TestCase):
 #         scan = scan_threshold_fast.FastThresholdScan()
 #         scan.start(**scan_threshold_fast.local_configuration)
 #         scan.close()
-# 
+#
 #     def test_register_test(self):
 #         from bdaq53.scans import test_registers
 #         scan = test_registers.RegisterTest()
 #         scan.start(**test_registers.local_configuration)
 #         scan.close()
-# 
+#
 #     def test_threshold_noise_tuning(self):
 #         from bdaq53.scans import tune_local_threshold_noise
 #         scan = tune_local_threshold_noise.NoiseTuning()
-- 
GitLab