Added tool to find best config parameters

configs/so_optimizer.json

+{
+    "targetChipNumber": 4,
+    "repetitions": 2,
+    "parameters":
+    [
+        {
+            "name": "SldoAnalogTrim",
+            "min": 20,
+            "max": 26,
+            "step": 1
+        },
+        {
+            "name": "SldoDigitalTrim",
+            "min": 19,
+            "max": 27,
+            "step": 1
+        }
+    ]
+}

libDCS/parameterOptimizer.py

+import numpy as np
+from random import randint
+from prettytable import PrettyTable
+from time import sleep
+import json
+import argparse
+import subprocess
+import sys, os
+import signal
+
+
+def signal_handler(sig, frame):
+    """Capture SIGINT (ctrl C)"""
+
+    print('done')
+    sys.exit(0)
+signal.signal(signal.SIGINT, signal_handler)
+
+
+def count0(enMaskPath):
+    """Counts disabled pixels in EnMask
+
+    Args:
+        enMaskPath (str): path to enMask
+
+    Returns:
+        int: number of bad pixels (-1 if scan failed)
+    """
+
+    badPixels = 0
+
+    try:
+        with open(enMaskPath) as file1:
+
+            data = json.load(file1)
+            for y in data["Data"]:
+                badPixels += y.count(0)
+
+        return float(badPixels)
+    except:
+        return -1
+
+
+def whereToGoNext(x, y, M):
+    candidateX = []
+    candidateY = []
+
+    #print(x, y, M[x][y])
+    radius = 1
+    while candidateX == []:
+        for a in range(-radius, radius):
+            for b in range(-radius, radius):
+                if x + a > -1 and y + b > -1:
+                    try:
+                        if M[x + a][y + b] == ' ':
+                            candidateX.append(x + a)
+                            candidateY.append(y + b)
+                    except:
+                        pass
+
+        if radius >= max(len(M), len(M[0])): 
+            print("scan finished")
+            return None, None
+        radius += 1
+
+    i = randint(0, len(candidateX) -1)
+    return candidateX[i], candidateY[i]
+
+
+def updatecfg(args, chipName, name_v1, name_v2, v1range, v2range, xtest, ytest):
+    """Modifies name_v1 and name_v2 in Yarr's cfg"""
+
+    chipCfg = args.basepath + "/configs/rd53a/" + args.module + "/" + chipName + ".json"
+    with open(chipCfg, 'r') as f:
+        cfg = json.load(f)
+        cfg["RD53A"]["GlobalConfig"][name_v1] = v1range[xtest]
+        cfg["RD53A"]["GlobalConfig"][name_v2] = v2range[ytest]
+    
+    with open(chipCfg, 'w') as f:
+        json.dump(cfg, f, indent=4)
+
+def find(args):
+    """Finds a minimum amongst some setting parameters"""
+
+    optCfg = args.json
+
+    # Read CL arguments
+    with open(optCfg) as f:
+        cfg = json.load(f)
+
+        targetChipNumber = cfg["targetChipNumber"]
+        nReps = cfg["repetitions"]
+
+        # var 1
+        name_v1 = cfg["parameters"][0]["name"]
+        min_v1 = int(cfg["parameters"][0]["min"])
+        max_v1 = int(cfg["parameters"][0]["max"])
+        stp_v1 = int(cfg["parameters"][0]["step"])
+
+        # var 2
+        name_v2 = cfg["parameters"][1]["name"]
+        min_v2 = int(cfg["parameters"][1]["min"])
+        max_v2 = int(cfg["parameters"][1]["max"])
+        stp_v2 = int(cfg["parameters"][1]["step"])
+    
+    # Build matrix
+    v1range = [int(x) for x in range(min_v1, max_v1+1, stp_v1)]
+    v2range = [int(y) for y in range(min_v2, max_v2+1, stp_v2)]
+
+    results = [[' ' for a in range(len(v2range))] for b in range(len(v1range))]
+
+    # Randomly decide the three first testing points
+    xtest0 = []
+    ytest0 = []
+
+    for i in range(3):
+        xtest0.append(randint(0, len(v1range) - 1))
+        ytest0.append(randint(0, len(v2range) - 1))
+
+    # get path to so_modules.json
+    cfgIndex = args.basepath + "/configs/so_index.json"
+    with open(cfgIndex) as f:
+        cfg = json.load(f)
+        soCfgPath = cfg["module_config"]
+
+    # Get chip Name
+    with open(soCfgPath) as f:
+        cfg = json.load(f)
+        chipName = cfg["modules"]["KEKQ08"]["chips"][targetChipNumber-1]["Parameter"]["Name"]
+    
+    # Enable only target chip in yarr's connect. config
+    connCfg = args.basepath + "/configs/rd53a/" + args.module + "/connectivity.json"
+    with open(connCfg, 'r') as f:
+        cfg = json.load(f)
+        for i in range(len(cfg["chips"])):
+            if i == targetChipNumber - 1:
+                cfg["chips"][i]["enable"] = 1
+            else:
+                cfg["chips"][i]["enable"] = 0
+
+    with open(connCfg, 'w') as f:
+        json.dump(cfg, f, indent=4)
+
+
+    # Create some environmental variables
+    os.environ['SCANCFG'] = args.basepath + "/configs/so_yarr.json"
+
+    with open(args.basepath + "/configs/so_yarr.json") as f:
+        cfg = json.load(f)
+        yarrBase =cfg["YARR_directory"]
+
+    os.environ['YARRBASE'] = yarrBase
+    os.environ['SOBASE'] = args.basepath
+    os.environ['SO_RUNNUMBER'] = "optimizer"
+    os.environ['TIMEFILE'] = "/dev/null"
+
+    # power cycle
+    if args.powercycle:
+        os.system("python3 libDCS/qaqc.py -e " + args.equip + " -c " + args.channel + " power-off")
+        os.system("python3 libDCS/qaqc.py -e " + args.equip + " -c " + args.channel + " power-on")
+    
+    # Run first 3 digital scans
+    for i in range(3):
+        
+        # change yarr chip cfg
+        updatecfg(args, chipName, name_v1, name_v2, v1range, v2range, xtest0[i], ytest0[i])
+
+        print()
+
+        cmd = args.basepath + "/scanLauncher.sh -m " + args.module + " -s [\"std_digitalscan\",1]"
+
+        # run first digitalScan
+        total = 0
+        for j in range(nReps):
+            print("[ info ][po] Running a digital scan at %i %i..." % (v1range[xtest0[i]], v2range[ytest0[i]]))
+            process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)
+            output, error = process.communicate()
+            enMaskPath = args.basepath + "/data/optimizer/last_scan/" + chipName + "_EnMask.json"
+
+            count = count0(enMaskPath)
+            total += count
+
+            if count < 0:
+                total = -1; nReps = 1
+                break
+
+        # Fill the result matrix
+        results[xtest0[i]][ytest0[i]] = total / nReps
+
+
+        # prepare table
+        t = PrettyTable([' '] + [k for k in v2range])
+        for i in range(len(results)):
+            t.add_row([v1range[i]] + results[i])
+
+        t.border = False
+
+        print("")
+        print(t)
+
+    # Three previous points 
+    # TODO: Make this a single 2d array?
+    prev3x = [xtest0[-3], xtest0[-2], xtest0[-1]]
+    prev3y = [ytest0[-3], ytest0[-2], ytest0[-1]]
+    
+    # Update values
+    prev3validY = [prev3x[k] for k in range(len(prev3x)) if results[prev3x[k]][prev3y[k]] > -1]
+    prev3validX = [prev3y[k] for k in range(len(prev3y)) if results[prev3x[k]][prev3y[k]] > -1]
+    lookaround_x = min(prev3validX) if prev3validX != [] else randint(0, len(v1range) - 1)
+    lookaround_y = min(prev3validY) if prev3validY != [] else randint(0, len(v2range) - 1)
+    xtest1, ytest1 = whereToGoNext(lookaround_x, lookaround_y, results)
+
+    updatecfg(args, chipName, name_v1, name_v2, v1range, v2range, xtest1, ytest1)
+
+
+    print("[ info ][po] Entering the loop...")
+    while True:
+
+        # power cycle
+        if args.powercycle:
+            os.system("python3 libDCS/qaqc.py -e " + args.equip + " -c " + args.channel + " power-off")
+            os.system("python3 libDCS/qaqc.py -e " + args.equip + " -c " + args.channel + " power-on")
+
+        print()
+
+        # If the matrix is filled
+        if xtest1 == None:
+            return
+
+        # If we have already tested this point
+        elif results[xtest1][ytest1] != ' ':
+
+            # Find another one random
+            xtest1 = randint(0, len(v1range))
+            ytest1 = randint(0, len(v2range))
+
+            continue
+
+
+
+        # Run digitalScan
+        total = 0
+        for j in range(nReps):
+            print("[ info ][po] Testing %i %i..." % (v1range[xtest1], v2range[ytest1]))
+            process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)
+            output, error = process.communicate()
+            enMaskPath = args.basepath + "/data/optimizer/last_scan/" + chipName + "_EnMask.json"
+
+            count = count0(enMaskPath)
+            total += count
+
+            if count < 0:
+                total = -1; nReps = 1
+                break
+
+        # Fill the result matrix
+        results[xtest1][ytest1] = total / nReps
+
+
+        # Update list of previous 3 tested points
+        prev3x = [prev3x[-2], prev3x[-1], xtest1]
+        prev3y = [prev3y[-2], prev3y[-1], ytest1]
+
+
+        # Decide where to go next
+        prev3validY = [prev3x[k] for k in range(len(prev3x)) if results[prev3x[k]][prev3y[k]] > -1]
+        prev3validX = [prev3y[k] for k in range(len(prev3y)) if results[prev3x[k]][prev3y[k]] > -1]
+        xtest1, ytest1 = whereToGoNext(lookaround_x, lookaround_y, results)
+
+
+        # prepare table
+        print("haxis: %s; vaxis: %s" % (name_v2, name_v1))
+
+        t = PrettyTable(['#'] + [k for k in v2range])
+        for i in range(len(results)):
+            t.add_row([v1range[i]] + results[i])
+
+        t.border = False
+
+        print("")
+        print(t)
+        
+        sleep(0.5)
+
+
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "-b",
+        "--basepath",
+        default=".",
+        help="Path to the SO",
+    )
+    parser.add_argument(
+        "-e",
+        "--equip",
+        help="Configuration file with the power supply definition",
+    )
+    parser.add_argument(
+        "-c",
+        "--channel",
+        help="channel name"
+    )
+    parser.add_argument(
+        "-j",
+        "--json",
+        help="path to so_optimizer.json"
+    )
+    parser.add_argument(
+        "-m",
+        "--module",
+        help="module name"
+    )
+    parser.add_argument(
+        "-p",
+        "--powercycle",
+        action='store_true',
+        help="powercycle"
+    )
+
+    args = parser.parse_args()
+
+    if not args.json:
+        print("option '-j' is mandatory")
+        parser.print_help()
+        sys.exit(1)
+
+    if not args.json:
+        print("option '-c' is mandatory")
+        parser.print_help()
+        sys.exit(1)
+
+    if not args.module:
+        print("option '-m' is mandatory")
+        parser.print_help()
+        sys.exit(1)
+
+    if args.powercycle:
+        if not args.equip or not args.channel:
+            print("options '-e' and '-c' are mandatory when using '-p'")
+            parser.print_help()
+            sys.exit(1)           
+
+
+    find(args)
\ No newline at end of file

scanLauncher.sh

@@ -104,7 +104,14 @@ else
         targetAmpOrCharge=""
     fi
 fi
+# Build Yarr's "-m" option (reset px masks, default 0)
+reset_masks=0
+if echo "${scanName}" | grep "digital\|analog\|noise" >>/dev/null; then
+    if [[ $(echo $scan | jq '. | length') == 2 ]]; then
+        reset_masks=$(echo $scan | jq -r '.[1]')
+    fi
 
+fi
 ###### Log file  ######################
 RunNumber=$(cat $HOME/.yarr/runCounter)
 RunNumber=$(( RunNumber +  1))
@@ -133,16 +140,14 @@ if [[ $dcsMonitor == 1 ]] ;then
 
     $SOBASE/libDCS/read_dcs_background.sh $PSCFG $INFLUXDBDCSCFG &
 fi
-targetCharge=$(jq -j '.scan.target_charge' $SCANCFG)
 
-# Mask option (Yarr's -m)
-reset_masks=$(jq -j '.common_config.reset_pixel_masks' $SCANCFG)
+targetCharge=$(jq -j '.scan.target_charge' $SCANCFG)
 
 ctrlFile="$SOBASE/configs/rd53a/$module_id/controller.json"
 cnctFile="$SOBASE/configs/rd53a/$module_id/connectivity.json"
 
 #ctrlFile="/home/mario/work/yr/configs/controller/emuCfg.json"
-comm="$YARRBASE/bin/scanConsole -r $ctrlFile -c $cnctFile -p -t ${targetAmpOrCharge} ${targetToT} -s ${scanPath} -m $reset_masks $localdbOpt $qcOpt -o $SOBASE/data/$SO_RUNNUMBER" #> /dev/null 2>1
+comm="$YARRBASE/bin/scanConsole -r $ctrlFile -c $cnctFile -p -t ${targetAmpOrCharge} ${targetToT} -s ${scanPath} -m $reset_masks -o $SOBASE/data/$SO_RUNNUMBER $localdbOpt $qcOpt" #> /dev/null 2>1
 echo "[ info ][sl] Calling Yarr's scanConsole"
 echo "[ info ][sl] $comm"
 
@@ -178,7 +183,6 @@ echo -ne "$scanName     $elp\n" >> $TIMEFILE
 
 
 
-
 # stop taking data
 if [[ $dcsMonitor == 1 ]] ;then
     echo "[ info ][sl] Stopping data taking"