scanConsole.cpp 17 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
// #################################
// # Author: Timon Heim
// # Email: timon.heim at cern.ch
// # Project: Yarr
// # Description: Command line scan tool
// # Comment: To be used instead of gui
// ################################

#include <string>
#include <chrono>
#include <thread>
#include <vector>
Timon Heim's avatar
Timon Heim committed
13
#include <iomanip>
Jonathan Debus's avatar
Jonathan Debus committed
14
#include <map>
15

Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
16
#include "logging.h"
17
#include "LoggingConfig.h"
18

19
#include "ScanHelper.h"
20
#include "ScanOpts.h"
21

Timon Heim's avatar
Timon Heim committed
22
#include "HwController.h"
23
#include "AllChips.h"
Timon Heim's avatar
Timon Heim committed
24
#include "AllProcessors.h"
25
#include "AllStdActions.h"
26
#include "Bookkeeper.h"
Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
27
#include "FeedbackBase.h"
28
#include "ScanBase.h"
29
#include "DBHandler.h"
Jonathan Debus's avatar
Jonathan Debus committed
30

31
#include "storage.hpp"
32

33
34
#include "yarr.h"

35
auto logger = logging::make_log("scanConsole");
Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
36

37
int main(int argc, char *argv[]) {
38
39
    ScanOpts scanOpts;
    spdlog::set_pattern(scanOpts.defaultLogPattern);
40

Timon Heim's avatar
Timon Heim committed
41
    spdlog::info("-> Parsing command line parameters ...");
42

43
44
    int res=ScanHelper::parseOptions(argc,argv,scanOpts);
    if(res<=0) exit(res);
45

46
    unsigned runCounter=0;
47
    std::string strippedScan;
48
49
50
51
    std::string dataDir;
    json scanLog;
    std::unique_ptr<HwController> hwCtrl;
    std::unique_ptr<Bookkeeper> bookie;
52
    std::map<unsigned, std::array<std::string, 2>> feCfgMap;
53
    std::unique_ptr<ScanBase> scanBase;
54
55
56
    std::map<unsigned, std::unique_ptr<DataProcessor> > histogrammers;
    std::map<unsigned, std::vector<std::unique_ptr<DataProcessor>> > analyses;
    std::map<unsigned, std::unique_ptr<DataProcessor> > procs;
57
58
59
    std::string chipType;
    std::string timestampStr;
    std::time_t now;
60

61
62
63
    // Get new run number
    runCounter = ScanHelper::newRunCounter();

64
65
    // parsing all json files
    json loggerConfig;
66
    json chipConfig;
67
    json dbCfg;
68
    json ctrlCfg;
69
70
71
    json userCfg;
    json siteCfg;
    json scanCfg;
72

73
    json scanConsoleConfig;
Matthias Wittgen's avatar
Matthias Wittgen committed
74
    std::unique_ptr<DBHandler> database;
75

76
    if(!scanOpts.logCfgPath.empty()) {
77
78
        loggerConfig = ScanHelper::openJsonFile(scanOpts.logCfgPath);
        loggerConfig["outputDir"]=scanOpts.outputDir;
79
80
    } else {
        // default log setting
81
82
83
84
        loggerConfig["pattern"] = scanOpts.defaultLogPattern;
        loggerConfig["log_config"][0]["name"] = "all";
        loggerConfig["log_config"][0]["level"] = "info";
        loggerConfig["outputDir"]="";
85
    }
86
87
    spdlog::info("Configuring logger ...");
    logging::setupLoggers(loggerConfig);
88

89
    ScanHelper::banner(logger,"Welcome to the YARR Scan Console!");
90

Matthias Wittgen's avatar
Matthias Wittgen committed
91
    int result = ScanHelper::loadConfigFile(scanOpts, true, scanConsoleConfig);
92
93
94
    if(result<0) {
        spdlog::error("Failed to read configs");
        exit(-1);
95
96
    }

97
98
99
    ctrlCfg=scanConsoleConfig["ctrlConfig"];
    chipConfig=scanConsoleConfig["chipConfig"];
    scanCfg=scanConsoleConfig["scanCfg"];
100

101

102
103
104
105
106
107
108
109
    if(scanOpts.dbUse) {
        try {
            dbCfg = ScanHelper::openJsonFile(scanOpts.dbCfgPath);
            userCfg = ScanHelper::openJsonFile(scanOpts.dbUserCfgPath);
            siteCfg = ScanHelper::openJsonFile(scanOpts.dbSiteCfgPath);
        }catch (std::runtime_error &e) {
            logger->critical("#ERROR# failed to load database config: {}", e.what());
            return -1;
110
111
        }
    }
112

113
    // end of configuration section
114

115
116
117
    // create outdir directory
    dataDir=scanOpts.outputDir;
    strippedScan=ScanHelper::createOutputDir(scanOpts.scanType,runCounter,scanOpts.outputDir);
118

119
120
    if(scanOpts.scan_config_provided) {
        logger->info("Scan Type/Config {}", scanOpts.scanType);
121
122
123
    } else {
        logger->info("No scan configuration provided, will only configure front-ends");
    }
124

125
    logger->info("Connectivity:");
126
    for(std::string const& sTmp : scanOpts.cConfigPaths){
127
        logger->info("    {}", sTmp);
Jonathan Debus's avatar
Jonathan Debus committed
128
    }
129
130
131
132
    logger->info("Target ToT: {}", scanOpts.target_tot);
    logger->info("Target Charge: {}", scanOpts.target_charge);
    logger->info("Output Plots: {}", scanOpts.doPlots);
    logger->info("Output Directory: {}", scanOpts.outputDir);
133

Timon Heim's avatar
Timon Heim committed
134
    // Make symlink
Matthias Wittgen's avatar
Matthias Wittgen committed
135
    if(scanOpts.doOutput) ScanHelper::createSymlink(dataDir,strippedScan,runCounter);
136

137
    // Timestamp
138
    now = std::time(nullptr);
139
140
    timestampStr = ScanHelper::timestamp(now);
    logger->info("Timestamp: {}", timestampStr);
141
    logger->info("Run Number: {}", runCounter);
142
143

    // Add to scan log
144
    scanLog["yarr_version"] = yarr::version::get();
145
146
    scanLog["exec"] = scanOpts.commandLineStr;
    scanLog["timestamp"] = timestampStr;
Arisa Kubota's avatar
Arisa Kubota committed
147
    scanLog["startTime"] = (int)now;
148
    scanLog["runNumber"] = runCounter;
149
150
    scanLog["targetCharge"] = scanOpts.target_charge;
    scanLog["targetTot"] = scanOpts.target_tot;
151
    scanLog["testType"] = strippedScan;
152

153
    ScanHelper::banner(logger,"Init Hardware");
Timon Heim's avatar
Timon Heim committed
154

155
    logger->info("-> Opening controller config: {}", scanOpts.ctrlCfgPath);
Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
156

157
158
159
    try {
        hwCtrl = ScanHelper::loadController(ctrlCfg);
    } catch (std::runtime_error &e) {
160
        logger->critical("Error opening or loading controller config: {}", e.what());
Timon Heim's avatar
Timon Heim committed
161
        return -1;
162
    }
163
164
    // Add to scan log
    scanLog["ctrlCfg"] = ctrlCfg;
165
    scanLog["ctrlStatus"] = hwCtrl->getStatus();
166

167
    hwCtrl->setupMode();
168

169
170
    // Disable trigger in-case
    hwCtrl->setTrigEnable(0);
171

172
    bookie=std::make_unique<Bookkeeper>(&*hwCtrl, &*hwCtrl);
173

Timon Heim's avatar
Timon Heim committed
174

175
176
    bookie->setTargetTot(scanOpts.target_tot);
    bookie->setTargetCharge(scanOpts.target_charge);
Jonathan Debus's avatar
Jonathan Debus committed
177

178
    ScanHelper::banner(logger,"Loading Configs");
179
180


181
182
    // Loop chip configs
    for(json const& config : chipConfig){
183
        try {
184
            chipType = ScanHelper::buildChips(config, *bookie, &*hwCtrl, feCfgMap);
185
        } catch (std::runtime_error &e) {
186
            logger->critical("#ERROR# loading chip config: {}", e.what());
187
            return -1;
188
        }
Arisa Kubota's avatar
Arisa Kubota committed
189
        scanLog["connectivity"].push_back(config);
190
    }
191

Matthias Wittgen's avatar
Matthias Wittgen committed
192

193
    // Initial setting local DBHandler
194
    if (scanOpts.dbUse) {
Matthias Wittgen's avatar
Matthias Wittgen committed
195
        database = std::make_unique<DBHandler>();
196
197
198
        ScanHelper::banner(logger,"Set Database");
        database->initialize(scanOpts.dbCfgPath, scanOpts.progName, scanOpts.setQCMode, scanOpts.setInteractiveMode);
        if (database->checkConfigs(scanOpts.dbUserCfgPath, scanOpts.dbSiteCfgPath, scanOpts.cConfigPaths)==1)
199
200
201
202
            return -1;
        scanLog["dbCfg"] = dbCfg;
        scanLog["userCfg"] = userCfg;
        scanLog["siteCfg"] = siteCfg;
203
    }
204

205
    // Reset masks
206
    if (scanOpts.mask_opt == 1) {
207
208
        for (unsigned id=0; id<bookie->getNumOfEntries(); id++) {
            FrontEnd *fe = bookie->getEntry(id).fe;
209
            fe->enableAll();
210
        }
211
    }
212
213
214
215
    for (unsigned id=0; id<bookie->getNumOfEntries(); id++) {
        FrontEnd *fe = bookie->getEntry(id).fe;
        auto feCfg = dynamic_cast<FrontEndCfg*>(fe);
        ScanHelper::writeFeConfig(feCfg, scanOpts.outputDir + feCfgMap.at(id)[1] + ".before");
216
    }
217
218
219
    bookie->initGlobalFe(StdDict::getFrontEnd(chipType).release());
    bookie->getGlobalFe()->makeGlobal();
    bookie->getGlobalFe()->init(&*hwCtrl, 0, 0);
220

221
    ScanHelper::banner(logger,"Configure FEs");
222

223
    std::chrono::steady_clock::time_point cfg_start = std::chrono::steady_clock::now();
224
225
226

    // Before configuring each FE, broadcast reset to all tx channels
    // Enable all tx channels
227
    hwCtrl->setCmdEnable(bookie->getTxMaskUnique());
228
229
230
231
232

    // send global/broadcast reset command to all frontends
    if(scanOpts.doResetBeforeScan) {
        bookie->getGlobalFe()->resetAll();
    }
233

234
235
    for (unsigned id=0; id<bookie->getNumOfEntries(); id++) {
        FrontEnd *fe = bookie->getEntry(id).fe;
236
237
        auto feCfg = dynamic_cast<FrontEndCfg*>(fe);
        logger->info("Configuring {}", feCfg->getName());
238
        // Select correct channel
239
        hwCtrl->setCmdEnable(feCfg->getTxChannel());
240
241
242
        // Configure
        fe->configure();
        // Wait for fifo to be empty
243
        std::this_thread::sleep_for(std::chrono::microseconds(100));
Timon Heim's avatar
Timon Heim committed
244
        while(!hwCtrl->isCmdEmpty());
245
246
    }
    std::chrono::steady_clock::time_point cfg_end = std::chrono::steady_clock::now();
247
    logger->info("Sent configuration to all FEs in {} ms!",
Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
248
                 std::chrono::duration_cast<std::chrono::milliseconds>(cfg_end-cfg_start).count());
249

250
251
252
    // Wait for rx to sync with FE stream
    // TODO Check RX sync
    std::this_thread::sleep_for(std::chrono::microseconds(1000));
253
    hwCtrl->flushBuffer();
254
255
    int comCheckErrors = 0;
    int idCheckErrors = 0;
256
257
    for (unsigned id=0; id<bookie->getNumOfEntries(); id++) {
        FrontEnd *fe = bookie->getEntry(id).fe;
258
259
        auto feCfg = dynamic_cast<FrontEndCfg*>(fe);
        logger->info("Checking com {}", feCfg->getName());
260
        // Select correct channel
261
262
        hwCtrl->setCmdEnable(feCfg->getTxChannel());
        hwCtrl->setRxEnable(feCfg->getRxChannel());
263
        hwCtrl->checkRxSync(); // Must be done per fe (Aurora link) and after setRxEnable().
264
265

        // check communication with FE by reading back a register
Rafael Goncalves Gama's avatar
Rafael Goncalves Gama committed
266
        if (fe->checkCom() != 1) {
267
268
269
            logger->error("Can't establish communication, aborting!");
            comCheckErrors++;
            continue;
270
        }
271
272
273

        // check that the current FE name is valid
        if (!fe->hasValidName()) {
274
275
            logger->error("Invalid chip name, aborting!");
            idCheckErrors++;
276
        }
277
278
279
280
281
282
283

    }

    if (comCheckErrors > 0 || idCheckErrors > 0) {
        logger->critical("Could not establish correct communication or read incorrect efuse id from at least one FE! Abortin!");
        return -1;
    } else {
Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
284
        logger->info("... success!");
285
    }
286

287
    // at this point, if we're not running a scan we should just exit
288
    if(!scanOpts.scan_config_provided) {
289
290
291
        return 0;
    }

292
    // Enable all active channels
Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
293
    logger->info("Enabling Tx channels");
294
295
    hwCtrl->setCmdEnable(bookie->getTxMask());
    for (uint32_t channel : bookie->getTxMask()) {
Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
296
        logger->info("Enabling Tx channel {}", channel);
297
    }
Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
298
    logger->info("Enabling Rx channels");
299
300
    hwCtrl->setRxEnable(bookie->getRxMask());
    for (uint32_t channel : bookie->getRxMask()) {
Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
301
        logger->info("Enabling Rx channel {}", channel);
302
    }
303

Timon Heim's avatar
Timon Heim committed
304
    //hwCtrl->runMode();
305

306
    ScanHelper::banner(logger,"Setup Scan");
307

308
    // Make backup of scan config
309

310
    // Create backup of current config
311
    if (scanOpts.scanType.find("json") != std::string::npos) {
312
        // TODO fix folder
313
314
        std::ifstream cfgFile(scanOpts.scanType);
        std::ofstream backupCfgFile(scanOpts.outputDir + strippedScan + ".json");
315
316
317
318
319
        backupCfgFile << cfgFile.rdbuf();
        backupCfgFile.close();
        cfgFile.close();
    }

320
    // For sending feedback data
321
    FeedbackClipboardMap fbData;
322

323
    // TODO Make this nice
324
    try {
325
        scanBase = ScanHelper::buildScan(scanCfg, *bookie, &fbData);
326
    } catch (const char *msg) {
327
        logger->warn("No scan to run, exiting with msg: {}", msg);
328
329
        return 0;
    }
330

331
    // TODO not to use the raw pointer!
332
    try {
333
334
        ScanHelper::buildRawDataProcs(procs, *bookie, chipType);
        ScanHelper::buildHistogrammers(histogrammers, scanCfg, *bookie, scanBase.get(), scanOpts.outputDir);
335
        ScanHelper::buildAnalyses(analyses, scanCfg, *bookie, scanBase.get(),
Timon Heim's avatar
Timon Heim committed
336
                                  &fbData, scanOpts.mask_opt, scanOpts.outputDir);
337
338
339
340
    } catch (const char *msg) {
        logger->error("{}", msg);
        return -1;
    }
341
    
342

Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
343
    logger->info("Running pre scan!");
344
345
    scanBase->init();
    scanBase->preScan();
346

347
    // Run from downstream to upstream
Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
348
    logger->info("Starting histogrammer and analysis threads:");
349
350
    for (unsigned id=0; id<bookie->getNumOfEntries(); id++) {
        FrontEnd *fe = bookie->getEntry(id).fe;
351
        if (fe->isActive()) {
352
          for (auto& ana : analyses[id]) {
353
354
355
            ana->init();
            ana->run();
          }
356

357
358
          histogrammers[id]->init();
          histogrammers[id]->run();
359
          
360
361
          procs[id]->init();
          procs[id]->run();
362

Timon Heim's avatar
Timon Heim committed
363
          logger->info(" .. started threads of Fe {}", dynamic_cast<FrontEndCfg*>(fe)->getRxChannel());
364
365
        }
    }
366
    
367
368
    // Now the all downstream processors are ready --> Run scan

369
    ScanHelper::banner(logger,"Scan");
370

Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
371
    logger->info("Starting scan!");
372
    std::chrono::steady_clock::time_point scan_start = std::chrono::steady_clock::now();
373
374
    scanBase->run();
    scanBase->postScan();
Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
375
    logger->info("Scan done!");
376
377

    // Join from upstream to downstream.
378
379
    for (unsigned id=0; id<bookie->getNumOfEntries(); id++) {
        FrontEnd *fe = bookie->getEntry(id).fe;
380
        if (fe->isActive()) {
381
          fe->clipRawData.finish();
382
383
        }
    }
384

385
    std::chrono::steady_clock::time_point scan_done = std::chrono::steady_clock::now();
Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
386
    logger->info("Waiting for processors to finish ...");
387
388
389
390
391
392
    // Join DataProcessor
    // Join histogrammers
    for( auto& proc : procs ) {
      proc.second->join();
    }
    
393
    std::chrono::steady_clock::time_point processor_done = std::chrono::steady_clock::now();
Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
394
    logger->info("Processor done, waiting for histogrammer ...");
395

396
397
    for (unsigned id=0; id<bookie->getNumOfEntries(); id++) {
        FrontEnd *fe = bookie->getEntry(id).fe;
398
        if (fe->isActive()) {
399
            fe->clipData.finish();
400
401
        }
    }
402

403
404
405
406
    // Join histogrammers
    for( auto& histogrammer : histogrammers ) {
      histogrammer.second->join();
    }
407

Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
408
    logger->info("Processor done, waiting for analysis ...");
409

410
411
    for (unsigned id=0; id<bookie->getNumOfEntries(); id++) {
        FrontEnd *fe = bookie->getEntry(id).fe;
412
        if (fe->isActive()) {
413
            fe->clipHisto.finish();
414
415
416
        }
    }

417
418
    // Join analyses
    for( auto& ana : analyses ) {
419
      FrontEnd *fe = bookie->getEntry(ana.first).fe;
420
421
422
      for (unsigned i=0; i<ana.second.size(); i++) {
        ana.second[i]->join();
        // Also declare done for its output ClipBoard
423
        fe->clipResult.at(i)->finish();
424
      }
425
    }
426

427
    std::chrono::steady_clock::time_point all_done = std::chrono::steady_clock::now();
Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
428
    logger->info("All done!");
429

430
431
    // Joining is done.

432
433
    hwCtrl->disableCmd();
    hwCtrl->disableRx();
434

435
    ScanHelper::banner(logger,"Timing");
436

437
438
439
440
    logger->info("-> Configuration: {} ms", std::chrono::duration_cast<std::chrono::milliseconds>(cfg_end-cfg_start).count());
    logger->info("-> Scan:          {} ms", std::chrono::duration_cast<std::chrono::milliseconds>(scan_done-scan_start).count());
    logger->info("-> Processing:    {} ms", std::chrono::duration_cast<std::chrono::milliseconds>(processor_done-scan_done).count());
    logger->info("-> Analysis:      {} ms", std::chrono::duration_cast<std::chrono::milliseconds>(all_done-processor_done).count());
441

442
443
444
445
    scanLog["stopwatch"]["config"] = (uint32_t) std::chrono::duration_cast<std::chrono::milliseconds>(cfg_end-cfg_start).count();
    scanLog["stopwatch"]["scan"] = (uint32_t) std::chrono::duration_cast<std::chrono::milliseconds>(scan_done-scan_start).count();
    scanLog["stopwatch"]["processing"] = (uint32_t) std::chrono::duration_cast<std::chrono::milliseconds>(processor_done-scan_done).count();
    scanLog["stopwatch"]["analysis"] = (uint32_t) std::chrono::duration_cast<std::chrono::milliseconds>(all_done-processor_done).count();
446

447
    ScanHelper::banner(logger,"Cleanup");
Timon Heim's avatar
Timon Heim committed
448

449
450
    // Call constructor (eg shutdown Emu threads)
    hwCtrl.reset();
451
    scanLog["finishTime"] = (int)std::time(nullptr);
Matthias Wittgen's avatar
Matthias Wittgen committed
452
    ScanHelper::writeScanLog(scanLog, scanOpts.outputDir + "scanLog.json");
453

454
    // Cleanup
455
    //delete scanBase;
456
457
    for (unsigned id=0; id<bookie->getNumOfEntries(); id++) {
        FrontEnd *fe = bookie->getEntry(id).fe;
Matthias Wittgen's avatar
Matthias Wittgen committed
458
459
        if(!fe->isActive()) continue;
        auto feCfg = dynamic_cast<FrontEndCfg*>(fe);
Timon Heim's avatar
Timon Heim committed
460

Matthias Wittgen's avatar
Matthias Wittgen committed
461
462
        // Save config
        if (!feCfg->isLocked()) {
463
            const std::string &filename=feCfgMap.at(id)[0];
Matthias Wittgen's avatar
Matthias Wittgen committed
464
465
466
467
468
469
            logger->info("Saving config of FE {} to {}",
                         feCfg->getName(), filename);
            ScanHelper::writeFeConfig(feCfg, filename);
        } else {
            logger->warn("Not saving config for FE {} as it is protected!", feCfg->getName());
        }
Timon Heim's avatar
Timon Heim committed
470

Matthias Wittgen's avatar
Matthias Wittgen committed
471
        // Save extra config in data folder
472
        ScanHelper::writeFeConfig(feCfg, scanOpts.outputDir + feCfgMap.at(id)[1] + ".after");
Matthias Wittgen's avatar
Matthias Wittgen committed
473
474
475
476
477

        // Plot
        // store output results (if any)
        if(analyses.empty()) continue;
        logger->info("-> Storing output results of FE {}", feCfg->getRxChannel());
478
        auto &output = *(fe->clipResult.back());
Matthias Wittgen's avatar
Matthias Wittgen committed
479
480
481
482
483
484
485
486
487
488
489
490
        std::string name = feCfg->getName();
        if (output.empty()) {
            logger->warn(
                    "There were no results for chip {}, this usually means that the chip did not send any data at all.",
                    name);
            continue;
        }
        while(!output.empty()) {
            auto histo = output.popData();
            // only create the image files if asked to
            if(scanOpts.doPlots) {
                histo->plot(name, scanOpts.outputDir);
491
            }
Matthias Wittgen's avatar
Matthias Wittgen committed
492
493
494
            // always dump the data
            histo->toFile(name, scanOpts.outputDir);
        } // while
495
    } // i
Bruce Joseph Gallop's avatar
Bruce Joseph Gallop committed
496
    logger->info("Finishing run: {}", runCounter);
497
498
499
500
    if(scanOpts.doPlots) {
        bool ok = ScanHelper::lsdir(dataDir+ "last_scan/");
        if(!ok)
            logger->info("Find plots in: {}last_scan", dataDir);
501
    }
502

503
    // Register test info into database
504
505
    if (scanOpts.dbUse) {
        database->cleanUp("scan", scanOpts.outputDir, false, false);
506
507
    }

508
509
510
    return 0;
}