Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision
  • add-types
  • ci-run-on-release
  • dispatch-check
  • main
  • v0.1.0
  • v0.2.0
  • v0.3.0
  • v0.3.1
  • v0.4.0
  • v0.5.0
  • v0.5.1
  • v1.0.0
12 results

Target

Select target project
  • wotsubo/PSBoardDataBase
1 result
Select Git revision
  • add-types
  • ci-run-on-release
  • dispatch-check
  • main
  • v0.1.0
  • v0.2.0
  • v0.3.0
  • v0.3.1
  • v0.4.0
  • v0.5.0
  • v0.5.1
  • v1.0.0
12 results
Show changes
......@@ -100,10 +100,12 @@ function get_skew_and_riseup(file::T) where {T <: AbstractString}
skew = missing
is_raised = false
let
_time, high = _parse_line(popfirst!(lines))
last_low_time = let
time, high = _parse_line(popfirst!(lines))
if high == 0
last_low_time = time
time
else
missing
end
end
......
......@@ -88,7 +88,7 @@ function create_database_from_exported_csvs(
add_qaqc_single_result(db, single_result_df, runlist_table)
add_qaqc_dispatch(db, dispatch_table)
add_qaqc_runlist_from_masterlogs(db, masterlog_dir)
add_qaqc_100test_result(db, extra_100test_result_df)
add_qaqc_100test_result(db, extra_100test_result_df, slavelog_dir)
add_skew_from_slave_clk_logs(db, slavelog_dir)
add_slavelog_result(db, slavelog_dir)
......
......@@ -25,7 +25,7 @@ end
Fill qaqc_campaigns table in `db`.
"""
function insert_qaqc_campaign_id(db::SQLite.DB)
campaigns = [1, 2, 3, 4, 5, 6]
campaigns = [1, 2, 3, 4, 5, 6, 7]
dates = [
(DateTime(2024, 7, 22), DateTime(2024, 7, 24)),
(DateTime(2024, 8, 6), DateTime(2024, 8, 9)),
......@@ -33,6 +33,7 @@ function insert_qaqc_campaign_id(db::SQLite.DB)
(DateTime(2024, 9, 30), DateTime(2024, 10, 4)),
(DateTime(2024, 11, 11), DateTime(2024, 11, 14)),
(DateTime(2024, 12, 9), DateTime(2024, 12, 12)),
(DateTime(2025, 1, 20), DateTime(2024, 1, 23)),
]
stmt_insert_campaigns = DBInterface.prepare(
db,
......@@ -215,8 +216,10 @@ function get_campaign_id_from_run_id(runid::Integer)
4
elseif runid < 354
5
elseif runid < Inf # TODO: update this at the end of 6th campaign
elseif runid < 425
6
elseif 448 runid < Inf # TODO: update this at the end of 6th campaign
7
else
@error "Fix this function"
DomainError("runid $(runid) is not registered to the software")
......@@ -605,15 +608,20 @@ function get_num_tests_for_extra_runs(runid::Int64)
end
"""
add_qaqc_100test_result(db::SQLite.DB, table::DataFrame) -> nothing
add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::AbstractString) -> nothing
Fill `qaqc_extra_run_results` table in `db` from `table` DataFrame,
which is converted from a raw exported CSV.
# Args
- `table`: 100 test result table, prepared with [`prepare_100test_table`](@ref)
- `logs_dir`: where slave log files located in certain format
# Detail
- skips psboards in `resistance_test_passed` with `passed == false`
"""
function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame)
function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame, logs_dir::AbstractString)
position_id_map =
["B-$i-$j" for i in 0:1 for j in 1:9] |> enumerate .|> (x -> begin
(i, s) = x
......@@ -622,22 +630,25 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame)
table = prepare_100test_table(table)
stmt_search_runid = DBInterface.prepare(
db,
sql"""
SELECT id
FROM qaqc_runs
WHERE id = :runid
""",
)
stmt_search_resistance_error = DBInterface.prepare(
db,
sql"""
SELECT psb_id
FROM qaqc_resistance_check
WHERE psb_id = :psboard_id AND passed = 0
""",
)
qaqc_run_ids =
DBInterface.execute(
db,
sql"""
SELECT id
FROM qaqc_runs
""",
) |> Tables.columntable |> (t -> t.id)
resistance_error_psb_list =
DBInterface.execute(
db,
sql"""
SELECT psb_id
FROM qaqc_resistance_check
WHERE passed = 0
""",
) |>
Tables.columntable |>
(t -> t.psb_id)
stmt_insert_result = DBInterface.prepare(
db,
sql"""
......@@ -659,7 +670,8 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame)
bcid_fail,
invalid_register_value,
power_out_of_range,
note
note,
is_slavelog_valid
)
VALUES (
:runid,
......@@ -678,47 +690,61 @@ function add_qaqc_100test_result(db::SQLite.DB, table::DataFrame)
:bcid_fail,
:invalid_register_value,
:power_out_of_range,
:note
:note,
:is_slavelog_valid
)
""",
)
lock_db = ReentrantLock()
for row in eachrow(table)
if DBInterface.execute(stmt_search_runid, (; runid = row.runid)) |> isempty
Threads.@threads for row in eachrow(table)
if ismissing(row.runid) || !(row.runid in qaqc_run_ids)
# search for resistance error
if !isempty(
DBInterface.execute(
stmt_search_resistance_error,
(; psboard_id = row.motherboard_id),
),
)
if row.motherboard_id in resistance_error_psb_list
continue
end
error("Runid $(row.runid) not found in `qaqc_runs` table.")
end
DBInterface.execute(
stmt_insert_result,
(
runid = row.runid,
psboard_id = row.motherboard_id,
position = position_id_map[row.position],
num_tests = get_num_tests_for_extra_runs(row.runid),
insufficient_reset_with_10 = row.var"10回reset足りず",
reset_failed_though_reconfig_done = row.var"reconfig_done = 0なのにresetしていない",
always_hit_flag_true = row.var"always_hit_flag",
dac_is_0 = row.var"DAC = 0",
bcid_shift = row.var"DAC = 0",
efficiency_99percent = row.var"efficiency 99%",
bcid_fail_111 = row.var"BCID 0:0:0",
bcid_fail_000 = row.var"BCID 1:1:1",
low_efficiency = row.var"low efficiency",
bcid_fail = row.var"BCID fail",
invalid_register_value = row.var"invalid register values",
power_out_of_range = row.var"power out of range",
note = row.Column20,
),
)
is_slavelog_valid = try
SlaveLogParser.parse_slavelog_file(
joinpath(
logs_dir,
"main",
"$(row.motherboard_id)_$(row.runid)_longrun.txt",
),
)
true
catch e
@debug "Failed to parse slave log due to $(e)" catch_backtrace()
false
end
lock(lock_db) do
DBInterface.execute(
stmt_insert_result,
(
runid = row.runid,
psboard_id = row.motherboard_id,
position = position_id_map[row.position],
num_tests = get_num_tests_for_extra_runs(row.runid),
insufficient_reset_with_10 = row.var"10回reset足りず",
reset_failed_though_reconfig_done = row.var"reconfig_done = 0なのにresetしていない",
always_hit_flag_true = row.var"always_hit_flag",
dac_is_0 = row.var"DAC = 0",
bcid_shift = row.var"DAC = 0",
efficiency_99percent = row.var"efficiency 99%",
bcid_fail_111 = row.var"BCID 0:0:0",
bcid_fail_000 = row.var"BCID 1:1:1",
low_efficiency = row.var"low efficiency",
bcid_fail = row.var"BCID fail",
invalid_register_value = row.var"invalid register values",
power_out_of_range = row.var"power out of range",
note = row.Column20,
is_slavelog_valid,
),
)
end
end
nothing
......@@ -739,7 +765,7 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
db,
sql"""
UPDATE qaqc_single_run_results
SET lvds_tx_skew = :skew
SET lvds_tx_skew = :skew, is_slaveclocklog_valid = :is_slaveclocklog_valid
WHERE runid = :runid AND psboard_id = :psbid
""",
)
......@@ -757,9 +783,27 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
if m[:psbid] == "630" && m[:runid] == "190"
@debug "skipping... (psbid=630 runid=190 is broken)"
DBInterface.execute(
stmt_insert_skew_to_single_result,
(
skew = missing,
is_slaveclocklog_valid = false,
runid = m[:runid],
psbid = m[:psbid],
),
)
continue
elseif m[:psbid] == "627" && m[:runid] == "344"
@debug "skipping... (psbid=627 runid=344 is broken)"
DBInterface.execute(
stmt_insert_skew_to_single_result,
(
skew = missing,
is_slaveclocklog_valid = false,
runid = m[:runid],
psbid = m[:psbid],
),
)
continue
end
......@@ -772,7 +816,12 @@ function add_skew_from_slave_clk_logs(db::SQLite.DB, logs_dir::AbstractString)
DBInterface.execute(
stmt_insert_skew_to_single_result,
(skew = ClockParser.get_skew(file), runid = m[:runid], psbid = m[:psbid]),
(
skew = ClockParser.get_skew(file),
is_slaveclocklog_valid = true,
runid = m[:runid],
psbid = m[:psbid],
),
)
end
end
......@@ -787,8 +836,14 @@ Extract QAQC results from slave log files for single runs.
Slave log files are expected to located in certain format under `logs_dir`.
"""
function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
exclude_runs =
((runid = 51, reason = "clock only"), (runid = 175, reason = "broken files"))
exclude_runs = (
(runid = 51, psbid = nothing, reason = "clock only"),
(runid = 175, psbid = nothing, reason = "broken files"),
(runid = 437, psbid = 1215, reason = "PSBID 1215 is not completed"), # debug 6.5
(runid = 439, psbid = 703, reason = "PSBID 703 is not completed"), # debug 6.5
(runid = 434, psbid = 723, reason = "PSBID 723 is not completed"),
)
@assert eltype(exclude_runs) != Any
stmt_insert_slave_result_to_single_result = DBInterface.prepare(
db,
......@@ -797,7 +852,8 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
SET
power_3v3d = :power_3v3d,
power_3v3a = :power_3v3a,
power_n3va = :power_n3va
power_n3va = :power_n3va,
is_slavelog_valid = :is_slavelog_valid
WHERE
runid = :runid AND psboard_id = :psbid
""",
......@@ -823,23 +879,40 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
# exclusion
exclude_cond = Iterators.filter(exclude_runs) do cond
runid == cond.runid
exclude_cond = filter(exclude_runs) do cond
runid_matched = runid == cond.runid
psbid_matched = isnothing(cond.psbid) || cond.psbid == psbid
runid_matched && psbid_matched
end
if !isempty(exclude_cond)
@debug "skipping runid = $(runid) for $(first(exclude_cond).reason)"
DBInterface.execute(
stmt_insert_slave_result_to_single_result,
(;
power_3v3d = missing,
power_3v3a = missing,
power_n3va = missing,
is_slavelog_valid = false,
runid,
psbid,
),
)
continue
end
if !(runid in runids)
@debug "runid: $(runid) not in run list (psbid: $(psbid)). Parsing slave log to test its format."
slave_result = SlaveLogParser.parse_slavelog_file(file)
@debug "runid: $(runid) not in run list (psbid: $(psbid))"
continue
end
# main
slave_result = SlaveLogParser.parse_slavelog_file(file)
slave_result = try
SlaveLogParser.parse_slavelog_file(file)
catch e
throw(error("Failed to parse slave log file: $(file)\n$(e)"))
end
@assert length(slave_result.power) == 1 "Too many power results for single run"
......@@ -849,6 +922,7 @@ function add_slavelog_result(db::SQLite.DB, logs_dir::AbstractString)
power_3v3d = slave_result.power[1].result_3v3d,
power_3v3a = slave_result.power[1].result_3v3a,
power_n3va = slave_result.power[1].result_n3va,
is_slavelog_valid = true,
runid,
psbid,
),
......
......@@ -26,6 +26,8 @@ CREATE TABLE qaqc_single_run_results (
power_3v3a REAL,
power_n3va REAL,
note TEXT,
is_slavelog_valid BOOLEAN,
is_slaveclocklog_valid BOOLEAN,
FOREIGN KEY("runid") REFERENCES "qaqc_runs"("id"),
FOREIGN KEY("psboard_id") REFERENCES "ps_boards"("id"),
FOREIGN KEY("position") REFERENCES "qaqc_positions"("id")
......@@ -86,6 +88,7 @@ CREATE TABLE qaqc_extra_run_results (
invalid_register_value INTEGER,
power_out_of_range INTEGER,
note TEXT,
is_slavelog_valid BOOLEAN,
FOREIGN KEY("runid") REFERENCES "qaqc_runs"("id"),
FOREIGN KEY("psboard_id") REFERENCES "ps_boards"("id"),
FOREIGN KEY("position") REFERENCES "qaqc_positions"("id")
......
......@@ -243,7 +243,7 @@ true || include("../src/PSBoardDataBase.jl")
extra_100test_result_df =
CSV.read(PSBoardDataBase.DownloadCSVs.download_hundred_run_csv(), DataFrame)
@test PSBoardDataBase.add_qaqc_100test_result(db, extra_100test_result_df) |>
@test PSBoardDataBase.add_qaqc_100test_result(db, extra_100test_result_df, "input/slavelogs/") |>
isnothing
@test PSBoardDataBase.add_skew_from_slave_clk_logs(db, "input/slavelogs/") |>
......