diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6a03d741d5e3ecbd16a429ee1c8b58ae5cdbdf74..3382e6d153fd24c54fa947b93e6815d0e92dfe93 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -91,7 +91,6 @@ integration_tests: - pip install pytest pytest-cov - pytest test/integration --cov=lhcsmapi.api.analysis.qh --junitxml=report.xml --cov-report xml:cov.xml --cov-report term - mypy: stage: test script: diff --git a/lhcsmqh/analyses/commons.py b/lhcsmqh/analyses/commons.py index f1ca719388371281f385137860892e91423922cc..4d22bae8e619186f0b5b0f058955f9e89ab37518 100644 --- a/lhcsmqh/analyses/commons.py +++ b/lhcsmqh/analyses/commons.py @@ -88,12 +88,16 @@ class VoltageCurrentEvent(Event): i_hds_ref: List of reference current signals synchronised to the decay start r_hds: List of calculated resistance signals r_hds_ref: List of reference resistance signals + timestamp_decay: Index of the decay start + timestamp_decay_ref: Index of the reference decay start """ i_hds: list[pd.DataFrame] i_hds_ref: list[pd.DataFrame] r_hds: list[pd.DataFrame] r_hds_ref: list[pd.DataFrame] + decay_index: int + decay_index_ref: int def __eq__(self, other) -> bool: return ( @@ -103,6 +107,8 @@ class VoltageCurrentEvent(Event): and _check_list_dfs(self.i_hds_ref, other.i_hds_ref) and _check_list_dfs(self.r_hds, other.r_hds) and _check_list_dfs(self.r_hds_ref, other.r_hds_ref) + and self.decay_index == other.decay_index + and self.decay_index_ref == other.decay_index_ref ) @@ -241,14 +247,11 @@ def extract_decay(hds_dfs, index_decay_start): def preprocess_voltage_current( - u_hds_dfs: list[pd.DataFrame], i_hds_dfs: list[pd.DataFrame], current_offset: float, mean_start_value: float + u_hds_dfs: list[pd.DataFrame], i_hds_dfs: list[pd.DataFrame], current_offset: float, index_decay_start: float ) -> tuple[list[pd.DataFrame], list[pd.DataFrame]]: # Subtract current offset i_hds_no_offset_dfs = [i_hds_df - current_offset for i_hds_df in i_hds_dfs if not i_hds_df.empty] - # Find start of decay - index_decay_start = get_decay_start_index(i_hds_dfs, mean_start_value) - # Extract decay only u_hds_decay_dfs = extract_decay(u_hds_dfs, index_decay_start) i_hds_no_offset_decay_dfs = extract_decay(i_hds_no_offset_dfs, index_decay_start) @@ -284,6 +287,43 @@ def calculate_capacitance(tau_df, first_r_df): return tau_df_transposed.T +def analyze_voltage_current_event(voltage_current_event: VoltageCurrentEvent, discharge_level: int): + current_offset = 0.085 + + first_last_u_comp_df, first_r_comp_df, tau_comp_df, capacitance_comp_df = ( + analyze_single_qh_voltage_current_with_ref( + voltage_current_event.circuit_type, + discharge_level, + voltage_current_event.source, + voltage_current_event.timestamp, + voltage_current_event.u_hds, + voltage_current_event.i_hds, + voltage_current_event.u_hds_ref, + voltage_current_event.i_hds_ref, + voltage_current_event.decay_index, + voltage_current_event.decay_index_ref, + current_offset=current_offset, + ) + ) + + is_qh_ok = ( + first_last_u_comp_df["result"].all() + and first_r_comp_df["result"].all() + and tau_comp_df["result"].all() + and capacitance_comp_df["result"].all() + ) + + return VoltageCurrentResult( + voltage_current_event.source, + voltage_current_event.timestamp, + first_last_u_comp_df, + tau_comp_df, + is_qh_ok, + first_r_comp_df, + capacitance_comp_df, + ) + + def analyze_single_qh_voltage_current_with_ref( circuit_type: str, discharge_level: int, @@ -293,41 +333,16 @@ def analyze_single_qh_voltage_current_with_ref( i_hds_dfs: list[pd.DataFrame], u_hds_ref_dfs: list[pd.DataFrame], i_hds_ref_dfs: list[pd.DataFrame], + timestamp_decay: int, + timestamp_decay_ref: int, current_offset, - mean_start_value=50, -) -> tuple[ - pd.DataFrame, - pd.DataFrame, - pd.DataFrame, - pd.DataFrame, - list[pd.DataFrame], - list[pd.DataFrame], - list[pd.DataFrame], - list[pd.DataFrame], - list[pd.DataFrame], - list[pd.DataFrame], -]: +) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]: # Extract decay - u_hds_decay_dfs, i_hds_decay_dfs = preprocess_voltage_current( - u_hds_dfs, i_hds_dfs, current_offset, mean_start_value - ) + u_hds_decay_dfs, i_hds_decay_dfs = preprocess_voltage_current(u_hds_dfs, i_hds_dfs, current_offset, timestamp_decay) u_hds_decay_ref_dfs, i_hds_decay_ref_dfs = preprocess_voltage_current( - u_hds_ref_dfs, i_hds_ref_dfs, current_offset, mean_start_value + u_hds_ref_dfs, i_hds_ref_dfs, current_offset, timestamp_decay_ref ) - # Synchronize time of the raw signal from PM to 0 - # # For plotting - i_index_to_sync = i_hds_decay_dfs[0].index[0] - i_index_to_sync_ref = i_hds_decay_ref_dfs[0].index[0] - u_index_to_sync = u_hds_decay_dfs[0].index[0] - u_index_to_sync_ref = u_hds_decay_ref_dfs[0].index[0] - - u_hds_sync_dfs = [SignalIndexConversion.synchronize_df(hds_df, u_index_to_sync) for hds_df in u_hds_dfs] - i_hds_sync_dfs = [SignalIndexConversion.synchronize_df(hds_df, i_index_to_sync) for hds_df in i_hds_dfs] - - u_hds_ref_sync_dfs = [SignalIndexConversion.synchronize_df(hds_df, u_index_to_sync_ref) for hds_df in u_hds_ref_dfs] - i_hds_ref_sync_dfs = [SignalIndexConversion.synchronize_df(hds_df, i_index_to_sync_ref) for hds_df in i_hds_ref_dfs] - # # For feature engineering u_hds_decay_sync_dfs = SignalIndexConversion.synchronize_dfs(u_hds_decay_dfs) i_hds_decay_sync_dfs = SignalIndexConversion.synchronize_dfs(i_hds_decay_dfs) @@ -383,18 +398,7 @@ def analyze_single_qh_voltage_current_with_ref( capacitance_df, capacitance_ref_df, circuit_type, "QH", precision=3 ) - return ( - first_last_u_comp_df, - first_r_comp_df, - tau_comp_df, - capacitance_comp_df, - r_hds_dfs, - r_hds_ref_dfs, - u_hds_sync_dfs, - i_hds_sync_dfs, - u_hds_ref_sync_dfs, - i_hds_ref_sync_dfs, - ) + return first_last_u_comp_df, first_r_comp_df, tau_comp_df, capacitance_comp_df def find_source_timestamp_qh(circuit_type, circuit_name, start_time, stop_time): @@ -414,7 +418,7 @@ def find_source_timestamp_qh(circuit_type, circuit_name, start_time, stop_time): processing.EventProcessing(events) .filter_source(circuit_type, circuit_name, "QH") .sort_values(by=["timestamp", "source"]) - .drop_duplicates(column=["source", "timestamp"]) + .drop_duplicates(column=["source", "timestamp"]) # type: ignore .get_dataframe() ) events["circuit_type"] = circuit_type @@ -423,10 +427,7 @@ def find_source_timestamp_qh(circuit_type, circuit_name, start_time, stop_time): return events -def query_single_qh_event_pm(circuit_type, circuit_name, source, timestamp, signals, is_ref): - if is_ref: - meta_circuit_type = signal_metadata.get_circuit_type_for_circuit_name(circuit_name) - timestamp = reference.get_quench_heater_reference_discharge(meta_circuit_type, source, timestamp) +def query_single_qh_event_pm(circuit_type, circuit_name, source, timestamp, signals): pm_params = resolver.get_params_for_pm_signals( circuit_type, circuit_name, "QH", timestamp, signals=signals, wildcard={"CELL": source} @@ -444,16 +445,50 @@ def query_single_qh_event_pm(circuit_type, circuit_name, source, timestamp, sign ) -def query_qh_pm(circuit_type, circuit_name, events, signals, is_ref): - signals_dfs = [] +def __query_event(circuit_name: str, source: str, timestamp: int) -> Event: + circuit_type = signal_metadata.get_circuit_type_for_circuit_name(circuit_name) + reference_timestamp = reference.get_quench_heater_reference_discharge(circuit_type, source, timestamp) + u_hds = query_single_qh_event_pm(circuit_type, circuit_name, source, timestamp, "U_HDS") + u_hds_ref = query_single_qh_event_pm(circuit_type, circuit_name, source, reference_timestamp, "U_HDS") + + return Event( + source=source, + timestamp=timestamp, + circuit_type=circuit_type, + circuit_name=circuit_name, + u_hds=u_hds, + u_hds_ref=u_hds_ref, + ) + + +def query_voltage_event(circuit_name: str, source: str, timestamp: int) -> VoltageEvent: + event = __query_event(circuit_name, source, timestamp) + return VoltageEvent(**vars(event)) + - for _, row in events.iterrows(): - source = row["source"] - timestamp = row["timestamp"] +def query_voltage_current_event( + circuit_name: str, source: str, timestamp: int, discharge_level: float +) -> VoltageCurrentEvent: + circuit_type = signal_metadata.get_circuit_type_for_circuit_name(circuit_name) + reference_timestamp = reference.get_quench_heater_reference_discharge(circuit_type, source, timestamp) + mean_start_value = 15 if discharge_level < 450 else 50 - signals_dfs.append(query_single_qh_event_pm(circuit_type, circuit_name, source, timestamp, signals, is_ref)) + event = __query_event(circuit_name, source, timestamp) + i_hds = query_single_qh_event_pm(circuit_type, circuit_name, source, timestamp, "I_HDS") + r_hds = calculate_resistance(event.u_hds, i_hds) - return signals_dfs + i_hds_ref = query_single_qh_event_pm(circuit_type, circuit_name, source, reference_timestamp, "I_HDS") + r_hds_ref = calculate_resistance(event.u_hds_ref, i_hds_ref) + + return VoltageCurrentEvent( + **vars(event), + i_hds=i_hds, + i_hds_ref=i_hds_ref, + r_hds=r_hds, + r_hds_ref=r_hds_ref, + decay_index=get_decay_start_index(i_hds, mean_start_value), + decay_index_ref=get_decay_start_index(i_hds_ref, mean_start_value), + ) def query_single_qh_event_nxcals(spark, circuit_type, circuit_name, source, timestamp, duration, signals): @@ -488,6 +523,20 @@ def fill_value_at_location_0_with_latest_preceding(df: pd.DataFrame) -> pd.DataF return df.sort_index() +def analyze_voltage_event(voltage_event: VoltageEvent, discharge_level: float) -> VoltageResult: + first_last_u_comp_df, tau_u_comp_df = analyze_single_qh_voltage_with_ref( + voltage_event.u_hds, + voltage_event.u_hds_ref, + voltage_event.circuit_name, + voltage_event.timestamp, + discharge_level, + ) + + is_qh_ok = first_last_u_comp_df["result"].all() and tau_u_comp_df["result"].all() + + return VoltageResult(voltage_event.source, voltage_event.timestamp, first_last_u_comp_df, tau_u_comp_df, is_qh_ok) + + def analyze_single_qh_voltage_with_ref( u_dfs: list[pd.DataFrame], u_dfs_ref: list[pd.DataFrame], diff --git a/lhcsmqh/analyses/quench_heater_ccc.py b/lhcsmqh/analyses/quench_heater_ccc.py index db0aa36ea1549ec5a6c5cbe3a5bac858932fcca5..4f86e5a6104eca272aa44b50dc8212cec5e2debb 100644 --- a/lhcsmqh/analyses/quench_heater_ccc.py +++ b/lhcsmqh/analyses/quench_heater_ccc.py @@ -45,11 +45,8 @@ class QHCCCAnalysis(analysis.Analysis): if initial_charge_check: self._charge_check_level = 800 - self.voltage_current_events: list[commons.VoltageCurrentEvent] = [] - self.voltage_current_results: list[commons.VoltageCurrentResult] = [] - self.voltage_events: list[commons.VoltageEvent] = [] - self.voltage_results: list[commons.VoltageResult] = [] - self.low_charge_events: pd.DataFrame | None = None + self.events: list[commons.VoltageEvent | commons.VoltageCurrentEvent] = [] + self.results: list[commons.VoltageResult | commons.VoltageCurrentResult] = [] @property def initial_charge_check(self): @@ -75,206 +72,48 @@ class QHCCCAnalysis(analysis.Analysis): return qh_max_charge def query(self): - source_timestamp_qds_df = pd.DataFrame() + self.low_charge_events = pd.DataFrame(columns=["source", "timestamp"]) for circuit_type, detailed_circuit_types in commons.DETAILED_CIRCUIT_TYPES_MAP.items(): - circuit_names = signal_metadata.get_circuit_names(detailed_circuit_types) + circuit_names = signal_metadata.get_circuit_names(detailed_circuit_types) # type: ignore if circuit_type == GenericCircuitType.RQ: circuit_names = circuit_names[0:7] # RQFs and RQDs are the same from the powering point of view for circuit_name in circuit_names: - meta_circuit_type = signal_metadata.get_circuit_type_for_circuit_name(circuit_name) - source_timestamp_qds_df_i = commons.find_source_timestamp_qh( - meta_circuit_type, circuit_name, self._start_time, self._stop_time + source_timestamp_qds_df = commons.find_source_timestamp_qh( + signal_metadata.get_circuit_type_for_circuit_name(circuit_name), + circuit_name, + self._start_time, + self._stop_time, ) - if not source_timestamp_qds_df_i.empty: - source_timestamp_qds_df_i["circuit_type"] = circuit_type - source_timestamp_qds_df_i["circuit_name"] = circuit_name - source_timestamp_qds_df = pd.concat([source_timestamp_qds_df, source_timestamp_qds_df_i]) - - if not source_timestamp_qds_df.empty: - source_timestamp_qds_df["datetime"] = source_timestamp_qds_df.apply( - lambda row: Time.to_string(row["timestamp"]), axis=1 - ) - source_timestamp_qds_df = source_timestamp_qds_df[ - ["source", "timestamp", "datetime", "circuit_type", "circuit_name"] - ] - - if self._initial_charge_check: - source_timestamp_qds_df["max_charge"] = source_timestamp_qds_df.apply( - lambda row: self._get_qh_max_charge_from_nxcals(row.source, row.circuit_name, row.timestamp), axis=1 - ) - - self._source_timestamp_qds_df = source_timestamp_qds_df - - if not self._source_timestamp_qds_df.empty: - self._groups = source_timestamp_qds_df.groupby("circuit_name") - else: - self._groups = [] - - self._u_signals_dfs = {} - self._u_signals_dfs_ref = {} - self._i_signals_dfs = {} - self._i_signals_dfs_ref = {} - - for circuit_name, source_timestamp_df_i in self._groups: - circuit_type = source_timestamp_df_i["circuit_type"].values[0] - meta_circuit_type = signal_metadata.get_circuit_type_for_circuit_name(circuit_name) - - self._u_signals_dfs[circuit_name] = commons.query_qh_pm( - meta_circuit_type, circuit_name, source_timestamp_df_i, "U_HDS", False - ) - - self._u_signals_dfs_ref[circuit_name] = commons.query_qh_pm( - meta_circuit_type, circuit_name, source_timestamp_df_i, "U_HDS", True - ) - - # Current signals is only useful for RB and RQ circuit types - if circuit_type in ["RB", "RQ"]: - self._i_signals_dfs[circuit_name] = commons.query_qh_pm( - meta_circuit_type, circuit_name, source_timestamp_df_i, "I_HDS", False - ) - - self._i_signals_dfs_ref[circuit_name] = commons.query_qh_pm( - meta_circuit_type, circuit_name, source_timestamp_df_i, "I_HDS", True - ) - - def _analyze_voltage(self, circuit_type, circuit_name, source_timestamp_df_i) -> list[commons.VoltageResult]: - result = [] - u_signals = self._u_signals_dfs[circuit_name] - u_signals_ref = self._u_signals_dfs_ref[circuit_name] - - for i, row in source_timestamp_df_i.iterrows(): - timestamp = row["timestamp"] - if u_signals[i]: - self.voltage_events.append( - commons.VoltageEvent( - row["source"], timestamp, circuit_type, circuit_name, u_signals[i], u_signals_ref[i] - ) - ) - - first_last_u_comp_df, tau_u_comp_df = commons.analyze_single_qh_voltage_with_ref( - u_signals[i], u_signals_ref[i], circuit_name, timestamp, self._NOMINAL_VOLTAGE - ) - - is_qh_ok = first_last_u_comp_df["result"].all() and tau_u_comp_df["result"].all() - - result.append( - commons.VoltageResult(row["source"], timestamp, first_last_u_comp_df, tau_u_comp_df, is_qh_ok) - ) - else: - self.voltage_events.append( - commons.VoltageEvent(row["source"], timestamp, circuit_type, circuit_name, [], []) - ) - result.append(commons.VoltageResult(row["source"], timestamp, None, None, True)) # type: ignore - - return result - - def _analyze_voltage_current(self, circuit_type, circuit_name, source_timestamp_df_i): - result = [] - - u_signals = self._u_signals_dfs[circuit_name] - u_signals_ref = self._u_signals_dfs_ref[circuit_name] - i_signals = self._i_signals_dfs[circuit_name] - i_signals_ref = self._i_signals_dfs_ref[circuit_name] - - for i, row in source_timestamp_df_i.iterrows(): - if u_signals[i] and i_signals[i]: - ( - first_last_u_comp_df, - first_r_comp_df, - tau_comp_df, - capacitance_comp_df, - r_hds_dfs, - r_hds_ref_dfs, - u_sync_signals, - i_sync_signals, - u_sync_signals_ref, - i_sync_signals_ref, - ) = commons.analyze_single_qh_voltage_current_with_ref( - circuit_type, - self._NOMINAL_VOLTAGE, - row["source"], - row["timestamp"], - u_signals[i], - i_signals[i], - u_signals_ref[i], - i_signals_ref[i], - 0.085 if circuit_type == "RB" else 0.025, - ) - self.voltage_current_events.append( - commons.VoltageCurrentEvent( - row["source"], - row["timestamp"], - circuit_type, - circuit_name, - u_sync_signals, - u_sync_signals_ref, - i_sync_signals, - i_sync_signals_ref, - r_hds_dfs, - r_hds_ref_dfs, - ) - ) - - is_qh_ok = ( - first_last_u_comp_df["result"].all() - and first_r_comp_df["result"].all() - and tau_comp_df["result"].all() - ) - - result.append( - commons.VoltageCurrentResult( - row["source"], - row["timestamp"], - first_last_u_comp_df, - tau_comp_df, - is_qh_ok, - first_r_comp_df, - capacitance_comp_df, - ) - ) - else: - result.append( - commons.VoltageCurrentResult(row["source"], row["timestamp"], None, None, True, None, None) - ) - self.voltage_current_events.append( - commons.VoltageCurrentEvent( - row["source"], row["timestamp"], circuit_type, circuit_name, [], [], [], [], [], [] - ) - ) - - return result + for source, timestamp in source_timestamp_qds_df[["source", "timestamp"]].values: + if ( + self._initial_charge_check is True + and self._get_qh_max_charge_from_nxcals(source, circuit_name, timestamp) + < self._charge_check_level + ): + self.low_charge_events = pd.concat( + [self.low_charge_events, pd.DataFrame({"source": source, "timestamp": timestamp})], + ignore_index=True, + ) + else: + self.events.append( + commons.query_voltage_current_event(circuit_name, source, timestamp, self._NOMINAL_VOLTAGE) + if circuit_type in (GenericCircuitType.RB, GenericCircuitType.RQ) + else commons.query_voltage_event(circuit_name, source, timestamp) + ) def analyze(self): - if self._initial_charge_check and "max_charge" in self._source_timestamp_qds_df.columns: - self.low_charge_events = self._source_timestamp_qds_df[ - self._source_timestamp_qds_df["max_charge"] < self._charge_check_level - ] - - # We only analyze high charge events - self._source_timestamp_qds_df = self._source_timestamp_qds_df[ - self._source_timestamp_qds_df["max_charge"] >= self._charge_check_level - ] - - if not self._source_timestamp_qds_df.empty: - for circuit_name, source_timestamp_df_i in self._groups: - circuit_type = source_timestamp_df_i["circuit_type"].values[0] - - if circuit_type in ("RB", "RQ"): - self.voltage_current_results.extend( - self._analyze_voltage_current(circuit_type, circuit_name, source_timestamp_df_i) - ) - else: - self.voltage_results.extend( - self._analyze_voltage(circuit_type, circuit_name, source_timestamp_df_i) - ) - - self.events_number = len(self.voltage_current_events) + len(self.voltage_events) + self.results = [ + ( + commons.analyze_voltage_current_event(event, self._NOMINAL_VOLTAGE) + if isinstance(event, commons.VoltageCurrentEvent) + else commons.analyze_voltage_event(event, self._NOMINAL_VOLTAGE) + ) + for event in self.events + ] def get_analysis_output(self) -> bool: - return all(voltage_current_result.is_qh_ok for voltage_current_result in self.voltage_current_results) and all( - voltage_result.is_qh_ok for voltage_result in self.voltage_results - ) + return all(result.is_qh_ok for result in self.results) diff --git a/lhcsmqh/analyses/quench_heater_voltage_analysis.py b/lhcsmqh/analyses/quench_heater_voltage_analysis.py index e8e3f361b07f868c8a56863bef5819d004aaab6f..76cbdfc5f80f2a31b7e561b6a5793e8ef6161ec2 100644 --- a/lhcsmqh/analyses/quench_heater_voltage_analysis.py +++ b/lhcsmqh/analyses/quench_heater_voltage_analysis.py @@ -57,42 +57,15 @@ class QuenchHeaterVoltageAnalysis(analysis.Analysis): def query(self, discharges=None): self._events = discharges if discharges is not None else self.search_discharges() - self._signals_dfs = commons.query_qh_pm(self._circuit_type, self._circuit_name, self._events, "U_HDS", False) - self._signals_dfs_ref = commons.query_qh_pm(self._circuit_type, self._circuit_name, self._events, "U_HDS", True) - - def get_analysis_output(self) -> bool: - return all(voltage_result.is_qh_ok for voltage_result in self.voltage_results) + self.voltage_events = [ + commons.query_voltage_event(self._circuit_name, source, timestamp) + for source, timestamp in self._events[["source", "timestamp"]].values + ] def analyze(self): - for index, (u_dfs, u_dfs_ref) in enumerate(zip(self._signals_dfs, self._signals_dfs_ref)): - # logging here? - # the notebook prints some timestamp - - timestamp_event = None - - if "timestamp" in self._events.columns and index in self._events.index: - timestamp_event = self._events.at[index, "timestamp"] - - source_event = self._events.at[index, "source"] + self.voltage_results = [ + commons.analyze_voltage_event(voltage_event, self._discharge_level) for voltage_event in self.voltage_events + ] - if not u_dfs: - self.voltage_events.append( - commons.VoltageEvent(source_event, timestamp_event, self._circuit_type, self._circuit_name, [], []) - ) - self.voltage_results.append(commons.VoltageResult(source_event, timestamp_event, None, None, True)) - continue - - first_last_u_comp_df, tau_u_comp_df = commons.analyze_single_qh_voltage_with_ref( - u_dfs, u_dfs_ref, self._circuit_name, timestamp_event, self._discharge_level - ) - - is_qh_ok = first_last_u_comp_df["result"].all() and tau_u_comp_df["result"].all() - - self.voltage_events.append( - commons.VoltageEvent( - source_event, timestamp_event, self._circuit_type, self._circuit_name, u_dfs, u_dfs_ref - ) - ) - self.voltage_results.append( - commons.VoltageResult(source_event, timestamp_event, first_last_u_comp_df, tau_u_comp_df, is_qh_ok) - ) + def get_analysis_output(self) -> bool: + return all(voltage_result.is_qh_ok for voltage_result in self.voltage_results) diff --git a/lhcsmqh/analyses/quench_heater_voltage_current_analysis.py b/lhcsmqh/analyses/quench_heater_voltage_current_analysis.py index 148182c1a2b1bc3ae1c3d5c01bfa6847044d221f..56bbaaeeb800b12e529260722df728c4f3ae434c 100644 --- a/lhcsmqh/analyses/quench_heater_voltage_current_analysis.py +++ b/lhcsmqh/analyses/quench_heater_voltage_current_analysis.py @@ -42,10 +42,6 @@ class QuenchHeaterVoltageCurrentAnalysis(analysis.Analysis): self._start_time = Time.to_unix_timestamp(start_time) self._stop_time = Time.to_unix_timestamp(stop_time) self._discharge_level = discharge_level - self._signals_dfs = None - self._signals_dfs_ref = None - self.voltage_current_events: list[commons.VoltageCurrentEvent] = [] - self.voltage_current_results: list[commons.VoltageCurrentResult] = [] super().__init__(identifier) @@ -57,124 +53,20 @@ class QuenchHeaterVoltageCurrentAnalysis(analysis.Analysis): def query(self, discharges=None): self._events = discharges if discharges is not None else self.search_discharges() - self._clean_query_results() - with multiprocessing.Pool(8) as pool: - qh_source_timestamp_dfs = pool.starmap( - _query_task, - zip( - [self._circuit_type] * len(self._events), - [self._circuit_name] * len(self._events), - self._events["source"].values, - self._events["timestamp"].values, - [False] * len(self._events), - ), + with multiprocessing.Pool() as pool: + self.voltage_current_events = pool.starmap( + commons.query_voltage_current_event, + [ + (self._circuit_name, source, timestamp, self._discharge_level) + for source, timestamp in self._events[["source", "timestamp"]].values + ], ) - qh_source_timestamp_dfs_ref = pool.starmap( - _query_task, - zip( - [self._circuit_type] * len(self._events), - [self._circuit_name] * len(self._events), - self._events["source"].values, - self._events["timestamp"].values, - [True] * len(self._events), - ), - ) - - self._signals_dfs = {(source, timestamp): dfs for (source, timestamp, dfs) in qh_source_timestamp_dfs} - self._signals_dfs_ref = {(source, timestamp): dfs for (source, timestamp, dfs) in qh_source_timestamp_dfs_ref} - def analyze(self): - - for _, row in self._events.iterrows(): - source = row["source"] - timestamp = row["timestamp"] - - u_hds_dfs = list(filter(lambda df: "U_HDS" in df.columns.values[0], self._signals_dfs[(source, timestamp)])) - i_hds_dfs = list(filter(lambda df: "I_HDS" in df.columns.values[0], self._signals_dfs[(source, timestamp)])) - - if not u_hds_dfs or not i_hds_dfs: - self.voltage_current_events.append( - commons.VoltageCurrentEvent( - source, timestamp, self._circuit_type, self._circuit_name, None, None, None, None, None, None - ) - ) - self.voltage_current_results.append( - commons.VoltageCurrentResult(source, timestamp, None, None, True, None, None) - ) - continue - - u_hds_dfs_ref = list( - filter(lambda df: "U_HDS" in df.columns.values[0], self._signals_dfs_ref[(source, timestamp)]) - ) - i_hds_dfs_ref = list( - filter(lambda df: "I_HDS" in df.columns.values[0], self._signals_dfs_ref[(source, timestamp)]) - ) - - mean_start_value = 15 if self._discharge_level < 450 else 50 - - ( - first_last_u_comp_df, - first_r_comp_df, - tau_comp_df, - capacitance_comp_df, - r_hds_dfs, - r_hds_ref_dfs, - u_sync_signals, - i_sync_signals, - u_sync_signals_ref, - i_sync_signals_ref, - ) = commons.analyze_single_qh_voltage_current_with_ref( - self._circuit_type, - self._discharge_level, - source, - timestamp, - u_hds_dfs, - i_hds_dfs, - u_hds_dfs_ref, - i_hds_dfs_ref, - current_offset=0.085, - mean_start_value=mean_start_value, - ) - - is_qh_ok = ( - first_last_u_comp_df["result"].all() - and first_r_comp_df["result"].all() - and tau_comp_df["result"].all() - and capacitance_comp_df["result"].all() - ) - self.voltage_current_events.append( - commons.VoltageCurrentEvent( - source, - timestamp, - self._circuit_type, - self._circuit_name, - u_sync_signals, - u_sync_signals_ref, - i_sync_signals, - i_sync_signals_ref, - r_hds_dfs, - r_hds_ref_dfs, - ) - ) - self.voltage_current_results.append( - commons.VoltageCurrentResult( - source, timestamp, first_last_u_comp_df, tau_comp_df, is_qh_ok, first_r_comp_df, capacitance_comp_df - ) - ) + self.voltage_current_results = [ + commons.analyze_voltage_current_event(voltage_current_event, self._discharge_level) + for voltage_current_event in self.voltage_current_events + ] def get_analysis_output(self) -> bool: return all(voltage_current_result.is_qh_ok for voltage_current_result in self.voltage_current_results) - - def _clean_query_results(self): - self.voltage_current_events = [] - self.voltage_current_results = [] - if self._signals_dfs is not None: - del self._signals_dfs - if self._signals_dfs_ref is not None: - del self._signals_dfs_ref - - -def _query_task(circuit_type, circuit_name, source, timestamp, is_ref): - result = commons.query_single_qh_event_pm(circuit_type, circuit_name, source, timestamp, ["U_HDS", "I_HDS"], is_ref) - return source, timestamp, result diff --git a/lhcsmqh/notebooks/HWC_QHD_PM_LIST_CCC.ipynb b/lhcsmqh/notebooks/HWC_QHD_PM_LIST_CCC.ipynb index 45831abbbe9efb18b189e58b948dc35afe284cad..86767fc32e4f6097790167b3824cfc69779145d8 100644 --- a/lhcsmqh/notebooks/HWC_QHD_PM_LIST_CCC.ipynb +++ b/lhcsmqh/notebooks/HWC_QHD_PM_LIST_CCC.ipynb @@ -226,7 +226,7 @@ "# the inline backend makes all the plots uninteractive\n", "%matplotlib inline\n", "plt.close(\"all\")\n", - "if qhccc.events_number > 0:\n", + "if len(qhccc.results) > 0:\n", "\n", " pd.set_option(\"display.max_columns\", None)\n", " pd.set_option(\"display.max_rows\", None)\n", @@ -246,13 +246,6 @@ " Time.sleep(5)\n", " !{sys.executable} -m jupyter nbconvert --to html $path_to_notebook --output-dir $report_destination_path_template --output $html_filename --TemplateExporter.exclude_input=True" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/lhcsmqh/output/_common.py b/lhcsmqh/output/_common.py index 5524e879ef30bfa0108ead2a0e1d96c3c6607437..9e06769d3bfe9d4943e0eb88716b866ef0f9f8f4 100644 --- a/lhcsmqh/output/_common.py +++ b/lhcsmqh/output/_common.py @@ -30,14 +30,13 @@ def get_summary_output( } for event, result in zip(events, results) ) - df_result["datetime"] = df_result["datetime"].astype("datetime64[ms]") # RQs are in ns whereas other circuits in ms return [output_types.HTMLOutput(df_result.to_html())] def get_summary_with_links_output( events: Sequence[analyses.Event], results: Sequence[analyses.Result] ) -> list[output_types.Output]: - def get_row(event, result): + def get_row(event: analyses.Event, result: analyses.Result): return [ event.source, int(event.timestamp), @@ -50,7 +49,6 @@ def get_summary_with_links_output( columns = ["source", "timestamp", "datetime", "analysis_result", "link_to_analysis"] df_result = pd.DataFrame(rows, columns=columns) - df_result["datetime"] = df_result["datetime"].astype("datetime64[ms]") # RQs are in ns whereas other circuits in ms return [output_types.HTMLOutput("<a id=Table>Table</a>"), output_types.HTMLOutput(df_result.to_html(escape=False))] @@ -144,8 +142,8 @@ def get_voltage_current_output( fig, ax = plt.subplots(1, 3, figsize=(20, 7)) for u_hds, u_hds_ref in zip(event.u_hds, event.u_hds_ref): - u_hds.plot(ax=ax[0]) - u_hds_ref.plot(ax=ax[0], style="--") + u_hds.set_index(u_hds.index - u_hds.index[event.decay_index]).loc[0:].plot(ax=ax[0]) + u_hds_ref.set_index(u_hds_ref.index - u_hds_ref.index[event.decay_index_ref]).loc[0:].plot(ax=ax[0], style="--") ax[0].grid(True) ax[0].tick_params(labelsize=FONT_SIZE) @@ -153,8 +151,8 @@ def get_voltage_current_output( ax[0].set_ylabel("U_HDS, [V]", fontsize=FONT_SIZE) for i_hds, i_hds_ref in zip(event.i_hds, event.i_hds_ref): - i_hds.plot(ax=ax[1], title=title) - i_hds_ref.plot(ax=ax[1], style="--") + i_hds.set_index(i_hds.index - i_hds.index[event.decay_index]).loc[0:].plot(ax=ax[1], title=title) + i_hds_ref.set_index(i_hds_ref.index - i_hds_ref.index[event.decay_index_ref]).loc[0:].plot(ax=ax[1], style="--") ax[1].title.set_size(FONT_SIZE * 4 / 3) ax[1].grid(True) @@ -163,8 +161,8 @@ def get_voltage_current_output( ax[1].set_ylabel("I_HDS, [A]", fontsize=FONT_SIZE) for r_hds, r_hds_ref in zip(event.r_hds, event.r_hds_ref): - r_hds.plot(ax=ax[2]) - r_hds_ref.plot(ax=ax[2], style="--") + r_hds.set_index(r_hds.index - r_hds.index[event.decay_index]).loc[0:].plot(ax=ax[2]) + r_hds_ref.set_index(r_hds_ref.index - r_hds_ref.index[event.decay_index_ref]).loc[0:].plot(ax=ax[2], style="--") ax[2].grid(True) ax[2].tick_params(labelsize=FONT_SIZE) @@ -220,7 +218,7 @@ def get_output_wrapper( ) -> list[output_types.Output]: outputs: list[output_types.Output] = [] - if len(events) == 0: + if len(results) == 0: outputs.append( output_types.HTMLOutput( '<h2><span style="background-color: #3DCF1A">   ' diff --git a/lhcsmqh/output/quench_heater_ccc_output.py b/lhcsmqh/output/quench_heater_ccc_output.py index 5e80c0a06b5670b378f3e3b8af03ba90494d109e..820893b5fdda67e88479ca2834e419188e3abe49 100644 --- a/lhcsmqh/output/quench_heater_ccc_output.py +++ b/lhcsmqh/output/quench_heater_ccc_output.py @@ -23,9 +23,7 @@ def get_summary(analysis: QHCCCAnalysis) -> list[output_types.Output]: Returns: A list of Output instances with only one HTMLOutput.""" - events: list[commons.Event] = [*analysis.voltage_events, *analysis.voltage_current_events] - results: list[commons.Result] = [*analysis.voltage_results, *analysis.voltage_current_results] - return output_common.get_summary_output(events, results) + return output_common.get_summary_output(analysis.events, analysis.results) def get_output(analysis: QHCCCAnalysis) -> list[output_types.Output]: @@ -39,11 +37,11 @@ def get_output(analysis: QHCCCAnalysis) -> list[output_types.Output]: output_list: list[output_types.Output] = [] - if analysis.initial_charge_check and analysis.low_charge_events is not None: + if analysis.initial_charge_check and not analysis.low_charge_events.empty: output_list.append(output_types.TextOutput("Low level charge events:")) output_list.append(output_types.HTMLOutput(analysis.low_charge_events.to_html())) - if len(analysis.voltage_events) + len(analysis.voltage_current_events) == 0: + if len(analysis.results) == 0: output_list.append( output_types.HTMLOutput( '<h2><span style="background-color: #3DCF1A">   ' @@ -52,12 +50,16 @@ def get_output(analysis: QHCCCAnalysis) -> list[output_types.Output]: ) return output_list + voltage_events = [event for event in analysis.events if isinstance(event, commons.VoltageEvent)] + voltage_current_events = [event for event in analysis.events if isinstance(event, commons.VoltageCurrentEvent)] + voltage_results = [result for result in analysis.results if isinstance(result, commons.VoltageResult)] + voltage_current_results = [ + result for result in analysis.results if isinstance(result, commons.VoltageCurrentResult) + ] output_list.extend( - output_common.map_two_arg_func( - output_common.get_voltage_output, analysis.voltage_events, analysis.voltage_results - ) + output_common.map_two_arg_func(output_common.get_voltage_output, voltage_events, voltage_results) + output_common.map_two_arg_func( - output_common.get_voltage_current_output, analysis.voltage_current_events, analysis.voltage_current_results + output_common.get_voltage_current_output, voltage_current_events, voltage_current_results ) ) diff --git a/pyproject.toml b/pyproject.toml index 1083a6629c28b7737ad993ea40c0d2dda76838ae..2074054f3057995f2537a81183141eed93b97d0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,5 +63,4 @@ select = [ warn_unused_configs = true warn_redundant_casts = true warn_no_return = true -# TODO: Uncomment the following line when refactoring is done -# check_untyped_defs = true \ No newline at end of file +check_untyped_defs = true \ No newline at end of file diff --git a/test/integration/test_qh_ccc.py b/test/integration/test_qh_ccc.py index e3d4ed0ae99d1e8e007dbc25d8d924026960ec7c..45eed8ae5bbe2a16b8c54b8cef89328fc0f4f1f4 100644 --- a/test/integration/test_qh_ccc.py +++ b/test/integration/test_qh_ccc.py @@ -587,14 +587,14 @@ voltage_current_results = [ "ref": { "18R2:U_HDS_1:tau_charge": 0.08228129379948082, "18R2:U_HDS_2:tau_charge": 0.08260873643120471, - "18R2:I_HDS_1:tau_charge": 0.07297708077127855, - "18R2:I_HDS_2:tau_charge": 0.07318586569514528, + "18R2:I_HDS_1:tau_charge": 0.07274595818520692, + "18R2:I_HDS_2:tau_charge": 0.072953758005772, }, "act": { "18R2:U_HDS_1:tau_charge": 0.08328160074983856, "18R2:U_HDS_2:tau_charge": 0.083464569352813, - "18R2:I_HDS_1:tau_charge": 0.0737712434857393, - "18R2:I_HDS_2:tau_charge": 0.07395414775476841, + "18R2:I_HDS_1:tau_charge": 0.07353598109018604, + "18R2:I_HDS_2:tau_charge": 0.0737185207152869, }, "diff": { "18R2:U_HDS_1:tau_charge": 0.003, @@ -613,8 +613,8 @@ voltage_current_results = [ is_qh_ok=True, first_r_comp=pd.DataFrame( { - "ref": {"18R2:R_HDS_1:first20mean": 10.970921018587077, "18R2:R_HDS_2:first20mean": 11.014593851410535}, - "act": {"18R2:R_HDS_1:first20mean": 10.988526972404937, "18R2:R_HDS_2:first20mean": 11.012254867114663}, + "ref": {"18R2:R_HDS_1:first20mean": 10.978842043920086, "18R2:R_HDS_2:first20mean": 11.022597465304987}, + "act": {"18R2:R_HDS_1:first20mean": 10.996457287920506, "18R2:R_HDS_2:first20mean": 11.020212967470988}, "diff": {"18R2:R_HDS_1:first20mean": 0.5, "18R2:R_HDS_2:first20mean": 0.5}, "result": {"18R2:R_HDS_1:first20mean": True, "18R2:R_HDS_2:first20mean": True}, } @@ -622,12 +622,12 @@ voltage_current_results = [ capacitance_comp=pd.DataFrame( { "ref": { - "18R2_C_HDS_1:capacitance": 0.0074999440484603596, - "18R2_C_HDS_2:capacitance": 0.00749993486329283, + "18R2_C_HDS_1:capacitance": 0.007494532981740723, + "18R2_C_HDS_2:capacitance": 0.007494489088549782, }, "act": { - "18R2_C_HDS_1:capacitance": 0.007578959487379922, - "18R2_C_HDS_2:capacitance": 0.0075792442474301065, + "18R2_C_HDS_1:capacitance": 0.007573493768881595, + "18R2_C_HDS_2:capacitance": 0.007573770996910885, }, "diff": {"18R2_C_HDS_1:capacitance": nan, "18R2_C_HDS_2:capacitance": nan}, "result": {"18R2_C_HDS_1:capacitance": nan, "18R2_C_HDS_2:capacitance": nan}, @@ -670,14 +670,14 @@ voltage_current_results = [ "ref": { "29R7:U_HDS_1:tau_charge": 0.0810654593155952, "29R7:U_HDS_2:tau_charge": 0.08115745755466233, - "29R7:I_HDS_1:tau_charge": 0.07184079996529523, - "29R7:I_HDS_2:tau_charge": 0.07225868679788079, + "29R7:I_HDS_1:tau_charge": 0.0716098896009452, + "29R7:I_HDS_2:tau_charge": 0.07202313629191724, }, "act": { "29R7:U_HDS_1:tau_charge": 0.08223054220288487, "29R7:U_HDS_2:tau_charge": 0.08215291321952317, - "29R7:I_HDS_1:tau_charge": 0.0727587355253246, - "29R7:I_HDS_2:tau_charge": 0.0728522663977371, + "29R7:I_HDS_1:tau_charge": 0.07252638545750353, + "29R7:I_HDS_2:tau_charge": 0.0726169312592807, }, "diff": { "29R7:U_HDS_1:tau_charge": 0.003, @@ -696,8 +696,8 @@ voltage_current_results = [ is_qh_ok=True, first_r_comp=pd.DataFrame( { - "ref": {"29R7:R_HDS_1:first20mean": 10.81556434898576, "29R7:R_HDS_2:first20mean": 10.962061384576465}, - "act": {"29R7:R_HDS_1:first20mean": 10.818035463648254, "29R7:R_HDS_2:first20mean": 10.92488589559435}, + "ref": {"29R7:R_HDS_1:first20mean": 10.823373491486715, "29R7:R_HDS_2:first20mean": 10.97014105731522}, + "act": {"29R7:R_HDS_1:first20mean": 10.825742866517423, "29R7:R_HDS_2:first20mean": 10.932778999765421}, "diff": {"29R7:R_HDS_1:first20mean": 0.5, "29R7:R_HDS_2:first20mean": 0.5}, "result": {"29R7:R_HDS_1:first20mean": True, "29R7:R_HDS_2:first20mean": True}, } @@ -705,12 +705,12 @@ voltage_current_results = [ capacitance_comp=pd.DataFrame( { "ref": { - "29R7_C_HDS_1:capacitance": 0.007495259304078496, - "29R7_C_HDS_2:capacitance": 0.007403485047880706, + "29R7_C_HDS_1:capacitance": 0.007489851420110231, + "29R7_C_HDS_2:capacitance": 0.007398032270564479, }, "act": { - "29R7_C_HDS_1:capacitance": 0.007601245390552047, - "29R7_C_HDS_2:capacitance": 0.007519795996464618, + "29R7_C_HDS_1:capacitance": 0.0075958336731988116, + "29R7_C_HDS_2:capacitance": 0.007514366952929889, }, "diff": {"29R7_C_HDS_1:capacitance": nan, "29R7_C_HDS_2:capacitance": nan}, "result": {"29R7_C_HDS_1:capacitance": nan, "29R7_C_HDS_2:capacitance": nan}, @@ -740,5 +740,7 @@ def test_acceptance_ccc( # assert assert analysis.get_analysis_output() is True - assert analysis.voltage_results == voltage_results - assert analysis.voltage_current_results == voltage_current_results + assert [result for result in analysis.results if isinstance(result, VoltageResult)] == voltage_results + assert [ + result for result in analysis.results if isinstance(result, VoltageCurrentResult) + ] == voltage_current_results diff --git a/test/unit/test_output.py b/test/unit/test_output.py index 873b07e17dd02e4a73438b0241489324887a13c2..7e1557969d6371f06bbf81f76da95e4cfbad7193 100644 --- a/test/unit/test_output.py +++ b/test/unit/test_output.py @@ -118,6 +118,8 @@ voltage_current_events = [ i_hds_ref=MagicMock(), r_hds=MagicMock(), r_hds_ref=MagicMock(), + decay_index=MagicMock(), + decay_index_ref=MagicMock(), ), commons.VoltageCurrentEvent( source="A30R7", @@ -130,6 +132,8 @@ voltage_current_events = [ i_hds_ref=MagicMock(), r_hds=MagicMock(), r_hds_ref=MagicMock(), + decay_index=MagicMock(), + decay_index_ref=MagicMock(), ), commons.VoltageCurrentEvent( source="B31R7", @@ -142,6 +146,8 @@ voltage_current_events = [ i_hds_ref=MagicMock(), r_hds=MagicMock(), r_hds_ref=MagicMock(), + decay_index=MagicMock(), + decay_index_ref=MagicMock(), ), ] @@ -600,10 +606,8 @@ voltage_current_results = [ def test_qh_ccc_output(): # arrange analysis = analyses.QHCCCAnalysis("", 0, 1, False) - analysis.voltage_events = voltage_events - analysis.voltage_results = voltage_results - analysis.voltage_current_events = voltage_current_events - analysis.voltage_current_results = voltage_current_results + analysis.events = voltage_events + voltage_current_events + analysis.results = voltage_results + voltage_current_results # act output = quench_heater_ccc_output.get_output(analysis) @@ -615,10 +619,8 @@ def test_qh_ccc_output(): def test_qh_ccc_summary(): # arrange analysis = analyses.QHCCCAnalysis("", 0, 1, False) - analysis.voltage_events = voltage_events - analysis.voltage_results = voltage_results - analysis.voltage_current_events = voltage_current_events - analysis.voltage_current_results = voltage_current_results + analysis.events = voltage_events + voltage_current_events + analysis.results = voltage_results + voltage_current_results # act output = quench_heater_ccc_output.get_summary(analysis)