Commit 80679723 authored by Aleksandra Mnich's avatar Aleksandra Mnich
Browse files

[SIGMON-356] merge dev

parents 9ad20589 17e6ce2e
Pipeline #4155115 failed with stages
in 7 minutes and 43 seconds
......@@ -94,16 +94,6 @@ sonar:
script:
- bash CI/sonar.sh
deploy_production:
# Deploy package
stage: deploy
script:
- pip install twine
- python setup.py sdist
- twine upload --verbose --skip-existing dist/*
only:
- tags
deploy_production_eos:
stage: deploy
variables:
......@@ -209,6 +199,18 @@ deploy_pro_lhcsmnb_eos:
before_script: [ ]
after_script: [ ]
build_wheel:
stage: build
extends: .acc_py_build_wheel
only:
- tags
deploy_accpy:
stage: deploy
extends: .acc_py_release_wheel
dependencies:
- build_wheel
deploy_lhcsmanalysis_eos:
stage: deploy_analysis
variables:
......@@ -216,12 +218,11 @@ deploy_lhcsmanalysis_eos:
"VENV_PATH": "/eos/project/l/lhcsm/venv_${CI_COMMIT_BRANCH}"
"SPARK3_REQUIREMENTS_FILE": swan-nxcals-spark3-requirements.txt
"SCRIPT_FILENAME": "${CI_COMMIT_BRANCH}.sh"
"ANALYSIS_BRANCH": "${CI_COMMIT_BRANCH}"
only:
- branches
image: gitlab-registry.cern.ch/ci-tools/ci-web-deployer
script:
- bash CI/commons_deployment_setup.sh
- bash CI/eos_setup.sh
- bash CI/deploy_analysis.sh
before_script: [ ]
after_script: [ ]
......
#!/bin/bash
# This script is triggered by CI to clone a chosen branch of lhc-sm-analysis and copy it's contents to the venv on EOS.
# The project lhc-sm-analysis consists of several packages with sigmon analysis.
# This script is triggered by CI to clone a chosen branch of the lhc-sm-analysis repositories and copy their contents to the venv on EOS.
# The lhc-sm-analysis group contains several repositories with sigmon analysis packages.
# Contents of each package are copied to be accessible under a path /eos/project/l/lhcsm/venv_${CI_COMMIT_BRANCH}/lhcsmapi/api/
# set ANALYSIS_BRANCH if not set
export ANALYSIS_BRANCH="${ANALYSIS_BRANCH:master}"
export ANALYSIS_BRANCH="${ANALYSIS_BRANCH:-$CI_COMMIT_BRANCH}"
# check if branch exists
git ls-remote --exit-code --heads https://:@gitlab.cern.ch:8443/LHCData/lhc-sm-analysis.git "${ANALYSIS_BRANCH}"
if [ "$?" == "2" ]; then
echo "Branch '${ANALYSIS_BRANCH}' not found in lhc-sm-analysis, using 'master' instead"
export ANALYSIS_BRANCH='master'
fi;
yum install git -y
yum install jq -y
git clone --single-branch --branch $ANALYSIS_BRANCH https://:@gitlab.cern.ch:8443/LHCData/lhc-sm-analysis.git
echo "${EOS_ACCOUNT_PASSWORD}" | kinit "${EOS_ACCOUNT_USERNAME}@CERN.CH"
analysis_dirs=($(find lhc-sm-analysis -maxdepth 1 -mindepth 1 -type d | grep -v 'CI' | grep -v 'git'))
REPOS=$(curl -s https://gitlab.cern.ch/api/v4/groups/152156 | jq -r ".projects[].http_url_to_repo")
EOS_PATH=/eos/project/l/lhcsm/venv_${CI_COMMIT_BRANCH}/lhcsmapi/api/
for REPO in ${REPOS}; do
rsync="/usr/bin/rsync"
if [ ! -x $rsync ]
then
echo ERROR: $rsync not found
exit 1
fi
# extract the name
DIR=($(echo $REPO | cut -d/ -f6 | cut -d. -f1))
# SSH will be used to connect to LXPLUS and there check if the EOS folder exists
ssh="/usr/bin/ssh"
if [ ! -x $ssh ]
then
echo ERROR: $ssh not found
exit 1
fi
# check if branch exists
git ls-remote --exit-code --heads "${REPO}" "${ANALYSIS_BRANCH}"
if [ "$?" == "2" ]; then
echo "Branch '${ANALYSIS_BRANCH}' not found in ${DIR}, using 'dev' instead"
export ANALYSIS_BRANCH='dev'
fi;
# Copy contents of each directory to EOS
git clone --single-branch --branch $ANALYSIS_BRANCH "${REPO}"
for dir in "${analysis_dirs[@]}"; do
$rsync -abvuz -e "ssh -o StrictHostKeyChecking=no -o GSSAPIAuthentication=yes -o GSSAPITrustDNS=yes -o GSSAPIDelegateCredentials=yes" $dir/lhcsmapi/api/analysis $EOS_ACCOUNT_USERNAME@lxplus.cern.ch:$EOS_PATH/
EOS_PATH=/eos/project/l/lhcsm/venv_${CI_COMMIT_BRANCH}/lhcsmapi/api/
rsync="/usr/bin/rsync"
if [ ! -x $rsync ]
then
echo ERROR: $rsync not found
exit 1
fi
# SSH will be used to connect to LXPLUS and there check if the EOS folder exists
ssh="/usr/bin/ssh"
if [ ! -x $ssh ]
then
echo ERROR: $ssh not found
exit 1
fi
# Copy contents to EOS
$rsync -abvuz -e "ssh -o StrictHostKeyChecking=no -o GSSAPIAuthentication=yes -o GSSAPITrustDNS=yes -o GSSAPIDelegateCredentials=yes" $DIR/lhcsmapi/api/analysis $EOS_ACCOUNT_USERNAME@lxplus.cern.ch:$EOS_PATH/
if [ $? -ne 0 ]
then
echo ERROR: Rsync to \"$EOS_PATH\" via lxplus.cern.ch, failed
......
......@@ -212,7 +212,8 @@ def get_magnets_visualisation(circuit_type: str, circuit_name: str, results_tabl
# join with results_table
if 'timestamp_iqps' in results_table.columns:
sub_results_table = results_table[['Position', 'I_Q_M', 'timestamp_iqps']]
sub_results_table['datetime_iqps'] = results_table['timestamp_iqps'].apply(lambda col: Time.to_string_short(col))
sub_results_table['datetime_iqps'] = results_table['timestamp_iqps'].apply(
lambda col: Time.to_string_short(col))
else:
sub_results_table = results_table[['Position', 'I_Q_M']]
sub_results_table['datetime_iqps'] = np.nan
......@@ -490,4 +491,4 @@ def draw_schematic(circuit_name: str,
fig.update_shapes(dict(xref='x', yref='y'))
fig.layout.update(showlegend=False)
fig.show()
\ No newline at end of file
fig.show()
......@@ -18,7 +18,8 @@ def compare_features_to_reference(features_df: pd.DataFrame,
*,
wildcard=None,
timestamp=None,
nominal_voltage=900) -> pd.DataFrame:
nominal_voltage=900,
precision=6) -> pd.DataFrame:
""" Method comparing input features with reference features given as a [min, max] range of values for:
- Quench Heaters
......@@ -39,6 +40,8 @@ def compare_features_to_reference(features_df: pd.DataFrame,
during the operation)
:param nominal_voltage: nominal QH voltage (10, 300, 900); 900 by default
:type nominal_voltage: int
:param precision: floating point precision to which values are compared
:type precision: int
:return: True if features are within min/max range from reference features, otherwise False
"""
......@@ -61,8 +64,11 @@ def compare_features_to_reference(features_df: pd.DataFrame,
comparison_df = pd.concat([reference_df[common_columns], features_df[common_columns]], sort=True)
comparison_df_t = comparison_df.T
comparison_df_t['result'] = comparison_df_t.apply(lambda row:
(row['act'] >= row['min']) and (row['act'] <= row['max']),
def _round(floating):
return round(floating, precision)
comparison_df_t['result'] = comparison_df_t.apply(lambda row: (_round(row['act']) >= _round(row['min'])) and
(_round(row['act']) <= _round(row['max'])),
axis=1)
return comparison_df_t
......@@ -73,7 +79,8 @@ def compare_difference_of_features_to_reference(features_df: pd.DataFrame,
circuit_type: str,
system: str,
wildcard=None,
nominal_voltage=900) -> pd.DataFrame:
nominal_voltage=900,
precision=6) -> pd.DataFrame:
""" Method comparing the difference of the input features and the reference ones to an acceptance range
:param features_df: input features
......@@ -86,6 +93,8 @@ def compare_difference_of_features_to_reference(features_df: pd.DataFrame,
:param wildcard: wildcard in order to replace reference feature name
:param nominal_voltage: nominal QH voltage (10, 300, 900); 900 by default
:type nominal_voltage: int
:param precision: floating point precision to which values are compared
:type precision: int
:return: a DataFrame with comparison of features
"""
......@@ -110,7 +119,7 @@ def compare_difference_of_features_to_reference(features_df: pd.DataFrame,
def compare(row):
if np.isnan(row['diff']):
return np.nan
return round(abs(row['act'] - row['ref']), 6) <= row['diff']
return round(abs(row['act'] - row['ref']), precision) <= row['diff']
comparison_df = comparison_df.T
comparison_df['result'] = comparison_df.apply(compare, axis=1)
......
......@@ -309,7 +309,7 @@ def _filter_u_diode(dataframe: pd.DataFrame, circuit_name: str, source_qds: str)
if circuit_name.startswith('RQ'):
crate_name, signal_name = MappingMetadata.get_crate_and_diode_signal_name_for_rq(circuit_name, magnet_name)
if signal_name == 'U_REF_N1':
source_name = f'DQQDS.{crate_name}.{magnet_name}:U_REF_N1'
source_name = f'DQQDS.{crate_name}.{circuit_name}'
return dataframe.filter(regex=f'{source_name}:{signal_name}')
......
......@@ -1237,9 +1237,14 @@ class QdsRqAnalysis(QdsAnalysis):
ax[1][1].set_xlim((-0.1, 0.1))
plt.show()
def analyze_qds_run3(self, source_timestamp_qds_df: pd.DataFrame, circuit_names: List[str],
iqps_analog_dfs: List[pd.DataFrame], iqps_digital_dfs: List[pd.DataFrame],
u_nqps_rqd_dfs: List[pd.DataFrame], u_nqps_rqf_dfs: List[pd.DataFrame], start_point=100) -> None:
def analyze_qds_run3(self,
source_timestamp_qds_df: pd.DataFrame,
circuit_names: List[str],
iqps_analog_dfs: List[pd.DataFrame],
iqps_digital_dfs: List[pd.DataFrame],
u_nqps_rqd_dfs: List[pd.DataFrame],
u_nqps_rqf_dfs: List[pd.DataFrame],
start_point=100) -> None:
""" Method analyzing QDS signals in RQ circuit. It is plotting all signals and calculating:
- quench origin (RQD/RQF, INT/EXT)
- time derivative of the U_QS0 signal
......
......@@ -215,7 +215,8 @@ class QuenchHeaterVoltageAnalysis(CircuitAnalysis):
circuit_type,
'QH',
wildcard={wildcard_key: source_qh},
nominal_voltage=nominal_voltage)
nominal_voltage=nominal_voltage,
precision=1)
@staticmethod
def calculate_char_time(timestamp_qh, u_hds_dfs: List[pd.DataFrame]) -> pd.DataFrame:
......@@ -256,7 +257,8 @@ class QuenchHeaterVoltageAnalysis(CircuitAnalysis):
circuit_type,
'QH',
wildcard={wildcard_key: source_qh},
nominal_voltage=nominal_voltage)
nominal_voltage=nominal_voltage,
precision=3)
@staticmethod
def plot_voltage_with_ref(source_qh: str, timestamp_qh: int, timestamp_ref: int, u_hds_dfs: List[pd.DataFrame],
......@@ -294,13 +296,13 @@ class QuenchHeaterVoltageAnalysis(CircuitAnalysis):
:return: None
"""
row_color_dct = {False: 'background-color: red', True: ''}
display_html(first_last_u_comp_df.style.set_table_attributes("style='display:inline'") \
.set_caption('Comparison of the inititial and final voltage') \
.apply(lambda s: first_last_u_comp_df['result'].map(row_color_dct))._repr_html_() \
+ tau_comp_df.style.set_table_attributes("style='display:inline'") \
.set_caption('Comparison of the discharge characteristic time to the reference') \
.apply(lambda s: tau_comp_df['result'].map(row_color_dct))._repr_html_()
, raw=True)
display_html(first_last_u_comp_df.style.set_table_attributes("style='display:inline'").set_caption(
'Comparison of the inititial and final voltage').apply(
lambda s: first_last_u_comp_df['result'].map(row_color_dct)).set_precision(1).render() +
tau_comp_df.style.set_table_attributes("style='display:inline'").set_caption(
'Comparison of the discharge characteristic time to the reference').apply(
lambda s: tau_comp_df['result'].map(row_color_dct)).set_precision(3).render(),
raw=True)
def get_analysis_result(self):
print(self.analysis_result.toJSON())
......
......@@ -170,8 +170,11 @@ class QuenchHeaterVoltageCurrentAnalysis(QuenchHeaterVoltageAnalysis):
capacitance_df = self.calculate_capacitance(tau_df, first_r_df)
capacitance_ref_df = self.calculate_capacitance(tau_ref_df, first_r_ref_df)
capacitance_comp_df = comparison.compare_difference_of_features_to_reference(
capacitance_df, capacitance_ref_df, self.circuit_type, 'QH')
capacitance_comp_df = comparison.compare_difference_of_features_to_reference(capacitance_df,
capacitance_ref_df,
self.circuit_type,
'QH',
precision=3)
# Display comparison
self.display_features(first_last_u_comp_df, first_r_comp_df, tau_comp_df, capacitance_comp_df)
......@@ -282,18 +285,18 @@ class QuenchHeaterVoltageCurrentAnalysis(QuenchHeaterVoltageAnalysis):
"""
row_color_dct = {False: 'background-color: red', True: '', np.nan: ''}
display_html(first_last_u_comp_df.style.set_table_attributes("style='display:inline'").set_caption(
'Comparison of the inititial and final voltage').apply(lambda s: first_last_u_comp_df['result'] \
.map(row_color_dct))._repr_html_()
+ first_r_comp_df.style.set_table_attributes("style='display:inline'").set_caption(
'Comparison of the initial resisitance to the reference').apply(lambda s: first_r_comp_df['result'] \
.map(row_color_dct))._repr_html_()
+ tau_comp_df.style.set_table_attributes("style='display:inline'").set_caption(
'Comparison of the discharge characteristic time to the reference').apply(lambda s: tau_comp_df['result'] \
.map(row_color_dct))._repr_html_()
+ capacitance_comp_df.style.set_table_attributes("style='display:inline'").set_caption(
'Comparison of the estimated capacitance to the reference').apply(lambda s: capacitance_comp_df['result'] \
.map(row_color_dct))._repr_html_()
, raw=True)
'Comparison of the inititial and final voltage').apply(
lambda s: first_last_u_comp_df['result'].map(row_color_dct)).set_precision(1).render() +
first_r_comp_df.style.set_table_attributes("style='display:inline'").set_caption(
'Comparison of the initial resistance to the reference').apply(
lambda s: first_r_comp_df['result'].map(row_color_dct)).set_precision(2).render() +
tau_comp_df.style.set_table_attributes("style='display:inline'").set_caption(
'Comparison of the discharge characteristic time to the reference').apply(
lambda s: tau_comp_df['result'].map(row_color_dct)).set_precision(3).render() +
capacitance_comp_df.style.set_table_attributes("style='display:inline'").set_caption(
'Comparison of the estimated capacitance to the reference').apply(
lambda s: capacitance_comp_df['result'].map(row_color_dct)).set_precision(3).render(),
raw=True)
@staticmethod
def synchronize_raw_signal_to_decay(hds_dfs: List[pd.DataFrame], index_to_sync: float) -> List[pd.DataFrame]:
......@@ -451,7 +454,8 @@ class QuenchHeaterVoltageCurrentAnalysis(QuenchHeaterVoltageAnalysis):
circuit_type,
'QH',
wildcard={'CELL': source_qh},
nominal_voltage=nominal_voltage)
nominal_voltage=nominal_voltage,
precision=2)
@staticmethod
def calculate_capacitance(tau_df: pd.DataFrame, first_r_df: pd.DataFrame) -> pd.DataFrame:
......
from .analysis import Analysis, AnalysisManager, AnalysisError
from .analysis import Analysis, AnalysisManager, AnalysisError
......@@ -26,7 +26,8 @@ REQUIREMENTS: dict = {
'plotly',
'pyspark',
'numpy',
'nxcals>=1.0.29'
'nxcals>=1.0.29',
'pyarrow'
],
'test': ['pytest', 'sphinx', 'sphinx-rtd-theme'],
'dev': [
......
......@@ -186,10 +186,7 @@ class DataQueryTest(unittest.TestCase):
variable = 'v1'
input_dataframe = self.spark.createDataFrame([], StructType([]))
expected_result = pd.DataFrame(
pd.DataFrame({
variable: pd.Series(dtype='str')
}))
expected_result = pd.DataFrame(pd.DataFrame({variable: pd.Series(dtype='str')}))
# act
actual_result = transform_variables_dataset_to_pandas(input_dataframe, variable)
# assert
......@@ -529,17 +526,19 @@ def test_query_pm_events_with_parameters(mock_get):
pd.testing.assert_frame_equal(source_timestamp_ref_df, source_timestamp_act_df)
@mock.patch('requests.get', side_effect=mocked_pmdbsignal_requests_get)
def test_query_raw_pm_signals(mock_get):
system = 'FGC'
class_name = '51_self_pmd'
source = 'RPTE.UA47.RB.A45'
timestamp = 1426220469520000000
signal_name = ['STATUS.I_MEAS']
signal_name = ['STATUS.I_MEAS']
response = query_raw_pm_data(system, class_name, source, timestamp, signal_name)
assert signal_name[0] in response
def _mocked_pmdata_requests_get(*args, **kwargs):
path = Path(os.path.dirname(__file__))
......@@ -548,13 +547,11 @@ def _mocked_pmdata_requests_get(*args, **kwargs):
self.text = text
self.status_code = status_code
json_path = os.path.join(
path.parent, 'resources/dbsignal/post_mortem/'
'pm_data_mock.json')
json_path = os.path.join(path.parent, 'resources/dbsignal/post_mortem/' 'pm_data_mock.json')
with open(json_path) as json_file:
return MockedRequest(json.dumps(json.load(json_file)))
@mock.patch('requests.get', side_effect=_mocked_pmdata_requests_get)
def test_query_raw_pm_data(mock_get):
system = 'LBDS'
......@@ -563,4 +560,4 @@ def test_query_raw_pm_data(mock_get):
timestamp = 1430431203968406625
response = query_raw_pm_data(system, class_name, source, timestamp)
assert response['acquisitionStamp'] == timestamp
\ No newline at end of file
assert response['acquisitionStamp'] == timestamp
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment