Commit 86136f57 authored by Aleksandra Mnich's avatar Aleksandra Mnich
Browse files

reference notebooks fixes

parent f6138075
Pipeline #3175405 passed with stage
in 26 seconds
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
%% Cell type:markdown id: tags:
%% Cell type:code id:2b3664d3 tags:
``` python
"""Manual spark configuration based on the default Spark configuration from the NXCALS bundle
and https://gitlab.cern.ch/msobiesz/spark-pipelines/-/blob/master/NXCALS-example-for-DAaaS.ipynb.
Used unless the spark context is already created. (outside of SWAN service or pyspark)
"""
if 'spark' not in locals() and 'spark' not in globals():
import os
from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession
nxcals_jars = os.getenv('NXCALS_JARS')
conf = SparkConf()
conf.set('spark.master', 'yarn')
conf.set("spark.driver.host", "spark-runner.cern.ch")
conf.set("spark.driver.port", '5001')
conf.set("spark.blockManager.port", '5101')
conf.set("spark.ui.port", '5201')
conf.set('spark.executorEnv.PYTHONPATH', os.getenv('PYTHONPATH'))
conf.set('spark.executorEnv.LD_LIBRARY_PATH', os.getenv('LD_LIBRARY_PATH'))
conf.set('spark.executorEnv.JAVA_HOME', os.getenv('JAVA_HOME'))
conf.set('spark.executorEnv.SPARK_HOME', os.getenv('SPARK_HOME'))
conf.set('spark.executorEnv.SPARK_EXTRA_CLASSPATH', os.getenv('SPARK_DIST_CLASSPATH'))
conf.set('spark.driver.extraClassPath', nxcals_jars)
conf.set('spark.executor.extraClassPath', nxcals_jars)
conf.set('spark.driver.extraJavaOptions',
"-Dservice.url=https://cs-ccr-nxcals5.cern.ch:19093,https://cs-ccr-nxcals5.cern.ch:19094,"
"https://cs-ccr-nxcals6.cern.ch:19093,https://cs-ccr-nxcals6.cern.ch:19094,"
"https://cs-ccr-nxcals7.cern.ch:19093,https://cs-ccr-nxcals7.cern.ch:19094,"
"https://cs-ccr-nxcals8.cern.ch:19093,https://cs-ccr-nxcals8.cern.ch:19094,"
"https://cs-ccr-nxcalsstr4.cern.ch:19093,https://cs-ccr-nxcalsstr5.cern.ch:19093")
sc = SparkContext(conf=conf)
spark = SparkSession(sc)
```
%% Cell type:markdown id:86e93e96 tags:
# Notebook to list QHD PM timestamps by circuit type and time range
%% Cell type:markdown id: tags:
%% Cell type:markdown id:6e01e66c tags:
# 0. Initialise Working Environment
%% Cell type:code id: tags:
%% Cell type:code id:ebe73a7a tags:
``` python
import os, sys, warnings
import pandas as pd
from IPython.display import display, HTML, Javascript, clear_output, Markdown
#
import lhcsmapi
from lhcsmapi.Time import Time
from lhcsmapi.Timer import Timer
from lhcsmapi.pyedsl.QueryBuilder import QueryBuilder
from lhcsmapi.analysis.RbCircuitQuery import RbCircuitQuery
from lhcsmapi.analysis.RqCircuitQuery import RqCircuitQuery
from lhcsmapi.analysis.IpqCircuitQuery import IpqCircuitQuery
from lhcsmapi.analysis.IpdCircuitQuery import IpdCircuitQuery
from lhcsmapi.analysis.ItCircuitQuery import ItCircuitQuery
from lhcsmapi.metadata.SignalMetadata import SignalMetadata
from lhcsmapi.analysis.report_template import apply_report_template
analysis_start_time = Time.get_analysis_start_time()
lhcsmapi.get_lhcsmapi_version()
lhcsmapi.get_lhcsmhwc_version('../__init__.py')
```
%%%% Output: stream
Analysis executed with lhc-sm-api version: 1.5.17
Analysis executed with lhc-sm-hwc notebooks version: 1.5.65
Analysis executed with lhc-sm-hwc notebooks version: 1.5.66
%%%% Output: stream
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
%% Cell type:markdown id: tags:
%% Cell type:markdown id:61c76b35 tags:
# 1. User Input (circuit types: RB, RQ, IPQ, IPD, IT)
%% Cell type:code id: tags:
%% Cell type:code id:2d5e6653 tags:parameters
``` python
detailed_circuit_types = {
'RB': ['RB'],
'RQ': ['RQ'],
'IPQ': ['IPQ2', 'IPQ4', 'IPQ8'],
'IPD': ['IPD2', 'IPD2_B1B2'],
'IT': ['IT']
}
start_time = '2021-10-18 07:00:00'
stop_time = '2021-10-20 23:01:00'
```
%% Cell type:code id: tags:
%% Cell type:code id:b095af0e tags:injected-parameters
``` python
# Parameters
start_time = "2021-10-18 07:00:00"
stop_time = "2021-10-20 23:01:00"
parametrized_marker = None
```
%% Cell type:code id:8f3fb7b2 tags:
``` python
print('start_time = ', start_time)
print('stop_time = ', stop_time)
```
%%%% Output: stream
start_time = 2021-10-18 07:00:00
stop_time = 2021-10-20 23:01:00
%% Cell type:markdown id: tags:
%% Cell type:markdown id:5ad1ec68 tags:
# 2. Search for PMs
%% Cell type:code id: tags:
%% Cell type:code id:09ed9696 tags:
``` python
source_timestamp_qds_df = pd.DataFrame()
for circuit_type in detailed_circuit_types:
print(f"{circuit_type}")
circuits = SignalMetadata.get_circuit_names(detailed_circuit_types[circuit_type])
if circuit_type == 'RQ':
circuits = circuits[0:7]
for circuit_name in circuits:
meta_circuit_type = circuit_type
if circuit_type == 'IPQ':
meta_circuit_type = SignalMetadata.get_circuit_type_for_circuit_name(circuit_name)
elif circuit_type == 'IPD':
if circuit_type == 'IPQ' or circuit_type == 'IPD':
meta_circuit_type = SignalMetadata.get_circuit_type_for_circuit_name(circuit_name)
source_timestamp_qds_df_i = QueryBuilder().with_pm() \
.with_duration(t_start=start_time, t_end=stop_time) \
.with_circuit_type(meta_circuit_type) \
.with_metadata(circuit_name=circuit_name, system='QH', source='*') \
.event_query() \
.filter_source(circuit_name, 'QH') \
.sort_values(by='timestamp').df
if source_timestamp_qds_df_i.empty == False:
print(source_timestamp_qds_df_i)
source_timestamp_qds_df_i['circuit_type'] = source_timestamp_qds_df_i.apply(lambda row: circuit_type, axis=1)
source_timestamp_qds_df = pd.concat([source_timestamp_qds_df, source_timestamp_qds_df_i], ignore_index=True)
if source_timestamp_qds_df.empty == False:
source_timestamp_qds_df['datetime'] = source_timestamp_qds_df.apply(lambda row: Time.to_string(row['timestamp']), axis=1)
```
%%%% Output: stream
RB
%%%% Output: stream
source timestamp
0 A28L3 1634542949913000000
1 B29L3 1634542949913000000
2 C28L3 1634542949913000000
source timestamp
0 B15R5 1634605470606000000
1 B15R5 1634607003034000000
2 B15R5 1634646052393000000
%%%% Output: stream
RQ
%%%% Output: stream
source timestamp
0 27L3 1634542916538649604
IPQ
%%%% Output: stream
IPD
%%%% Output: stream
IT
%% Cell type:markdown id: tags:
%% Cell type:markdown id:763ab95e tags:
# 3. The list of QHD PM Timestamps, if any
%% Cell type:code id: tags:
%% Cell type:code id:53accbff tags:
``` python
if source_timestamp_qds_df.empty == False:
display(source_timestamp_qds_df)
warnings.warn('WARNING: To be checked that QH discharges has been accepted by QHDA-notebooks!', stacklevel=2)
warnings.warn('WARNINIG: To be checked that QH discharges has been accepted by QHDA-notebooks!', stacklevel=2)
else:
print('There were no QH discharges on selected time range!')
```
%%%% Output: display_data
%%%% Output: stream
WARNING: To be checked that QH discharges has been accepted by QHDA-notebooks!
WARNINIG: To be checked that QH discharges has been accepted by QHDA-notebooks!
%% Cell type:markdown id: tags:
%% Cell type:markdown id:ef492ad3 tags:
# 4. Save html-report
%% Cell type:code id: tags:
%% Cell type:raw id:223bacf3 tags:ignore
``` python
if source_timestamp_qds_df.empty == False:
# pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
analysis_start_time = Time.get_analysis_start_time()
date_time_qhd_pm_list = Time.to_datetime(start_time).strftime("%Y-%m-%d-%Hh%M")
!mkdir -p /eos/project/m/mp3/LHC_QHs
file_name = "LHC_QHD_PM_LIST-{}-{}".format(date_time_qhd_pm_list, analysis_start_time)
apply_report_template()
file_name_html = file_name + '.html'
full_path = '/eos/project/m/mp3/LHC_QHs/{}'.format(file_name_html)
print('Compact notebook report saved to (Windows): ' + '\\\\cernbox-smb' + full_path.replace('/', '\\'))
display(Javascript('IPython.notebook.save_notebook();'))
Time.sleep(5)
#!{sys.executable} -m jupyter nbconvert --to html $'HWC_QHD_PM_LIST.ipynb' --output /eos/project/m/mp3/LHC_QHs/$file_name_html --TemplateExporter.exclude_input=True --TagRemovePreprocessor.remove_all_outputs_tags='["skip_output"]' --TagRemovePreprocessor.remove_cell_tags='["skip_cell"]'
```
%%%% Output: display_data
%%%% Output: display_data
%%%% Output: stream
Compact notebook report saved to (Windows): \\cernbox-smb\eos\project\m\mp3\LHC_QHs\LHC_QHD_PM_LIST-2021-10-18-07h00-2021-10-25-15h20.html
%%%% Output: display_data
!{sys.executable} -m jupyter nbconvert --to html $'HWC_QHD_PM_LIST.ipynb' --output /eos/project/m/mp3/LHC_QHs/$file_name_html --TemplateExporter.exclude_input=True --TagRemovePreprocessor.remove_all_outputs_tags='["skip_output"]' --TagRemovePreprocessor.remove_cell_tags='["skip_cell"]'
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment