Commit 86136f57 authored by Aleksandra Mnich's avatar Aleksandra Mnich
Browse files

reference notebooks fixes

parent f6138075
Pipeline #3175405 passed with stage
in 26 seconds
This diff is collapsed.
This diff is collapsed.
%% Cell type:markdown id: tags: %% Cell type:code id:2b3664d3 tags:
``` python
"""Manual spark configuration based on the default Spark configuration from the NXCALS bundle
and https://gitlab.cern.ch/msobiesz/spark-pipelines/-/blob/master/NXCALS-example-for-DAaaS.ipynb.
Used unless the spark context is already created. (outside of SWAN service or pyspark)
"""
if 'spark' not in locals() and 'spark' not in globals():
import os
from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession
nxcals_jars = os.getenv('NXCALS_JARS')
conf = SparkConf()
conf.set('spark.master', 'yarn')
conf.set("spark.driver.host", "spark-runner.cern.ch")
conf.set("spark.driver.port", '5001')
conf.set("spark.blockManager.port", '5101')
conf.set("spark.ui.port", '5201')
conf.set('spark.executorEnv.PYTHONPATH', os.getenv('PYTHONPATH'))
conf.set('spark.executorEnv.LD_LIBRARY_PATH', os.getenv('LD_LIBRARY_PATH'))
conf.set('spark.executorEnv.JAVA_HOME', os.getenv('JAVA_HOME'))
conf.set('spark.executorEnv.SPARK_HOME', os.getenv('SPARK_HOME'))
conf.set('spark.executorEnv.SPARK_EXTRA_CLASSPATH', os.getenv('SPARK_DIST_CLASSPATH'))
conf.set('spark.driver.extraClassPath', nxcals_jars)
conf.set('spark.executor.extraClassPath', nxcals_jars)
conf.set('spark.driver.extraJavaOptions',
"-Dservice.url=https://cs-ccr-nxcals5.cern.ch:19093,https://cs-ccr-nxcals5.cern.ch:19094,"
"https://cs-ccr-nxcals6.cern.ch:19093,https://cs-ccr-nxcals6.cern.ch:19094,"
"https://cs-ccr-nxcals7.cern.ch:19093,https://cs-ccr-nxcals7.cern.ch:19094,"
"https://cs-ccr-nxcals8.cern.ch:19093,https://cs-ccr-nxcals8.cern.ch:19094,"
"https://cs-ccr-nxcalsstr4.cern.ch:19093,https://cs-ccr-nxcalsstr5.cern.ch:19093")
sc = SparkContext(conf=conf)
spark = SparkSession(sc)
```
%% Cell type:markdown id:86e93e96 tags:
# Notebook to list QHD PM timestamps by circuit type and time range # Notebook to list QHD PM timestamps by circuit type and time range
%% Cell type:markdown id: tags: %% Cell type:markdown id:6e01e66c tags:
# 0. Initialise Working Environment # 0. Initialise Working Environment
%% Cell type:code id: tags: %% Cell type:code id:ebe73a7a tags:
``` python ``` python
import os, sys, warnings import os, sys, warnings
import pandas as pd import pandas as pd
from IPython.display import display, HTML, Javascript, clear_output, Markdown from IPython.display import display, HTML, Javascript, clear_output, Markdown
...@@ -28,15 +66,15 @@ ...@@ -28,15 +66,15 @@
analysis_start_time = Time.get_analysis_start_time() analysis_start_time = Time.get_analysis_start_time()
lhcsmapi.get_lhcsmapi_version() lhcsmapi.get_lhcsmapi_version()
lhcsmapi.get_lhcsmhwc_version('../__init__.py') lhcsmapi.get_lhcsmhwc_version('../__init__.py')
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id:61c76b35 tags:
# 1. User Input (circuit types: RB, RQ, IPQ, IPD, IT) # 1. User Input (circuit types: RB, RQ, IPQ, IPD, IT)
%% Cell type:code id: tags: %% Cell type:code id:2d5e6653 tags:parameters
``` python ``` python
detailed_circuit_types = { detailed_circuit_types = {
'RB': ['RB'], 'RB': ['RB'],
'RQ': ['RQ'], 'RQ': ['RQ'],
...@@ -48,22 +86,32 @@ ...@@ -48,22 +86,32 @@
start_time = '2021-10-18 07:00:00' start_time = '2021-10-18 07:00:00'
stop_time = '2021-10-20 23:01:00' stop_time = '2021-10-20 23:01:00'
``` ```
%% Cell type:code id: tags: %% Cell type:code id:b095af0e tags:injected-parameters
``` python
# Parameters
start_time = "2021-10-18 07:00:00"
stop_time = "2021-10-20 23:01:00"
parametrized_marker = None
```
%% Cell type:code id:8f3fb7b2 tags:
``` python ``` python
print('start_time = ', start_time) print('start_time = ', start_time)
print('stop_time = ', stop_time) print('stop_time = ', stop_time)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id:5ad1ec68 tags:
# 2. Search for PMs # 2. Search for PMs
%% Cell type:code id: tags: %% Cell type:code id:09ed9696 tags:
``` python ``` python
source_timestamp_qds_df = pd.DataFrame() source_timestamp_qds_df = pd.DataFrame()
for circuit_type in detailed_circuit_types: for circuit_type in detailed_circuit_types:
...@@ -72,16 +120,13 @@ ...@@ -72,16 +120,13 @@
if circuit_type == 'RQ': if circuit_type == 'RQ':
circuits = circuits[0:7] circuits = circuits[0:7]
for circuit_name in circuits: for circuit_name in circuits:
meta_circuit_type = circuit_type meta_circuit_type = circuit_type
if circuit_type == 'IPQ': if circuit_type == 'IPQ' or circuit_type == 'IPD':
meta_circuit_type = SignalMetadata.get_circuit_type_for_circuit_name(circuit_name)
elif circuit_type == 'IPD':
meta_circuit_type = SignalMetadata.get_circuit_type_for_circuit_name(circuit_name) meta_circuit_type = SignalMetadata.get_circuit_type_for_circuit_name(circuit_name)
source_timestamp_qds_df_i = QueryBuilder().with_pm() \ source_timestamp_qds_df_i = QueryBuilder().with_pm() \
.with_duration(t_start=start_time, t_end=stop_time) \ .with_duration(t_start=start_time, t_end=stop_time) \
.with_circuit_type(meta_circuit_type) \ .with_circuit_type(meta_circuit_type) \
.with_metadata(circuit_name=circuit_name, system='QH', source='*') \ .with_metadata(circuit_name=circuit_name, system='QH', source='*') \
.event_query() \ .event_query() \
...@@ -95,34 +140,33 @@ ...@@ -95,34 +140,33 @@
if source_timestamp_qds_df.empty == False: if source_timestamp_qds_df.empty == False:
source_timestamp_qds_df['datetime'] = source_timestamp_qds_df.apply(lambda row: Time.to_string(row['timestamp']), axis=1) source_timestamp_qds_df['datetime'] = source_timestamp_qds_df.apply(lambda row: Time.to_string(row['timestamp']), axis=1)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id:763ab95e tags:
# 3. The list of QHD PM Timestamps, if any # 3. The list of QHD PM Timestamps, if any
%% Cell type:code id: tags: %% Cell type:code id:53accbff tags:
``` python ``` python
if source_timestamp_qds_df.empty == False: if source_timestamp_qds_df.empty == False:
display(source_timestamp_qds_df) display(source_timestamp_qds_df)
warnings.warn('WARNING: To be checked that QH discharges has been accepted by QHDA-notebooks!', stacklevel=2) warnings.warn('WARNINIG: To be checked that QH discharges has been accepted by QHDA-notebooks!', stacklevel=2)
else: else:
print('There were no QH discharges on selected time range!') print('There were no QH discharges on selected time range!')
``` ```
%%%% Output: display_data %%%% Output: display_data
%% Cell type:markdown id: tags: %% Cell type:markdown id:ef492ad3 tags:
# 4. Save html-report # 4. Save html-report
%% Cell type:code id: tags: %% Cell type:raw id:223bacf3 tags:ignore
``` python
if source_timestamp_qds_df.empty == False: if source_timestamp_qds_df.empty == False:
# pd.set_option('display.max_columns', None) # pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None) pd.set_option('display.max_rows', None)
analysis_start_time = Time.get_analysis_start_time() analysis_start_time = Time.get_analysis_start_time()
date_time_qhd_pm_list = Time.to_datetime(start_time).strftime("%Y-%m-%d-%Hh%M") date_time_qhd_pm_list = Time.to_datetime(start_time).strftime("%Y-%m-%d-%Hh%M")
...@@ -134,16 +178,6 @@ ...@@ -134,16 +178,6 @@
file_name_html = file_name + '.html' file_name_html = file_name + '.html'
full_path = '/eos/project/m/mp3/LHC_QHs/{}'.format(file_name_html) full_path = '/eos/project/m/mp3/LHC_QHs/{}'.format(file_name_html)
print('Compact notebook report saved to (Windows): ' + '\\\\cernbox-smb' + full_path.replace('/', '\\')) print('Compact notebook report saved to (Windows): ' + '\\\\cernbox-smb' + full_path.replace('/', '\\'))
display(Javascript('IPython.notebook.save_notebook();')) display(Javascript('IPython.notebook.save_notebook();'))
Time.sleep(5) Time.sleep(5)
#!{sys.executable} -m jupyter nbconvert --to html $'HWC_QHD_PM_LIST.ipynb' --output /eos/project/m/mp3/LHC_QHs/$file_name_html --TemplateExporter.exclude_input=True --TagRemovePreprocessor.remove_all_outputs_tags='["skip_output"]' --TagRemovePreprocessor.remove_cell_tags='["skip_cell"]' !{sys.executable} -m jupyter nbconvert --to html $'HWC_QHD_PM_LIST.ipynb' --output /eos/project/m/mp3/LHC_QHs/$file_name_html --TemplateExporter.exclude_input=True --TagRemovePreprocessor.remove_all_outputs_tags='["skip_output"]' --TagRemovePreprocessor.remove_cell_tags='["skip_cell"]'
```
%%%% Output: display_data
%%%% Output: display_data
%%%% Output: display_data
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment