Commit 86136f57 authored by Aleksandra Mnich's avatar Aleksandra Mnich
Browse files

reference notebooks fixes

parent f6138075
Pipeline #3175405 passed with stage
in 26 seconds
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
%% Cell type:markdown id: tags:
%% Cell type:code id:2b3664d3 tags:
``` python
"""Manual spark configuration based on the default Spark configuration from the NXCALS bundle
and https://gitlab.cern.ch/msobiesz/spark-pipelines/-/blob/master/NXCALS-example-for-DAaaS.ipynb.
Used unless the spark context is already created. (outside of SWAN service or pyspark)
"""
if 'spark' not in locals() and 'spark' not in globals():
import os
from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession
nxcals_jars = os.getenv('NXCALS_JARS')
conf = SparkConf()
conf.set('spark.master', 'yarn')
conf.set("spark.driver.host", "spark-runner.cern.ch")
conf.set("spark.driver.port", '5001')
conf.set("spark.blockManager.port", '5101')
conf.set("spark.ui.port", '5201')
conf.set('spark.executorEnv.PYTHONPATH', os.getenv('PYTHONPATH'))
conf.set('spark.executorEnv.LD_LIBRARY_PATH', os.getenv('LD_LIBRARY_PATH'))
conf.set('spark.executorEnv.JAVA_HOME', os.getenv('JAVA_HOME'))
conf.set('spark.executorEnv.SPARK_HOME', os.getenv('SPARK_HOME'))
conf.set('spark.executorEnv.SPARK_EXTRA_CLASSPATH', os.getenv('SPARK_DIST_CLASSPATH'))
conf.set('spark.driver.extraClassPath', nxcals_jars)
conf.set('spark.executor.extraClassPath', nxcals_jars)
conf.set('spark.driver.extraJavaOptions',
"-Dservice.url=https://cs-ccr-nxcals5.cern.ch:19093,https://cs-ccr-nxcals5.cern.ch:19094,"
"https://cs-ccr-nxcals6.cern.ch:19093,https://cs-ccr-nxcals6.cern.ch:19094,"
"https://cs-ccr-nxcals7.cern.ch:19093,https://cs-ccr-nxcals7.cern.ch:19094,"
"https://cs-ccr-nxcals8.cern.ch:19093,https://cs-ccr-nxcals8.cern.ch:19094,"
"https://cs-ccr-nxcalsstr4.cern.ch:19093,https://cs-ccr-nxcalsstr5.cern.ch:19093")
sc = SparkContext(conf=conf)
spark = SparkSession(sc)
```
%% Cell type:markdown id:86e93e96 tags:
# Notebook to list QHD PM timestamps by circuit type and time range
%% Cell type:markdown id: tags:
%% Cell type:markdown id:6e01e66c tags:
# 0. Initialise Working Environment
%% Cell type:code id: tags:
%% Cell type:code id:ebe73a7a tags:
``` python
import os, sys, warnings
import pandas as pd
from IPython.display import display, HTML, Javascript, clear_output, Markdown
......@@ -28,15 +66,15 @@
analysis_start_time = Time.get_analysis_start_time()
lhcsmapi.get_lhcsmapi_version()
lhcsmapi.get_lhcsmhwc_version('../__init__.py')
```
%% Cell type:markdown id: tags:
%% Cell type:markdown id:61c76b35 tags:
# 1. User Input (circuit types: RB, RQ, IPQ, IPD, IT)
%% Cell type:code id: tags:
%% Cell type:code id:2d5e6653 tags:parameters
``` python
detailed_circuit_types = {
'RB': ['RB'],
'RQ': ['RQ'],
......@@ -48,22 +86,32 @@
start_time = '2021-10-18 07:00:00'
stop_time = '2021-10-20 23:01:00'
```
%% Cell type:code id: tags:
%% Cell type:code id:b095af0e tags:injected-parameters
``` python
# Parameters
start_time = "2021-10-18 07:00:00"
stop_time = "2021-10-20 23:01:00"
parametrized_marker = None
```
%% Cell type:code id:8f3fb7b2 tags:
``` python
print('start_time = ', start_time)
print('stop_time = ', stop_time)
```
%% Cell type:markdown id: tags:
%% Cell type:markdown id:5ad1ec68 tags:
# 2. Search for PMs
%% Cell type:code id: tags:
%% Cell type:code id:09ed9696 tags:
``` python
source_timestamp_qds_df = pd.DataFrame()
for circuit_type in detailed_circuit_types:
......@@ -72,16 +120,13 @@
if circuit_type == 'RQ':
circuits = circuits[0:7]
for circuit_name in circuits:
meta_circuit_type = circuit_type
if circuit_type == 'IPQ':
meta_circuit_type = SignalMetadata.get_circuit_type_for_circuit_name(circuit_name)
elif circuit_type == 'IPD':
if circuit_type == 'IPQ' or circuit_type == 'IPD':
meta_circuit_type = SignalMetadata.get_circuit_type_for_circuit_name(circuit_name)
source_timestamp_qds_df_i = QueryBuilder().with_pm() \
.with_duration(t_start=start_time, t_end=stop_time) \
.with_circuit_type(meta_circuit_type) \
.with_metadata(circuit_name=circuit_name, system='QH', source='*') \
.event_query() \
......@@ -95,34 +140,33 @@
if source_timestamp_qds_df.empty == False:
source_timestamp_qds_df['datetime'] = source_timestamp_qds_df.apply(lambda row: Time.to_string(row['timestamp']), axis=1)
```
%% Cell type:markdown id: tags:
%% Cell type:markdown id:763ab95e tags:
# 3. The list of QHD PM Timestamps, if any
%% Cell type:code id: tags:
%% Cell type:code id:53accbff tags:
``` python
if source_timestamp_qds_df.empty == False:
display(source_timestamp_qds_df)
warnings.warn('WARNING: To be checked that QH discharges has been accepted by QHDA-notebooks!', stacklevel=2)
warnings.warn('WARNINIG: To be checked that QH discharges has been accepted by QHDA-notebooks!', stacklevel=2)
else:
print('There were no QH discharges on selected time range!')
```
%%%% Output: display_data
%% Cell type:markdown id: tags:
%% Cell type:markdown id:ef492ad3 tags:
# 4. Save html-report
%% Cell type:code id: tags:
%% Cell type:raw id:223bacf3 tags:ignore
``` python
if source_timestamp_qds_df.empty == False:
# pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
analysis_start_time = Time.get_analysis_start_time()
date_time_qhd_pm_list = Time.to_datetime(start_time).strftime("%Y-%m-%d-%Hh%M")
......@@ -134,16 +178,6 @@
file_name_html = file_name + '.html'
full_path = '/eos/project/m/mp3/LHC_QHs/{}'.format(file_name_html)
print('Compact notebook report saved to (Windows): ' + '\\\\cernbox-smb' + full_path.replace('/', '\\'))
display(Javascript('IPython.notebook.save_notebook();'))
Time.sleep(5)
#!{sys.executable} -m jupyter nbconvert --to html $'HWC_QHD_PM_LIST.ipynb' --output /eos/project/m/mp3/LHC_QHs/$file_name_html --TemplateExporter.exclude_input=True --TagRemovePreprocessor.remove_all_outputs_tags='["skip_output"]' --TagRemovePreprocessor.remove_cell_tags='["skip_cell"]'
```
%%%% Output: display_data
%%%% Output: display_data
%%%% Output: display_data
!{sys.executable} -m jupyter nbconvert --to html $'HWC_QHD_PM_LIST.ipynb' --output /eos/project/m/mp3/LHC_QHs/$file_name_html --TemplateExporter.exclude_input=True --TagRemovePreprocessor.remove_all_outputs_tags='["skip_output"]' --TagRemovePreprocessor.remove_cell_tags='["skip_cell"]'
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment