Skip to content
GitLab
Commit c142ba6c authored by Aleksandra Mnich's avatar Aleksandra Mnich
Browse files

Merge branch 'notebook_ci_fix' into 'dev' 

reference notebooks fixes

See merge request !57
parents f6138075 86136f57
Pipeline #3176871 failed with stages
in 145 minutes and 11 seconds
Expand all Hide whitespace changes
Inline Side-by-side
test/resources/result_AN_RB_FPA.ipynb
View file @ c142ba6c
This diff is collapsed.
test/resources/result_AN_RQ_FPA.ipynb
View file @ c142ba6c
This diff is collapsed.
test/resources/result_HWC_QHD_PM_LIST.ipynb
View file @ c142ba6c
%% Cell type:markdown id: tags:
%% Cell type:code id:2b3664d3 tags:
``` python
"""Manual spark configuration based on the default Spark configuration from the NXCALS bundle
and https://gitlab.cern.ch/msobiesz/spark-pipelines/-/blob/master/NXCALS-example-for-DAaaS.ipynb.
Used unless the spark context is already created. (outside of SWAN service or pyspark)
"""
if 'spark' not in locals() and 'spark' not in globals():
import os
from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession
nxcals_jars = os.getenv('NXCALS_JARS')
conf = SparkConf()
conf.set('spark.master', 'yarn')
conf.set("spark.driver.host", "spark-runner.cern.ch")
conf.set("spark.driver.port", '5001')
conf.set("spark.blockManager.port", '5101')
conf.set("spark.ui.port", '5201')
conf.set('spark.executorEnv.PYTHONPATH', os.getenv('PYTHONPATH'))
conf.set('spark.executorEnv.LD_LIBRARY_PATH', os.getenv('LD_LIBRARY_PATH'))
conf.set('spark.executorEnv.JAVA_HOME', os.getenv('JAVA_HOME'))
conf.set('spark.executorEnv.SPARK_HOME', os.getenv('SPARK_HOME'))
conf.set('spark.executorEnv.SPARK_EXTRA_CLASSPATH', os.getenv('SPARK_DIST_CLASSPATH'))
conf.set('spark.driver.extraClassPath', nxcals_jars)
conf.set('spark.executor.extraClassPath', nxcals_jars)
conf.set('spark.driver.extraJavaOptions',
"-Dservice.url=https://cs-ccr-nxcals5.cern.ch:19093,https://cs-ccr-nxcals5.cern.ch:19094,"
"https://cs-ccr-nxcals6.cern.ch:19093,https://cs-ccr-nxcals6.cern.ch:19094,"
"https://cs-ccr-nxcals7.cern.ch:19093,https://cs-ccr-nxcals7.cern.ch:19094,"
"https://cs-ccr-nxcals8.cern.ch:19093,https://cs-ccr-nxcals8.cern.ch:19094,"
"https://cs-ccr-nxcalsstr4.cern.ch:19093,https://cs-ccr-nxcalsstr5.cern.ch:19093")
sc = SparkContext(conf=conf)
spark = SparkSession(sc)
```
%% Cell type:markdown id:86e93e96 tags:
# Notebook to list QHD PM timestamps by circuit type and time range
%% Cell type:markdown id: tags:
%% Cell type:markdown id:6e01e66c tags:
# 0. Initialise Working Environment
%% Cell type:code id: tags:
%% Cell type:code id:ebe73a7a tags:
``` python
import os, sys, warnings
import pandas as pd
from IPython.display import display, HTML, Javascript, clear_output, Markdown
#
import lhcsmapi
from lhcsmapi.Time import Time
from lhcsmapi.Timer import Timer
from lhcsmapi.pyedsl.QueryBuilder import QueryBuilder
from lhcsmapi.analysis.RbCircuitQuery import RbCircuitQuery
from lhcsmapi.analysis.RqCircuitQuery import RqCircuitQuery
from lhcsmapi.analysis.IpqCircuitQuery import IpqCircuitQuery
from lhcsmapi.analysis.IpdCircuitQuery import IpdCircuitQuery
from lhcsmapi.analysis.ItCircuitQuery import ItCircuitQuery
from lhcsmapi.metadata.SignalMetadata import SignalMetadata
from lhcsmapi.analysis.report_template import apply_report_template
analysis_start_time = Time.get_analysis_start_time()
lhcsmapi.get_lhcsmapi_version()
lhcsmapi.get_lhcsmhwc_version('../__init__.py')
```
%% Output
Analysis executed with lhc-sm-api version: 1.5.17
Analysis executed with lhc-sm-hwc notebooks version: 1.5.65
Analysis executed with lhc-sm-hwc notebooks version: 1.5.66
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
%% Cell type:markdown id: tags:
%% Cell type:markdown id:61c76b35 tags:
# 1. User Input (circuit types: RB, RQ, IPQ, IPD, IT)
%% Cell type:code id: tags:
%% Cell type:code id:2d5e6653 tags:parameters
``` python
detailed_circuit_types = {
'RB': ['RB'],
'RQ': ['RQ'],
'IPQ': ['IPQ2', 'IPQ4', 'IPQ8'],
'IPD': ['IPD2', 'IPD2_B1B2'],
'IT': ['IT']
}
start_time = '2021-10-18 07:00:00'
stop_time = '2021-10-20 23:01:00'
```
%% Cell type:code id: tags:
%% Cell type:code id:b095af0e tags:injected-parameters
``` python
# Parameters
start_time = "2021-10-18 07:00:00"
stop_time = "2021-10-20 23:01:00"
parametrized_marker = None
```
%% Cell type:code id:8f3fb7b2 tags:
``` python
print('start_time = ', start_time)
print('stop_time = ', stop_time)
```
%% Output
start_time = 2021-10-18 07:00:00
stop_time = 2021-10-20 23:01:00
%% Cell type:markdown id: tags:
%% Cell type:markdown id:5ad1ec68 tags:
# 2. Search for PMs
%% Cell type:code id: tags:
%% Cell type:code id:09ed9696 tags:
``` python
source_timestamp_qds_df = pd.DataFrame()
for circuit_type in detailed_circuit_types:
print(f"{circuit_type}")
circuits = SignalMetadata.get_circuit_names(detailed_circuit_types[circuit_type])
if circuit_type == 'RQ':
circuits = circuits[0:7]
for circuit_name in circuits:
meta_circuit_type = circuit_type
if circuit_type == 'IPQ':
if circuit_type == 'IPQ' or circuit_type == 'IPD':
meta_circuit_type = SignalMetadata.get_circuit_type_for_circuit_name(circuit_name)
elif circuit_type == 'IPD':
meta_circuit_type = SignalMetadata.get_circuit_type_for_circuit_name(circuit_name)
source_timestamp_qds_df_i