Skip to content

GitLab

Commit c142ba6c authored by Aleksandra Mnich's avatar Aleksandra Mnich
Browse files

Merge branch 'notebook_ci_fix' into 'dev' 

reference notebooks fixes

See merge request !57
parents f6138075 86136f57
Pipeline #3176871 failed with stages
in 145 minutes and 11 seconds
Expand all Hide whitespace changes
Inline Side-by-side
test/resources/result_AN_RB_FPA.ipynb
View file @ c142ba6c
This diff is collapsed.
test/resources/result_AN_RQ_FPA.ipynb
View file @ c142ba6c
This diff is collapsed.
test/resources/result_HWC_QHD_PM_LIST.ipynb
View file @ c142ba6c
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "2b3664d3",
"metadata": {
"execution": {
"iopub.execute_input": "2021-10-27T21:41:02.645122Z",
"iopub.status.busy": "2021-10-27T21:41:02.644194Z",
"iopub.status.idle": "2021-10-27T21:41:39.944927Z",
"shell.execute_reply": "2021-10-27T21:41:39.943923Z"
},
"papermill": {
"duration": 37.359841,
"end_time": "2021-10-27T21:41:39.945145",
"exception": false,
"start_time": "2021-10-27T21:41:02.585304",
"status": "completed"
}
},
"outputs": [],
"source": [
"\"\"\"Manual spark configuration based on the default Spark configuration from the NXCALS bundle\n",
"and https://gitlab.cern.ch/msobiesz/spark-pipelines/-/blob/master/NXCALS-example-for-DAaaS.ipynb.\n",
"Used unless the spark context is already created. (outside of SWAN service or pyspark)\n",
"\"\"\"\n",
"if 'spark' not in locals() and 'spark' not in globals():\n",
" import os\n",
" from pyspark import SparkContext, SparkConf\n",
" from pyspark.sql import SparkSession\n",
"\n",
" nxcals_jars = os.getenv('NXCALS_JARS')\n",
" conf = SparkConf()\n",
" conf.set('spark.master', 'yarn')\n",
" conf.set(\"spark.driver.host\", \"spark-runner.cern.ch\")\n",
" conf.set(\"spark.driver.port\", '5001')\n",
" conf.set(\"spark.blockManager.port\", '5101')\n",
" conf.set(\"spark.ui.port\", '5201')\n",
" conf.set('spark.executorEnv.PYTHONPATH', os.getenv('PYTHONPATH'))\n",
" conf.set('spark.executorEnv.LD_LIBRARY_PATH', os.getenv('LD_LIBRARY_PATH'))\n",
" conf.set('spark.executorEnv.JAVA_HOME', os.getenv('JAVA_HOME'))\n",
" conf.set('spark.executorEnv.SPARK_HOME', os.getenv('SPARK_HOME'))\n",
" conf.set('spark.executorEnv.SPARK_EXTRA_CLASSPATH', os.getenv('SPARK_DIST_CLASSPATH'))\n",
" conf.set('spark.driver.extraClassPath', nxcals_jars)\n",
" conf.set('spark.executor.extraClassPath', nxcals_jars)\n",
" conf.set('spark.driver.extraJavaOptions',\n",
" \"-Dservice.url=https://cs-ccr-nxcals5.cern.ch:19093,https://cs-ccr-nxcals5.cern.ch:19094,\"\n",
" \"https://cs-ccr-nxcals6.cern.ch:19093,https://cs-ccr-nxcals6.cern.ch:19094,\"\n",
" \"https://cs-ccr-nxcals7.cern.ch:19093,https://cs-ccr-nxcals7.cern.ch:19094,\"\n",
" \"https://cs-ccr-nxcals8.cern.ch:19093,https://cs-ccr-nxcals8.cern.ch:19094,\"\n",
" \"https://cs-ccr-nxcalsstr4.cern.ch:19093,https://cs-ccr-nxcalsstr5.cern.ch:19093\")\n",
"\n",
" sc = SparkContext(conf=conf)\n",
" spark = SparkSession(sc)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"id": "86e93e96",
"metadata": {
"papermill": {
"duration": 0.043237,
"end_time": "2021-10-27T21:41:40.048851",
"exception": false,
"start_time": "2021-10-27T21:41:40.005614",
"status": "completed"
},
"tags": []
},
"source": [
"# Notebook to list QHD PM timestamps by circuit type and time range"
]
},
{
"cell_type": "markdown",
"metadata": {},
"id": "6e01e66c",
"metadata": {
"papermill": {
"duration": 0.043459,
"end_time": "2021-10-27T21:41:40.136848",
"exception": false,
"start_time": "2021-10-27T21:41:40.093389",
"status": "completed"
},
"tags": []
},
"source": [
"# 0. Initialise Working Environment"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"execution_count": 2,
"id": "ebe73a7a",
"metadata": {
"execution": {
"iopub.execute_input": "2021-10-27T21:41:40.236170Z",
"iopub.status.busy": "2021-10-27T21:41:40.235298Z",
"iopub.status.idle": "2021-10-27T21:41:45.929222Z",
"shell.execute_reply": "2021-10-27T21:41:45.929995Z"
},
"papermill": {
"duration": 5.749358,
"end_time": "2021-10-27T21:41:45.930320",
"exception": false,
"start_time": "2021-10-27T21:41:40.180962",
"status": "completed"
},
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Analysis executed with lhc-sm-api version: 1.5.17\n",
"Analysis executed with lhc-sm-hwc notebooks version: 1.5.65\n"
"Analysis executed with lhc-sm-hwc notebooks version: 1.5.66\n"
]
},
{
......@@ -76,15 +167,43 @@
},
{
"cell_type": "markdown",
"metadata": {},
"id": "61c76b35",
"metadata": {
"papermill": {
"duration": 0.039167,
"end_time": "2021-10-27T21:41:46.011951",
"exception": false,
"start_time": "2021-10-27T21:41:45.972784",
"status": "completed"
},
"tags": []
},
"source": [
"# 1. User Input (circuit types: RB, RQ, IPQ, IPD, IT)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"execution_count": 3,
"id": "2d5e6653",
"metadata": {
"execution": {
"iopub.execute_input": "2021-10-27T21:41:46.097084Z",
"iopub.status.busy": "2021-10-27T21:41:46.096250Z",
"iopub.status.idle": "2021-10-27T21:41:46.099544Z",
"shell.execute_reply": "2021-10-27T21:41:46.099002Z"
},
"papermill": {
"duration": 0.048872,
"end_time": "2021-10-27T21:41:46.099750",
"exception": false,
"start_time": "2021-10-27T21:41:46.050878",
"status": "completed"
},
"tags": [
"parameters"
]
},
"outputs": [],
"source": [
"detailed_circuit_types = {\n",
......@@ -101,8 +220,54 @@
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"execution_count": 4,
"id": "b095af0e",
"metadata": {
"execution": {
"iopub.execute_input": "2021-10-27T21:41:46.188975Z",
"iopub.status.busy": "2021-10-27T21:41:46.188167Z",
"iopub.status.idle": "2021-10-27T21:41:46.192141Z",
"shell.execute_reply": "2021-10-27T21:41:46.191154Z"
},
"papermill": {
"duration": 0.04905,
"end_time": "2021-10-27T21:41:46.192344",
"exception": false,
"start_time": "2021-10-27T21:41:46.143294",
"status": "completed"
},
"tags": [
"injected-parameters"
]
},
"outputs": [],
"source": [
"# Parameters\n",
"start_time = \"2021-10-18 07:00:00\"\n",
"stop_time = \"2021-10-20 23:01:00\"\n",
"parametrized_marker = None\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "8f3fb7b2",
"metadata": {
"execution": {
"iopub.execute_input": "2021-10-27T21:41:46.287901Z",
"iopub.status.busy": "2021-10-27T21:41:46.287093Z",
"iopub.status.idle": "2021-10-27T21:41:46.289713Z",
"shell.execute_reply": "2021-10-27T21:41:46.290214Z"
},
"papermill": {
"duration": 0.053989,
"end_time": "2021-10-27T21:41:46.290392",
"exception": false,
"start_time": "2021-10-27T21:41:46.236403",
"status": "completed"
},
"tags": []
},
"outputs": [
{
"name": "stdout",
......@@ -120,23 +285,54 @@
},
{
"cell_type": "markdown",
"metadata": {},
"id": "5ad1ec68",
"metadata": {
"papermill": {
"duration": 0.042729,
"end_time": "2021-10-27T21:41:46.376602",
"exception": false,
"start_time": "2021-10-27T21:41:46.333873",
"status": "completed"
},
"tags": []
},
"source": [
"# 2. Search for PMs"
]
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 6,
"id": "09ed9696",
"metadata": {
"scrolled": false
"execution": {
"iopub.execute_input": "2021-10-27T21:41:46.476179Z",
"iopub.status.busy": "2021-10-27T21:41:46.475307Z",
"iopub.status.idle": "2021-10-27T21:41:52.090267Z",
"shell.execute_reply": "2021-10-27T21:41:52.089513Z"
},
"papermill": {
"duration": 5.665805,
"end_time": "2021-10-27T21:41:52.090551",
"exception": false,
"start_time": "2021-10-27T21:41:46.424746",
"status": "completed"
},
"scrolled": false,
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"RB\n",
"RB\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" source timestamp\n",
"0 A28L3 1634542949913000000\n",
"1 B29L3 1634542949913000000\n",
......@@ -144,12 +340,36 @@
" source timestamp\n",
"0 B15R5 1634605470606000000\n",
"1 B15R5 1634607003034000000\n",
"2 B15R5 1634646052393000000\n",
"RQ\n",
"2 B15R5 1634646052393000000\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"RQ\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" source timestamp\n",
"0 27L3 1634542916538649604\n",
"IPQ\n",
"IPD\n",
"IPQ\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"IPD\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"IT\n"
]
}
......@@ -165,11 +385,8 @@
"\n",
" for circuit_name in circuits:\n",
" meta_circuit_type = circuit_type\n",
" if circuit_type == 'IPQ': \n",
" if circuit_type == 'IPQ' or circuit_type == 'IPD': \n",
" meta_circuit_type = SignalMetadata.get_circuit_type_for_circuit_name(circuit_name)\n",
" elif circuit_type == 'IPD': \n",
" meta_circuit_type = SignalMetadata.get_circuit_type_for_circuit_name(circuit_name)\n",
" \n",
" \n",
" source_timestamp_qds_df_i = QueryBuilder().with_pm() \\\n",
" .with_duration(t_start=start_time, t_end=stop_time) \\\n",
......@@ -190,15 +407,41 @@
},
{
"cell_type": "markdown",
"metadata": {},
"id": "763ab95e",
"metadata": {
"papermill": {
"duration": 0.051634,
"end_time": "2021-10-27T21:41:52.196288",
"exception": false,
"start_time": "2021-10-27T21:41:52.144654",
"status": "completed"
},
"tags": []
},
"source": [
"# 3. The list of QHD PM Timestamps, if any"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"execution_count": 7,
"id": "53accbff",
"metadata": {
"execution": {
"iopub.execute_input": "2021-10-27T21:41:52.310862Z",
"iopub.status.busy": "2021-10-27T21:41:52.310077Z",
"iopub.status.idle": "2021-10-27T21:41:52.322775Z",
"shell.execute_reply": "2021-10-27T21:41:52.322069Z"
},
"papermill": {
"duration": 0.07574,
"end_time": "2021-10-27T21:41:52.322965",
"exception": false,
"start_time": "2021-10-27T21:41:52.247225",
"status": "completed"
},
"tags": []
},
"outputs": [
{
"data": {
......@@ -308,74 +551,52 @@
"name": "stderr",
"output_type": "stream",
"text": [
"WARNING: To be checked that QH discharges has been accepted by QHDA-notebooks!\n"
"WARNINIG: To be checked that QH discharges has been accepted by QHDA-notebooks!\n"
]
}
],
"source": [
"if source_timestamp_qds_df.empty == False:\n",
" display(source_timestamp_qds_df)\n",
" warnings.warn('WARNING: To be checked that QH discharges has been accepted by QHDA-notebooks!', stacklevel=2)\n",
" warnings.warn('WARNINIG: To be checked that QH discharges has been accepted by QHDA-notebooks!', stacklevel=2)\n",
"else:\n",
" print('There were no QH discharges on selected time range!')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"id": "ef492ad3",
"metadata": {
"papermill": {
"duration": 0.057517,
"end_time": "2021-10-27T21:41:52.436373",
"exception": false,
"start_time": "2021-10-27T21:41:52.378856",
"status": "completed"
},
"tags": []
},
"source": [
"# 4. Save html-report"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<style>.container { width:95% !important; }</style>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<style>div.text_cell_render { padding: 1pt; } div#notebook p, div#notebook, div#notebook li, p { font-size: 9pt; line-height: 135%; margin: 0; } .rendered_html h1, .rendered_html h1:first-child { font-size: 14pt; margin: 7pt 0; } .rendered_html h2, .rendered_html h2:first-child { font-size: 12pt; margin: 6pt 0; } .rendered_html h3, .rendered_html h3:first-child { font-size: 10pt; margin: 6pt 0; } </style>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Compact notebook report saved to (Windows): \\\\cernbox-smb\\eos\\project\\m\\mp3\\LHC_QHs\\LHC_QHD_PM_LIST-2021-10-18-07h00-2021-10-25-15h20.html\n"
]
"cell_type": "raw",
"execution_count": null,
"id": "223bacf3",
"metadata": {
"papermill": {
"duration": 0.05694,
"end_time": "2021-10-27T21:41:52.549521",
"exception": false,
"start_time": "2021-10-27T21:41:52.492581",
"status": "completed"
},
{
"data": {
"application/javascript": [
"IPython.notebook.save_notebook();"
],
"text/plain": [
"<IPython.core.display.Javascript object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"tags": [
"ignore"
]
},
"outputs": [],
"source": [
"if source_timestamp_qds_df.empty == False:\n",
" # pd.set_option('display.max_columns', None)\n",
......@@ -392,11 +613,12 @@
" print('Compact notebook report saved to (Windows): ' + '\\\\\\\\cernbox-smb' + full_path.replace('/', '\\\\'))\n",
" display(Javascript('IPython.notebook.save_notebook();'))\n",
" Time.sleep(5)\n",
" #!{sys.executable} -m jupyter nbconvert --to html $'HWC_QHD_PM_LIST.ipynb' --output /eos/project/m/mp3/LHC_QHs/$file_name_html --TemplateExporter.exclude_input=True --TagRemovePreprocessor.remove_all_outputs_tags='[\"skip_output\"]' --TagRemovePreprocessor.remove_cell_tags='[\"skip_cell\"]'"
" !{sys.executable} -m jupyter nbconvert --to html $'HWC_QHD_PM_LIST.ipynb' --output /eos/project/m/mp3/LHC_QHs/$file_name_html --TemplateExporter.exclude_input=True --TagRemovePreprocessor.remove_all_outputs_tags='[\"skip_output\"]' --TagRemovePreprocessor.remove_cell_tags='[\"skip_cell\"]'"
]
}
],
"metadata": {
"celltoolbar": "Tags",
"kernelspec": {
"display_name": "Python 3",
"language": "python",
......@@ -414,6 +636,22 @@
"pygments_lexer": "ipython3",
"version": "3.8.6"
},
"papermill": {
"default_parameters": {},
"duration": 53.4518,
"end_time": "2021-10-27T21:41:53.223268",
"environment_variables": {},
"exception": null,
"input_path": "/builds/LHCData/lhc-sm-hwc/test/../qh/HWC_QHD_PM_LIST.ipynb",
"output_path": "results/result_HWC_QHD_PM_LIST.ipynb",
"parameters": {
"parametrized_marker": null,
"start_time": "2021-10-18 07:00:00",
"stop_time": "2021-10-20 23:01:00"
},
"start_time": "2021-10-27T21:40:59.771468",
"version": "2.3.3"
},
"sparkconnect": {
"bundled_options": [],
"list_of_options": []
......@@ -433,5 +671,5 @@
}
},
"nbformat": 4,
"nbformat_minor": 4
"nbformat_minor": 5
}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment