{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "2b3664d3", "metadata": { "execution": { "iopub.execute_input": "2021-10-27T21:41:02.645122Z", "iopub.status.busy": "2021-10-27T21:41:02.644194Z", "iopub.status.idle": "2021-10-27T21:41:39.944927Z", "shell.execute_reply": "2021-10-27T21:41:39.943923Z" }, "papermill": { "duration": 37.359841, "end_time": "2021-10-27T21:41:39.945145", "exception": false, "start_time": "2021-10-27T21:41:02.585304", "status": "completed" } }, "outputs": [], "source": [ "\"\"\"Manual spark configuration based on the default Spark configuration from the NXCALS bundle\n", "and https://gitlab.cern.ch/msobiesz/spark-pipelines/-/blob/master/NXCALS-example-for-DAaaS.ipynb.\n", "Used unless the spark context is already created. (outside of SWAN service or pyspark)\n", "\"\"\"\n", "if 'spark' not in locals() and 'spark' not in globals():\n", " import os\n", " from pyspark import SparkContext, SparkConf\n", " from pyspark.sql import SparkSession\n", "\n", " nxcals_jars = os.getenv('NXCALS_JARS')\n", " conf = SparkConf()\n", " conf.set('spark.master', 'yarn')\n", " conf.set(\"spark.driver.host\", \"spark-runner.cern.ch\")\n", " conf.set(\"spark.driver.port\", '5001')\n", " conf.set(\"spark.blockManager.port\", '5101')\n", " conf.set(\"spark.ui.port\", '5201')\n", " conf.set('spark.executorEnv.PYTHONPATH', os.getenv('PYTHONPATH'))\n", " conf.set('spark.executorEnv.LD_LIBRARY_PATH', os.getenv('LD_LIBRARY_PATH'))\n", " conf.set('spark.executorEnv.JAVA_HOME', os.getenv('JAVA_HOME'))\n", " conf.set('spark.executorEnv.SPARK_HOME', os.getenv('SPARK_HOME'))\n", " conf.set('spark.executorEnv.SPARK_EXTRA_CLASSPATH', os.getenv('SPARK_DIST_CLASSPATH'))\n", " conf.set('spark.driver.extraClassPath', nxcals_jars)\n", " conf.set('spark.executor.extraClassPath', nxcals_jars)\n", " conf.set('spark.driver.extraJavaOptions',\n", " \"-Dservice.url=https://cs-ccr-nxcals5.cern.ch:19093,https://cs-ccr-nxcals5.cern.ch:19094,\"\n", " \"https://cs-ccr-nxcals6.cern.ch:19093,https://cs-ccr-nxcals6.cern.ch:19094,\"\n", " \"https://cs-ccr-nxcals7.cern.ch:19093,https://cs-ccr-nxcals7.cern.ch:19094,\"\n", " \"https://cs-ccr-nxcals8.cern.ch:19093,https://cs-ccr-nxcals8.cern.ch:19094,\"\n", " \"https://cs-ccr-nxcalsstr4.cern.ch:19093,https://cs-ccr-nxcalsstr5.cern.ch:19093\")\n", "\n", " sc = SparkContext(conf=conf)\n", " spark = SparkSession(sc)\n" ] }, { "cell_type": "markdown", "id": "86e93e96", "metadata": { "papermill": { "duration": 0.043237, "end_time": "2021-10-27T21:41:40.048851", "exception": false, "start_time": "2021-10-27T21:41:40.005614", "status": "completed" }, "tags": [] }, "source": [ "# Notebook to list QHD PM timestamps by circuit type and time range" ] }, { "cell_type": "markdown", "id": "6e01e66c", "metadata": { "papermill": { "duration": 0.043459, "end_time": "2021-10-27T21:41:40.136848", "exception": false, "start_time": "2021-10-27T21:41:40.093389", "status": "completed" }, "tags": [] }, "source": [ "# 0. Initialise Working Environment" ] }, { "cell_type": "code", "execution_count": 2, "id": "ebe73a7a", "metadata": { "execution": { "iopub.execute_input": "2021-10-27T21:41:40.236170Z", "iopub.status.busy": "2021-10-27T21:41:40.235298Z", "iopub.status.idle": "2021-10-27T21:41:45.929222Z", "shell.execute_reply": "2021-10-27T21:41:45.929995Z" }, "papermill": { "duration": 5.749358, "end_time": "2021-10-27T21:41:45.930320", "exception": false, "start_time": "2021-10-27T21:41:40.180962", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Analysis executed with lhc-sm-api version: 1.5.17\n", "Analysis executed with lhc-sm-hwc notebooks version: 1.5.66\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n", "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n", "`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n", "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n", "`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n", "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n", "`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n", "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n", "`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n", "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n", "`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n", "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n", "`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n", "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n", "`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n", "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n", "`np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n", "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n" ] } ], "source": [ "import os, sys, warnings\n", "import pandas as pd\n", "from IPython.display import display, HTML, Javascript, clear_output, Markdown\n", "#\n", "import lhcsmapi\n", "from lhcsmapi.Time import Time\n", "from lhcsmapi.Timer import Timer\n", "from lhcsmapi.pyedsl.QueryBuilder import QueryBuilder\n", "from lhcsmapi.analysis.RbCircuitQuery import RbCircuitQuery\n", "from lhcsmapi.analysis.RqCircuitQuery import RqCircuitQuery\n", "from lhcsmapi.analysis.IpqCircuitQuery import IpqCircuitQuery\n", "from lhcsmapi.analysis.IpdCircuitQuery import IpdCircuitQuery\n", "from lhcsmapi.analysis.ItCircuitQuery import ItCircuitQuery\n", "from lhcsmapi.metadata.SignalMetadata import SignalMetadata\n", "from lhcsmapi.analysis.report_template import apply_report_template\n", "\n", "analysis_start_time = Time.get_analysis_start_time()\n", "lhcsmapi.get_lhcsmapi_version()\n", "lhcsmapi.get_lhcsmhwc_version('../__init__.py')" ] }, { "cell_type": "markdown", "id": "61c76b35", "metadata": { "papermill": { "duration": 0.039167, "end_time": "2021-10-27T21:41:46.011951", "exception": false, "start_time": "2021-10-27T21:41:45.972784", "status": "completed" }, "tags": [] }, "source": [ "# 1. User Input (circuit types: RB, RQ, IPQ, IPD, IT)" ] }, { "cell_type": "code", "execution_count": 3, "id": "2d5e6653", "metadata": { "execution": { "iopub.execute_input": "2021-10-27T21:41:46.097084Z", "iopub.status.busy": "2021-10-27T21:41:46.096250Z", "iopub.status.idle": "2021-10-27T21:41:46.099544Z", "shell.execute_reply": "2021-10-27T21:41:46.099002Z" }, "papermill": { "duration": 0.048872, "end_time": "2021-10-27T21:41:46.099750", "exception": false, "start_time": "2021-10-27T21:41:46.050878", "status": "completed" }, "tags": [ "parameters" ] }, "outputs": [], "source": [ "detailed_circuit_types = {\n", " 'RB': ['RB'],\n", " 'RQ': ['RQ'],\n", " 'IPQ': ['IPQ2', 'IPQ4', 'IPQ8'],\n", " 'IPD': ['IPD2', 'IPD2_B1B2'],\n", " 'IT': ['IT']\n", "}\n", "\n", "start_time = '2021-10-18 07:00:00'\n", "stop_time = '2021-10-20 23:01:00'\n" ] }, { "cell_type": "code", "execution_count": 4, "id": "b095af0e", "metadata": { "execution": { "iopub.execute_input": "2021-10-27T21:41:46.188975Z", "iopub.status.busy": "2021-10-27T21:41:46.188167Z", "iopub.status.idle": "2021-10-27T21:41:46.192141Z", "shell.execute_reply": "2021-10-27T21:41:46.191154Z" }, "papermill": { "duration": 0.04905, "end_time": "2021-10-27T21:41:46.192344", "exception": false, "start_time": "2021-10-27T21:41:46.143294", "status": "completed" }, "tags": [ "injected-parameters" ] }, "outputs": [], "source": [ "# Parameters\n", "start_time = \"2021-10-18 07:00:00\"\n", "stop_time = \"2021-10-20 23:01:00\"\n", "parametrized_marker = None\n" ] }, { "cell_type": "code", "execution_count": 5, "id": "8f3fb7b2", "metadata": { "execution": { "iopub.execute_input": "2021-10-27T21:41:46.287901Z", "iopub.status.busy": "2021-10-27T21:41:46.287093Z", "iopub.status.idle": "2021-10-27T21:41:46.289713Z", "shell.execute_reply": "2021-10-27T21:41:46.290214Z" }, "papermill": { "duration": 0.053989, "end_time": "2021-10-27T21:41:46.290392", "exception": false, "start_time": "2021-10-27T21:41:46.236403", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "start_time = 2021-10-18 07:00:00\n", "stop_time = 2021-10-20 23:01:00\n" ] } ], "source": [ "print('start_time = ', start_time)\n", "print('stop_time = ', stop_time)" ] }, { "cell_type": "markdown", "id": "5ad1ec68", "metadata": { "papermill": { "duration": 0.042729, "end_time": "2021-10-27T21:41:46.376602", "exception": false, "start_time": "2021-10-27T21:41:46.333873", "status": "completed" }, "tags": [] }, "source": [ "# 2. Search for PMs" ] }, { "cell_type": "code", "execution_count": 6, "id": "09ed9696", "metadata": { "execution": { "iopub.execute_input": "2021-10-27T21:41:46.476179Z", "iopub.status.busy": "2021-10-27T21:41:46.475307Z", "iopub.status.idle": "2021-10-27T21:41:52.090267Z", "shell.execute_reply": "2021-10-27T21:41:52.089513Z" }, "papermill": { "duration": 5.665805, "end_time": "2021-10-27T21:41:52.090551", "exception": false, "start_time": "2021-10-27T21:41:46.424746", "status": "completed" }, "scrolled": false, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "RB\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " source timestamp\n", "0 A28L3 1634542949913000000\n", "1 B29L3 1634542949913000000\n", "2 C28L3 1634542949913000000\n", " source timestamp\n", "0 B15R5 1634605470606000000\n", "1 B15R5 1634607003034000000\n", "2 B15R5 1634646052393000000\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "RQ\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " source timestamp\n", "0 27L3 1634542916538649604\n", "IPQ\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "IPD\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "IT\n" ] } ], "source": [ "source_timestamp_qds_df = pd.DataFrame()\n", "\n", "for circuit_type in detailed_circuit_types:\n", " print(f\"{circuit_type}\")\n", " circuits = SignalMetadata.get_circuit_names(detailed_circuit_types[circuit_type])\n", " if circuit_type == 'RQ':\n", " circuits = circuits[0:7]\n", "\n", " for circuit_name in circuits:\n", " meta_circuit_type = circuit_type\n", " if circuit_type == 'IPQ' or circuit_type == 'IPD': \n", " meta_circuit_type = SignalMetadata.get_circuit_type_for_circuit_name(circuit_name)\n", " \n", " source_timestamp_qds_df_i = QueryBuilder().with_pm() \\\n", " .with_duration(t_start=start_time, t_end=stop_time) \\\n", " .with_circuit_type(meta_circuit_type) \\\n", " .with_metadata(circuit_name=circuit_name, system='QH', source='*') \\\n", " .event_query() \\\n", " .filter_source(circuit_name, 'QH') \\\n", " .sort_values(by='timestamp').df\n", "\n", " if source_timestamp_qds_df_i.empty == False:\n", " print(source_timestamp_qds_df_i)\n", " source_timestamp_qds_df_i['circuit_type'] = source_timestamp_qds_df_i.apply(lambda row: circuit_type, axis=1)\n", " source_timestamp_qds_df = pd.concat([source_timestamp_qds_df, source_timestamp_qds_df_i], ignore_index=True)\n", "\n", "if source_timestamp_qds_df.empty == False:\n", " source_timestamp_qds_df['datetime'] = source_timestamp_qds_df.apply(lambda row: Time.to_string(row['timestamp']), axis=1)" ] }, { "cell_type": "markdown", "id": "763ab95e", "metadata": { "papermill": { "duration": 0.051634, "end_time": "2021-10-27T21:41:52.196288", "exception": false, "start_time": "2021-10-27T21:41:52.144654", "status": "completed" }, "tags": [] }, "source": [ "# 3. The list of QHD PM Timestamps, if any" ] }, { "cell_type": "code", "execution_count": 7, "id": "53accbff", "metadata": { "execution": { "iopub.execute_input": "2021-10-27T21:41:52.310862Z", "iopub.status.busy": "2021-10-27T21:41:52.310077Z", "iopub.status.idle": "2021-10-27T21:41:52.322775Z", "shell.execute_reply": "2021-10-27T21:41:52.322069Z" }, "papermill": { "duration": 0.07574, "end_time": "2021-10-27T21:41:52.322965", "exception": false, "start_time": "2021-10-27T21:41:52.247225", "status": "completed" }, "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
sourcetimestampcircuit_typedatetime
0A28L31634542949913000000RB2021-10-18 09:42:29.913000+02:00
1B29L31634542949913000000RB2021-10-18 09:42:29.913000+02:00
2C28L31634542949913000000RB2021-10-18 09:42:29.913000+02:00
3B15R51634605470606000000RB2021-10-19 03:04:30.606000+02:00
4B15R51634607003034000000RB2021-10-19 03:30:03.034000+02:00
5B15R51634646052393000000RB2021-10-19 14:20:52.393000+02:00
627L31634542916538649604RQ2021-10-18 09:41:56.538649604+02:00
\n", "
" ], "text/plain": [ " source timestamp circuit_type \\\n", "0 A28L3 1634542949913000000 RB \n", "1 B29L3 1634542949913000000 RB \n", "2 C28L3 1634542949913000000 RB \n", "3 B15R5 1634605470606000000 RB \n", "4 B15R5 1634607003034000000 RB \n", "5 B15R5 1634646052393000000 RB \n", "6 27L3 1634542916538649604 RQ \n", "\n", " datetime \n", "0 2021-10-18 09:42:29.913000+02:00 \n", "1 2021-10-18 09:42:29.913000+02:00 \n", "2 2021-10-18 09:42:29.913000+02:00 \n", "3 2021-10-19 03:04:30.606000+02:00 \n", "4 2021-10-19 03:30:03.034000+02:00 \n", "5 2021-10-19 14:20:52.393000+02:00 \n", "6 2021-10-18 09:41:56.538649604+02:00 " ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "WARNINIG: To be checked that QH discharges has been accepted by QHDA-notebooks!\n" ] } ], "source": [ "if source_timestamp_qds_df.empty == False:\n", " display(source_timestamp_qds_df)\n", " warnings.warn('WARNINIG: To be checked that QH discharges has been accepted by QHDA-notebooks!', stacklevel=2)\n", "else:\n", " print('There were no QH discharges on selected time range!')" ] }, { "cell_type": "markdown", "id": "ef492ad3", "metadata": { "papermill": { "duration": 0.057517, "end_time": "2021-10-27T21:41:52.436373", "exception": false, "start_time": "2021-10-27T21:41:52.378856", "status": "completed" }, "tags": [] }, "source": [ "# 4. Save html-report" ] }, { "cell_type": "raw", "execution_count": null, "id": "223bacf3", "metadata": { "papermill": { "duration": 0.05694, "end_time": "2021-10-27T21:41:52.549521", "exception": false, "start_time": "2021-10-27T21:41:52.492581", "status": "completed" }, "tags": [ "ignore" ] }, "outputs": [], "source": [ "if source_timestamp_qds_df.empty == False:\n", " # pd.set_option('display.max_columns', None)\n", " pd.set_option('display.max_rows', None)\n", " analysis_start_time = Time.get_analysis_start_time()\n", " date_time_qhd_pm_list = Time.to_datetime(start_time).strftime(\"%Y-%m-%d-%Hh%M\")\n", " !mkdir -p /eos/project/m/mp3/LHC_QHs\n", "\n", " file_name = \"LHC_QHD_PM_LIST-{}-{}\".format(date_time_qhd_pm_list, analysis_start_time)\n", " \n", " apply_report_template()\n", " file_name_html = file_name + '.html'\n", " full_path = '/eos/project/m/mp3/LHC_QHs/{}'.format(file_name_html)\n", " print('Compact notebook report saved to (Windows): ' + '\\\\\\\\cernbox-smb' + full_path.replace('/', '\\\\'))\n", " display(Javascript('IPython.notebook.save_notebook();'))\n", " Time.sleep(5)\n", " !{sys.executable} -m jupyter nbconvert --to html $'HWC_QHD_PM_LIST.ipynb' --output /eos/project/m/mp3/LHC_QHs/$file_name_html --TemplateExporter.exclude_input=True --TagRemovePreprocessor.remove_all_outputs_tags='[\"skip_output\"]' --TagRemovePreprocessor.remove_cell_tags='[\"skip_cell\"]'" ] } ], "metadata": { "celltoolbar": "Tags", "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.6" }, "papermill": { "default_parameters": {}, "duration": 53.4518, "end_time": "2021-10-27T21:41:53.223268", "environment_variables": {}, "exception": null, "input_path": "/builds/LHCData/lhc-sm-hwc/test/../qh/HWC_QHD_PM_LIST.ipynb", "output_path": "results/result_HWC_QHD_PM_LIST.ipynb", "parameters": { "parametrized_marker": null, "start_time": "2021-10-18 07:00:00", "stop_time": "2021-10-20 23:01:00" }, "start_time": "2021-10-27T21:40:59.771468", "version": "2.3.3" }, "sparkconnect": { "bundled_options": [], "list_of_options": [] }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": false, "sideBar": false, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 5 }