AD_system_demo_1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "false-lafayette",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Demo-Notebook-for-TimeSeries-Anomaly-Detection\" data-toc-modified-id=\"Demo-Notebook-for-TimeSeries-Anomaly-Detection-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Demo Notebook for TimeSeries Anomaly Detection</a></span></li><li><span><a href=\"#Installation-of-libraries-and-imports\" data-toc-modified-id=\"Installation-of-libraries-and-imports-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Installation of libraries and imports</a></span><ul class=\"toc-item\"><li><span><a href=\"#Installation-of-adcern-and-others-libraries\" data-toc-modified-id=\"Installation-of-adcern-and-others-libraries-2.1\"><span class=\"toc-item-num\">2.1&nbsp;&nbsp;</span>Installation of adcern and others libraries</a></span></li><li><span><a href=\"#Imports\" data-toc-modified-id=\"Imports-2.2\"><span class=\"toc-item-num\">2.2&nbsp;&nbsp;</span>Imports</a></span></li></ul></li><li><span><a href=\"#Init-of-configuration-files---ETL\" data-toc-modified-id=\"Init-of-configuration-files---ETL-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>Init of configuration files - ETL</a></span><ul class=\"toc-item\"><li><span><a href=\"#Creation\" data-toc-modified-id=\"Creation-3.1\"><span class=\"toc-item-num\">3.1&nbsp;&nbsp;</span>Creation</a></span></li><li><span><a href=\"#Reading-the-json\" data-toc-modified-id=\"Reading-the-json-3.2\"><span class=\"toc-item-num\">3.2&nbsp;&nbsp;</span>Reading the json</a></span></li></ul></li><li><span><a href=\"#ETL-steps-(Extract,-Transform,-Load)\" data-toc-modified-id=\"ETL-steps-(Extract,-Transform,-Load)-4\"><span class=\"toc-item-num\">4&nbsp;&nbsp;</span>ETL steps (Extract, Transform, Load)</a></span><ul class=\"toc-item\"><li><span><a href=\"#Compute-Normalization\" data-toc-modified-id=\"Compute-Normalization-4.1\"><span class=\"toc-item-num\">4.1&nbsp;&nbsp;</span>Compute Normalization</a></span></li><li><span><a href=\"#Transform-Data\" data-toc-modified-id=\"Transform-Data-4.2\"><span class=\"toc-item-num\">4.2&nbsp;&nbsp;</span>Transform Data</a></span></li><li><span><a href=\"#Copy-Locally\" data-toc-modified-id=\"Copy-Locally-4.3\"><span class=\"toc-item-num\">4.3&nbsp;&nbsp;</span>Copy Locally</a></span></li></ul></li><li><span><a href=\"#Visualization-of-downloaded-time-series\" data-toc-modified-id=\"Visualization-of-downloaded-time-series-5\"><span class=\"toc-item-num\">5&nbsp;&nbsp;</span>Visualization of downloaded time series</a></span><ul class=\"toc-item\"><li><span><a href=\"#Reading-time-series-with-pandas-and-host-definition\" data-toc-modified-id=\"Reading-time-series-with-pandas-and-host-definition-5.1\"><span class=\"toc-item-num\">5.1&nbsp;&nbsp;</span>Reading time series with pandas and host definition</a></span></li><li><span><a href=\"#Reconstruction-function\" data-toc-modified-id=\"Reconstruction-function-5.2\"><span class=\"toc-item-num\">5.2&nbsp;&nbsp;</span>Reconstruction function</a></span></li><li><span><a href=\"#Normalized-data-visualization\" data-toc-modified-id=\"Normalized-data-visualization-5.3\"><span class=\"toc-item-num\">5.3&nbsp;&nbsp;</span>Normalized data visualization</a></span></li><li><span><a href=\"#How-to-retrieve-the-original-data-without-normalization\" data-toc-modified-id=\"How-to-retrieve-the-original-data-without-normalization-5.4\"><span class=\"toc-item-num\">5.4&nbsp;&nbsp;</span>How to retrieve the original data without normalization</a></span></li><li><span><a href=\"#Original-data-visualization\" data-toc-modified-id=\"Original-data-visualization-5.5\"><span class=\"toc-item-num\">5.5&nbsp;&nbsp;</span>Original data visualization</a></span></li></ul></li><li><span><a href=\"#Init-of-configuration-file---ANALYSIS\" data-toc-modified-id=\"Init-of-configuration-file---ANALYSIS-6\"><span class=\"toc-item-num\">6&nbsp;&nbsp;</span>Init of configuration file - ANALYSIS</a></span><ul class=\"toc-item\"><li><span><a href=\"#Creation\" data-toc-modified-id=\"Creation-6.1\"><span class=\"toc-item-num\">6.1&nbsp;&nbsp;</span>Creation</a></span></li><li><span><a href=\"#Reading-the-json\" data-toc-modified-id=\"Reading-the-json-6.2\"><span class=\"toc-item-num\">6.2&nbsp;&nbsp;</span>Reading the json</a></span></li></ul></li><li><span><a href=\"#ANALYSIS-to-produce-anomaly-scores\" data-toc-modified-id=\"ANALYSIS-to-produce-anomaly-scores-7\"><span class=\"toc-item-num\">7&nbsp;&nbsp;</span>ANALYSIS to produce anomaly scores</a></span><ul class=\"toc-item\"><li><span><a href=\"#Analysis\" data-toc-modified-id=\"Analysis-7.1\"><span class=\"toc-item-num\">7.1&nbsp;&nbsp;</span>Analysis</a></span></li></ul></li><li><span><a href=\"#Visualization-of-the-results\" data-toc-modified-id=\"Visualization-of-the-results-8\"><span class=\"toc-item-num\">8&nbsp;&nbsp;</span>Visualization of the results</a></span><ul class=\"toc-item\"><li><span><a href=\"#Reading-the-scores\" data-toc-modified-id=\"Reading-the-scores-8.1\"><span class=\"toc-item-num\">8.1&nbsp;&nbsp;</span>Reading the scores</a></span></li><li><span><a href=\"#Visualization-of-both-downloaded-data-and-scores\" data-toc-modified-id=\"Visualization-of-both-downloaded-data-and-scores-8.2\"><span class=\"toc-item-num\">8.2&nbsp;&nbsp;</span>Visualization of both downloaded data and scores</a></span></li></ul></li><li><span><a href=\"#ETL-Steps-not-used\" data-toc-modified-id=\"ETL-Steps-not-used-9\"><span class=\"toc-item-num\">9&nbsp;&nbsp;</span>ETL Steps not used</a></span><ul class=\"toc-item\"><li><span><a href=\"#Data-Presence\" data-toc-modified-id=\"Data-Presence-9.1\"><span class=\"toc-item-num\">9.1&nbsp;&nbsp;</span>Data Presence</a></span></li><li><span><a href=\"#Check-Normalization\" data-toc-modified-id=\"Check-Normalization-9.2\"><span class=\"toc-item-num\">9.2&nbsp;&nbsp;</span>Check Normalization</a></span></li></ul></li><li><span><a href=\"#Adding-a-model\" data-toc-modified-id=\"Adding-a-model-10\"><span class=\"toc-item-num\">10&nbsp;&nbsp;</span>Adding a model</a></span><ul class=\"toc-item\"><li><span><a href=\"#Changing-config-file---Analysis\" data-toc-modified-id=\"Changing-config-file---Analysis-10.1\"><span class=\"toc-item-num\">10.1&nbsp;&nbsp;</span>Changing config file - Analysis</a></span></li><li><span><a href=\"#Reading-the-json\" data-toc-modified-id=\"Reading-the-json-10.2\"><span class=\"toc-item-num\">10.2&nbsp;&nbsp;</span>Reading the json</a></span></li><li><span><a href=\"#Analysis-of-multiple-models\" data-toc-modified-id=\"Analysis-of-multiple-models-10.3\"><span class=\"toc-item-num\">10.3&nbsp;&nbsp;</span>Analysis of multiple models</a></span></li><li><span><a href=\"#Checking-new-results\" data-toc-modified-id=\"Checking-new-results-10.4\"><span class=\"toc-item-num\">10.4&nbsp;&nbsp;</span>Checking new results</a></span></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "pressing-plasma",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Demo Notebook for TimeSeries Anomaly Detection\n",
    "This notebook shows the Anomaly Detection system for TimeSeries in action using the tools provided in this  repository.\n",
    "\n",
    "**Note that we recommend to run this notebook using swan004.cern.ch with the following configuration:**\n",
    "- Software stack: Other releases, **97a**\n",
    "- Spark cluster: **General Purpose (Analytix)**\n",
    "\n",
    "---\n",
    "**REMEMBER TO ACTIVATE THE ANALYTIX CLUSTER IN THE SWAN CONFIGURATION!!**\n",
    "Activating it means to activate spark (click the spark icon in the upper part of the notebook to enable spark before starting to run the notebook cells)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "naval-specific",
   "metadata": {},
   "source": [
    "# Installation of libraries and imports"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "stopped-praise",
   "metadata": {},
   "source": [
    "## Installation of adcern and others libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "vocal-participation",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T18:33:36.844986Z",
     "start_time": "2021-05-03T18:20:48.908597Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting git+https://:****@gitlab.cern.ch:8443/cloud-infrastructure/data-analytics.git@qa-v0.4\n",
      "  Cloning https://:****@gitlab.cern.ch:8443/cloud-infrastructure/data-analytics.git (to revision qa-v0.4) to /tmp/pip-req-build-6x76eylt\n",
      "  Running command git clone -q https://:@gitlab.cern.ch:8443/cloud-infrastructure/data-analytics.git /tmp/pip-req-build-6x76eylt\n",
      "  Running command git checkout -b qa-v0.4 --track origin/qa-v0.4\n",
      "  Switched to a new branch 'qa-v0.4'\n",
      "  Branch qa-v0.4 set up to track remote branch qa-v0.4 from origin.\n",
      "Collecting setuptools==41.0.0 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c8/b0/cc6b7ba28d5fb790cf0d5946df849233e32b8872b6baca10c9e002ff5b41/setuptools-41.0.0-py2.py3-none-any.whl (575kB)\n",
      "\u001b[K     |████████████████████████████████| 583kB 8.1MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: pbr!=2.1.0,>=2.0.0 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from data-analytics==0.4.0.0rc5.dev83) (5.2.1)\n",
      "Collecting matplotlib==3.2.1 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b2/c2/71fcf957710f3ba1f09088b35776a799ba7dd95f7c2b195ec800933b276b/matplotlib-3.2.1-cp37-cp37m-manylinux1_x86_64.whl (12.4MB)\n",
      "\u001b[K     |████████████████████████████████| 12.4MB 10.4MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting numpy==1.18.5 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d6/c6/58e517e8b1fb192725cfa23c01c2e60e4e6699314ee9684a1c5f5c9b27e1/numpy-1.18.5-cp37-cp37m-manylinux1_x86_64.whl (20.1MB)\n",
      "\u001b[K     |████████████████████████████████| 20.1MB 26.3MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting pandas==1.0.4 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a4/5f/1b6e0efab4bfb738478919d40b0e3e1a06e3d9996da45eb62a77e9a090d9/pandas-1.0.4-cp37-cp37m-manylinux1_x86_64.whl (10.1MB)\n",
      "\u001b[K     |████████████████████████████████| 10.1MB 24.1MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting requests==2.21.0 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/e3/20f3d364d6c8e5d2353c72a67778eb189176f08e873c9900e10c0287b84b/requests-2.21.0-py2.py3-none-any.whl (57kB)\n",
      "\u001b[K     |████████████████████████████████| 61kB 9.9MB/s  eta 0:00:01\n",
      "\u001b[?25hCollecting pyspark==2.4.6 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e9/e4/5c15ab8d354c4e3528510821865e6748209a9b0ff6a1788f4cd36cc2a5dc/pyspark-2.4.6.tar.gz (218.4MB)\n",
      "\u001b[K     |████████████████████████████████| 218.4MB 174kB/s  eta 0:00:01   |                                | 296kB 21.8MB/s eta 0:00:11     |███████████████████▊            | 134.4MB 38.2MB/s eta 0:00:03     |██████████████████████          | 150.2MB 38.2MB/s eta 0:00:02     |████████████████████████        | 164.0MB 34.9MB/s eta 0:00:02\n",
      "\u001b[?25hCollecting scipy==1.4.1 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/dd/82/c1fe128f3526b128cfd185580ba40d01371c5d299fcf7f77968e22dfcc2e/scipy-1.4.1-cp37-cp37m-manylinux1_x86_64.whl (26.1MB)\n",
      "\u001b[K     |████████████████████████████████| 26.1MB 16.1MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: six==1.12.0 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from data-analytics==0.4.0.0rc5.dev83) (1.12.0)\n",
      "Collecting seaborn==0.10.1 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c7/e6/54aaaafd0b87f51dfba92ba73da94151aa3bc179e5fe88fc5dfb3038e860/seaborn-0.10.1-py3-none-any.whl (215kB)\n",
      "\u001b[K     |████████████████████████████████| 225kB 22.7MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting statsmodels==0.11.1 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7b/6a/0bf4184c3fb6f9f43df997b88de5784b4cb2f6bd19a5dc213463971076cf/statsmodels-0.11.1-cp37-cp37m-manylinux1_x86_64.whl (8.7MB)\n",
      "\u001b[K     |████████████████████████████████| 8.7MB 19.2MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting scikit-learn==0.23.1 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b8/7e/74e707b66490d4eb05f702966ad0990881127acecf9d5cdcef3c95ec6c16/scikit_learn-0.23.1-cp37-cp37m-manylinux1_x86_64.whl (6.8MB)\n",
      "\u001b[K     |████████████████████████████████| 6.8MB 49.3MB/s eta 0:00:01                    | 266kB 49.3MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: tables==3.6.1 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from data-analytics==0.4.0.0rc5.dev83) (3.6.1)\n",
      "Collecting dictdiffer==0.8.1 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "  Downloading https://files.pythonhosted.org/packages/97/92/350b6b6ec39c5f87d98d04c91a50c498518716a05368e6dea88b5c69b590/dictdiffer-0.8.1-py2.py3-none-any.whl\n",
      "Collecting elasticsearch==7.7.1 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4a/33/d0ed32e077f7dc860153fa866fc52ac312886c9890962ff29379aa753dd1/elasticsearch-7.7.1-py2.py3-none-any.whl (99kB)\n",
      "\u001b[K     |████████████████████████████████| 102kB 3.2MB/s ta 0:00:011     |███████████████████▉            | 61kB 3.8MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting pyyaml==5.3.1 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/64/c2/b80047c7ac2478f9501676c988a5411ed5572f35d1beff9cae07d321512c/PyYAML-5.3.1.tar.gz (269kB)\n",
      "\u001b[K     |████████████████████████████████| 276kB 19.4MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting pyod==0.8.0 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/cf/68/99df05e5666248e9c10359457e2da1b89943f5ac96749ceb1c131001eb88/pyod-0.8.0.tar.gz (93kB)\n",
      "\u001b[K     |████████████████████████████████| 102kB 21.2MB/s ta 0:00:01\n",
      "\u001b[?25hCollecting pyarrow==0.17.1 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c4/dd/4d2ce1c64a94e5f35ae622cdfdda6eeab927cf6570d826dcfe09e9ba08f8/pyarrow-0.17.1-cp37-cp37m-manylinux2010_x86_64.whl (64.2MB)\n",
      "\u001b[K     |████████████████████████████████| 64.2MB 32.1MB/s eta 0:00:01    |███████████▋                    | 23.2MB 5.6MB/s eta 0:00:08     |████████████████████████████▌   | 57.2MB 32.1MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting numba==0.50.1 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9d/2f/2b3ad106afafd684d31cd24dd624f15d2d725aba394752932d08d5283e10/numba-0.50.1-cp37-cp37m-manylinux1_x86_64.whl (2.6MB)\n",
      "\u001b[K     |████████████████████████████████| 2.6MB 39.3MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting llvmlite==0.33.0 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0a/28/0a35b3c2685bf2ea327cef5577bdf91f387f0f4594417a2a05a1d42fb7c2/llvmlite-0.33.0-cp37-cp37m-manylinux1_x86_64.whl (18.3MB)\n",
      "\u001b[K     |████████████████████████████████| 18.3MB 21.3MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting click==7.1.2 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d2/3d/fa76db83bf75c4f8d338c2fd15c8d33fdd7ad23a9b5e57eb6c5de26b430e/click-7.1.2-py2.py3-none-any.whl (82kB)\n",
      "\u001b[K     |████████████████████████████████| 92kB 24.6MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting cliff==3.4.0 (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/06/03b1f92d46546a18eabf33ff7f37ef422c18c93d5a926bf590fee32ebe75/cliff-3.4.0-py3-none-any.whl (76kB)\n",
      "\u001b[K     |████████████████████████████████| 81kB 12.9MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting pandasticsearch (from data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/09/d0/c46cf54ba2a58e3b5057cb95098546cc393de6a390018c22f1a6afd5dfed/pandasticsearch-0.5.3.tar.gz (251kB)\n",
      "\u001b[K     |████████████████████████████████| 256kB 25.0MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: python-dateutil>=2.1 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from matplotlib==3.2.1->data-analytics==0.4.0.0rc5.dev83) (2.8.0)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from matplotlib==3.2.1->data-analytics==0.4.0.0rc5.dev83) (2.4.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from matplotlib==3.2.1->data-analytics==0.4.0.0rc5.dev83) (0.10.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from matplotlib==3.2.1->data-analytics==0.4.0.0rc5.dev83) (1.0.1)\n",
      "Requirement already satisfied: pytz>=2017.2 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from pandas==1.0.4->data-analytics==0.4.0.0rc5.dev83) (2019.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from requests==2.21.0->data-analytics==0.4.0.0rc5.dev83) (2019.3.9)\n",
      "Collecting urllib3<1.25,>=1.21.1 (from requests==2.21.0->data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/01/11/525b02e4acc0c747de8b6ccdab376331597c569c42ea66ab0a1dbd36eca2/urllib3-1.24.3-py2.py3-none-any.whl (118kB)\n",
      "\u001b[K     |████████████████████████████████| 122kB 27.1MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: idna<2.9,>=2.5 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from requests==2.21.0->data-analytics==0.4.0.0rc5.dev83) (2.8)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from requests==2.21.0->data-analytics==0.4.0.0rc5.dev83) (3.0.4)\n",
      "Requirement already satisfied: py4j==0.10.7 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from pyspark==2.4.6->data-analytics==0.4.0.0rc5.dev83) (0.10.7)\n",
      "Requirement already satisfied: patsy>=0.5 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from statsmodels==0.11.1->data-analytics==0.4.0.0rc5.dev83) (0.5.1)\n",
      "Requirement already satisfied: joblib>=0.11 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from scikit-learn==0.23.1->data-analytics==0.4.0.0rc5.dev83) (0.14.0)\n",
      "Collecting threadpoolctl>=2.0.0 (from scikit-learn==0.23.1->data-analytics==0.4.0.0rc5.dev83)\n",
      "  Downloading https://files.pythonhosted.org/packages/f7/12/ec3f2e203afa394a149911729357aa48affc59c20e2c1c8297a60f33f133/threadpoolctl-2.1.0-py3-none-any.whl\n",
      "Requirement already satisfied: numexpr>=2.6.2 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from tables==3.6.1->data-analytics==0.4.0.0rc5.dev83) (2.6.9)\n",
      "Collecting combo (from pyod==0.8.0->data-analytics==0.4.0.0rc5.dev83)\n",
      "  Downloading https://files.pythonhosted.org/packages/12/ae/66029dcaa88ccca77f454dbb29c1178c751ec24fc771ed475a992b49a02d/combo-0.1.2.tar.gz\n",
      "Collecting suod (from pyod==0.8.0->data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4e/8a/255ed2c959abab7c712b10fe710e454d5a8e3461c6ae60e426349a8eb6a5/suod-0.0.6.tar.gz (2.1MB)\n",
      "\u001b[K     |████████████████████████████████| 2.1MB 37.0MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting stevedore>=2.0.1 (from cliff==3.4.0->data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/49/b602307aeac3df3384ff1fcd05da9c0376c622a6c48bb5325f28ab165b57/stevedore-3.3.0-py3-none-any.whl (49kB)\n",
      "\u001b[K     |████████████████████████████████| 51kB 14.6MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: PrettyTable<0.8,>=0.7.2 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from cliff==3.4.0->data-analytics==0.4.0.0rc5.dev83) (0.7.2)\n",
      "Collecting cmd2!=0.8.3,>=0.8.0 (from cliff==3.4.0->data-analytics==0.4.0.0rc5.dev83)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/15/8b/15061b32332bb35ea2a2f6263d0f616779d576e82739ec8e7fcf3c94abf5/cmd2-1.5.0-py3-none-any.whl (133kB)\n",
      "\u001b[K     |████████████████████████████████| 143kB 27.4MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: psutil in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from suod->pyod==0.8.0->data-analytics==0.4.0.0rc5.dev83) (5.6.2)\n",
      "Collecting importlib-metadata>=1.7.0; python_version < \"3.8\" (from stevedore>=2.0.1->cliff==3.4.0->data-analytics==0.4.0.0rc5.dev83)\n",
      "  Downloading https://files.pythonhosted.org/packages/8e/e2/49966924c93909d47612bb47d911448140a2f6c1390aec2f4c1afbe3748f/importlib_metadata-4.0.1-py3-none-any.whl\n",
      "Collecting colorama>=0.3.7 (from cmd2!=0.8.3,>=0.8.0->cliff==3.4.0->data-analytics==0.4.0.0rc5.dev83)\n",
      "  Downloading https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl\n",
      "Requirement already satisfied: attrs>=16.3.0 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from cmd2!=0.8.3,>=0.8.0->cliff==3.4.0->data-analytics==0.4.0.0rc5.dev83) (19.3.0)\n",
      "Collecting pyperclip>=1.6 (from cmd2!=0.8.3,>=0.8.0->cliff==3.4.0->data-analytics==0.4.0.0rc5.dev83)\n",
      "  Downloading https://files.pythonhosted.org/packages/a7/2c/4c64579f847bd5d539803c8b909e54ba087a79d01bb3aba433a95879a6c5/pyperclip-1.8.2.tar.gz\n",
      "Requirement already satisfied: wcwidth>=0.1.7 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from cmd2!=0.8.3,>=0.8.0->cliff==3.4.0->data-analytics==0.4.0.0rc5.dev83) (0.1.7)\n",
      "Requirement already satisfied: zipp>=0.5 in /cvmfs/sft.cern.ch/lcg/views/LCG_97apython3/x86_64-centos7-gcc8-opt/lib/python3.7/site-packages (from importlib-metadata>=1.7.0; python_version < \"3.8\"->stevedore>=2.0.1->cliff==3.4.0->data-analytics==0.4.0.0rc5.dev83) (0.5.1)\n",
      "Collecting typing-extensions>=3.6.4; python_version < \"3.8\" (from importlib-metadata>=1.7.0; python_version < \"3.8\"->stevedore>=2.0.1->cliff==3.4.0->data-analytics==0.4.0.0rc5.dev83)\n",
      "  Downloading https://files.pythonhosted.org/packages/2e/35/6c4fff5ab443b57116cb1aad46421fb719bed2825664e8fe77d66d99bcbc/typing_extensions-3.10.0.0-py3-none-any.whl\n",
      "Building wheels for collected packages: data-analytics, pyspark, pyyaml, pyod, pandasticsearch, combo, suod, pyperclip\n",
      "  Building wheel for data-analytics (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /tmp/pip-ephem-wheel-cache-j16jwrx9/wheels/15/c2/a5/b8cdd4a27351e39866586a51b8ea2c523b42e64a859737ea5e\n",
      "  Building wheel for pyspark (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /tmp/smetaj/.cache/pip/wheels/1e/5e/6a/17e906c94ec7246f260330a66e44a06a0809033ba2738a74a8\n",
      "  Building wheel for pyyaml (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /tmp/smetaj/.cache/pip/wheels/a7/c1/ea/cf5bd31012e735dc1dfea3131a2d5eae7978b251083d6247bd\n",
      "  Building wheel for pyod (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /tmp/smetaj/.cache/pip/wheels/ba/a6/81/2dd042e240090f3603a686b897d03402219a86e3f61bc71184\n",
      "  Building wheel for pandasticsearch (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /tmp/smetaj/.cache/pip/wheels/33/d3/38/984341c95870a77b5b09b5fa604f4dbd32389af4492cef36c4\n",
      "  Building wheel for combo (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /tmp/smetaj/.cache/pip/wheels/01/d9/bf/d1a371a5f0844cd8a53c04c14daa89974c93f429dda9dceb86\n",
      "  Building wheel for suod (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /tmp/smetaj/.cache/pip/wheels/c5/d7/c1/6c778aee7fccfe3c054ea9bab92c5994ae3a0f6bba7078541e\n",
      "  Building wheel for pyperclip (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /tmp/smetaj/.cache/pip/wheels/25/af/b8/3407109267803f4015e1ee2ff23be0c8c19ce4008665931ee1\n",
      "Successfully built data-analytics pyspark pyyaml pyod pandasticsearch combo suod pyperclip\n",
      "\u001b[31mERROR: tensorflow 1.14.0 requires google-pasta>=0.1.6, which is not installed.\u001b[0m\n",
      "\u001b[31mERROR: caniusepython3 7.1.0 requires argparse, which is not installed.\u001b[0m\n",
      "\u001b[31mERROR: itkwidgets 0.25.3 has requirement ipywidgets>=7.5.1, but you'll have ipywidgets 7.4.2 which is incompatible.\u001b[0m\n",
      "\u001b[31mERROR: ipympl 0.4.1 has requirement ipywidgets>=7.5.0, but you'll have ipywidgets 7.4.2 which is incompatible.\u001b[0m\n",
      "\u001b[31mERROR: hepdata-converter 0.1.35 has requirement matplotlib<3.0.0, but you'll have matplotlib 3.2.1 which is incompatible.\u001b[0m\n",
      "\u001b[31mERROR: distributed 1.28.1 has requirement dask>=0.18.0, but you'll have dask 0+unknown which is incompatible.\u001b[0m\n",
      "\u001b[31mERROR: suod 0.0.6 has requirement joblib>=0.14.1, but you'll have joblib 0.14.0 which is incompatible.\u001b[0m\n",
      "Installing collected packages: setuptools, numpy, matplotlib, pandas, urllib3, requests, pyspark, scipy, seaborn, statsmodels, threadpoolctl, scikit-learn, dictdiffer, elasticsearch, pyyaml, llvmlite, numba, combo, suod, pyod, pyarrow, click, typing-extensions, importlib-metadata, stevedore, colorama, pyperclip, cmd2, cliff, pandasticsearch, data-analytics\n",
      "\u001b[33m  WARNING: The scripts f2py, f2py3 and f2py3.7 are installed in '/eos/user/s/smetaj/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
      "\u001b[33m  WARNING: The script plasma_store is installed in '/eos/user/s/smetaj/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
      "\u001b[33m  WARNING: The scripts data_mining and elaborate_scores are installed in '/eos/user/s/smetaj/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
      "Successfully installed click cliff cmd2 colorama combo data-analytics dictdiffer elasticsearch importlib-metadata llvmlite matplotlib numba numpy pandas pandasticsearch pyarrow pyod pyperclip pyspark pyyaml requests scikit-learn scipy seaborn setuptools statsmodels stevedore suod threadpoolctl typing-extensions urllib3\n"
     ]
    }
   ],
   "source": [
    "# Set the variable to True the first time to download the libraries.\n",
    "# Note that with @branch you can install a specific branch\n",
    "\n",
    "first_time = False\n",
    "if first_time:\n",
    "    # If you install the libraries it takes some minutes \n",
    "    # (with a fresh installation 12 minutes)\n",
    "    !pip install --user git+https://:@gitlab.cern.ch:8443/cloud-infrastructure/data-analytics.git@qa-v0.4"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "abstract-puzzle",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "agricultural-discrimination",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T18:35:30.857508Z",
     "start_time": "2021-05-03T18:35:19.489449Z"
    }
   },
   "outputs": [],
   "source": [
    "# AD System Libraries ----------------------------------------------------\n",
    "import adcern.cmd.data_mining as DM\n",
    "import etl.spark_etl.etl_pipeline as PL\n",
    "\n",
    "# To pass command line parameters and to use other functions -------------\n",
    "import sys, os, re, json\n",
    "\n",
    "# To run the visualization function --------------------------------------\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from dateutil import tz\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import plotly.graph_objects as go\n",
    "from plotly.subplots import make_subplots\n",
    "\n",
    "# To read more parquet files with pandas ---------------------------------\n",
    "import glob"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "timely-parish",
   "metadata": {},
   "source": [
    "# Init of configuration files - ETL"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "informed-bikini",
   "metadata": {},
   "source": [
    "Our ETL and analysis methods work with json configuration files in input. In these files we save all the paths, the dates and the hyperparameters used then by the methods.\n",
    "\n",
    "Usually we have the files already saved for static tests or we create them in our production pipeline; **for the purpose of this notebook instead, we create the json files starting from a python dict.**\n",
    "\n",
    "Note that we need 2 config files: \n",
    "- A config file about the training part.\n",
    "- A config file about the data used by the trained model to infer the scores. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "durable-parliament",
   "metadata": {},
   "source": [
    "## Creation\n",
    "Note that the 2 config files share most of the parameters so we will have:\n",
    "- json_data: containing all the main information\n",
    "- json_data_train, json_data_inference: containing specific paths for the 2 different purposes\n",
    "\n",
    "**Note also that you have to be sure that you have the writing rights for all the paths contained here, in particular *HDFS_folder_with_write_rights* should point to your hdfs personal folder to ensure that**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "laughing-aging",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T18:37:04.581683Z",
     "start_time": "2021-05-03T18:37:04.577078Z"
    }
   },
   "outputs": [],
   "source": [
    "# if you want to run the notebook on more data you can change \n",
    "# this boolean variable to change the dates in the \n",
    "# following configuration json.\n",
    "\n",
    "longer_run_with_more_data = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "contrary-thong",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T19:28:54.536642Z",
     "start_time": "2021-05-03T19:28:54.266918Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Folder to save data and results: /user/smetaj/\n",
      "Json files saved.\n"
     ]
    }
   ],
   "source": [
    "demo_name = 'demo_AD_System'\n",
    "json_file_train = 'demo_config_train.json'\n",
    "json_file_inference = 'demo_config_inference.json'\n",
    "username = os.environ['USER']\n",
    "HDFS_folder_with_write_rights = '/user/' + username + '/'\n",
    "\n",
    "print(\"Folder to save data and results: \" + HDFS_folder_with_write_rights)\n",
    "\n",
    "collectd_path = \"/project/monitoring/collectd/\"\n",
    "                                  \n",
    "json_data = {}\n",
    "\n",
    "# Absolute path identifier of the cell/hostgroup that you want to mine.\n",
    "# Note that it is in a list format, but only one hostgroup is supported so far.\n",
    "json_data['hostgroups'] = []\n",
    "json_data['hostgroups'].append('cloud_compute/level2/main/gva_shared_017')\n",
    "\n",
    "# The pattern of the names of your data folders and \".metadata\" files.\n",
    "json_data['code_project_name'] = demo_name\n",
    "\n",
    "# Local area of your VM where to save your data and metadata data are saved in\n",
    "# folders with one parquet only. Metadata are saved in file with the same name\n",
    "# of the resepctive foler plus the \".metadata\" extension.\n",
    "json_data['local_cache_folder'] = '/eos/user/' + username[:1] + \\\n",
    "    '/' + username + '/' + demo_name + '/local_cache_'\n",
    "\n",
    "# HDFS Area where Spark saves the aggregated data of your cell.\n",
    "# Note that the saving can create multiple file depending on the number of\n",
    "# partitions that the workers were using.\n",
    "json_data['hdfs_out_folder'] = HDFS_folder_with_write_rights + \\\n",
    "    demo_name + '/raw_parquet_'\n",
    "\n",
    "# HDFS Area where Spark saves the aggregated data of your cell.\n",
    "# Note that here we force it to be one partiotion only.\n",
    "json_data['hdfs_cache_folder'] = HDFS_folder_with_write_rights + \\\n",
    "    demo_name + '/compressed_'\n",
    "\n",
    "# HDFS Area where Spark saves the normalization coefficients computed on the\n",
    "# normalziation chunk of data between the normalization dates.\n",
    "json_data['normalization_out_folder'] = HDFS_folder_with_write_rights + \\\n",
    "    demo_name + '/normalization/'\n",
    "\n",
    "# ----------------------------------------------------------------------------\n",
    "# ----------------------------------------------------------------------------\n",
    "\n",
    "# Wether you want to overwrite (true) or not (false) the raw data in HDFS.\n",
    "# If not sure leave true.\n",
    "json_data['overwrite_on_hdfs'] = True\n",
    "\n",
    "# Wether you want to overwrite (true) or not (false) the noramlization\n",
    "# coefficeints in HDFS. If not sure leave true.\n",
    "json_data['overwrite_normalization'] = True\n",
    "\n",
    "# The level of aggregation of your raw time series data.\n",
    "# The aggregator is typically the mean operator.\n",
    "# e.g. if 5 it means that we summarize the data every 5 min, and the values\n",
    "# with timestamp 7.45 will represent the mean of the previous 5 minutes from\n",
    "# 7.40 to 7.45 but that value will have 7.45 as timestamp\n",
    "json_data['aggregate_every_n_minutes'] = 10\n",
    "\n",
    "# The length of your windows of data.\n",
    "# e.g. if aggregate_every_n_minutes = 10 and history_steps = 6 it means that\n",
    "# every windows is summarizing 6 * 10 = 60 minutes\n",
    "json_data['history_steps'] = 48\n",
    "\n",
    "# The number of step you want to move your window.\n",
    "# e.g. if aggregate_every_n_minutes = 10 and history_steps = 2 it means that\n",
    "# you will get a window of data that is translated of 10 * 2 = 20 min with\n",
    "# respect to the previous.\n",
    "# Note that if slide_steps has the same value of history_steps you have non-\n",
    "# overlapping windows.\n",
    "json_data['slide_steps'] = 1\n",
    "\n",
    "# Used to create windows with future steps. If not sure keep this to 0.\n",
    "json_data['future_steps'] = 0\n",
    "\n",
    "# ----------------------------------------------------------------------------\n",
    "# ----------------------------------------------------------------------------\n",
    "\n",
    "# Dates representing the start/end of the data and noramlization chunks.\n",
    "# - start_date -> the starting date of data chunk of ETL\n",
    "# - end_date -> the ending date of data chunk of ETL\n",
    "# - start_date_normalization -> the starting date of the chunk of data used\n",
    "#   to learn noramlization coefficeints (typically this chunk preceeds the\n",
    "#   chunk of data)\n",
    "# - end_date_normalization -> the ending date of the chunk of data used\n",
    "#   to learn noramlization coefficeints\n",
    "# Note that the upper extremum is excluded (i.e. data will stop at the 23:59\n",
    "# of the day preeceeding the date_end_excluded)\n",
    "if longer_run_with_more_data:\n",
    "    json_data['date_start'] = \"2021-03-01\"\n",
    "    json_data['date_end_excluded'] = \"2021-03-20\"\n",
    "else:\n",
    "    json_data['date_start'] = \"2021-04-01\"\n",
    "    json_data['date_end_excluded'] = \"2021-04-08\"    \n",
    "json_data['date_start_normalization'] = json_data['date_start']\n",
    "json_data['date_end_normalization_excluded'] = json_data['date_end_excluded']\n",
    "\n",
    "# ----------------------------------------------------------------------------\n",
    "# ----------------------------------------------------------------------------\n",
    "\n",
    "# List of plugins to mine.\n",
    "# Note that it is a dictionary where every key represents the name your plugin\n",
    "# have and the value is a dictionary with:\n",
    "# 'plugin_instance', 'type' 'type_instance', 'plugin_name'\n",
    "# the value asigned to these key is defining an and-filter.\n",
    "# you will get only the data that have all those attributes\n",
    "# ('plugin_instance', 'type' 'type_instance', 'plugin_name') in and with the\n",
    "# specified value.\n",
    "# Note that if you do not want to filter on one attribute do not express it.\n",
    "json_data['selected_plugins'] = {\n",
    "    'cpu__percent_idle': {\n",
    "        'plugin_data_path': collectd_path + 'cpu',\n",
    "        'plugin_filter': \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"\n",
    "    }, \n",
    "    'load_longterm': {\n",
    "        'plugin_data_path': collectd_path + 'load',\n",
    "        'plugin_filter': \"value_instance == 'longterm' and plugin == 'load'\"\n",
    "    },\n",
    "    'disk_io_time': {\n",
    "        'plugin_data_path': collectd_path + 'cloud',\n",
    "        'plugin_filter': \"value_instance == 'io_time' and plugin == 'disk'\"\n",
    "    }\n",
    "}\n",
    "\n",
    "json_data_train = json_data.copy()\n",
    "json_data_train['local_cache_folder'] += 'train/'\n",
    "json_data_train['hdfs_out_folder'] += 'train/'\n",
    "json_data_train['hdfs_cache_folder'] += 'train/'\n",
    "\n",
    "json_data_inference = json_data.copy()\n",
    "json_data_inference['local_cache_folder'] += 'inference/'\n",
    "json_data_inference['hdfs_out_folder'] += 'inference/'\n",
    "json_data_inference['hdfs_cache_folder'] += 'inference/'\n",
    "# The imporant change is that we want to have NON OVERLAPPING windows\n",
    "# in the inference!\n",
    "json_data_inference['slide_steps'] = 48\n",
    "\n",
    "if longer_run_with_more_data:\n",
    "    json_data_inference['date_start'] = \"2021-03-20\"\n",
    "    json_data_inference['date_end_excluded'] = \"2021-04-20\"\n",
    "else:\n",
    "    json_data_inference['date_start'] = \"2021-04-08\"\n",
    "    json_data_inference['date_end_excluded'] = \"2021-04-14\"   \n",
    "\n",
    "with open(json_file_train, 'w') as outfile:\n",
    "    json.dump(json_data_train, outfile, indent=4)\n",
    "\n",
    "with open(json_file_inference, 'w') as outfile:\n",
    "    json.dump(json_data_inference, outfile, indent=4)\n",
    "    \n",
    "print(\"Json files saved.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "daily-nebraska",
   "metadata": {},
   "source": [
    "## Reading the json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "distinguished-works",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T18:40:53.541935Z",
     "start_time": "2021-05-03T18:40:53.523494Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "RESOURCE DETAILS: demo_config_train.json\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "{\n",
      "    \"hostgroups\": [\n",
      "        \"cloud_compute/level2/main/gva_shared_017\"\n",
      "    ],\n",
      "    \"code_project_name\": \"demo_AD_System\",\n",
      "    \"local_cache_folder\": \"/eos/user/s/smetaj/demo_AD_System/local_cache_train/\",\n",
      "    \"hdfs_out_folder\": \"/user/smetaj/demo_AD_System/raw_parquet_train/\",\n",
      "    \"hdfs_cache_folder\": \"/user/smetaj/demo_AD_System/compressed_train/\",\n",
      "    \"normalization_out_folder\": \"/user/smetaj/demo_AD_System/normalization/\",\n",
      "    \"overwrite_on_hdfs\": true,\n",
      "    \"overwrite_normalization\": true,\n",
      "    \"aggregate_every_n_minutes\": 10,\n",
      "    \"history_steps\": 48,\n",
      "    \"slide_steps\": 1,\n",
      "    \"future_steps\": 0,\n",
      "    \"date_start\": \"2021-04-01\",\n",
      "    \"date_end_excluded\": \"2021-04-08\",\n",
      "    \"date_start_normalization\": \"2021-04-01\",\n",
      "    \"date_end_normalization_excluded\": \"2021-04-08\",\n",
      "    \"selected_plugins\": {\n",
      "        \"cpu__percent_idle\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cpu\",\n",
      "            \"plugin_filter\": \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"\n",
      "        },\n",
      "        \"load_longterm\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/load\",\n",
      "            \"plugin_filter\": \"value_instance == 'longterm' and plugin == 'load'\"\n",
      "        },\n",
      "        \"disk_io_time\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cloud\",\n",
      "            \"plugin_filter\": \"value_instance == 'io_time' and plugin == 'disk'\"\n",
      "        }\n",
      "    }\n",
      "}\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "RESOURCE DETAILS: demo_config_inference.json\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "{\n",
      "    \"hostgroups\": [\n",
      "        \"cloud_compute/level2/main/gva_shared_017\"\n",
      "    ],\n",
      "    \"code_project_name\": \"demo_AD_System\",\n",
      "    \"local_cache_folder\": \"/eos/user/s/smetaj/demo_AD_System/local_cache_inference/\",\n",
      "    \"hdfs_out_folder\": \"/user/smetaj/demo_AD_System/raw_parquet_inference/\",\n",
      "    \"hdfs_cache_folder\": \"/user/smetaj/demo_AD_System/compressed_inference/\",\n",
      "    \"normalization_out_folder\": \"/user/smetaj/demo_AD_System/normalization/\",\n",
      "    \"overwrite_on_hdfs\": true,\n",
      "    \"overwrite_normalization\": true,\n",
      "    \"aggregate_every_n_minutes\": 10,\n",
      "    \"history_steps\": 48,\n",
      "    \"slide_steps\": 48,\n",
      "    \"future_steps\": 0,\n",
      "    \"date_start\": \"2021-04-08\",\n",
      "    \"date_end_excluded\": \"2021-04-19\",\n",
      "    \"date_start_normalization\": \"2021-04-01\",\n",
      "    \"date_end_normalization_excluded\": \"2021-04-08\",\n",
      "    \"selected_plugins\": {\n",
      "        \"cpu__percent_idle\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cpu\",\n",
      "            \"plugin_filter\": \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"\n",
      "        },\n",
      "        \"load_longterm\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/load\",\n",
      "            \"plugin_filter\": \"value_instance == 'longterm' and plugin == 'load'\"\n",
      "        },\n",
      "        \"disk_io_time\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cloud\",\n",
      "            \"plugin_filter\": \"value_instance == 'io_time' and plugin == 'disk'\"\n",
      "        }\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Let's use the read_resource function in the data_mining library to be \n",
    "# sure that the format is correct and to print the files.\n",
    "\n",
    "_ = DM.read_resource(resource_file=json_file_train)\n",
    "_ = DM.read_resource(resource_file=json_file_inference)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "monthly-drive",
   "metadata": {},
   "source": [
    "# ETL steps (Extract, Transform, Load)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "curious-instruction",
   "metadata": {},
   "source": [
    "What we want to achieve here is to reproduce the steps to download and normalize the data (following the order in the graph below).\n",
    "\n",
    "![](https://mattermost.web.cern.ch/files/z18wtsurhtfrzr8eumtao5ybmc/public?h=UB1juU5YHeFJopbeSHrYD2QAlM3TxLRHvsihgtFVB-k)\n",
    "\n",
    "For every step, we will call the specific function of the data_mining library (for both the json files in most of the cases). \n",
    "\n",
    "**Note that we use the click python library (it permits to use python functions through the command line but it has a default behavior that calls exit(0) at the end of them). To avoid the exit(0) problem just use the standalone_mode=False option.**\n",
    "\n",
    "Moreover, we will skip the first 2 steps of the pipeline that we show in the image above (data_presence and check_normalization) because they are used in production pipelines to check and avoid the re-processing of already processed time intervals. For the purpose of this example, we will force the reprocessing if data are available"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "palestinian-investing",
   "metadata": {},
   "source": [
    "## Compute Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "preliminary-shade",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T18:47:02.085990Z",
     "start_time": "2021-05-03T18:45:46.333302Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "PREPARING SPARK:\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK CONTEXT: <SparkContext master=yarn appName=pyspark_shell_swan>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK OBJECT: <pyspark.sql.session.SparkSession object at 0x7f05fb323510>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "RESOURCE DETAILS: demo_config_train.json\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "{\n",
      "    \"hostgroups\": [\n",
      "        \"cloud_compute/level2/main/gva_shared_017\"\n",
      "    ],\n",
      "    \"code_project_name\": \"demo_AD_System\",\n",
      "    \"local_cache_folder\": \"/eos/user/s/smetaj/demo_AD_System/local_cache_train/\",\n",
      "    \"hdfs_out_folder\": \"/user/smetaj/demo_AD_System/raw_parquet_train/\",\n",
      "    \"hdfs_cache_folder\": \"/user/smetaj/demo_AD_System/compressed_train/\",\n",
      "    \"normalization_out_folder\": \"/user/smetaj/demo_AD_System/normalization/\",\n",
      "    \"overwrite_on_hdfs\": true,\n",
      "    \"overwrite_normalization\": true,\n",
      "    \"aggregate_every_n_minutes\": 10,\n",
      "    \"history_steps\": 48,\n",
      "    \"slide_steps\": 1,\n",
      "    \"future_steps\": 0,\n",
      "    \"date_start\": \"2021-04-01\",\n",
      "    \"date_end_excluded\": \"2021-04-08\",\n",
      "    \"date_start_normalization\": \"2021-04-01\",\n",
      "    \"date_end_normalization_excluded\": \"2021-04-08\",\n",
      "    \"selected_plugins\": {\n",
      "        \"cpu__percent_idle\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cpu\",\n",
      "            \"plugin_filter\": \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"\n",
      "        },\n",
      "        \"load_longterm\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/load\",\n",
      "            \"plugin_filter\": \"value_instance == 'longterm' and plugin == 'load'\"\n",
      "        },\n",
      "        \"disk_io_time\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cloud\",\n",
      "            \"plugin_filter\": \"value_instance == 'io_time' and plugin == 'disk'\"\n",
      "        }\n",
      "    }\n",
      "}\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "COMPUTE NORMALIZATION COEFFICIENTS:\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "Start: (2021, 4, 1) - End (2021, 4, 7)\n",
      "{'plugin_data_path': '/project/monitoring/collectd/cpu', 'plugin_filter': \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"}\n",
      "{'plugin_data_path': '/project/monitoring/collectd/load', 'plugin_filter': \"value_instance == 'longterm' and plugin == 'load'\"}\n",
      "{'plugin_data_path': '/project/monitoring/collectd/cloud', 'plugin_filter': \"value_instance == 'io_time' and plugin == 'disk'\"}\n",
      "/project/monitoring/collectd/cloud/2021/04/01/\n",
      "/project/monitoring/collectd/cloud/2021/04/02/\n",
      "/project/monitoring/collectd/cloud/2021/04/03/\n",
      "/project/monitoring/collectd/cloud/2021/04/04/\n",
      "/project/monitoring/collectd/cloud/2021/04/05/\n",
      "/project/monitoring/collectd/cloud/2021/04/06/\n",
      "/project/monitoring/collectd/cloud/2021/04/07/\n",
      "/project/monitoring/collectd/cpu/2021/04/01/\n",
      "/project/monitoring/collectd/cpu/2021/04/02/\n",
      "/project/monitoring/collectd/cpu/2021/04/03/\n",
      "/project/monitoring/collectd/cpu/2021/04/04/\n",
      "/project/monitoring/collectd/cpu/2021/04/05/\n",
      "/project/monitoring/collectd/cpu/2021/04/06/\n",
      "/project/monitoring/collectd/cpu/2021/04/07/\n",
      "/project/monitoring/collectd/load/2021/04/01/\n",
      "/project/monitoring/collectd/load/2021/04/02/\n",
      "/project/monitoring/collectd/load/2021/04/03/\n",
      "/project/monitoring/collectd/load/2021/04/04/\n",
      "/project/monitoring/collectd/load/2021/04/05/\n",
      "/project/monitoring/collectd/load/2021/04/06/\n",
      "/project/monitoring/collectd/load/2021/04/07/\n",
      "filter_str:  ( type == 'percent' and type_instance == 'idle' and plugin == 'cpu' ) or ( value_instance == 'longterm' and plugin == 'load' ) or ( value_instance == 'io_time' and plugin == 'disk' )\n",
      "NEW id_norm:  0a8f66\n",
      "Normalization Saved successfully\n",
      "Coefficient preparation shared:  0\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SUCCESS (inspect the first 40 rows):\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "                                  hostgroup             plugin    mean  stddev\n",
      "0  cloud_compute/level2/main/gva_shared_017  cpu__percent_idle  76.387  16.884\n",
      "1  cloud_compute/level2/main/gva_shared_017      load_longterm   0.269   0.253\n",
      "2  cloud_compute/level2/main/gva_shared_017       disk_io_time  56.666  71.620\n"
     ]
    }
   ],
   "source": [
    "# We must download firstly the normalization coefficients that we will use\n",
    "# to normalize the real data that we will download later.\n",
    "\n",
    "# Note that the function will use the same path for both both train and \n",
    "# inference (because the hostgroup, the dates and the plugins are the same).\n",
    "\n",
    "sys.argv = ['', '--resource_file', json_file_train]\n",
    "DM.compute_normalization(standalone_mode=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "proud-conspiracy",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-15T08:08:23.318591Z",
     "start_time": "2021-04-15T08:08:23.267079Z"
    }
   },
   "source": [
    "## Transform Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "excessive-family",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T18:48:27.814673Z",
     "start_time": "2021-05-03T18:47:02.090004Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "PREPARING SPARK:\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK CONTEXT: <SparkContext master=yarn appName=pyspark_shell_swan>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK OBJECT: <pyspark.sql.session.SparkSession object at 0x7f05fb414f10>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "RESOURCE DETAILS: demo_config_train.json\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "{\n",
      "    \"hostgroups\": [\n",
      "        \"cloud_compute/level2/main/gva_shared_017\"\n",
      "    ],\n",
      "    \"code_project_name\": \"demo_AD_System\",\n",
      "    \"local_cache_folder\": \"/eos/user/s/smetaj/demo_AD_System/local_cache_train/\",\n",
      "    \"hdfs_out_folder\": \"/user/smetaj/demo_AD_System/raw_parquet_train/\",\n",
      "    \"hdfs_cache_folder\": \"/user/smetaj/demo_AD_System/compressed_train/\",\n",
      "    \"normalization_out_folder\": \"/user/smetaj/demo_AD_System/normalization/\",\n",
      "    \"overwrite_on_hdfs\": true,\n",
      "    \"overwrite_normalization\": true,\n",
      "    \"aggregate_every_n_minutes\": 10,\n",
      "    \"history_steps\": 48,\n",
      "    \"slide_steps\": 1,\n",
      "    \"future_steps\": 0,\n",
      "    \"date_start\": \"2021-04-01\",\n",
      "    \"date_end_excluded\": \"2021-04-08\",\n",
      "    \"date_start_normalization\": \"2021-04-01\",\n",
      "    \"date_end_normalization_excluded\": \"2021-04-08\",\n",
      "    \"selected_plugins\": {\n",
      "        \"cpu__percent_idle\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cpu\",\n",
      "            \"plugin_filter\": \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"\n",
      "        },\n",
      "        \"load_longterm\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/load\",\n",
      "            \"plugin_filter\": \"value_instance == 'longterm' and plugin == 'load'\"\n",
      "        },\n",
      "        \"disk_io_time\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cloud\",\n",
      "            \"plugin_filter\": \"value_instance == 'io_time' and plugin == 'disk'\"\n",
      "        }\n",
      "    }\n",
      "}\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "DOWNLOAD DATA - LONG MINING PROCESS...\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "Start: (2021, 4, 1) - End (2021, 4, 7)\n",
      "Reading normalization:  0a8f66\n",
      "{'plugin_data_path': '/project/monitoring/collectd/cpu', 'plugin_filter': \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"}\n",
      "{'plugin_data_path': '/project/monitoring/collectd/load', 'plugin_filter': \"value_instance == 'longterm' and plugin == 'load'\"}\n",
      "{'plugin_data_path': '/project/monitoring/collectd/cloud', 'plugin_filter': \"value_instance == 'io_time' and plugin == 'disk'\"}\n",
      "/project/monitoring/collectd/cloud/2021/04/01/\n",
      "/project/monitoring/collectd/cloud/2021/04/02/\n",
      "/project/monitoring/collectd/cloud/2021/04/03/\n",
      "/project/monitoring/collectd/cloud/2021/04/04/\n",
      "/project/monitoring/collectd/cloud/2021/04/05/\n",
      "/project/monitoring/collectd/cloud/2021/04/06/\n",
      "/project/monitoring/collectd/cloud/2021/04/07/\n",
      "/project/monitoring/collectd/cpu/2021/04/01/\n",
      "/project/monitoring/collectd/cpu/2021/04/02/\n",
      "/project/monitoring/collectd/cpu/2021/04/03/\n",
      "/project/monitoring/collectd/cpu/2021/04/04/\n",
      "/project/monitoring/collectd/cpu/2021/04/05/\n",
      "/project/monitoring/collectd/cpu/2021/04/06/\n",
      "/project/monitoring/collectd/cpu/2021/04/07/\n",
      "/project/monitoring/collectd/load/2021/04/01/\n",
      "/project/monitoring/collectd/load/2021/04/02/\n",
      "/project/monitoring/collectd/load/2021/04/03/\n",
      "/project/monitoring/collectd/load/2021/04/04/\n",
      "/project/monitoring/collectd/load/2021/04/05/\n",
      "/project/monitoring/collectd/load/2021/04/06/\n",
      "/project/monitoring/collectd/load/2021/04/07/\n",
      "filter_str:  ( type == 'percent' and type_instance == 'idle' and plugin == 'cpu' ) or ( value_instance == 'longterm' and plugin == 'load' ) or ( value_instance == 'io_time' and plugin == 'disk' )\n",
      "['cpu__percent_idle', 'load_longterm', 'disk_io_time']\n",
      "Deleting any previous/old remainders in /user/smetaj/demo_AD_System/raw_parquet_train//demo_AD_System ...\n",
      "Error while deleting directory:  [Errno 2] No such file or directory: '/user/smetaj/demo_AD_System/raw_parquet_train//demo_AD_System'\n",
      "Saving the big guy (window dataframe) in: /user/smetaj/demo_AD_System/raw_parquet_train//demo_AD_System ..\n",
      "Saved successfully: /user/smetaj/demo_AD_System/raw_parquet_train//demo_AD_System\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SUCCESS - DATA AGGREGATED IN HDFS.\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Here we download the data that we will normalize using the coefficients \n",
    "# produced by the previous step.\n",
    "\n",
    "# Note that we have to download both the train and the inference datasets,\n",
    "# so we will have 2 different download_data calls.\n",
    "    \n",
    "sys.argv = ['', '--resource_file', json_file_train]\n",
    "DM.transform_data(standalone_mode=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "composite-electron",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T19:31:32.804102Z",
     "start_time": "2021-05-03T19:29:24.608877Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "PREPARING SPARK:\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK CONTEXT: <SparkContext master=yarn appName=pyspark_shell_swan>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK OBJECT: <pyspark.sql.session.SparkSession object at 0x7f064a5f4e50>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "RESOURCE DETAILS: demo_config_inference.json\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "{\n",
      "    \"hostgroups\": [\n",
      "        \"cloud_compute/level2/main/gva_shared_017\"\n",
      "    ],\n",
      "    \"code_project_name\": \"demo_AD_System\",\n",
      "    \"local_cache_folder\": \"/eos/user/s/smetaj/demo_AD_System/local_cache_inference/\",\n",
      "    \"hdfs_out_folder\": \"/user/smetaj/demo_AD_System/raw_parquet_inference/\",\n",
      "    \"hdfs_cache_folder\": \"/user/smetaj/demo_AD_System/compressed_inference/\",\n",
      "    \"normalization_out_folder\": \"/user/smetaj/demo_AD_System/normalization/\",\n",
      "    \"overwrite_on_hdfs\": true,\n",
      "    \"overwrite_normalization\": true,\n",
      "    \"aggregate_every_n_minutes\": 10,\n",
      "    \"history_steps\": 48,\n",
      "    \"slide_steps\": 48,\n",
      "    \"future_steps\": 0,\n",
      "    \"date_start\": \"2021-04-08\",\n",
      "    \"date_end_excluded\": \"2021-04-14\",\n",
      "    \"date_start_normalization\": \"2021-04-01\",\n",
      "    \"date_end_normalization_excluded\": \"2021-04-08\",\n",
      "    \"selected_plugins\": {\n",
      "        \"cpu__percent_idle\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cpu\",\n",
      "            \"plugin_filter\": \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"\n",
      "        },\n",
      "        \"load_longterm\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/load\",\n",
      "            \"plugin_filter\": \"value_instance == 'longterm' and plugin == 'load'\"\n",
      "        },\n",
      "        \"disk_io_time\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cloud\",\n",
      "            \"plugin_filter\": \"value_instance == 'io_time' and plugin == 'disk'\"\n",
      "        }\n",
      "    }\n",
      "}\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "DOWNLOAD DATA - LONG MINING PROCESS...\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "Start: (2021, 4, 8) - End (2021, 4, 13)\n",
      "Reading normalization:  0a8f66\n",
      "{'plugin_data_path': '/project/monitoring/collectd/cpu', 'plugin_filter': \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"}\n",
      "{'plugin_data_path': '/project/monitoring/collectd/load', 'plugin_filter': \"value_instance == 'longterm' and plugin == 'load'\"}\n",
      "{'plugin_data_path': '/project/monitoring/collectd/cloud', 'plugin_filter': \"value_instance == 'io_time' and plugin == 'disk'\"}\n",
      "/project/monitoring/collectd/cloud/2021/04/08/\n",
      "/project/monitoring/collectd/cloud/2021/04/09/\n",
      "/project/monitoring/collectd/cloud/2021/04/10/\n",
      "/project/monitoring/collectd/cloud/2021/04/11/\n",
      "/project/monitoring/collectd/cloud/2021/04/12/\n",
      "/project/monitoring/collectd/cloud/2021/04/13/\n",
      "/project/monitoring/collectd/cpu/2021/04/08/\n",
      "/project/monitoring/collectd/cpu/2021/04/09/\n",
      "/project/monitoring/collectd/cpu/2021/04/10/\n",
      "/project/monitoring/collectd/cpu/2021/04/11/\n",
      "/project/monitoring/collectd/cpu/2021/04/12/\n",
      "/project/monitoring/collectd/cpu/2021/04/13/\n",
      "/project/monitoring/collectd/load/2021/04/08/\n",
      "/project/monitoring/collectd/load/2021/04/09/\n",
      "/project/monitoring/collectd/load/2021/04/10/\n",
      "/project/monitoring/collectd/load/2021/04/11/\n",
      "/project/monitoring/collectd/load/2021/04/12/\n",
      "/project/monitoring/collectd/load/2021/04/13/\n",
      "filter_str:  ( type == 'percent' and type_instance == 'idle' and plugin == 'cpu' ) or ( value_instance == 'longterm' and plugin == 'load' ) or ( value_instance == 'io_time' and plugin == 'disk' )\n",
      "['cpu__percent_idle', 'load_longterm', 'disk_io_time']\n",
      "Deleting any previous/old remainders in /user/smetaj/demo_AD_System/raw_parquet_inference//demo_AD_System ...\n",
      "Error while deleting directory:  [Errno 2] No such file or directory: '/user/smetaj/demo_AD_System/raw_parquet_inference//demo_AD_System'\n",
      "Saving the big guy (window dataframe) in: /user/smetaj/demo_AD_System/raw_parquet_inference//demo_AD_System ..\n",
      "Saved successfully: /user/smetaj/demo_AD_System/raw_parquet_inference//demo_AD_System\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SUCCESS - DATA AGGREGATED IN HDFS.\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sys.argv = ['', '--resource_file', json_file_inference]\n",
    "DM.transform_data(standalone_mode=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "liberal-scotland",
   "metadata": {},
   "source": [
    "## Copy Locally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "executive-native",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T18:50:13.991759Z",
     "start_time": "2021-05-03T18:49:47.942324Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "PREPARING SPARK:\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK CONTEXT: <SparkContext master=yarn appName=pyspark_shell_swan>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK OBJECT: <pyspark.sql.session.SparkSession object at 0x7f05fb2f5b10>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "RESOURCE DETAILS: demo_config_train.json\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "{\n",
      "    \"hostgroups\": [\n",
      "        \"cloud_compute/level2/main/gva_shared_017\"\n",
      "    ],\n",
      "    \"code_project_name\": \"demo_AD_System\",\n",
      "    \"local_cache_folder\": \"/eos/user/s/smetaj/demo_AD_System/local_cache_train/\",\n",
      "    \"hdfs_out_folder\": \"/user/smetaj/demo_AD_System/raw_parquet_train/\",\n",
      "    \"hdfs_cache_folder\": \"/user/smetaj/demo_AD_System/compressed_train/\",\n",
      "    \"normalization_out_folder\": \"/user/smetaj/demo_AD_System/normalization/\",\n",
      "    \"overwrite_on_hdfs\": true,\n",
      "    \"overwrite_normalization\": true,\n",
      "    \"aggregate_every_n_minutes\": 10,\n",
      "    \"history_steps\": 48,\n",
      "    \"slide_steps\": 1,\n",
      "    \"future_steps\": 0,\n",
      "    \"date_start\": \"2021-04-01\",\n",
      "    \"date_end_excluded\": \"2021-04-08\",\n",
      "    \"date_start_normalization\": \"2021-04-01\",\n",
      "    \"date_end_normalization_excluded\": \"2021-04-08\",\n",
      "    \"selected_plugins\": {\n",
      "        \"cpu__percent_idle\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cpu\",\n",
      "            \"plugin_filter\": \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"\n",
      "        },\n",
      "        \"load_longterm\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/load\",\n",
      "            \"plugin_filter\": \"value_instance == 'longterm' and plugin == 'load'\"\n",
      "        },\n",
      "        \"disk_io_time\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cloud\",\n",
      "            \"plugin_filter\": \"value_instance == 'io_time' and plugin == 'disk'\"\n",
      "        }\n",
      "    }\n",
      "}\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "CACHE NEW DATA:\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "ASSUMPTION: no data was in cache before and if anything is there, it will be deleted.\n",
      "Deleting any old remainders...\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "CACHE CREATION\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "1617228000\n",
      "SUCCESS\n",
      "Deleting the raw data saved in /user/smetaj/demo_AD_System/raw_parquet_train/demo_AD_System ...\n",
      "Save the config file locally: /eos/user/s/smetaj/demo_AD_System/local_cache_train/demo_AD_System.metadata\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SUCCESS - CACHED DATA AVAILABLE LOCALLY\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# The final step of the ETL procedure is to save the HDFS downloaded \n",
    "# files into EOS, in this way the data are easier to be accessed and used.\n",
    "\n",
    "# As always we must do the same for both the datasets.\n",
    "\n",
    "sys.argv = ['', '--resource_file', json_file_train]\n",
    "DM.copy_locally(standalone_mode=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "amber-product",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T19:31:43.239907Z",
     "start_time": "2021-05-03T19:31:32.810420Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "PREPARING SPARK:\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK CONTEXT: <SparkContext master=yarn appName=pyspark_shell_swan>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK OBJECT: <pyspark.sql.session.SparkSession object at 0x7f05745ae950>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "RESOURCE DETAILS: demo_config_inference.json\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "{\n",
      "    \"hostgroups\": [\n",
      "        \"cloud_compute/level2/main/gva_shared_017\"\n",
      "    ],\n",
      "    \"code_project_name\": \"demo_AD_System\",\n",
      "    \"local_cache_folder\": \"/eos/user/s/smetaj/demo_AD_System/local_cache_inference/\",\n",
      "    \"hdfs_out_folder\": \"/user/smetaj/demo_AD_System/raw_parquet_inference/\",\n",
      "    \"hdfs_cache_folder\": \"/user/smetaj/demo_AD_System/compressed_inference/\",\n",
      "    \"normalization_out_folder\": \"/user/smetaj/demo_AD_System/normalization/\",\n",
      "    \"overwrite_on_hdfs\": true,\n",
      "    \"overwrite_normalization\": true,\n",
      "    \"aggregate_every_n_minutes\": 10,\n",
      "    \"history_steps\": 48,\n",
      "    \"slide_steps\": 48,\n",
      "    \"future_steps\": 0,\n",
      "    \"date_start\": \"2021-04-08\",\n",
      "    \"date_end_excluded\": \"2021-04-14\",\n",
      "    \"date_start_normalization\": \"2021-04-01\",\n",
      "    \"date_end_normalization_excluded\": \"2021-04-08\",\n",
      "    \"selected_plugins\": {\n",
      "        \"cpu__percent_idle\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cpu\",\n",
      "            \"plugin_filter\": \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"\n",
      "        },\n",
      "        \"load_longterm\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/load\",\n",
      "            \"plugin_filter\": \"value_instance == 'longterm' and plugin == 'load'\"\n",
      "        },\n",
      "        \"disk_io_time\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cloud\",\n",
      "            \"plugin_filter\": \"value_instance == 'io_time' and plugin == 'disk'\"\n",
      "        }\n",
      "    }\n",
      "}\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "CACHE NEW DATA:\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "ASSUMPTION: no data was in cache before and if anything is there, it will be deleted.\n",
      "Deleting any old remainders...\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "CACHE CREATION\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "1617832800\n",
      "SUCCESS\n",
      "Deleting the raw data saved in /user/smetaj/demo_AD_System/raw_parquet_inference/demo_AD_System ...\n",
      "Save the config file locally: /eos/user/s/smetaj/demo_AD_System/local_cache_inference/demo_AD_System.metadata\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SUCCESS - CACHED DATA AVAILABLE LOCALLY\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sys.argv = ['', '--resource_file', json_file_inference]\n",
    "DM.copy_locally(standalone_mode=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "clinical-puppy",
   "metadata": {},
   "source": [
    "# Visualization of downloaded time series"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "loved-victoria",
   "metadata": {},
   "source": [
    "## Reading time series with pandas and host definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "positive-people",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T19:31:43.591850Z",
     "start_time": "2021-05-03T19:31:43.246446Z"
    }
   },
   "outputs": [],
   "source": [
    "# Now we have downloaded the data. \n",
    "# We also want some tools to visualize the time series.\n",
    "\n",
    "# in local_path we have our data saved in EOS\n",
    "local_path_train = json_data[\"local_cache_folder\"] + 'train/' + json_data[\"code_project_name\"]\n",
    "local_path_inference = json_data[\"local_cache_folder\"] + 'inference/' + json_data[\"code_project_name\"]\n",
    "\n",
    "# nr_timeseries will be equal to the number of plugins that we have downloaded\n",
    "nr_timeseries = len(json_data[\"selected_plugins\"])\n",
    "\n",
    "# finally df will be the pandas dataframe containing the data\n",
    "# (it is eaisier to print them with pandas)\n",
    "df_train = pd.read_parquet(local_path_train)\n",
    "df_inference = pd.read_parquet(local_path_inference)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "relevant-barbados",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T19:31:43.631050Z",
     "start_time": "2021-05-03T19:31:43.597536Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>hostname</th>\n",
       "      <th>hostgroup</th>\n",
       "      <th>cpu__percent_idle_h0</th>\n",
       "      <th>cpu__percent_idle_h1</th>\n",
       "      <th>cpu__percent_idle_h2</th>\n",
       "      <th>cpu__percent_idle_h3</th>\n",
       "      <th>cpu__percent_idle_h4</th>\n",
       "      <th>cpu__percent_idle_h5</th>\n",
       "      <th>cpu__percent_idle_h6</th>\n",
       "      <th>...</th>\n",
       "      <th>load_longterm_h39</th>\n",
       "      <th>load_longterm_h40</th>\n",
       "      <th>load_longterm_h41</th>\n",
       "      <th>load_longterm_h42</th>\n",
       "      <th>load_longterm_h43</th>\n",
       "      <th>load_longterm_h44</th>\n",
       "      <th>load_longterm_h45</th>\n",
       "      <th>load_longterm_h46</th>\n",
       "      <th>load_longterm_h47</th>\n",
       "      <th>ts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1618063200</td>\n",
       "      <td>p06492044x27120.cern.ch</td>\n",
       "      <td>cloud_compute/level2/main/gva_shared_017</td>\n",
       "      <td>-0.215828</td>\n",
       "      <td>-0.051549</td>\n",
       "      <td>-0.346833</td>\n",
       "      <td>-0.495070</td>\n",
       "      <td>0.012011</td>\n",
       "      <td>-0.048024</td>\n",
       "      <td>-0.049717</td>\n",
       "      <td>...</td>\n",
       "      <td>0.889130</td>\n",
       "      <td>1.045257</td>\n",
       "      <td>1.454985</td>\n",
       "      <td>1.672815</td>\n",
       "      <td>1.432148</td>\n",
       "      <td>1.741700</td>\n",
       "      <td>2.517194</td>\n",
       "      <td>2.421937</td>\n",
       "      <td>1.307202</td>\n",
       "      <td>2021-04-10 16:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1618034400</td>\n",
       "      <td>p06492044s46921.cern.ch</td>\n",
       "      <td>cloud_compute/level2/main/gva_shared_017</td>\n",
       "      <td>0.345407</td>\n",
       "      <td>0.192619</td>\n",
       "      <td>0.194089</td>\n",
       "      <td>0.131184</td>\n",
       "      <td>0.289174</td>\n",
       "      <td>0.194735</td>\n",
       "      <td>0.251950</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.290733</td>\n",
       "      <td>-0.306522</td>\n",
       "      <td>-0.267896</td>\n",
       "      <td>-0.259140</td>\n",
       "      <td>-0.253557</td>\n",
       "      <td>-0.181621</td>\n",
       "      <td>-0.111413</td>\n",
       "      <td>-0.103557</td>\n",
       "      <td>-0.121400</td>\n",
       "      <td>2021-04-10 08:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1618092000</td>\n",
       "      <td>p06428644v93101.cern.ch</td>\n",
       "      <td>cloud_compute/level2/main/gva_shared_017</td>\n",
       "      <td>0.790763</td>\n",
       "      <td>0.820482</td>\n",
       "      <td>0.706678</td>\n",
       "      <td>0.763613</td>\n",
       "      <td>0.742577</td>\n",
       "      <td>0.671884</td>\n",
       "      <td>0.776097</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.621739</td>\n",
       "      <td>-0.657115</td>\n",
       "      <td>-0.627470</td>\n",
       "      <td>-0.652833</td>\n",
       "      <td>-0.633289</td>\n",
       "      <td>-0.578854</td>\n",
       "      <td>-0.541722</td>\n",
       "      <td>-0.539526</td>\n",
       "      <td>-0.663812</td>\n",
       "      <td>2021-04-11 00:00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 148 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    timestamp                 hostname  \\\n",
       "0  1618063200  p06492044x27120.cern.ch   \n",
       "1  1618034400  p06492044s46921.cern.ch   \n",
       "2  1618092000  p06428644v93101.cern.ch   \n",
       "\n",
       "                                  hostgroup  cpu__percent_idle_h0  \\\n",
       "0  cloud_compute/level2/main/gva_shared_017             -0.215828   \n",
       "1  cloud_compute/level2/main/gva_shared_017              0.345407   \n",
       "2  cloud_compute/level2/main/gva_shared_017              0.790763   \n",
       "\n",
       "   cpu__percent_idle_h1  cpu__percent_idle_h2  cpu__percent_idle_h3  \\\n",
       "0             -0.051549             -0.346833             -0.495070   \n",
       "1              0.192619              0.194089              0.131184   \n",
       "2              0.820482              0.706678              0.763613   \n",
       "\n",
       "   cpu__percent_idle_h4  cpu__percent_idle_h5  cpu__percent_idle_h6  ...  \\\n",
       "0              0.012011             -0.048024             -0.049717  ...   \n",
       "1              0.289174              0.194735              0.251950  ...   \n",
       "2              0.742577              0.671884              0.776097  ...   \n",
       "\n",
       "   load_longterm_h39  load_longterm_h40  load_longterm_h41  load_longterm_h42  \\\n",
       "0           0.889130           1.045257           1.454985           1.672815   \n",
       "1          -0.290733          -0.306522          -0.267896          -0.259140   \n",
       "2          -0.621739          -0.657115          -0.627470          -0.652833   \n",
       "\n",
       "   load_longterm_h43  load_longterm_h44  load_longterm_h45  load_longterm_h46  \\\n",
       "0           1.432148           1.741700           2.517194           2.421937   \n",
       "1          -0.253557          -0.181621          -0.111413          -0.103557   \n",
       "2          -0.633289          -0.578854          -0.541722          -0.539526   \n",
       "\n",
       "   load_longterm_h47                   ts  \n",
       "0           1.307202  2021-04-10 16:00:00  \n",
       "1          -0.121400  2021-04-10 08:00:00  \n",
       "2          -0.663812  2021-04-11 00:00:00  \n",
       "\n",
       "[3 rows x 148 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Note that each row of df will be a window of data following the granularity definition in the configuration file. \n",
    "# In this example it consists of 48 points representing 8 hours of time interval with an aggregation of 10 minutes.\n",
    "# So for every day, for every hostname, we will have 3 rows (3 x 8hours = 24),\n",
    "# each row will be composed by timestamp, hostname, hostgroup, ts and\n",
    "# for each plugin we will have 48 columns, 1 for each 10 minutes.\n",
    "df_inference.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "innocent-percentage",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T19:31:43.642493Z",
     "start_time": "2021-05-03T19:31:43.637265Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['p06428644w45688.cern.ch',\n",
       " 'p06428644x11005.cern.ch',\n",
       " 'p06428644v24034.cern.ch',\n",
       " 'p06428644r01940.cern.ch',\n",
       " 'p06492044e17935.cern.ch']"
      ]
     },