AD_system_demo_1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "known-novel",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Demo-Notebook-for-TimeSeries-Anomaly-Detection\" data-toc-modified-id=\"Demo-Notebook-for-TimeSeries-Anomaly-Detection-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Demo Notebook for TimeSeries Anomaly Detection</a></span></li><li><span><a href=\"#Installation-of-libraries-and-imports\" data-toc-modified-id=\"Installation-of-libraries-and-imports-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Installation of libraries and imports</a></span><ul class=\"toc-item\"><li><span><a href=\"#Installation-of-adcern-and-others-libraries\" data-toc-modified-id=\"Installation-of-adcern-and-others-libraries-2.1\"><span class=\"toc-item-num\">2.1&nbsp;&nbsp;</span>Installation of adcern and others libraries</a></span></li><li><span><a href=\"#Imports\" data-toc-modified-id=\"Imports-2.2\"><span class=\"toc-item-num\">2.2&nbsp;&nbsp;</span>Imports</a></span></li></ul></li><li><span><a href=\"#Init-of-configuration-files---ETL\" data-toc-modified-id=\"Init-of-configuration-files---ETL-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>Init of configuration files - ETL</a></span><ul class=\"toc-item\"><li><span><a href=\"#Creation\" data-toc-modified-id=\"Creation-3.1\"><span class=\"toc-item-num\">3.1&nbsp;&nbsp;</span>Creation</a></span></li><li><span><a href=\"#Reading-the-json\" data-toc-modified-id=\"Reading-the-json-3.2\"><span class=\"toc-item-num\">3.2&nbsp;&nbsp;</span>Reading the json</a></span></li></ul></li><li><span><a href=\"#ETL-steps-(Extract,-Transform,-Load)\" data-toc-modified-id=\"ETL-steps-(Extract,-Transform,-Load)-4\"><span class=\"toc-item-num\">4&nbsp;&nbsp;</span>ETL steps (Extract, Transform, Load)</a></span><ul class=\"toc-item\"><li><span><a href=\"#Compute-Normalization\" data-toc-modified-id=\"Compute-Normalization-4.1\"><span class=\"toc-item-num\">4.1&nbsp;&nbsp;</span>Compute Normalization</a></span></li><li><span><a href=\"#Transform-Data\" data-toc-modified-id=\"Transform-Data-4.2\"><span class=\"toc-item-num\">4.2&nbsp;&nbsp;</span>Transform Data</a></span></li><li><span><a href=\"#Copy-Locally\" data-toc-modified-id=\"Copy-Locally-4.3\"><span class=\"toc-item-num\">4.3&nbsp;&nbsp;</span>Copy Locally</a></span></li></ul></li><li><span><a href=\"#Visualization-of-downloaded-time-series\" data-toc-modified-id=\"Visualization-of-downloaded-time-series-5\"><span class=\"toc-item-num\">5&nbsp;&nbsp;</span>Visualization of downloaded time series</a></span><ul class=\"toc-item\"><li><span><a href=\"#Reading-time-series-with-pandas-and-host-definition\" data-toc-modified-id=\"Reading-time-series-with-pandas-and-host-definition-5.1\"><span class=\"toc-item-num\">5.1&nbsp;&nbsp;</span>Reading time series with pandas and host definition</a></span></li><li><span><a href=\"#Reconstruction-function\" data-toc-modified-id=\"Reconstruction-function-5.2\"><span class=\"toc-item-num\">5.2&nbsp;&nbsp;</span>Reconstruction function</a></span></li><li><span><a href=\"#Normalized-data-visualization\" data-toc-modified-id=\"Normalized-data-visualization-5.3\"><span class=\"toc-item-num\">5.3&nbsp;&nbsp;</span>Normalized data visualization</a></span></li><li><span><a href=\"#How-to-retrieve-the-original-data-without-normalization\" data-toc-modified-id=\"How-to-retrieve-the-original-data-without-normalization-5.4\"><span class=\"toc-item-num\">5.4&nbsp;&nbsp;</span>How to retrieve the original data without normalization</a></span></li><li><span><a href=\"#Original-data-visualization\" data-toc-modified-id=\"Original-data-visualization-5.5\"><span class=\"toc-item-num\">5.5&nbsp;&nbsp;</span>Original data visualization</a></span></li></ul></li><li><span><a href=\"#Init-of-configuration-file---ANALYSIS\" data-toc-modified-id=\"Init-of-configuration-file---ANALYSIS-6\"><span class=\"toc-item-num\">6&nbsp;&nbsp;</span>Init of configuration file - ANALYSIS</a></span><ul class=\"toc-item\"><li><span><a href=\"#Creation\" data-toc-modified-id=\"Creation-6.1\"><span class=\"toc-item-num\">6.1&nbsp;&nbsp;</span>Creation</a></span></li><li><span><a href=\"#Reading-the-json\" data-toc-modified-id=\"Reading-the-json-6.2\"><span class=\"toc-item-num\">6.2&nbsp;&nbsp;</span>Reading the json</a></span></li></ul></li><li><span><a href=\"#ANALYSIS-to-produce-anomaly-scores\" data-toc-modified-id=\"ANALYSIS-to-produce-anomaly-scores-7\"><span class=\"toc-item-num\">7&nbsp;&nbsp;</span>ANALYSIS to produce anomaly scores</a></span><ul class=\"toc-item\"><li><span><a href=\"#Analysis\" data-toc-modified-id=\"Analysis-7.1\"><span class=\"toc-item-num\">7.1&nbsp;&nbsp;</span>Analysis</a></span></li></ul></li><li><span><a href=\"#Visualization-of-the-results\" data-toc-modified-id=\"Visualization-of-the-results-8\"><span class=\"toc-item-num\">8&nbsp;&nbsp;</span>Visualization of the results</a></span><ul class=\"toc-item\"><li><span><a href=\"#Reading-the-scores\" data-toc-modified-id=\"Reading-the-scores-8.1\"><span class=\"toc-item-num\">8.1&nbsp;&nbsp;</span>Reading the scores</a></span></li><li><span><a href=\"#Visualization-of-both-downloaded-data-and-scores\" data-toc-modified-id=\"Visualization-of-both-downloaded-data-and-scores-8.2\"><span class=\"toc-item-num\">8.2&nbsp;&nbsp;</span>Visualization of both downloaded data and scores</a></span></li></ul></li><li><span><a href=\"#ETL-Steps-not-used\" data-toc-modified-id=\"ETL-Steps-not-used-9\"><span class=\"toc-item-num\">9&nbsp;&nbsp;</span>ETL Steps not used</a></span><ul class=\"toc-item\"><li><span><a href=\"#Data-Presence\" data-toc-modified-id=\"Data-Presence-9.1\"><span class=\"toc-item-num\">9.1&nbsp;&nbsp;</span>Data Presence</a></span></li><li><span><a href=\"#Check-Normalization\" data-toc-modified-id=\"Check-Normalization-9.2\"><span class=\"toc-item-num\">9.2&nbsp;&nbsp;</span>Check Normalization</a></span></li></ul></li><li><span><a href=\"#Adding-a-model\" data-toc-modified-id=\"Adding-a-model-10\"><span class=\"toc-item-num\">10&nbsp;&nbsp;</span>Adding a model</a></span><ul class=\"toc-item\"><li><span><a href=\"#Changing-config-file---Analysis\" data-toc-modified-id=\"Changing-config-file---Analysis-10.1\"><span class=\"toc-item-num\">10.1&nbsp;&nbsp;</span>Changing config file - Analysis</a></span></li><li><span><a href=\"#Reading-the-json\" data-toc-modified-id=\"Reading-the-json-10.2\"><span class=\"toc-item-num\">10.2&nbsp;&nbsp;</span>Reading the json</a></span></li><li><span><a href=\"#Analysis-of-multiple-models\" data-toc-modified-id=\"Analysis-of-multiple-models-10.3\"><span class=\"toc-item-num\">10.3&nbsp;&nbsp;</span>Analysis of multiple models</a></span></li><li><span><a href=\"#Checking-new-results\" data-toc-modified-id=\"Checking-new-results-10.4\"><span class=\"toc-item-num\">10.4&nbsp;&nbsp;</span>Checking new results</a></span></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "understood-cross",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Demo Notebook for TimeSeries Anomaly Detection\n",
    "This notebook shows the Anomaly Detection system for TimeSeries in action using the tools provided in this  repository.\n",
    "\n",
    "**Note that we recommend to run this notebook using swan004.cern.ch with the following configuration:**\n",
    "- Software stack: Other releases, **97a**\n",
    "- Spark cluster: **General Purpose (Analytix)**\n",
    "\n",
    "---\n",
    "**REMEMBER TO ACTIVATE THE ANALYTIX CLUSTER IN THE SWAN CONFIGURATION!!**\n",
    "Activating it means to activate spark (click the spark icon in the upper part of the notebook to enable spark before starting to run the notebook cells)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "legendary-curve",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Installation of libraries and imports"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "union-teaching",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Installation of adcern and others libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "manual-accuracy",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T14:28:15.289755Z",
     "start_time": "2021-05-03T14:28:15.284123Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Set the variable to True the first time to download the libraries.\n",
    "# Note that with @branch you can install a specific branch\n",
    "\n",
    "first_time = False\n",
    "if first_time:\n",
    "    # If you install the libraries it takes some minutes\n",
    "    !pip install --user git+https://:@gitlab.cern.ch:8443/cloud-infrastructure/data-analytics.git"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "played-aluminum",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "romance-prevention",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T14:28:25.414952Z",
     "start_time": "2021-05-03T14:28:19.143582Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# AD System Libraries ----------------------------------------------------\n",
    "import adcern.cmd.data_mining as DM\n",
    "import etl.spark_etl.etl_pipeline as PL\n",
    "\n",
    "# To pass command line parameters and to use other functions -------------\n",
    "import sys, os, re, json\n",
    "\n",
    "# To run the visualization function --------------------------------------\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import plotly.graph_objects as go\n",
    "from plotly.subplots import make_subplots\n",
    "\n",
    "from dateutil import tz\n",
    "\n",
    "# To read more parquet files with pandas ---------------------------------\n",
    "import glob"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "superb-queens",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Init of configuration files - ETL"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "confused-navigation",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Our ETL and analysis methods work with json configuration files in input. In these files we save all the paths, the dates and the hyperparameters used then by the methods.\n",
    "\n",
    "Usually we have the files already saved for static tests or we create them in our production pipeline; **for the purpose of this notebook instead, we create the json files starting from a python dict.**\n",
    "\n",
    "Note that we need 2 config files: \n",
    "- A config file about the training part.\n",
    "- A config file about the data used by the trained model to infer the scores. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "possible-memory",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Creation\n",
    "Note that the 2 config files share most of the parameters so we will have:\n",
    "- json_data: containing all the main information\n",
    "- json_data_train, json_data_inference: containing specific paths for the 2 different purposes\n",
    "\n",
    "**Note also that you have to be sure that you have the writing rights for all the paths contained here, in particular *HDFS_folder_with_write_rights* should point to your hdfs personal folder to ensure that**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "spoken-finding",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T14:49:25.662860Z",
     "start_time": "2021-05-03T14:49:25.210257Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "demo_name = 'demo_AD_System'\n",
    "json_file_train = 'demo_config_train.json'\n",
    "json_file_inference = 'demo_config_inference.json'\n",
    "username = os.environ['USER']\n",
    "HDFS_folder_with_write_rights = '/user/' + username + '/'\n",
    "print(\"Folder to save data and results: \" + HDFS_folder_with_write_rights)\n",
    "\n",
    "collectd_path = \"/project/monitoring/collectd/\"\n",
    "                                  \n",
    "json_data = {}\n",
    "\n",
    "# Absolute path identifier of the cell/hostgroup that you want to mine.\n",
    "# Note that it is in a list format, but only one hostgroup is supported so far.\n",
    "json_data['hostgroups'] = []\n",
    "json_data['hostgroups'].append('cloud_compute/level2/main/gva_shared_017')\n",
    "\n",
    "# The pattern of the names of your data folders and \".metadata\" files.\n",
    "json_data['code_project_name'] = demo_name\n",
    "\n",
    "# Local area of your VM where to save your data and metadata data are saved in\n",
    "# folders with one parquet only. Metadata are saved in file with the same name\n",
    "# of the resepctive foler plus the \".metadata\" extension.\n",
    "json_data['local_cache_folder'] = '/eos/user/' + username[:1] + \\\n",
    "    '/' + username + '/' + demo_name + '/local_cache_'\n",
    "\n",
    "# HDFS Area where Spark saves the aggregated data of your cell.\n",
    "# Note that the saving can create multiple file depending on the number of\n",
    "# partitions that the workers were using.\n",
    "json_data['hdfs_out_folder'] = HDFS_folder_with_write_rights + \\\n",
    "    demo_name + '/raw_parquet_'\n",
    "\n",
    "# HDFS Area where Spark saves the aggregated data of your cell.\n",
    "# Note that here we force it to be one partiotion only.\n",
    "json_data['hdfs_cache_folder'] = HDFS_folder_with_write_rights + \\\n",
    "    demo_name + '/compressed_'\n",
    "\n",
    "# HDFS Area where Spark saves the normalization coefficients computed on the\n",
    "# normalziation chunk of data between the normalization dates.\n",
    "json_data['normalization_out_folder'] = HDFS_folder_with_write_rights + \\\n",
    "    demo_name + '/normalization/'\n",
    "\n",
    "# ----------------------------------------------------------------------------\n",
    "# ----------------------------------------------------------------------------\n",
    "\n",
    "# Wether you want to overwrite (true) or not (false) the raw data in HDFS.\n",
    "# If not sure leave true.\n",
    "json_data['overwrite_on_hdfs'] = True\n",
    "\n",
    "# Wether you want to overwrite (true) or not (false) the noramlization\n",
    "# coefficeints in HDFS. If not sure leave true.\n",
    "json_data['overwrite_normalization'] = True\n",
    "\n",
    "# The level of aggregation of your raw time series data.\n",
    "# The aggregator is typically the mean operator.\n",
    "# e.g. if 5 it means that we summarize the data every 5 min, and the values\n",
    "# with timestamp 7.45 will represent the mean of the previous 5 minutes from\n",
    "# 7.40 to 7.45 but that value will have 7.45 as timestamp\n",
    "json_data['aggregate_every_n_minutes'] = 10\n",
    "\n",
    "# The length of your windows of data.\n",
    "# e.g. if aggregate_every_n_minutes = 10 and history_steps = 6 it means that\n",
    "# every windows is summarizing 6 * 10 = 60 minutes\n",
    "json_data['history_steps'] = 48\n",
    "\n",
    "# The number of step you want to move your window.\n",
    "# e.g. if aggregate_every_n_minutes = 10 and history_steps = 2 it means that\n",
    "# you will get a window of data that is translated of 10 * 2 = 20 min with\n",
    "# respect to the previous.\n",
    "# Note that if slide_steps has the same value of history_steps you have non-\n",
    "# overlapping windows.\n",
    "json_data['slide_steps'] = 1\n",
    "\n",
    "# Used to create windows with future steps. If not sure keep this to 0.\n",
    "json_data['future_steps'] = 0\n",
    "\n",
    "# ----------------------------------------------------------------------------\n",
    "# ----------------------------------------------------------------------------\n",
    "\n",
    "# Dates representing the start/end of the data and noramlization chunks.\n",
    "# - start_date -> the starting date of data chunk of ETL\n",
    "# - end_date -> the ending date of data chunk of ETL\n",
    "# - start_date_normalization -> the starting date of the chunk of data used\n",
    "#   to learn noramlization coefficeints (typically this chunk preceeds the\n",
    "#   chunk of data)\n",
    "# - end_date_normalization -> the ending date of the chunk of data used\n",
    "#   to learn noramlization coefficeints\n",
    "# Note that the upper extremum is excluded (i.e. data will stop at the 23:59\n",
    "# of the day preeceeding the date_end_excluded)\n",
    "json_data['date_start'] = \"2021-03-01\"\n",
    "json_data['date_end_excluded'] = \"2021-03-20\"\n",
    "json_data['date_start_normalization'] = \"2021-03-01\"\n",
    "json_data['date_end_normalization_excluded'] = \"2021-03-20\"\n",
    "\n",
    "# ----------------------------------------------------------------------------\n",
    "# ----------------------------------------------------------------------------\n",
    "\n",
    "# List of plugins to mine.\n",
    "# Note that it is a dictionary where every key represents the name your plugin\n",
    "# have and the value is a dictionary with:\n",
    "# 'plugin_instance', 'type' 'type_instance', 'plugin_name'\n",
    "# the value asigned to these key is defining an and-filter.\n",
    "# you will get only the data that have all those attributes\n",
    "# ('plugin_instance', 'type' 'type_instance', 'plugin_name') in and with the\n",
    "# specified value.\n",
    "# Note that if you do not want to filter on one attribute do not express it.\n",
    "json_data['selected_plugins'] = {\n",
    "    'cpu__percent_idle': {\n",
    "        'plugin_data_path': collectd_path + 'cpu',\n",
    "        'plugin_filter': \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"\n",
    "    }, \n",
    "    'load_longterm': {\n",
    "        'plugin_data_path': collectd_path + 'load',\n",
    "        'plugin_filter': \"value_instance == 'longterm' and plugin == 'load'\"\n",
    "    },\n",
    "    'disk_io_time': {\n",
    "        'plugin_data_path': collectd_path + 'cloud',\n",
    "        'plugin_filter': \"value_instance == 'io_time' and plugin == 'disk'\"\n",
    "    }\n",
    "}\n",
    "\n",
    "json_data_train = json_data.copy()\n",
    "json_data_train['local_cache_folder'] += 'train/'\n",
    "json_data_train['hdfs_out_folder'] += 'train/'\n",
    "json_data_train['hdfs_cache_folder'] += 'train/'\n",
    "\n",
    "json_data_inference = json_data.copy()\n",
    "json_data_inference['local_cache_folder'] += 'inference/'\n",
    "json_data_inference['hdfs_out_folder'] += 'inference/'\n",
    "json_data_inference['hdfs_cache_folder'] += 'inference/'\n",
    "# The imporant change is that we want to have NON OVERLAPPING windows\n",
    "# in the inference!\n",
    "json_data_inference['slide_steps'] = 48\n",
    "# I want to infer only 24 hours!\n",
    "json_data_inference['date_start'] = \"2021-03-20\"\n",
    "json_data_inference['date_end_excluded'] = \"2021-04-20\"\n",
    "\n",
    "with open(json_file_train, 'w') as outfile:\n",
    "    json.dump(json_data_train, outfile, indent=4)\n",
    "\n",
    "with open(json_file_inference, 'w') as outfile:\n",
    "    json.dump(json_data_inference, outfile, indent=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "anticipated-messenger",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Reading the json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "empty-september",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T14:49:30.976439Z",
     "start_time": "2021-05-03T14:49:30.953248Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "RESOURCE DETAILS: demo_config_train.json\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "{\n",
      "    \"hostgroups\": [\n",
      "        \"cloud_compute/level2/main/gva_shared_017\"\n",
      "    ],\n",
      "    \"code_project_name\": \"demo_AD_System\",\n",
      "    \"local_cache_folder\": \"/eos/user/s/smetaj/demo_AD_System/local_cache_train/\",\n",
      "    \"hdfs_out_folder\": \"/user/smetaj/demo_AD_System/raw_parquet_train/\",\n",
      "    \"hdfs_cache_folder\": \"/user/smetaj/demo_AD_System/compressed_train/\",\n",
      "    \"normalization_out_folder\": \"/user/smetaj/demo_AD_System/normalization/\",\n",
      "    \"overwrite_on_hdfs\": true,\n",
      "    \"overwrite_normalization\": true,\n",
      "    \"aggregate_every_n_minutes\": 10,\n",
      "    \"history_steps\": 48,\n",
      "    \"slide_steps\": 1,\n",
      "    \"future_steps\": 0,\n",
      "    \"date_start\": \"2021-03-01\",\n",
      "    \"date_end_excluded\": \"2021-03-20\",\n",
      "    \"date_start_normalization\": \"2021-03-01\",\n",
      "    \"date_end_normalization_excluded\": \"2021-03-20\",\n",
      "    \"selected_plugins\": {\n",
      "        \"cpu__percent_idle\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cpu\",\n",
      "            \"plugin_filter\": \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"\n",
      "        },\n",
      "        \"load_longterm\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/load\",\n",
      "            \"plugin_filter\": \"value_instance == 'longterm' and plugin == 'load'\"\n",
      "        },\n",
      "        \"disk_io_time\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cloud\",\n",
      "            \"plugin_filter\": \"value_instance == 'io_time' and plugin == 'disk'\"\n",
      "        }\n",
      "    }\n",
      "}\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "RESOURCE DETAILS: demo_config_inference.json\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "{\n",
      "    \"hostgroups\": [\n",
      "        \"cloud_compute/level2/main/gva_shared_017\"\n",
      "    ],\n",
      "    \"code_project_name\": \"demo_AD_System\",\n",
      "    \"local_cache_folder\": \"/eos/user/s/smetaj/demo_AD_System/local_cache_inference/\",\n",
      "    \"hdfs_out_folder\": \"/user/smetaj/demo_AD_System/raw_parquet_inference/\",\n",
      "    \"hdfs_cache_folder\": \"/user/smetaj/demo_AD_System/compressed_inference/\",\n",
      "    \"normalization_out_folder\": \"/user/smetaj/demo_AD_System/normalization/\",\n",
      "    \"overwrite_on_hdfs\": true,\n",
      "    \"overwrite_normalization\": true,\n",
      "    \"aggregate_every_n_minutes\": 10,\n",
      "    \"history_steps\": 48,\n",
      "    \"slide_steps\": 48,\n",
      "    \"future_steps\": 0,\n",
      "    \"date_start\": \"2021-03-20\",\n",
      "    \"date_end_excluded\": \"2021-04-20\",\n",
      "    \"date_start_normalization\": \"2021-03-01\",\n",
      "    \"date_end_normalization_excluded\": \"2021-03-20\",\n",
      "    \"selected_plugins\": {\n",
      "        \"cpu__percent_idle\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cpu\",\n",
      "            \"plugin_filter\": \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"\n",
      "        },\n",
      "        \"load_longterm\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/load\",\n",
      "            \"plugin_filter\": \"value_instance == 'longterm' and plugin == 'load'\"\n",
      "        },\n",
      "        \"disk_io_time\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cloud\",\n",
      "            \"plugin_filter\": \"value_instance == 'io_time' and plugin == 'disk'\"\n",
      "        }\n",
      "    }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Let's use the read_resource function in the data_mining library to be \n",
    "# sure that the format is correct and to print the files.\n",
    "\n",
    "_ = DM.read_resource(resource_file=json_file_train)\n",
    "_ = DM.read_resource(resource_file=json_file_inference)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "focused-ballot",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# ETL steps (Extract, Transform, Load)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "figured-ground",
   "metadata": {
    "hidden": true
   },
   "source": [
    "What we want to achieve here is to reproduce the steps to download and normalize the data (following the order in the graph below).\n",
    "![](https://mattermost.web.cern.ch/files/skktqwaws7nkzpb16c73n11tac/public?h=KflwzsI6wpa58LJCUgTGH-r8dJHEskq_a4R_05QMOPg)\n",
    "\n",
    "For every step, we will call the specific function of the data_mining library (for both the json files in most of the cases). \n",
    "\n",
    "**Note that we use the click python library (it permits to use python functions through the command line but it has a default behavior that calls exit(0) at the end of them). To avoid the exit(0) problem just use the standalone_mode=False option.**\n",
    "\n",
    "Moreover, we will skip the first 2 steps of the pipeline that we show in the image above (data_presence and check_normalization) because they are used in production pipelines to check and avoid the re-processing of already processed time intervals. For the purpose of this example, we will force the reprocessing if data are available"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "played-faculty",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Compute Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "attempted-wrist",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T14:51:01.068276Z",
     "start_time": "2021-05-03T14:49:34.205805Z"
    },
    "hidden": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "PREPARING SPARK:\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK CONTEXT: <SparkContext master=yarn appName=pyspark_shell_swan>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK OBJECT: <pyspark.sql.session.SparkSession object at 0x7f40f5cfdd90>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "RESOURCE DETAILS: demo_config_train.json\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "{\n",
      "    \"hostgroups\": [\n",
      "        \"cloud_compute/level2/main/gva_shared_017\"\n",
      "    ],\n",
      "    \"code_project_name\": \"demo_AD_System\",\n",
      "    \"local_cache_folder\": \"/eos/user/s/smetaj/demo_AD_System/local_cache_train/\",\n",
      "    \"hdfs_out_folder\": \"/user/smetaj/demo_AD_System/raw_parquet_train/\",\n",
      "    \"hdfs_cache_folder\": \"/user/smetaj/demo_AD_System/compressed_train/\",\n",
      "    \"normalization_out_folder\": \"/user/smetaj/demo_AD_System/normalization/\",\n",
      "    \"overwrite_on_hdfs\": true,\n",
      "    \"overwrite_normalization\": true,\n",
      "    \"aggregate_every_n_minutes\": 10,\n",
      "    \"history_steps\": 48,\n",
      "    \"slide_steps\": 1,\n",
      "    \"future_steps\": 0,\n",
      "    \"date_start\": \"2021-03-01\",\n",
      "    \"date_end_excluded\": \"2021-03-20\",\n",
      "    \"date_start_normalization\": \"2021-03-01\",\n",
      "    \"date_end_normalization_excluded\": \"2021-03-20\",\n",
      "    \"selected_plugins\": {\n",
      "        \"cpu__percent_idle\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cpu\",\n",
      "            \"plugin_filter\": \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"\n",
      "        },\n",
      "        \"load_longterm\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/load\",\n",
      "            \"plugin_filter\": \"value_instance == 'longterm' and plugin == 'load'\"\n",
      "        },\n",
      "        \"disk_io_time\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cloud\",\n",
      "            \"plugin_filter\": \"value_instance == 'io_time' and plugin == 'disk'\"\n",
      "        }\n",
      "    }\n",
      "}\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "COMPUTE NORMALIZATION COEFFICIENTS:\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "Start: (2021, 3, 1) - End (2021, 3, 19)\n",
      "{'plugin_data_path': '/project/monitoring/collectd/cpu', 'plugin_filter': \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"}\n",
      "{'plugin_data_path': '/project/monitoring/collectd/load', 'plugin_filter': \"value_instance == 'longterm' and plugin == 'load'\"}\n",
      "{'plugin_data_path': '/project/monitoring/collectd/cloud', 'plugin_filter': \"value_instance == 'io_time' and plugin == 'disk'\"}\n",
      "/project/monitoring/collectd/cloud/2021/03/01/\n",
      "/project/monitoring/collectd/cloud/2021/03/02/\n",
      "/project/monitoring/collectd/cloud/2021/03/03/\n",
      "/project/monitoring/collectd/cloud/2021/03/04/\n",
      "/project/monitoring/collectd/cloud/2021/03/05/\n",
      "/project/monitoring/collectd/cloud/2021/03/06/\n",
      "/project/monitoring/collectd/cloud/2021/03/07/\n",
      "/project/monitoring/collectd/cloud/2021/03/08/\n",
      "/project/monitoring/collectd/cloud/2021/03/09/\n",
      "/project/monitoring/collectd/cloud/2021/03/10/\n",
      "/project/monitoring/collectd/cloud/2021/03/11/\n",
      "/project/monitoring/collectd/cloud/2021/03/12/\n",
      "/project/monitoring/collectd/cloud/2021/03/13/\n",
      "/project/monitoring/collectd/cloud/2021/03/14/\n",
      "/project/monitoring/collectd/cloud/2021/03/15/\n",
      "/project/monitoring/collectd/cloud/2021/03/16/\n",
      "/project/monitoring/collectd/cloud/2021/03/17/\n",
      "/project/monitoring/collectd/cloud/2021/03/18/\n",
      "/project/monitoring/collectd/cloud/2021/03/19/\n",
      "/project/monitoring/collectd/cpu/2021/03/01/\n",
      "/project/monitoring/collectd/cpu/2021/03/02/\n",
      "/project/monitoring/collectd/cpu/2021/03/03/\n",
      "/project/monitoring/collectd/cpu/2021/03/04/\n",
      "/project/monitoring/collectd/cpu/2021/03/05/\n",
      "/project/monitoring/collectd/cpu/2021/03/06/\n",
      "/project/monitoring/collectd/cpu/2021/03/07/\n",
      "/project/monitoring/collectd/cpu/2021/03/08/\n",
      "/project/monitoring/collectd/cpu/2021/03/09/\n",
      "/project/monitoring/collectd/cpu/2021/03/10/\n",
      "/project/monitoring/collectd/cpu/2021/03/11/\n",
      "/project/monitoring/collectd/cpu/2021/03/12/\n",
      "/project/monitoring/collectd/cpu/2021/03/13/\n",
      "/project/monitoring/collectd/cpu/2021/03/14/\n",
      "/project/monitoring/collectd/cpu/2021/03/15/\n",
      "/project/monitoring/collectd/cpu/2021/03/16/\n",
      "/project/monitoring/collectd/cpu/2021/03/17/\n",
      "/project/monitoring/collectd/cpu/2021/03/18/\n",
      "/project/monitoring/collectd/cpu/2021/03/19/\n",
      "/project/monitoring/collectd/load/2021/03/01/\n",
      "/project/monitoring/collectd/load/2021/03/02/\n",
      "/project/monitoring/collectd/load/2021/03/03/\n",
      "/project/monitoring/collectd/load/2021/03/04/\n",
      "/project/monitoring/collectd/load/2021/03/05/\n",
      "/project/monitoring/collectd/load/2021/03/06/\n",
      "/project/monitoring/collectd/load/2021/03/07/\n",
      "/project/monitoring/collectd/load/2021/03/08/\n",
      "/project/monitoring/collectd/load/2021/03/09/\n",
      "/project/monitoring/collectd/load/2021/03/10/\n",
      "/project/monitoring/collectd/load/2021/03/11/\n",
      "/project/monitoring/collectd/load/2021/03/12/\n",
      "/project/monitoring/collectd/load/2021/03/13/\n",
      "/project/monitoring/collectd/load/2021/03/14/\n",
      "/project/monitoring/collectd/load/2021/03/15/\n",
      "/project/monitoring/collectd/load/2021/03/16/\n",
      "/project/monitoring/collectd/load/2021/03/17/\n",
      "/project/monitoring/collectd/load/2021/03/18/\n",
      "/project/monitoring/collectd/load/2021/03/19/\n",
      "filter_str:  ( type == 'percent' and type_instance == 'idle' and plugin == 'cpu' ) or ( value_instance == 'longterm' and plugin == 'load' ) or ( value_instance == 'io_time' and plugin == 'disk' )\n",
      "NEW id_norm:  ef8bce\n",
      "Normalization Saved successfully\n",
      "Coefficient preparation shared:  0\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SUCCESS (inspect the first 40 rows):\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "                                  hostgroup             plugin    mean  stddev\n",
      "0  cloud_compute/level2/main/gva_shared_017  cpu__percent_idle  74.198  18.997\n",
      "1  cloud_compute/level2/main/gva_shared_017      load_longterm   0.302   0.294\n",
      "2  cloud_compute/level2/main/gva_shared_017       disk_io_time  60.421  69.629\n"
     ]
    }
   ],
   "source": [
    "# We must download firstly the normalization coefficients that we will use\n",
    "# to normalize the real data that we will download later.\n",
    "\n",
    "# Note that the function will use the same path for both both train and \n",
    "# inference (because the hostgroup, the dates and the plugins are the same).\n",
    "\n",
    "sys.argv = ['', '--resource_file', json_file_train]\n",
    "DM.compute_normalization(standalone_mode=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "nonprofit-furniture",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-04-15T08:08:23.318591Z",
     "start_time": "2021-04-15T08:08:23.267079Z"
    },
    "hidden": true
   },
   "source": [
    "## Transform Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "middle-convention",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T14:52:57.741856Z",
     "start_time": "2021-05-03T14:51:01.075438Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "PREPARING SPARK:\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK CONTEXT: <SparkContext master=yarn appName=pyspark_shell_swan>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK OBJECT: <pyspark.sql.session.SparkSession object at 0x7f4081d8a4d0>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "RESOURCE DETAILS: demo_config_train.json\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "{\n",
      "    \"hostgroups\": [\n",
      "        \"cloud_compute/level2/main/gva_shared_017\"\n",
      "    ],\n",
      "    \"code_project_name\": \"demo_AD_System\",\n",
      "    \"local_cache_folder\": \"/eos/user/s/smetaj/demo_AD_System/local_cache_train/\",\n",
      "    \"hdfs_out_folder\": \"/user/smetaj/demo_AD_System/raw_parquet_train/\",\n",
      "    \"hdfs_cache_folder\": \"/user/smetaj/demo_AD_System/compressed_train/\",\n",
      "    \"normalization_out_folder\": \"/user/smetaj/demo_AD_System/normalization/\",\n",
      "    \"overwrite_on_hdfs\": true,\n",
      "    \"overwrite_normalization\": true,\n",
      "    \"aggregate_every_n_minutes\": 10,\n",
      "    \"history_steps\": 48,\n",
      "    \"slide_steps\": 1,\n",
      "    \"future_steps\": 0,\n",
      "    \"date_start\": \"2021-03-01\",\n",
      "    \"date_end_excluded\": \"2021-03-20\",\n",
      "    \"date_start_normalization\": \"2021-03-01\",\n",
      "    \"date_end_normalization_excluded\": \"2021-03-20\",\n",
      "    \"selected_plugins\": {\n",
      "        \"cpu__percent_idle\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cpu\",\n",
      "            \"plugin_filter\": \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"\n",
      "        },\n",
      "        \"load_longterm\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/load\",\n",
      "            \"plugin_filter\": \"value_instance == 'longterm' and plugin == 'load'\"\n",
      "        },\n",
      "        \"disk_io_time\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cloud\",\n",
      "            \"plugin_filter\": \"value_instance == 'io_time' and plugin == 'disk'\"\n",
      "        }\n",
      "    }\n",
      "}\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "DOWNLOAD DATA - LONG MINING PROCESS...\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "Start: (2021, 3, 1) - End (2021, 3, 19)\n",
      "Reading normalization:  ef8bce\n",
      "{'plugin_data_path': '/project/monitoring/collectd/cpu', 'plugin_filter': \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"}\n",
      "{'plugin_data_path': '/project/monitoring/collectd/load', 'plugin_filter': \"value_instance == 'longterm' and plugin == 'load'\"}\n",
      "{'plugin_data_path': '/project/monitoring/collectd/cloud', 'plugin_filter': \"value_instance == 'io_time' and plugin == 'disk'\"}\n",
      "/project/monitoring/collectd/cloud/2021/03/01/\n",
      "/project/monitoring/collectd/cloud/2021/03/02/\n",
      "/project/monitoring/collectd/cloud/2021/03/03/\n",
      "/project/monitoring/collectd/cloud/2021/03/04/\n",
      "/project/monitoring/collectd/cloud/2021/03/05/\n",
      "/project/monitoring/collectd/cloud/2021/03/06/\n",
      "/project/monitoring/collectd/cloud/2021/03/07/\n",
      "/project/monitoring/collectd/cloud/2021/03/08/\n",
      "/project/monitoring/collectd/cloud/2021/03/09/\n",
      "/project/monitoring/collectd/cloud/2021/03/10/\n",
      "/project/monitoring/collectd/cloud/2021/03/11/\n",
      "/project/monitoring/collectd/cloud/2021/03/12/\n",
      "/project/monitoring/collectd/cloud/2021/03/13/\n",
      "/project/monitoring/collectd/cloud/2021/03/14/\n",
      "/project/monitoring/collectd/cloud/2021/03/15/\n",
      "/project/monitoring/collectd/cloud/2021/03/16/\n",
      "/project/monitoring/collectd/cloud/2021/03/17/\n",
      "/project/monitoring/collectd/cloud/2021/03/18/\n",
      "/project/monitoring/collectd/cloud/2021/03/19/\n",
      "/project/monitoring/collectd/cpu/2021/03/01/\n",
      "/project/monitoring/collectd/cpu/2021/03/02/\n",
      "/project/monitoring/collectd/cpu/2021/03/03/\n",
      "/project/monitoring/collectd/cpu/2021/03/04/\n",
      "/project/monitoring/collectd/cpu/2021/03/05/\n",
      "/project/monitoring/collectd/cpu/2021/03/06/\n",
      "/project/monitoring/collectd/cpu/2021/03/07/\n",
      "/project/monitoring/collectd/cpu/2021/03/08/\n",
      "/project/monitoring/collectd/cpu/2021/03/09/\n",
      "/project/monitoring/collectd/cpu/2021/03/10/\n",
      "/project/monitoring/collectd/cpu/2021/03/11/\n",
      "/project/monitoring/collectd/cpu/2021/03/12/\n",
      "/project/monitoring/collectd/cpu/2021/03/13/\n",
      "/project/monitoring/collectd/cpu/2021/03/14/\n",
      "/project/monitoring/collectd/cpu/2021/03/15/\n",
      "/project/monitoring/collectd/cpu/2021/03/16/\n",
      "/project/monitoring/collectd/cpu/2021/03/17/\n",
      "/project/monitoring/collectd/cpu/2021/03/18/\n",
      "/project/monitoring/collectd/cpu/2021/03/19/\n",
      "/project/monitoring/collectd/load/2021/03/01/\n",
      "/project/monitoring/collectd/load/2021/03/02/\n",
      "/project/monitoring/collectd/load/2021/03/03/\n",
      "/project/monitoring/collectd/load/2021/03/04/\n",
      "/project/monitoring/collectd/load/2021/03/05/\n",
      "/project/monitoring/collectd/load/2021/03/06/\n",
      "/project/monitoring/collectd/load/2021/03/07/\n",
      "/project/monitoring/collectd/load/2021/03/08/\n",
      "/project/monitoring/collectd/load/2021/03/09/\n",
      "/project/monitoring/collectd/load/2021/03/10/\n",
      "/project/monitoring/collectd/load/2021/03/11/\n",
      "/project/monitoring/collectd/load/2021/03/12/\n",
      "/project/monitoring/collectd/load/2021/03/13/\n",
      "/project/monitoring/collectd/load/2021/03/14/\n",
      "/project/monitoring/collectd/load/2021/03/15/\n",
      "/project/monitoring/collectd/load/2021/03/16/\n",
      "/project/monitoring/collectd/load/2021/03/17/\n",
      "/project/monitoring/collectd/load/2021/03/18/\n",
      "/project/monitoring/collectd/load/2021/03/19/\n",
      "filter_str:  ( type == 'percent' and type_instance == 'idle' and plugin == 'cpu' ) or ( value_instance == 'longterm' and plugin == 'load' ) or ( value_instance == 'io_time' and plugin == 'disk' )\n",
      "['cpu__percent_idle', 'load_longterm', 'disk_io_time']\n",
      "Deleting any previous/old remainders in /user/smetaj/demo_AD_System/raw_parquet_train//demo_AD_System ...\n",
      "Error while deleting directory:  [Errno 2] No such file or directory: '/user/smetaj/demo_AD_System/raw_parquet_train//demo_AD_System'\n",
      "Saving the big guy (window dataframe) in: /user/smetaj/demo_AD_System/raw_parquet_train//demo_AD_System ..\n",
      "Saved successfully: /user/smetaj/demo_AD_System/raw_parquet_train//demo_AD_System\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SUCCESS - DATA AGGREGATED IN HDFS.\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Here we download the data that we will normalize using the coefficients \n",
    "# produced by the previous step.\n",
    "\n",
    "# Note that we have to download both the train and the inference datasets,\n",
    "# so we will have 2 different download_data calls.\n",
    "    \n",
    "sys.argv = ['', '--resource_file', json_file_train]\n",
    "DM.transform_data(standalone_mode=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "theoretical-visit",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T14:55:39.191564Z",
     "start_time": "2021-05-03T14:52:57.746059Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "PREPARING SPARK:\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK CONTEXT: <SparkContext master=yarn appName=pyspark_shell_swan>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK OBJECT: <pyspark.sql.session.SparkSession object at 0x7f4081da5d10>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "RESOURCE DETAILS: demo_config_inference.json\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "{\n",
      "    \"hostgroups\": [\n",
      "        \"cloud_compute/level2/main/gva_shared_017\"\n",
      "    ],\n",
      "    \"code_project_name\": \"demo_AD_System\",\n",
      "    \"local_cache_folder\": \"/eos/user/s/smetaj/demo_AD_System/local_cache_inference/\",\n",
      "    \"hdfs_out_folder\": \"/user/smetaj/demo_AD_System/raw_parquet_inference/\",\n",
      "    \"hdfs_cache_folder\": \"/user/smetaj/demo_AD_System/compressed_inference/\",\n",
      "    \"normalization_out_folder\": \"/user/smetaj/demo_AD_System/normalization/\",\n",
      "    \"overwrite_on_hdfs\": true,\n",
      "    \"overwrite_normalization\": true,\n",
      "    \"aggregate_every_n_minutes\": 10,\n",
      "    \"history_steps\": 48,\n",
      "    \"slide_steps\": 48,\n",
      "    \"future_steps\": 0,\n",
      "    \"date_start\": \"2021-03-20\",\n",
      "    \"date_end_excluded\": \"2021-04-20\",\n",
      "    \"date_start_normalization\": \"2021-03-01\",\n",
      "    \"date_end_normalization_excluded\": \"2021-03-20\",\n",
      "    \"selected_plugins\": {\n",
      "        \"cpu__percent_idle\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cpu\",\n",
      "            \"plugin_filter\": \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"\n",
      "        },\n",
      "        \"load_longterm\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/load\",\n",
      "            \"plugin_filter\": \"value_instance == 'longterm' and plugin == 'load'\"\n",
      "        },\n",
      "        \"disk_io_time\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cloud\",\n",
      "            \"plugin_filter\": \"value_instance == 'io_time' and plugin == 'disk'\"\n",
      "        }\n",
      "    }\n",
      "}\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "DOWNLOAD DATA - LONG MINING PROCESS...\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "Start: (2021, 3, 20) - End (2021, 4, 19)\n",
      "Reading normalization:  ef8bce\n",
      "{'plugin_data_path': '/project/monitoring/collectd/cpu', 'plugin_filter': \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"}\n",
      "{'plugin_data_path': '/project/monitoring/collectd/load', 'plugin_filter': \"value_instance == 'longterm' and plugin == 'load'\"}\n",
      "{'plugin_data_path': '/project/monitoring/collectd/cloud', 'plugin_filter': \"value_instance == 'io_time' and plugin == 'disk'\"}\n",
      "not found:  /project/monitoring/collectd/load/2021/04/14/\n",
      "not found:  /project/monitoring/collectd/cpu/2021/04/14/\n",
      "not found:  /project/monitoring/collectd/cloud/2021/04/14/\n",
      "/project/monitoring/collectd/cloud/2021/03/20/\n",
      "/project/monitoring/collectd/cloud/2021/03/21/\n",
      "/project/monitoring/collectd/cloud/2021/03/22/\n",
      "/project/monitoring/collectd/cloud/2021/03/23/\n",
      "/project/monitoring/collectd/cloud/2021/03/24/\n",
      "/project/monitoring/collectd/cloud/2021/03/25/\n",
      "/project/monitoring/collectd/cloud/2021/03/26/\n",
      "/project/monitoring/collectd/cloud/2021/03/27/\n",
      "/project/monitoring/collectd/cloud/2021/03/28/\n",
      "/project/monitoring/collectd/cloud/2021/03/29/\n",
      "/project/monitoring/collectd/cloud/2021/03/30/\n",
      "/project/monitoring/collectd/cloud/2021/03/31/\n",
      "/project/monitoring/collectd/cloud/2021/04/01/\n",
      "/project/monitoring/collectd/cloud/2021/04/02/\n",
      "/project/monitoring/collectd/cloud/2021/04/03/\n",
      "/project/monitoring/collectd/cloud/2021/04/04/\n",
      "/project/monitoring/collectd/cloud/2021/04/05/\n",
      "/project/monitoring/collectd/cloud/2021/04/06/\n",
      "/project/monitoring/collectd/cloud/2021/04/07/\n",
      "/project/monitoring/collectd/cloud/2021/04/08/\n",
      "/project/monitoring/collectd/cloud/2021/04/09/\n",
      "/project/monitoring/collectd/cloud/2021/04/10/\n",
      "/project/monitoring/collectd/cloud/2021/04/11/\n",
      "/project/monitoring/collectd/cloud/2021/04/12/\n",
      "/project/monitoring/collectd/cloud/2021/04/13/\n",
      "/project/monitoring/collectd/cloud/2021/04/15/\n",
      "/project/monitoring/collectd/cloud/2021/04/16/\n",
      "/project/monitoring/collectd/cloud/2021/04/17/\n",
      "/project/monitoring/collectd/cloud/2021/04/18/\n",
      "/project/monitoring/collectd/cloud/2021/04/19/\n",
      "/project/monitoring/collectd/cpu/2021/03/20/\n",
      "/project/monitoring/collectd/cpu/2021/03/21/\n",
      "/project/monitoring/collectd/cpu/2021/03/22/\n",
      "/project/monitoring/collectd/cpu/2021/03/23/\n",
      "/project/monitoring/collectd/cpu/2021/03/24/\n",
      "/project/monitoring/collectd/cpu/2021/03/25/\n",
      "/project/monitoring/collectd/cpu/2021/03/26/\n",
      "/project/monitoring/collectd/cpu/2021/03/27/\n",
      "/project/monitoring/collectd/cpu/2021/03/28/\n",
      "/project/monitoring/collectd/cpu/2021/03/29/\n",
      "/project/monitoring/collectd/cpu/2021/03/30/\n",
      "/project/monitoring/collectd/cpu/2021/03/31/\n",
      "/project/monitoring/collectd/cpu/2021/04/01/\n",
      "/project/monitoring/collectd/cpu/2021/04/02/\n",
      "/project/monitoring/collectd/cpu/2021/04/03/\n",
      "/project/monitoring/collectd/cpu/2021/04/04/\n",
      "/project/monitoring/collectd/cpu/2021/04/05/\n",
      "/project/monitoring/collectd/cpu/2021/04/06/\n",
      "/project/monitoring/collectd/cpu/2021/04/07/\n",
      "/project/monitoring/collectd/cpu/2021/04/08/\n",
      "/project/monitoring/collectd/cpu/2021/04/09/\n",
      "/project/monitoring/collectd/cpu/2021/04/10/\n",
      "/project/monitoring/collectd/cpu/2021/04/11/\n",
      "/project/monitoring/collectd/cpu/2021/04/12/\n",
      "/project/monitoring/collectd/cpu/2021/04/13/\n",
      "/project/monitoring/collectd/cpu/2021/04/15/\n",
      "/project/monitoring/collectd/cpu/2021/04/16/\n",
      "/project/monitoring/collectd/cpu/2021/04/17/\n",
      "/project/monitoring/collectd/cpu/2021/04/18/\n",
      "/project/monitoring/collectd/cpu/2021/04/19/\n",
      "/project/monitoring/collectd/load/2021/03/20/\n",
      "/project/monitoring/collectd/load/2021/03/21/\n",
      "/project/monitoring/collectd/load/2021/03/22/\n",
      "/project/monitoring/collectd/load/2021/03/23/\n",
      "/project/monitoring/collectd/load/2021/03/24/\n",
      "/project/monitoring/collectd/load/2021/03/25/\n",
      "/project/monitoring/collectd/load/2021/03/26/\n",
      "/project/monitoring/collectd/load/2021/03/27/\n",
      "/project/monitoring/collectd/load/2021/03/28/\n",
      "/project/monitoring/collectd/load/2021/03/29/\n",
      "/project/monitoring/collectd/load/2021/03/30/\n",
      "/project/monitoring/collectd/load/2021/03/31/\n",
      "/project/monitoring/collectd/load/2021/04/01/\n",
      "/project/monitoring/collectd/load/2021/04/02/\n",
      "/project/monitoring/collectd/load/2021/04/03/\n",
      "/project/monitoring/collectd/load/2021/04/04/\n",
      "/project/monitoring/collectd/load/2021/04/05/\n",
      "/project/monitoring/collectd/load/2021/04/06/\n",
      "/project/monitoring/collectd/load/2021/04/07/\n",
      "/project/monitoring/collectd/load/2021/04/08/\n",
      "/project/monitoring/collectd/load/2021/04/09/\n",
      "/project/monitoring/collectd/load/2021/04/10/\n",
      "/project/monitoring/collectd/load/2021/04/11/\n",
      "/project/monitoring/collectd/load/2021/04/12/\n",
      "/project/monitoring/collectd/load/2021/04/13/\n",
      "/project/monitoring/collectd/load/2021/04/15/\n",
      "/project/monitoring/collectd/load/2021/04/16/\n",
      "/project/monitoring/collectd/load/2021/04/17/\n",
      "/project/monitoring/collectd/load/2021/04/18/\n",
      "/project/monitoring/collectd/load/2021/04/19/\n",
      "filter_str:  ( type == 'percent' and type_instance == 'idle' and plugin == 'cpu' ) or ( value_instance == 'longterm' and plugin == 'load' ) or ( value_instance == 'io_time' and plugin == 'disk' )\n",
      "['cpu__percent_idle', 'load_longterm', 'disk_io_time']\n",
      "Deleting any previous/old remainders in /user/smetaj/demo_AD_System/raw_parquet_inference//demo_AD_System ...\n",
      "Error while deleting directory:  [Errno 2] No such file or directory: '/user/smetaj/demo_AD_System/raw_parquet_inference//demo_AD_System'\n",
      "Saving the big guy (window dataframe) in: /user/smetaj/demo_AD_System/raw_parquet_inference//demo_AD_System ..\n",
      "Saved successfully: /user/smetaj/demo_AD_System/raw_parquet_inference//demo_AD_System\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SUCCESS - DATA AGGREGATED IN HDFS.\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sys.argv = ['', '--resource_file', json_file_inference]\n",
    "DM.transform_data(standalone_mode=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sonic-mounting",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Copy Locally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "desperate-pathology",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T14:56:27.466547Z",
     "start_time": "2021-05-03T14:55:39.195524Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "PREPARING SPARK:\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK CONTEXT: <SparkContext master=yarn appName=pyspark_shell_swan>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK OBJECT: <pyspark.sql.session.SparkSession object at 0x7f40a81eda10>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "RESOURCE DETAILS: demo_config_train.json\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "{\n",
      "    \"hostgroups\": [\n",
      "        \"cloud_compute/level2/main/gva_shared_017\"\n",
      "    ],\n",
      "    \"code_project_name\": \"demo_AD_System\",\n",
      "    \"local_cache_folder\": \"/eos/user/s/smetaj/demo_AD_System/local_cache_train/\",\n",
      "    \"hdfs_out_folder\": \"/user/smetaj/demo_AD_System/raw_parquet_train/\",\n",
      "    \"hdfs_cache_folder\": \"/user/smetaj/demo_AD_System/compressed_train/\",\n",
      "    \"normalization_out_folder\": \"/user/smetaj/demo_AD_System/normalization/\",\n",
      "    \"overwrite_on_hdfs\": true,\n",
      "    \"overwrite_normalization\": true,\n",
      "    \"aggregate_every_n_minutes\": 10,\n",
      "    \"history_steps\": 48,\n",
      "    \"slide_steps\": 1,\n",
      "    \"future_steps\": 0,\n",
      "    \"date_start\": \"2021-03-01\",\n",
      "    \"date_end_excluded\": \"2021-03-20\",\n",
      "    \"date_start_normalization\": \"2021-03-01\",\n",
      "    \"date_end_normalization_excluded\": \"2021-03-20\",\n",
      "    \"selected_plugins\": {\n",
      "        \"cpu__percent_idle\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cpu\",\n",
      "            \"plugin_filter\": \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"\n",
      "        },\n",
      "        \"load_longterm\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/load\",\n",
      "            \"plugin_filter\": \"value_instance == 'longterm' and plugin == 'load'\"\n",
      "        },\n",
      "        \"disk_io_time\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cloud\",\n",
      "            \"plugin_filter\": \"value_instance == 'io_time' and plugin == 'disk'\"\n",
      "        }\n",
      "    }\n",
      "}\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "CACHE NEW DATA:\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "ASSUMPTION: no data was in cache before and if anything is there, it will be deleted.\n",
      "Deleting any old remainders...\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "CACHE CREATION\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "1614553200\n",
      "SUCCESS\n",
      "Deleting the raw data saved in /user/smetaj/demo_AD_System/raw_parquet_train/demo_AD_System ...\n",
      "Save the config file locally: /eos/user/s/smetaj/demo_AD_System/local_cache_train/demo_AD_System.metadata\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SUCCESS - CACHED DATA AVAILABLE LOCALLY\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# The final step of the ETL procedure is to save the HDFS downloaded \n",
    "# files into EOS, in this way the data are easier to be accessed and used.\n",
    "\n",
    "# As always we must do the same for both the datasets.\n",
    "\n",
    "sys.argv = ['', '--resource_file', json_file_train]\n",
    "DM.copy_locally(standalone_mode=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "formal-enterprise",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T14:56:42.692287Z",
     "start_time": "2021-05-03T14:56:27.529326Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "PREPARING SPARK:\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK CONTEXT: <SparkContext master=yarn appName=pyspark_shell_swan>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SPARK OBJECT: <pyspark.sql.session.SparkSession object at 0x7f40abc32d90>\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "RESOURCE DETAILS: demo_config_inference.json\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "{\n",
      "    \"hostgroups\": [\n",
      "        \"cloud_compute/level2/main/gva_shared_017\"\n",
      "    ],\n",
      "    \"code_project_name\": \"demo_AD_System\",\n",
      "    \"local_cache_folder\": \"/eos/user/s/smetaj/demo_AD_System/local_cache_inference/\",\n",
      "    \"hdfs_out_folder\": \"/user/smetaj/demo_AD_System/raw_parquet_inference/\",\n",
      "    \"hdfs_cache_folder\": \"/user/smetaj/demo_AD_System/compressed_inference/\",\n",
      "    \"normalization_out_folder\": \"/user/smetaj/demo_AD_System/normalization/\",\n",
      "    \"overwrite_on_hdfs\": true,\n",
      "    \"overwrite_normalization\": true,\n",
      "    \"aggregate_every_n_minutes\": 10,\n",
      "    \"history_steps\": 48,\n",
      "    \"slide_steps\": 48,\n",
      "    \"future_steps\": 0,\n",
      "    \"date_start\": \"2021-03-20\",\n",
      "    \"date_end_excluded\": \"2021-04-20\",\n",
      "    \"date_start_normalization\": \"2021-03-01\",\n",
      "    \"date_end_normalization_excluded\": \"2021-03-20\",\n",
      "    \"selected_plugins\": {\n",
      "        \"cpu__percent_idle\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cpu\",\n",
      "            \"plugin_filter\": \"type == 'percent' and type_instance == 'idle' and plugin == 'cpu'\"\n",
      "        },\n",
      "        \"load_longterm\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/load\",\n",
      "            \"plugin_filter\": \"value_instance == 'longterm' and plugin == 'load'\"\n",
      "        },\n",
      "        \"disk_io_time\": {\n",
      "            \"plugin_data_path\": \"/project/monitoring/collectd/cloud\",\n",
      "            \"plugin_filter\": \"value_instance == 'io_time' and plugin == 'disk'\"\n",
      "        }\n",
      "    }\n",
      "}\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "CACHE NEW DATA:\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "ASSUMPTION: no data was in cache before and if anything is there, it will be deleted.\n",
      "Deleting any old remainders...\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "CACHE CREATION\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "1616194800\n",
      "SUCCESS\n",
      "Deleting the raw data saved in /user/smetaj/demo_AD_System/raw_parquet_inference/demo_AD_System ...\n",
      "Save the config file locally: /eos/user/s/smetaj/demo_AD_System/local_cache_inference/demo_AD_System.metadata\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n",
      "SUCCESS - CACHED DATA AVAILABLE LOCALLY\n",
      "\n",
      "pppppppppppppppppppppppppppppppppppppppppppppppppp\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sys.argv = ['', '--resource_file', json_file_inference]\n",
    "DM.copy_locally(standalone_mode=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "frank-viking",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Visualization of downloaded time series"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "decreased-conflict",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Reading time series with pandas and host definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "nonprofit-dragon",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T14:56:46.529305Z",
     "start_time": "2021-05-03T14:56:42.699288Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Now we have downloaded the data. \n",
    "# We also want some tools to visualize the time series.\n",
    "\n",
    "# in local_path we have our data saved in EOS\n",
    "local_path_train = json_data[\"local_cache_folder\"] + 'train/' + json_data[\"code_project_name\"]\n",
    "local_path_inference = json_data[\"local_cache_folder\"] + 'inference/' + json_data[\"code_project_name\"]\n",
    "\n",
    "# nr_timeseries will be equal to the number of plugins that we have downloaded\n",
    "nr_timeseries = len(json_data[\"selected_plugins\"])\n",
    "\n",
    "# finally df will be the pandas dataframe containing the data\n",
    "# (it is eaisier to print them with pandas)\n",
    "df_train = pd.read_parquet(local_path_train)\n",
    "df_inference = pd.read_parquet(local_path_inference)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "operating-running",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T14:56:46.579295Z",
     "start_time": "2021-05-03T14:56:46.533965Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>hostname</th>\n",
       "      <th>hostgroup</th>\n",
       "      <th>cpu__percent_idle_h0</th>\n",
       "      <th>cpu__percent_idle_h1</th>\n",
       "      <th>cpu__percent_idle_h2</th>\n",
       "      <th>cpu__percent_idle_h3</th>\n",
       "      <th>cpu__percent_idle_h4</th>\n",
       "      <th>cpu__percent_idle_h5</th>\n",
       "      <th>cpu__percent_idle_h6</th>\n",
       "      <th>...</th>\n",
       "      <th>load_longterm_h39</th>\n",
       "      <th>load_longterm_h40</th>\n",
       "      <th>load_longterm_h41</th>\n",
       "      <th>load_longterm_h42</th>\n",
       "      <th>load_longterm_h43</th>\n",
       "      <th>load_longterm_h44</th>\n",
       "      <th>load_longterm_h45</th>\n",
       "      <th>load_longterm_h46</th>\n",
       "      <th>load_longterm_h47</th>\n",
       "      <th>ts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1617289200</td>\n",
       "      <td>p06428644w55120.cern.ch</td>\n",
       "      <td>cloud_compute/level2/main/gva_shared_017</td>\n",
       "      <td>0.236088</td>\n",
       "      <td>0.227848</td>\n",
       "      <td>0.243771</td>\n",
       "      <td>0.223976</td>\n",
       "      <td>0.198013</td>\n",
       "      <td>0.207111</td>\n",
       "      <td>0.193086</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.441156</td>\n",
       "      <td>-0.387925</td>\n",
       "      <td>-0.323980</td>\n",
       "      <td>-0.426701</td>\n",
       "      <td>-0.444898</td>\n",
       "      <td>-0.464626</td>\n",
       "      <td>-0.497619</td>\n",
       "      <td>-0.472619</td>\n",
       "      <td>-0.382993</td>\n",
       "      <td>2021-04-01 17:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1616770800</td>\n",
       "      <td>p06492044x27120.cern.ch</td>\n",
       "      <td>cloud_compute/level2/main/gva_shared_017</td>\n",
       "      <td>-0.607309</td>\n",
       "      <td>-0.593825</td>\n",
       "      <td>-0.539714</td>\n",
       "      <td>-0.609813</td>\n",
       "      <td>-0.832071</td>\n",
       "      <td>-0.663781</td>\n",
       "      <td>-0.510553</td>\n",
       "      <td>...</td>\n",
       "      <td>1.719558</td>\n",
       "      <td>1.712075</td>\n",
       "      <td>1.800340</td>\n",
       "      <td>1.461054</td>\n",
       "      <td>1.266837</td>\n",
       "      <td>0.829422</td>\n",
       "      <td>0.421259</td>\n",
       "      <td>0.274320</td>\n",
       "      <td>0.477721</td>\n",
       "      <td>2021-03-26 16:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1617836400</td>\n",
       "      <td>p06428644r12066.cern.ch</td>\n",
       "      <td>cloud_compute/level2/main/gva_shared_017</td>\n",
       "      <td>-2.064705</td>\n",
       "      <td>-1.436896</td>\n",
       "      <td>-0.930914</td>\n",
       "      <td>-1.901045</td>\n",
       "      <td>-0.657402</td>\n",
       "      <td>-0.541086</td>\n",
       "      <td>-0.013392</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.697704</td>\n",
       "      <td>-0.810091</td>\n",
       "      <td>-0.843537</td>\n",
       "      <td>-0.904762</td>\n",
       "      <td>-0.924320</td>\n",
       "      <td>-0.895408</td>\n",
       "      <td>-0.839002</td>\n",
       "      <td>-0.762755</td>\n",
       "      <td>-0.623639</td>\n",
       "      <td>2021-04-08 01:00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 148 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    timestamp                 hostname  \\\n",
       "0  1617289200  p06428644w55120.cern.ch   \n",
       "1  1616770800  p06492044x27120.cern.ch   \n",
       "2  1617836400  p06428644r12066.cern.ch   \n",
       "\n",
       "                                  hostgroup  cpu__percent_idle_h0  \\\n",
       "0  cloud_compute/level2/main/gva_shared_017              0.236088   \n",
       "1  cloud_compute/level2/main/gva_shared_017             -0.607309   \n",
       "2  cloud_compute/level2/main/gva_shared_017             -2.064705   \n",
       "\n",
       "   cpu__percent_idle_h1  cpu__percent_idle_h2  cpu__percent_idle_h3  \\\n",
       "0              0.227848              0.243771              0.223976   \n",
       "1             -0.593825             -0.539714             -0.609813   \n",
       "2             -1.436896             -0.930914             -1.901045   \n",
       "\n",
       "   cpu__percent_idle_h4  cpu__percent_idle_h5  cpu__percent_idle_h6  ...  \\\n",
       "0              0.198013              0.207111              0.193086  ...   \n",
       "1             -0.832071             -0.663781             -0.510553  ...   \n",
       "2             -0.657402             -0.541086             -0.013392  ...   \n",
       "\n",
       "   load_longterm_h39  load_longterm_h40  load_longterm_h41  load_longterm_h42  \\\n",
       "0          -0.441156          -0.387925          -0.323980          -0.426701   \n",
       "1           1.719558           1.712075           1.800340           1.461054   \n",
       "2          -0.697704          -0.810091          -0.843537          -0.904762   \n",
       "\n",
       "   load_longterm_h43  load_longterm_h44  load_longterm_h45  load_longterm_h46  \\\n",
       "0          -0.444898          -0.464626          -0.497619          -0.472619   \n",
       "1           1.266837           0.829422           0.421259           0.274320   \n",
       "2          -0.924320          -0.895408          -0.839002          -0.762755   \n",
       "\n",
       "   load_longterm_h47                   ts  \n",
       "0          -0.382993  2021-04-01 17:00:00  \n",
       "1           0.477721  2021-03-26 16:00:00  \n",
       "2          -0.623639  2021-04-08 01:00:00  \n",
       "\n",
       "[3 rows x 148 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Note that each row of df will be a window of data following the granularity definition in the configuration file. \n",
    "# In this example it consists of 48 points representing 8 hours of time interval with an aggregation of 10 minutes.\n",
    "# So for every day, for every hostname, we will have 3 rows (3 x 8hours = 24),\n",
    "# each row will be composed by timestamp, hostname, hostgroup, ts and\n",
    "# for each plugin we will have 48 columns, 1 for each 10 minutes.\n",
    "df_inference.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "refined-chester",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T14:56:46.589975Z",
     "start_time": "2021-05-03T14:56:46.582582Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['p06428644y58212.cern.ch',\n",
       " 'p06428644w00836.cern.ch',\n",
       " 'p06428644y16173.cern.ch',\n",
       " 'p06428644u33510.cern.ch',\n",
       " 'p06428644w45688.cern.ch']"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We want to visualize the data about 1 host in particular \n",
    "# (let's print 5 random hosts we have in our pandas dataframe).\n",
    "\n",
    "hostnames_in_df = list(set(list(df_inference['hostname'])))\n",
    "hostnames_in_df[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "mental-norfolk",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-03T14:56:46.595739Z",
     "start_time": "2021-05-03T14:56:46.592808Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# From previous studies we know that in this specific hostgroup there is a\n",
    "# particular host that was anomalous in the specific period of time we are\n",
    "# studying, so let's initizialize the host_to_viz variable.\n",
    "\n",
    "host_to_viz = 'p06428644y10205.cern.ch'  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "found-effectiveness",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Reconstruction function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "continuous-transition",
   "metadata": {
    "ExecuteTime": {