From 5f355f55a4cd000e9512add4441aaba7aba1ba07 Mon Sep 17 00:00:00 2001
From: Raul Villar Ramos <raul.villar.ramos@cern.ch>
Date: Wed, 11 Mar 2020 16:47:03 +0100
Subject: [PATCH 01/21] OS8753_Handle_crash_reports

---
 rallytester/rallytester.py            |  9 ++++++++
 rallytester/tests/fixtures.py         | 25 +++++++++++++++++++++
 rallytester/tests/test_rallytester.py | 32 +++++++++++++++++++++++++++
 3 files changed, 66 insertions(+)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index 9bcbf2d..baf97fa 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -143,6 +143,15 @@ def execute_task(deployment, task, msgTrimmer=None):
             .format(msg, output, stderr, task_id, task_descriptor)
         )
 
+    # Check if the task is crashed
+    try:
+        if any(job["tasks"][0]["status"] == "crashed" for job in json_result):
+            return ("Task status is crashed."
+                    "Use: task report --uuid --json "
+                    )
+    except:
+        pass
+
     scenarios = ",".join([job['key']['name'] for job in json_result])
 
     # If there is any error in the results
diff --git a/rallytester/tests/fixtures.py b/rallytester/tests/fixtures.py
index 7601c7b..4de0a8d 100644
--- a/rallytester/tests/fixtures.py
+++ b/rallytester/tests/fixtures.py
@@ -372,3 +372,28 @@ TASK_RESULTS_FAIL2 = u"""
     }
 ]
 """
+
+TASK_RESULTS_CRASH = u"""
+[
+    {
+        "info": {
+            "generated_at": "2019-03-05T16:50:31",
+            "rally_version": "0.0.0",
+            "format_version": "1.1"
+        },
+        "tasks": [
+            {
+                "uuid": "5312f2e6-f00d-4a63-9162-40c895cdaea1",
+                "title": "",
+                "description": "",
+                "status": "crashed",
+                "tags": [],
+                "created_at": "2019-03-05T15:14:54",
+                "updated_at": "2019-03-05T15:14:56",
+                "pass_sla": true,
+                "subtasks": []
+            }
+        ]
+    }
+]
+"""
diff --git a/rallytester/tests/test_rallytester.py b/rallytester/tests/test_rallytester.py
index 89a6b8f..4759d5a 100644
--- a/rallytester/tests/test_rallytester.py
+++ b/rallytester/tests/test_rallytester.py
@@ -440,3 +440,35 @@ class TestRallytester(unittest.TestCase):
                 " Image '{\"name\": \"my_image\"}' not found",
                 rallytester.REASONS_PATTERN)
         )
+
+    @mock.patch('rallytester.rallytester.subprocess.Popen')
+    @mock.patch(func, return_value=io.StringIO(fixtures.TASK_RESULTS_CRASH))
+    @mock.patch('os.path.exists', return_value=True)
+    def test_execute_task_crash(self,
+                                            mock_os,
+                                            mock_open,
+                                            mock_sub):
+        mock_pr = mock_sub.return_value
+        type(mock_pr).returncode = mock.PropertyMock(return_value=0)
+
+        mock_pr.communicate.side_effect = [
+            (
+                'Using task: 5312f2e6-f00d-4a63-9162-40c895cdaea1',
+                None
+            ),
+            (
+                fixtures.TASK_RESULTS_CRASH,
+                None
+            )
+        ]
+
+        self.assertEqual(
+            (
+                    "Task status is crashed."
+                    "Use: task report --uuid --json "
+            ),
+            rallytester.execute_task(
+                deployment='fake_deployment',
+                task='fake_task'
+            )
+        )
-- 
GitLab


From a4ad6765f3688643ad00130bb1bc7153450f221d Mon Sep 17 00:00:00 2001
From: Raul Villar Ramos <raul.villar.ramos@cern.ch>
Date: Wed, 11 Mar 2020 16:57:14 +0100
Subject: [PATCH 02/21] OS8753_Handle_crash_reports_pep8

---
 rallytester/rallytester.py            |  2 +-
 rallytester/tests/test_rallytester.py | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index baf97fa..03ee58b 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -149,7 +149,7 @@ def execute_task(deployment, task, msgTrimmer=None):
             return ("Task status is crashed."
                     "Use: task report --uuid --json "
                     )
-    except:
+    except BaseException:
         pass
 
     scenarios = ",".join([job['key']['name'] for job in json_result])
diff --git a/rallytester/tests/test_rallytester.py b/rallytester/tests/test_rallytester.py
index 4759d5a..f128039 100644
--- a/rallytester/tests/test_rallytester.py
+++ b/rallytester/tests/test_rallytester.py
@@ -445,9 +445,9 @@ class TestRallytester(unittest.TestCase):
     @mock.patch(func, return_value=io.StringIO(fixtures.TASK_RESULTS_CRASH))
     @mock.patch('os.path.exists', return_value=True)
     def test_execute_task_crash(self,
-                                            mock_os,
-                                            mock_open,
-                                            mock_sub):
+                                mock_os,
+                                mock_open,
+                                mock_sub):
         mock_pr = mock_sub.return_value
         type(mock_pr).returncode = mock.PropertyMock(return_value=0)
 
@@ -464,8 +464,8 @@ class TestRallytester(unittest.TestCase):
 
         self.assertEqual(
             (
-                    "Task status is crashed."
-                    "Use: task report --uuid --json "
+                "Task status is crashed."
+                "Use: task report --uuid --json "
             ),
             rallytester.execute_task(
                 deployment='fake_deployment',
-- 
GitLab


From 3ab5e6aa5595c0c6fad92531d7e5ad25d927093a Mon Sep 17 00:00:00 2001
From: ravillar <raul.villar.ramos@cern.ch>
Date: Mon, 16 Mar 2020 12:44:34 +0100
Subject: [PATCH 03/21] OS8753_handle_crash_report

---
 rallytester/rallytester.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index 03ee58b..db5e315 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -143,14 +143,12 @@ def execute_task(deployment, task, msgTrimmer=None):
             .format(msg, output, stderr, task_id, task_descriptor)
         )
 
-    # Check if the task is crashed
-    try:
+    # Capture crash. Only crash contains 'tasks'
+    if any('tasks' in j for j in json_result):
         if any(job["tasks"][0]["status"] == "crashed" for job in json_result):
-            return ("Task status is crashed."
-                    "Use: task report --uuid --json "
-                    )
-    except BaseException:
-        pass
+                        return ("Task status is crashed."
+                                "Use: task report --uuid --json "
+                                )
 
     scenarios = ",".join([job['key']['name'] for job in json_result])
 
-- 
GitLab


From fd17658223be25d5421065cd2d1c1a0389b68f51 Mon Sep 17 00:00:00 2001
From: ravillar <raul.villar.ramos@cern.ch>
Date: Mon, 23 Mar 2020 16:45:25 +0100
Subject: [PATCH 04/21] OS8753_replace_results_by_reports

---
 rallytester/rallytester.py            |  51 +-
 rallytester/tests/fixtures.py         | 845 +++++++++++++++++---------
 rallytester/tests/test_rallytester.py |  19 +-
 3 files changed, 606 insertions(+), 309 deletions(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index db5e315..c553173 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -110,7 +110,7 @@ def execute_task(deployment, task, msgTrimmer=None):
 
     # Get results executing rally command
     process = subprocess.Popen([  # nosec
-        '/usr/bin/rally', 'task', 'results', task_id],
+        '/usr/bin/rally', 'task', 'report', task_id, '--json'],
         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     output, stderr = process.communicate()
 
@@ -143,26 +143,25 @@ def execute_task(deployment, task, msgTrimmer=None):
             .format(msg, output, stderr, task_id, task_descriptor)
         )
 
-    # Capture crash. Only crash contains 'tasks'
+    # Check if the task is crashed
     if any('tasks' in j for j in json_result):
         if any(job["tasks"][0]["status"] == "crashed" for job in json_result):
-                        return ("Task status is crashed."
-                                "Use: task report --uuid --json "
-                                )
+                        return ("Task status is crashed.")
+                        # return json here
 
-    scenarios = ",".join([job['key']['name'] for job in json_result])
+    scenarios = ",".join([job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['atomic_actions'][0]['name'] for job in json_result])
 
-    # If there is any error in the results
-    if any(job['result'] and
-            job['result'][0]['error'] for job in json_result):
+    # If there is any error in the reports
+    if any(job['tasks'] and 
+            job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'] for job in json_result):
         msg = ""
         tracebacks = ""
         for job in json_result:
-            if job['result'] and job['result'][0]['error']:
+            if job['tasks'] and job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error']:
                 # The real error is in the first result
                 # and the second element of the array
-                msg += '\n' + job['result'][0]['error'][1]
-                tracebacks += '\n' + job['result'][0]['error'][2]
+                msg += '\n' + job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'][1]
+                tracebacks += '\n' + job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'][2]
                 log.error('Task failed: {0}'.format(
                     _msg_enrich(msg.replace('\n', ' '), msgTrimmer)))
 
@@ -183,7 +182,7 @@ def execute_task(deployment, task, msgTrimmer=None):
     # to reimplement this part anyway to use `rally task report`
     # with a different format
     elif any(
-        not job['result'] and
+        not job['tasks'] and
         'sla' in job and
         any(sla['criterion'] == 'something_went_wrong' for sla in job['sla'])
         for job in json_result
@@ -210,6 +209,30 @@ def execute_task(deployment, task, msgTrimmer=None):
                                                      task_descriptor)
         )
 
+    if any('tasks' in j for j in json_result):
+        if any(job['tasks'][0]['subtasks'][0]['workloads'][0]['sla_results']['sla'][0]['success'] == False for job in json_result):
+        
+            msg = ""
+            tracebacks = ""
+            for job in json_result:
+                if job['tasks'] and job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error']:
+                    # The real error is in the first result
+                    # and the second element of the array
+                    msg += '\n' + job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'][1]
+                    tracebacks += '\n' + job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'][2]
+                    log.error('Task failed: {0}'.format(
+                        _msg_enrich(msg.replace('\n', ' '), msgTrimmer)))
+
+            return (
+                {
+                    'task': task,
+                    'scenarios': scenarios,
+                    'succeed': 0
+                },
+                "{0}\n{1}\nTask descriptor:\n{2}".format(msg, tracebacks,
+                                                        task_descriptor)
+            )
+
     # In any other case
     log.info('So far, so good, the task has been sucessfully completed')
     return (
@@ -217,7 +240,7 @@ def execute_task(deployment, task, msgTrimmer=None):
             'task': task,
             'scenarios': scenarios,
             'succeed': 1,
-            'duration': sum([job['result'][0]['duration']
+            'duration': sum([job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['duration']
                              for job in json_result])
         },
         None  # No errors
diff --git a/rallytester/tests/fixtures.py b/rallytester/tests/fixtures.py
index 4de0a8d..95960ae 100644
--- a/rallytester/tests/fixtures.py
+++ b/rallytester/tests/fixtures.py
@@ -74,324 +74,599 @@ TASK = u"""
 }
 """
 
-TASK_RESULTS = u"""
+TASK_REPORT_CRASH = u"""
 [
     {
-        "hooks": [],
-        "created_at": "2019-11-04T13:03:36",
-        "load_duration": 617.198,
-        "result": [
-            {
-                "timestamp": 1554987817.501209,
-                "error": [],
-                "duration": 617.1978030204773,
-                "output": {
-                    "additive": [],
-                    "complete": []
-                },
-                "idle_duration": 10.0,
-                "atomic_actions": {
-                    "cinder_v3.create_volume": 4.72377610206604,
-                    "nova.boot_server": 614.4812738895416,
-                    "nova.delete_server": 7.99204421043396
-                }
-            }
-        ],
-        "key": {
-            "kw": {
-                "runner": {
-                    "type": "constant",
-                    "concurrency": 1,
-                    "times": 1
-                },
-                "hooks": [],
-                "args": {
-                    "volume_size": 1,
-                    "flavor": {
-                        "name": "rally.nano"
-                    },
-                    "image": {
-                        "name": "Rally CirrOS IPv6"
-                    },
-                    "force_delete": false
-                },
-                "sla": {
-                    "failure_rate": {
-                        "max": 0
-                    }
-                },
-                "context": {}
-            },
-            "pos": 0,
-            "name": "Novaname",
-            "description": "Bootdesc"
+        "info": {
+            "generated_at": "2019-03-05T16:50:31",
+            "rally_version": "0.0.0",
+            "format_version": "1.1"
         },
-        "full_duration": 632.436,
-        "sla": [
+        "tasks": [
             {
-                "criterion": "failure_rate",
-                "detail": "Failure rate criteria'\
-                ' 0.00% <= 0.00% <= 0.00% - Passed",
-                "success": true
+                "uuid": "5312f2e6-f00d-4a63-9162-40c895cdaea1",
+                "title": "",
+                "description": "",
+                "status": "crashed",
+                "tags": [],
+                "created_at": "2019-03-05T15:14:54",
+                "updated_at": "2019-03-05T15:14:56",
+                "pass_sla": true,
+                "subtasks": []
             }
         ]
     }
 ]
 """
 
-TASK_RESULTS_FAIL = u"""
-[    {
-        "hooks": [],
-        "created_at": "2019-11-04T13:03:36",
-        "load_duration": 617.198,
-        "result": [
-            {
-                "timestamp": 1554987817.501209,
-                "error": [],
-                "duration": 617.1978030204773,
-                "output": {
-                    "additive": [],
-                    "complete": []
-                },
-                "idle_duration": 10.0,
-                "atomic_actions": {
-                    "cinder_v3.create_volume": 4.72377610206604,
-                    "nova.boot_server": 614.4812738895416,
-                    "nova.delete_server": 7.99204421043396
-                }
-            }
-        ],
-        "key": {
-            "kw": {
-                "runner": {
-                    "type": "constant",
-                    "concurrency": 1,
-                    "times": 1
-                },
-                "hooks": [],
-                "args": {
-                    "volume_size": 1,
-                    "flavor": {
-                        "name": "rally.nano"
-                    },
-                    "image": {
-                        "name": "Rally CirrOS IPv6"
-                    },
-                    "force_delete": false
-                },
-                "sla": {
-                    "failure_rate": {
-                        "max": 0
-                    }
-                },
-                "context": {}
-            },
-            "pos": 0,
-            "name": "Name",
-            "description": "desc"
-        },
-        "full_duration": 632.436,
-        "sla": [
-            {
-                "criterion": "failure_rate",
-                "detail": "Failure rate criteria'\
-                ' 0.00% <= 0.00% <= 0.00% - Passed",
-                "success": true
-            }
-        ]
-    },
+TASK_REPORT = u"""
+[
     {
-        "hooks": [],
-        "created_at": "2019-11-04T13:03:36",
-        "load_duration": 617.198,
-        "result": [
+        "info": {
+            "generated_at": "2020-03-16T16:07:30", 
+            "rally_version": "2.0.0", 
+            "format_version": "1.2"
+        }, 
+        "tasks": [
             {
-                "timestamp": 1554987817.501209,
-                "error": [
-                    "-----------",
-                    "Fake error1",
-                    "Fake error2"
-                ],
-                "duration": 617.1978030204773,
-                "output": {
-                    "additive": [],
-                    "complete": []
-                },
-                "idle_duration": 10.0,
-                "atomic_actions": {
-                    "cinder_v3.create_volume": 4.72377610206604,
-                    "nova.boot_server": 614.4812738895416,
-                    "nova.delete_server": 7.99204421043396
-                }
-            }
-        ],
-        "key": {
-            "kw": {
-                "runner": {
-                    "type": "constant",
-                    "concurrency": 1,
-                    "times": 1
-                },
-                "hooks": [],
-                "args": {
-                    "volume_size": 1,
-                    "flavor": {
-                        "name": "rally.nano"
-                    },
-                    "image": {
-                        "name": "Rally CirrOS IPv6"
-                    },
-                    "force_delete": false
-                },
-                "sla": {
-                    "failure_rate": {
-                        "max": 0
+                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d", 
+                "title": "", 
+                "description": "", 
+                "status": "finished", 
+                "tags": [], 
+                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708", 
+                "env_name": "teststack_cell02", 
+                "created_at": "2020-03-15T03:17:17", 
+                "updated_at": "2020-03-15T03:17:20", 
+                "pass_sla": true, 
+                "subtasks": [
+                    {
+                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12", 
+                        "title": "GlanceImages.list_images", 
+                        "description": "", 
+                        "status": "finished", 
+                        "created_at": "2020-03-15T03:17:18", 
+                        "updated_at": "2020-03-15T03:17:20", 
+                        "sla": {}, 
+                        "workloads": [
+                            {
+                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48", 
+                                "description": "List all images.", 
+                                "runner": {
+                                    "constant": {
+                                        "concurrency": 1, 
+                                        "times": 1
+                                    }
+                                }, 
+                                "hooks": [], 
+                                "scenario": {
+                                    "GlanceImages.list_images": {}
+                                }, 
+                                "min_duration": 0.1442, 
+                                "max_duration": 0.1442, 
+                                "start_time": 1584242238.24373, 
+                                "load_duration": 0.1442, 
+                                "full_duration": 0.380141, 
+                                "statistics": {
+                                    "durations": {
+                                        "total": {
+                                            "data": {
+                                                "success": "100.0%", 
+                                                "min": 0.144, 
+                                                "max": 0.144, 
+                                                "median": 0.144, 
+                                                "95%ile": 0.144, 
+                                                "iteration_count": 1, 
+                                                "avg": 0.144, 
+                                                "90%ile": 0.144
+                                            }, 
+                                            "display_name": "total", 
+                                            "name": "total", 
+                                            "count_per_iteration": 1, 
+                                            "children": [
+                                                {
+                                                    "data": {
+                                                        "success": "100.0%", 
+                                                        "min": 0.144, 
+                                                        "max": 0.144, 
+                                                        "median": 0.144, 
+                                                        "95%ile": 0.144, 
+                                                        "iteration_count": 1, 
+                                                        "avg": 0.144, 
+                                                        "90%ile": 0.144
+                                                    }, 
+                                                    "display_name": "duration", 
+                                                    "name": "duration", 
+                                                    "count_per_iteration": 1, 
+                                                    "children": []
+                                                }, 
+                                                {
+                                                    "data": {
+                                                        "success": "100.0%", 
+                                                        "min": 0.0, 
+                                                        "max": 0.0, 
+                                                        "median": 0.0, 
+                                                        "95%ile": 0.0, 
+                                                        "iteration_count": 1, 
+                                                        "avg": 0.0, 
+                                                        "90%ile": 0.0
+                                                    }, 
+                                                    "display_name": "idle_duration", 
+                                                    "name": "idle_duration", 
+                                                    "count_per_iteration": 1, 
+                                                    "children": []
+                                                }
+                                            ]
+                                        }, 
+                                        "atomics": [
+                                            {
+                                                "data": {
+                                                    "success": "100.0%", 
+                                                    "min": 0.144, 
+                                                    "max": 0.144, 
+                                                    "median": 0.144, 
+                                                    "95%ile": 0.144, 
+                                                    "iteration_count": 1, 
+                                                    "avg": 0.144, 
+                                                    "90%ile": 0.144
+                                                }, 
+                                                "display_name": "Novaname", 
+                                                "name": "Novaname", 
+                                                "count_per_iteration": 1, 
+                                                "children": []
+                                            }
+                                        ]
+                                    }
+                                }, 
+                                "data": [
+                                    {
+                                        "timestamp": 1584242238.243732, 
+                                        "error": [], 
+                                        "duration": 0.14420008659362793, 
+                                        "output": {
+                                            "additive": [], 
+                                            "complete": []
+                                        }, 
+                                        "idle_duration": 0.0, 
+                                        "atomic_actions": [
+                                            {
+                                                "finished_at": 1584242238.38778, 
+                                                "started_at": 1584242238.243803, 
+                                                "name": "Novaname", 
+                                                "children": []
+                                            }
+                                        ]
+                                    }
+                                ], 
+                                "failed_iteration_count": 0, 
+                                "total_iteration_count": 1, 
+                                "created_at": "2020-03-15T03:17:18", 
+                                "updated_at": "2020-03-15T03:17:20", 
+                                "contexts": {}, 
+                                "contexts_results": [
+                                    {
+                                        "setup": {
+                                            "finished_at": 1584242238.110536, 
+                                            "started_at": 1584242238.029151, 
+                                            "atomic_actions": [], 
+                                            "error": null
+                                        }, 
+                                        "cleanup": {
+                                            "finished_at": 1584242238.408476, 
+                                            "started_at": 1584242238.40758, 
+                                            "atomic_actions": [], 
+                                            "error": null
+                                        }, 
+                                        "plugin_cfg": {
+                                            "user_choice_method": "random"
+                                        }, 
+                                        "plugin_name": "users@openstack"
+                                    }
+                                ], 
+                                "position": 0, 
+                                "pass_sla": true, 
+                                "sla_results": {
+                                    "sla": [
+                                        {
+                                            "criterion": "failure_rate", 
+                                            "detail": "Failure rate criteria 0.00% <= 0.00% <= 0.00% - Passed", 
+                                            "success": true
+                                        }
+                                    ]
+                                }, 
+                                "sla": {
+                                    "failure_rate": {
+                                        "max": 0
+                                    }
+                                }
+                            }
+                        ]
                     }
-                },
-                "context": {}
-            },
-            "pos": 0,
-            "name": "name",
-            "description": "des"
-        },
-        "full_duration": 632.436,
-        "sla": [
-            {
-                "criterion": "failure_rate",
-                "detail": "Failure rate criteria'\
-                ' 0.00% <= 0.00% <= 0.00% - Passed",
-                "success": true
+                ]
             }
         ]
     }
 ]
 """
 
-TASK_RESULTS_FAIL2 = u"""
+TASK_REPORT_ERROR1 = u"""
 [
-        {
-        "hooks": [],
-        "created_at": "2019-11-04T13:03:36",
-        "load_duration": 617.198,
-        "result": [
-            {
-                "timestamp": 1554987817.501209,
-                "error": [],
-                "duration": 617.1978030204773,
-                "output": {
-                    "additive": [],
-                    "complete": []
-                },
-                "idle_duration": 10.0,
-                "atomic_actions": {
-                    "cinder_v3.create_volume": 4.72377610206604,
-                    "nova.boot_server": 614.4812738895416,
-                    "nova.delete_server": 7.99204421043396
-                }
-            }
-        ],
-        "key": {
-            "kw": {
-                "runner": {
-                    "type": "constant",
-                    "concurrency": 1,
-                    "times": 1
-                },
-                "hooks": [],
-                "args": {
-                    "volume_size": 1,
-                    "flavor": {
-                        "name": "rally.nano"
-                    },
-                    "image": {
-                        "name": "Rally CirrOS IPv6"
-                    },
-                    "force_delete": false
-                },
-                "sla": {
-                    "failure_rate": {
-                        "max": 0
-                    }
-                },
-                "context": {}
-            },
-            "pos": 0,
-            "name": "NovaServers",
-            "description": "Boot a server"
-        },
-        "full_duration": 632.436
-    },
     {
-        "hooks": [],
-        "created_at": "2019-11-04T13:03:36",
-        "load_duration": 617.198,
-        "result": [],
-        "key": {
-            "kw": {
-                "runner": {
-                    "type": "constant",
-                    "concurrency": 1,
-                    "times": 1
-                },
-                "hooks": [],
-                "args": {
-                    "volume_size": 1,
-                    "flavor": {
-                        "name": "rally.nano"
-                    },
-                    "image": {
-                        "name": "Rally CirrOS IPv6"
-                    },
-                    "force_delete": false
-                },
-                "sla": {
-                    "failure_rate": {
-                        "max": 0
-                    }
-                },
-                "context": {}
-            },
-            "pos": 0,
-            "name": "Novaname",
-            "description": "Bootdesc"
-        },
-        "full_duration": 632.436,
-        "sla": [
-            {
-                "criterion": "failure_rate",
-                "detail": "Failure rate criteria'\
-                ' 0.00% <= 0.00% <= 0.00% - Passed",
-                "success": true
-            },
+        "info": {
+            "generated_at": "2020-03-16T16:07:30", 
+            "rally_version": "2.0.0", 
+            "format_version": "1.2"
+        }, 
+        "tasks": [
             {
-                "criterion": "something_went_wrong",
-                "detail": "Frc",
-                "success": false
+                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d", 
+                "title": "", 
+                "description": "", 
+                "status": "finished", 
+                "tags": [], 
+                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708", 
+                "env_name": "teststack_cell02", 
+                "created_at": "2020-03-15T03:17:17", 
+                "updated_at": "2020-03-15T03:17:20", 
+                "pass_sla": true, 
+                "subtasks": [
+                    {
+                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12", 
+                        "title": "GlanceImages.list_images", 
+                        "description": "", 
+                        "status": "finished", 
+                        "created_at": "2020-03-15T03:17:18", 
+                        "updated_at": "2020-03-15T03:17:20", 
+                        "sla": {}, 
+                        "workloads": [
+                            {
+                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48", 
+                                "description": "List all images.", 
+                                "runner": {
+                                    "constant": {
+                                        "concurrency": 1, 
+                                        "times": 1
+                                    }
+                                }, 
+                                "hooks": [], 
+                                "scenario": {
+                                    "GlanceImages.list_images": {}
+                                }, 
+                                "min_duration": 0.1442, 
+                                "max_duration": 0.1442, 
+                                "start_time": 1584242238.24373, 
+                                "load_duration": 0.1442, 
+                                "full_duration": 0.380141, 
+                                "statistics": {
+                                    "durations": {
+                                        "total": {
+                                            "data": {
+                                                "success": "0.0%", 
+                                                "min": 0.144, 
+                                                "max": 0.144, 
+                                                "median": 0.144, 
+                                                "95%ile": 0.144, 
+                                                "iteration_count": 1, 
+                                                "avg": 0.144, 
+                                                "90%ile": 0.144
+                                            }, 
+                                            "display_name": "total", 
+                                            "name": "total", 
+                                            "count_per_iteration": 1, 
+                                            "children": [
+                                                {
+                                                    "data": {
+                                                        "success": "0.0%", 
+                                                        "min": 0.144, 
+                                                        "max": 0.144, 
+                                                        "median": 0.144, 
+                                                        "95%ile": 0.144, 
+                                                        "iteration_count": 1, 
+                                                        "avg": 0.144, 
+                                                        "90%ile": 0.144
+                                                    }, 
+                                                    "display_name": "duration", 
+                                                    "name": "duration", 
+                                                    "count_per_iteration": 1, 
+                                                    "children": []
+                                                }, 
+                                                {
+                                                    "data": {
+                                                        "success": "0.0%", 
+                                                        "min": 0.0, 
+                                                        "max": 0.0, 
+                                                        "median": 0.0, 
+                                                        "95%ile": 0.0, 
+                                                        "iteration_count": 1, 
+                                                        "avg": 0.0, 
+                                                        "90%ile": 0.0
+                                                    }, 
+                                                    "display_name": "idle_duration", 
+                                                    "name": "idle_duration", 
+                                                    "count_per_iteration": 1, 
+                                                    "children": []
+                                                }
+                                            ]
+                                        }, 
+                                        "atomics": [
+                                            {
+                                                "data": {
+                                                    "success": "0.0%", 
+                                                    "min": 0.144, 
+                                                    "max": 0.144, 
+                                                    "median": 0.144, 
+                                                    "95%ile": 0.144, 
+                                                    "iteration_count": 1, 
+                                                    "avg": 0.144, 
+                                                    "90%ile": 0.144
+                                                }, 
+                                                "display_name": "Novaname", 
+                                                "name": "Novaname", 
+                                                "count_per_iteration": 1, 
+                                                "children": []
+                                            }
+                                        ]
+                                    }
+                                }, 
+                                "data": [
+                                    {
+                                        "timestamp": 1584242238.243732, 
+                                        "error": [], 
+                                        "duration": 0.14420008659362793, 
+                                        "output": {
+                                            "additive": [], 
+                                            "complete": []
+                                        }, 
+                                        "idle_duration": 0.0, 
+                                        "atomic_actions": [
+                                            {
+                                                "finished_at": 1584242238.38778, 
+                                                "started_at": 1584242238.243803, 
+                                                "name": "Novaname", 
+                                                "children": []
+                                            }
+                                        ]
+                                    }
+                                ], 
+                                "failed_iteration_count": 0, 
+                                "total_iteration_count": 1, 
+                                "created_at": "2020-03-15T03:17:18", 
+                                "updated_at": "2020-03-15T03:17:20", 
+                                "contexts": {}, 
+                                "contexts_results": [
+                                    {
+                                        "setup": {
+                                            "finished_at": 1584242238.110536, 
+                                            "started_at": 1584242238.029151, 
+                                            "atomic_actions": [], 
+                                            "error": null
+                                        }, 
+                                        "cleanup": {
+                                            "finished_at": 1584242238.408476, 
+                                            "started_at": 1584242238.40758, 
+                                            "atomic_actions": [], 
+                                            "error": null
+                                        }, 
+                                        "plugin_cfg": {
+                                            "user_choice_method": "random"
+                                        }, 
+                                        "plugin_name": "users@openstack"
+                                    }
+                                ], 
+                                "position": 0, 
+                                "pass_sla": true, 
+                                "sla_results": {
+                                    "sla": [
+                                        {
+                                            "criterion": "failure_rate", 
+                                            "detail": "Fake error - FAIL", 
+                                            "success": false
+                                        }
+                                    ]
+                                }, 
+                                "sla": {
+                                    "failure_rate": {
+                                        "max": 0
+                                    }
+                                }
+                            }
+                        ]
+                    }
+                ]
             }
         ]
     }
 ]
 """
 
-TASK_RESULTS_CRASH = u"""
+TASK_REPORT_ERROR2 = u"""
 [
     {
         "info": {
-            "generated_at": "2019-03-05T16:50:31",
-            "rally_version": "0.0.0",
-            "format_version": "1.1"
-        },
+            "generated_at": "2020-03-16T16:07:30", 
+            "rally_version": "2.0.0", 
+            "format_version": "1.2"
+        }, 
         "tasks": [
             {
-                "uuid": "5312f2e6-f00d-4a63-9162-40c895cdaea1",
-                "title": "",
-                "description": "",
-                "status": "crashed",
-                "tags": [],
-                "created_at": "2019-03-05T15:14:54",
-                "updated_at": "2019-03-05T15:14:56",
-                "pass_sla": true,
-                "subtasks": []
+                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d", 
+                "title": "", 
+                "description": "", 
+                "status": "finished", 
+                "tags": [], 
+                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708", 
+                "env_name": "teststack_cell02", 
+                "created_at": "2020-03-15T03:17:17", 
+                "updated_at": "2020-03-15T03:17:20", 
+                "pass_sla": true, 
+                "subtasks": [
+                    {
+                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12", 
+                        "title": "GlanceImages.list_images", 
+                        "description": "", 
+                        "status": "finished", 
+                        "created_at": "2020-03-15T03:17:18", 
+                        "updated_at": "2020-03-15T03:17:20", 
+                        "sla": {}, 
+                        "workloads": [
+                            {
+                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48", 
+                                "description": "List all images.", 
+                                "runner": {
+                                    "constant": {
+                                        "concurrency": 1, 
+                                        "times": 1
+                                    }
+                                }, 
+                                "hooks": [], 
+                                "scenario": {
+                                    "GlanceImages.list_images": {}
+                                }, 
+                                "min_duration": 0.1442, 
+                                "max_duration": 0.1442, 
+                                "start_time": 1584242238.24373, 
+                                "load_duration": 0.1442, 
+                                "full_duration": 0.380141, 
+                                "statistics": {
+                                    "durations": {
+                                        "total": {
+                                            "data": {
+                                                "success": "100.0%", 
+                                                "min": 0.144, 
+                                                "max": 0.144, 
+                                                "median": 0.144, 
+                                                "95%ile": 0.144, 
+                                                "iteration_count": 1, 
+                                                "avg": 0.144, 
+                                                "90%ile": 0.144
+                                            }, 
+                                            "display_name": "total", 
+                                            "name": "total", 
+                                            "count_per_iteration": 1, 
+                                            "children": [
+                                                {
+                                                    "data": {
+                                                        "success": "100.0%", 
+                                                        "min": 0.144, 
+                                                        "max": 0.144, 
+                                                        "median": 0.144, 
+                                                        "95%ile": 0.144, 
+                                                        "iteration_count": 1, 
+                                                        "avg": 0.144, 
+                                                        "90%ile": 0.144
+                                                    }, 
+                                                    "display_name": "duration", 
+                                                    "name": "duration", 
+                                                    "count_per_iteration": 1, 
+                                                    "children": []
+                                                }, 
+                                                {
+                                                    "data": {
+                                                        "success": "100.0%", 
+                                                        "min": 0.0, 
+                                                        "max": 0.0, 
+                                                        "median": 0.0, 
+                                                        "95%ile": 0.0, 
+                                                        "iteration_count": 1, 
+                                                        "avg": 0.0, 
+                                                        "90%ile": 0.0
+                                                    }, 
+                                                    "display_name": "idle_duration", 
+                                                    "name": "idle_duration", 
+                                                    "count_per_iteration": 1, 
+                                                    "children": []
+                                                }
+                                            ]
+                                        }, 
+                                        "atomics": [
+                                            {
+                                                "data": {
+                                                    "success": "100.0%", 
+                                                    "min": 0.144, 
+                                                    "max": 0.144, 
+                                                    "median": 0.144, 
+                                                    "95%ile": 0.144, 
+                                                    "iteration_count": 1, 
+                                                    "avg": 0.144, 
+                                                    "90%ile": 0.144
+                                                }, 
+                                                "display_name": "Novaname", 
+                                                "name": "Novaname", 
+                                                "count_per_iteration": 1, 
+                                                "children": []
+                                            }
+                                        ]
+                                    }
+                                }, 
+                                "data": [
+                                    {
+                                        "timestamp": 1584242238.243732, 
+                                        "error": [
+                                            "-----------",
+                                            "Fake error1",
+                                            "Fake error2"
+                                        ], 
+                                        "duration": 0.14420008659362793, 
+                                        "output": {
+                                            "additive": [], 
+                                            "complete": []
+                                        }, 
+                                        "idle_duration": 0.0, 
+                                        "atomic_actions": [
+                                            {
+                                                "finished_at": 1584242238.38778, 
+                                                "started_at": 1584242238.243803, 
+                                                "name": "Novaname", 
+                                                "children": []
+                                            }
+                                        ]
+                                    }
+                                ], 
+                                "failed_iteration_count": 0, 
+                                "total_iteration_count": 1, 
+                                "created_at": "2020-03-15T03:17:18", 
+                                "updated_at": "2020-03-15T03:17:20", 
+                                "contexts": {}, 
+                                "contexts_results": [
+                                    {
+                                        "setup": {
+                                            "finished_at": 1584242238.110536, 
+                                            "started_at": 1584242238.029151, 
+                                            "atomic_actions": [], 
+                                            "error": null
+                                        }, 
+                                        "cleanup": {
+                                            "finished_at": 1584242238.408476, 
+                                            "started_at": 1584242238.40758, 
+                                            "atomic_actions": [], 
+                                            "error": null
+                                        }, 
+                                        "plugin_cfg": {
+                                            "user_choice_method": "random"
+                                        }, 
+                                        "plugin_name": "users@openstack"
+                                    }
+                                ], 
+                                "position": 0, 
+                                "pass_sla": true, 
+                                "sla_results": {
+                                    "sla": [
+                                        {
+                                            "criterion": "failure_rate", 
+                                            "detail": "Failure rate criteria 0.00% <= 0.00% <= 0.00% - Passed", 
+                                            "success": true
+                                        }
+                                    ]
+                                }, 
+                                "sla": {
+                                    "failure_rate": {
+                                        "max": 0
+                                    }
+                                }
+                            }
+                        ]
+                    }
+                ]
             }
         ]
     }
diff --git a/rallytester/tests/test_rallytester.py b/rallytester/tests/test_rallytester.py
index f128039..ff645be 100644
--- a/rallytester/tests/test_rallytester.py
+++ b/rallytester/tests/test_rallytester.py
@@ -282,7 +282,7 @@ class TestRallytester(unittest.TestCase):
                 None
             ),
             (
-                fixtures.TASK_RESULTS,
+                fixtures.TASK_REPORT,
                 None
             )
         ]
@@ -293,7 +293,7 @@ class TestRallytester(unittest.TestCase):
                     'task': 'fake_task',
                     'scenarios': 'Novaname',
                     'succeed': 1,
-                    'duration': 617.1978030204773,
+                    'duration': 0.14420008659362793,
                 },
                 None  # No errors
             ),
@@ -319,14 +319,14 @@ class TestRallytester(unittest.TestCase):
                 None
             ),
             (
-                fixtures.TASK_RESULTS_FAIL,
+                fixtures.TASK_REPORT_ERROR2,
                 None
             )
         ]
 
         self.assertEqual(
             (
-                {'task': 'fake_task', 'scenarios': 'Name,name', 'succeed': 0},
+                {'task': 'fake_task', 'scenarios': 'Novaname', 'succeed': 0},
                 '\nFake error1\n\nFake error2\n'
                 'Task descriptor:\n'
                 '\n'
@@ -365,7 +365,7 @@ class TestRallytester(unittest.TestCase):
                 None
             ),
             (
-                fixtures.TASK_RESULTS_FAIL2,
+                fixtures.TASK_REPORT_ERROR1,
                 None
             )
         ]
@@ -374,10 +374,10 @@ class TestRallytester(unittest.TestCase):
             (
                 {
                     'task': 'fake_task',
-                    'scenarios': 'NovaServers,Novaname',
+                    'scenarios': 'Novaname',
                     'succeed': 0,
                 },
-                '\nFrc\n\nTask descriptor:\n\n'
+                '\n\nTask descriptor:\n\n'
                 '{\n'
                 '    "NovaServers.list_servers": [\n'
                 '        {\n'
@@ -442,7 +442,7 @@ class TestRallytester(unittest.TestCase):
         )
 
     @mock.patch('rallytester.rallytester.subprocess.Popen')
-    @mock.patch(func, return_value=io.StringIO(fixtures.TASK_RESULTS_CRASH))
+    @mock.patch(func, return_value=io.StringIO(fixtures.TASK))
     @mock.patch('os.path.exists', return_value=True)
     def test_execute_task_crash(self,
                                 mock_os,
@@ -457,7 +457,7 @@ class TestRallytester(unittest.TestCase):
                 None
             ),
             (
-                fixtures.TASK_RESULTS_CRASH,
+                fixtures.TASK_REPORT_CRASH,
                 None
             )
         ]
@@ -465,7 +465,6 @@ class TestRallytester(unittest.TestCase):
         self.assertEqual(
             (
                 "Task status is crashed."
-                "Use: task report --uuid --json "
             ),
             rallytester.execute_task(
                 deployment='fake_deployment',
-- 
GitLab


From 97db6fd90905e3fd308e2e39c0307b880919f126 Mon Sep 17 00:00:00 2001
From: ravillar <raul.villar.ramos@cern.ch>
Date: Mon, 23 Mar 2020 18:51:26 +0100
Subject: [PATCH 05/21] OS8753_Fix_pep8

---
 rallytester/rallytester.py    |  40 +-
 rallytester/tests/fixtures.py | 664 +++++++++++++++++-----------------
 2 files changed, 366 insertions(+), 338 deletions(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index c553173..ca4817d 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -149,19 +149,25 @@ def execute_task(deployment, task, msgTrimmer=None):
                         return ("Task status is crashed.")
                         # return json here
 
-    scenarios = ",".join([job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['atomic_actions'][0]['name'] for job in json_result])
+    scenarios = ",".join(
+        [job['tasks'][0]['subtasks'][0]['workloads'][0]
+            ['data'][0]['atomic_actions'][0]['name'] for job in json_result])
 
     # If there is any error in the reports
-    if any(job['tasks'] and 
-            job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'] for job in json_result):
+    if any(job['tasks'] and
+            job['tasks'][0]['subtasks'][0]['workloads'][0]
+            ['data'][0]['error'] for job in json_result):
         msg = ""
         tracebacks = ""
         for job in json_result:
-            if job['tasks'] and job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error']:
+            if (job['tasks'] and job['tasks'][0]['subtasks'][0]['workloads'][0]
+                    ['data'][0]['error']):
                 # The real error is in the first result
                 # and the second element of the array
-                msg += '\n' + job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'][1]
-                tracebacks += '\n' + job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'][2]
+                msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads']
+                        [0]['data'][0]['error'][1])
+                tracebacks += ('\n' + job['tasks'][0]['subtasks'][0]
+                               ['workloads'][0]['data'][0]['error'][2])
                 log.error('Task failed: {0}'.format(
                     _msg_enrich(msg.replace('\n', ' '), msgTrimmer)))
 
@@ -210,16 +216,21 @@ def execute_task(deployment, task, msgTrimmer=None):
         )
 
     if any('tasks' in j for j in json_result):
-        if any(job['tasks'][0]['subtasks'][0]['workloads'][0]['sla_results']['sla'][0]['success'] == False for job in json_result):
-        
+        if (any(job['tasks'][0]['subtasks'][0]['workloads']
+                [0]['sla_results']['sla'][0]['success'] is False
+                for job in json_result)):
+
             msg = ""
             tracebacks = ""
             for job in json_result:
-                if job['tasks'] and job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error']:
+                if (job['tasks'] and job['tasks'][0]['subtasks']
+                        [0]['workloads'][0]['data'][0]['error']):
                     # The real error is in the first result
                     # and the second element of the array
-                    msg += '\n' + job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'][1]
-                    tracebacks += '\n' + job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'][2]
+                    msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads']
+                            [0]['data'][0]['error'][1])
+                    tracebacks += ('\n' + job['tasks'][0]['subtasks']
+                                   [0]['workloads'][0]['data'][0]['error'][2])
                     log.error('Task failed: {0}'.format(
                         _msg_enrich(msg.replace('\n', ' '), msgTrimmer)))
 
@@ -230,7 +241,7 @@ def execute_task(deployment, task, msgTrimmer=None):
                     'succeed': 0
                 },
                 "{0}\n{1}\nTask descriptor:\n{2}".format(msg, tracebacks,
-                                                        task_descriptor)
+                                                         task_descriptor)
             )
 
     # In any other case
@@ -240,8 +251,9 @@ def execute_task(deployment, task, msgTrimmer=None):
             'task': task,
             'scenarios': scenarios,
             'succeed': 1,
-            'duration': sum([job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['duration']
-                             for job in json_result])
+            'duration': sum([job['tasks'][0]['subtasks'][0]['workloads']
+                            [0]['data'][0]['duration']
+                            for job in json_result])
         },
         None  # No errors
     )
diff --git a/rallytester/tests/fixtures.py b/rallytester/tests/fixtures.py
index 95960ae..04f0626 100644
--- a/rallytester/tests/fixtures.py
+++ b/rallytester/tests/fixtures.py
@@ -103,177 +103,183 @@ TASK_REPORT = u"""
 [
     {
         "info": {
-            "generated_at": "2020-03-16T16:07:30", 
-            "rally_version": "2.0.0", 
+            "generated_at": "2020-03-16T16:07:30",
+            "rally_version": "2.0.0",
             "format_version": "1.2"
-        }, 
+        },
         "tasks": [
             {
-                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d", 
-                "title": "", 
-                "description": "", 
-                "status": "finished", 
-                "tags": [], 
-                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708", 
-                "env_name": "teststack_cell02", 
-                "created_at": "2020-03-15T03:17:17", 
-                "updated_at": "2020-03-15T03:17:20", 
-                "pass_sla": true, 
+                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d",
+                "title": "",
+                "description": "",
+                "status": "finished",
+                "tags": [],
+                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708",
+                "env_name": "teststack_cell02",
+                "created_at": "2020-03-15T03:17:17",
+                "updated_at": "2020-03-15T03:17:20",
+                "pass_sla": true,
                 "subtasks": [
                     {
-                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12", 
-                        "title": "GlanceImages.list_images", 
-                        "description": "", 
-                        "status": "finished", 
-                        "created_at": "2020-03-15T03:17:18", 
-                        "updated_at": "2020-03-15T03:17:20", 
-                        "sla": {}, 
+                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12",
+                        "title": "GlanceImages.list_images",
+                        "description": "",
+                        "status": "finished",
+                        "created_at": "2020-03-15T03:17:18",
+                        "updated_at": "2020-03-15T03:17:20",
+                        "sla": {},
                         "workloads": [
                             {
-                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48", 
-                                "description": "List all images.", 
+                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48",
+                                "description": "List all images.",
                                 "runner": {
                                     "constant": {
-                                        "concurrency": 1, 
+                                        "concurrency": 1,
                                         "times": 1
                                     }
-                                }, 
-                                "hooks": [], 
+                                },
+                                "hooks": [],
                                 "scenario": {
                                     "GlanceImages.list_images": {}
-                                }, 
-                                "min_duration": 0.1442, 
-                                "max_duration": 0.1442, 
-                                "start_time": 1584242238.24373, 
-                                "load_duration": 0.1442, 
-                                "full_duration": 0.380141, 
+                                },
+                                "min_duration": 0.1442,
+                                "max_duration": 0.1442,
+                                "start_time": 1584242238.24373,
+                                "load_duration": 0.1442,
+                                "full_duration": 0.380141,
                                 "statistics": {
                                     "durations": {
                                         "total": {
                                             "data": {
-                                                "success": "100.0%", 
-                                                "min": 0.144, 
-                                                "max": 0.144, 
-                                                "median": 0.144, 
-                                                "95%ile": 0.144, 
-                                                "iteration_count": 1, 
-                                                "avg": 0.144, 
+                                                "success": "100.0%",
+                                                "min": 0.144,
+                                                "max": 0.144,
+                                                "median": 0.144,
+                                                "95%ile": 0.144,
+                                                "iteration_count": 1,
+                                                "avg": 0.144,
                                                 "90%ile": 0.144
-                                            }, 
-                                            "display_name": "total", 
-                                            "name": "total", 
-                                            "count_per_iteration": 1, 
+                                            },
+                                            "display_name": "total",
+                                            "name": "total",
+                                            "count_per_iteration": 1,
                                             "children": [
                                                 {
                                                     "data": {
-                                                        "success": "100.0%", 
-                                                        "min": 0.144, 
-                                                        "max": 0.144, 
-                                                        "median": 0.144, 
-                                                        "95%ile": 0.144, 
-                                                        "iteration_count": 1, 
-                                                        "avg": 0.144, 
+                                                        "success": "100.0%",
+                                                        "min": 0.144,
+                                                        "max": 0.144,
+                                                        "median": 0.144,
+                                                        "95%ile": 0.144,
+                                                        "iteration_count": 1,
+                                                        "avg": 0.144,
                                                         "90%ile": 0.144
-                                                    }, 
-                                                    "display_name": "duration", 
-                                                    "name": "duration", 
-                                                    "count_per_iteration": 1, 
+                                                    },
+                                                    "display_name": "duration",
+                                                    "name": "duration",
+                                                    "count_per_iteration": 1,
                                                     "children": []
-                                                }, 
+                                                },
                                                 {
                                                     "data": {
-                                                        "success": "100.0%", 
-                                                        "min": 0.0, 
-                                                        "max": 0.0, 
-                                                        "median": 0.0, 
-                                                        "95%ile": 0.0, 
-                                                        "iteration_count": 1, 
-                                                        "avg": 0.0, 
+                                                        "success": "100.0%",
+                                                        "min": 0.0,
+                                                        "max": 0.0,
+                                                        "median": 0.0,
+                                                        "95%ile": 0.0,
+                                                        "iteration_count": 1,
+                                                        "avg": 0.0,
                                                         "90%ile": 0.0
-                                                    }, 
-                                                    "display_name": "idle_duration", 
-                                                    "name": "idle_duration", 
-                                                    "count_per_iteration": 1, 
+                                                    },
+                                                    "display_name":
+                                                        "idle_duration",
+                                                    "name": "idle_duration",
+                                                    "count_per_iteration": 1,
                                                     "children": []
                                                 }
                                             ]
-                                        }, 
+                                        },
                                         "atomics": [
                                             {
                                                 "data": {
-                                                    "success": "100.0%", 
-                                                    "min": 0.144, 
-                                                    "max": 0.144, 
-                                                    "median": 0.144, 
-                                                    "95%ile": 0.144, 
-                                                    "iteration_count": 1, 
-                                                    "avg": 0.144, 
+                                                    "success": "100.0%",
+                                                    "min": 0.144,
+                                                    "max": 0.144,
+                                                    "median": 0.144,
+                                                    "95%ile": 0.144,
+                                                    "iteration_count": 1,
+                                                    "avg": 0.144,
                                                     "90%ile": 0.144
-                                                }, 
-                                                "display_name": "Novaname", 
-                                                "name": "Novaname", 
-                                                "count_per_iteration": 1, 
+                                                },
+                                                "display_name": "Novaname",
+                                                "name": "Novaname",
+                                                "count_per_iteration": 1,
                                                 "children": []
                                             }
                                         ]
                                     }
-                                }, 
+                                },
                                 "data": [
                                     {
-                                        "timestamp": 1584242238.243732, 
-                                        "error": [], 
-                                        "duration": 0.14420008659362793, 
+                                        "timestamp": 1584242238.243732,
+                                        "error": [],
+                                        "duration": 0.14420008659362793,
                                         "output": {
-                                            "additive": [], 
+                                            "additive": [],
                                             "complete": []
-                                        }, 
-                                        "idle_duration": 0.0, 
+                                        },
+                                        "idle_duration": 0.0,
                                         "atomic_actions": [
                                             {
-                                                "finished_at": 1584242238.38778, 
-                                                "started_at": 1584242238.243803, 
-                                                "name": "Novaname", 
+                                                "finished_at":
+                                                    1584242238.38778,
+                                                "started_at":
+                                                    1584242238.243803,
+                                                "name": "Novaname",
                                                 "children": []
                                             }
                                         ]
                                     }
-                                ], 
-                                "failed_iteration_count": 0, 
-                                "total_iteration_count": 1, 
-                                "created_at": "2020-03-15T03:17:18", 
-                                "updated_at": "2020-03-15T03:17:20", 
-                                "contexts": {}, 
+                                ],
+                                "failed_iteration_count": 0,
+                                "total_iteration_count": 1,
+                                "created_at": "2020-03-15T03:17:18",
+                                "updated_at": "2020-03-15T03:17:20",
+                                "contexts": {},
                                 "contexts_results": [
                                     {
                                         "setup": {
-                                            "finished_at": 1584242238.110536, 
-                                            "started_at": 1584242238.029151, 
-                                            "atomic_actions": [], 
+                                            "finished_at": 1584242238.110536,
+                                            "started_at": 1584242238.029151,
+                                            "atomic_actions": [],
                                             "error": null
-                                        }, 
+                                        },
                                         "cleanup": {
-                                            "finished_at": 1584242238.408476, 
-                                            "started_at": 1584242238.40758, 
-                                            "atomic_actions": [], 
+                                            "finished_at": 1584242238.408476,
+                                            "started_at": 1584242238.40758,
+                                            "atomic_actions": [],
                                             "error": null
-                                        }, 
+                                        },
                                         "plugin_cfg": {
                                             "user_choice_method": "random"
-                                        }, 
+                                        },
                                         "plugin_name": "users@openstack"
                                     }
-                                ], 
-                                "position": 0, 
-                                "pass_sla": true, 
+                                ],
+                                "position": 0,
+                                "pass_sla": true,
                                 "sla_results": {
                                     "sla": [
                                         {
-                                            "criterion": "failure_rate", 
-                                            "detail": "Failure rate criteria 0.00% <= 0.00% <= 0.00% - Passed", 
+                                            "criterion": "failure_rate",
+                                            "detail":
+                                                "Failure rate criteria\
+                                                 0.00% <= 0.00% <= 0.00%\
+                                                  - Passed",
                                             "success": true
                                         }
                                     ]
-                                }, 
+                                },
                                 "sla": {
                                     "failure_rate": {
                                         "max": 0
@@ -293,177 +299,180 @@ TASK_REPORT_ERROR1 = u"""
 [
     {
         "info": {
-            "generated_at": "2020-03-16T16:07:30", 
-            "rally_version": "2.0.0", 
+            "generated_at": "2020-03-16T16:07:30",
+            "rally_version": "2.0.0",
             "format_version": "1.2"
-        }, 
+        },
         "tasks": [
             {
-                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d", 
-                "title": "", 
-                "description": "", 
-                "status": "finished", 
-                "tags": [], 
-                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708", 
-                "env_name": "teststack_cell02", 
-                "created_at": "2020-03-15T03:17:17", 
-                "updated_at": "2020-03-15T03:17:20", 
-                "pass_sla": true, 
+                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d",
+                "title": "",
+                "description": "",
+                "status": "finished",
+                "tags": [],
+                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708",
+                "env_name": "teststack_cell02",
+                "created_at": "2020-03-15T03:17:17",
+                "updated_at": "2020-03-15T03:17:20",
+                "pass_sla": true,
                 "subtasks": [
                     {
-                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12", 
-                        "title": "GlanceImages.list_images", 
-                        "description": "", 
-                        "status": "finished", 
-                        "created_at": "2020-03-15T03:17:18", 
-                        "updated_at": "2020-03-15T03:17:20", 
-                        "sla": {}, 
+                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12",
+                        "title": "GlanceImages.list_images",
+                        "description": "",
+                        "status": "finished",
+                        "created_at": "2020-03-15T03:17:18",
+                        "updated_at": "2020-03-15T03:17:20",
+                        "sla": {},
                         "workloads": [
                             {
-                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48", 
-                                "description": "List all images.", 
+                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48",
+                                "description": "List all images.",
                                 "runner": {
                                     "constant": {
-                                        "concurrency": 1, 
+                                        "concurrency": 1,
                                         "times": 1
                                     }
-                                }, 
-                                "hooks": [], 
+                                },
+                                "hooks": [],
                                 "scenario": {
                                     "GlanceImages.list_images": {}
-                                }, 
-                                "min_duration": 0.1442, 
-                                "max_duration": 0.1442, 
-                                "start_time": 1584242238.24373, 
-                                "load_duration": 0.1442, 
-                                "full_duration": 0.380141, 
+                                },
+                                "min_duration": 0.1442,
+                                "max_duration": 0.1442,
+                                "start_time": 1584242238.24373,
+                                "load_duration": 0.1442,
+                                "full_duration": 0.380141,
                                 "statistics": {
                                     "durations": {
                                         "total": {
                                             "data": {
-                                                "success": "0.0%", 
-                                                "min": 0.144, 
-                                                "max": 0.144, 
-                                                "median": 0.144, 
-                                                "95%ile": 0.144, 
-                                                "iteration_count": 1, 
-                                                "avg": 0.144, 
+                                                "success": "0.0%",
+                                                "min": 0.144,
+                                                "max": 0.144,
+                                                "median": 0.144,
+                                                "95%ile": 0.144,
+                                                "iteration_count": 1,
+                                                "avg": 0.144,
                                                 "90%ile": 0.144
-                                            }, 
-                                            "display_name": "total", 
-                                            "name": "total", 
-                                            "count_per_iteration": 1, 
+                                            },
+                                            "display_name": "total",
+                                            "name": "total",
+                                            "count_per_iteration": 1,
                                             "children": [
                                                 {
                                                     "data": {
-                                                        "success": "0.0%", 
-                                                        "min": 0.144, 
-                                                        "max": 0.144, 
-                                                        "median": 0.144, 
-                                                        "95%ile": 0.144, 
-                                                        "iteration_count": 1, 
-                                                        "avg": 0.144, 
+                                                        "success": "0.0%",
+                                                        "min": 0.144,
+                                                        "max": 0.144,
+                                                        "median": 0.144,
+                                                        "95%ile": 0.144,
+                                                        "iteration_count": 1,
+                                                        "avg": 0.144,
                                                         "90%ile": 0.144
-                                                    }, 
-                                                    "display_name": "duration", 
-                                                    "name": "duration", 
-                                                    "count_per_iteration": 1, 
+                                                    },
+                                                    "display_name": "duration",
+                                                    "name": "duration",
+                                                    "count_per_iteration": 1,
                                                     "children": []
-                                                }, 
+                                                },
                                                 {
                                                     "data": {
-                                                        "success": "0.0%", 
-                                                        "min": 0.0, 
-                                                        "max": 0.0, 
-                                                        "median": 0.0, 
-                                                        "95%ile": 0.0, 
-                                                        "iteration_count": 1, 
-                                                        "avg": 0.0, 
+                                                        "success": "0.0%",
+                                                        "min": 0.0,
+                                                        "max": 0.0,
+                                                        "median": 0.0,
+                                                        "95%ile": 0.0,
+                                                        "iteration_count": 1,
+                                                        "avg": 0.0,
                                                         "90%ile": 0.0
-                                                    }, 
-                                                    "display_name": "idle_duration", 
-                                                    "name": "idle_duration", 
-                                                    "count_per_iteration": 1, 
+                                                    },
+                                                    "display_name":
+                                                        "idle_duration",
+                                                    "name": "idle_duration",
+                                                    "count_per_iteration": 1,
                                                     "children": []
                                                 }
                                             ]
-                                        }, 
+                                        },
                                         "atomics": [
                                             {
                                                 "data": {
-                                                    "success": "0.0%", 
-                                                    "min": 0.144, 
-                                                    "max": 0.144, 
-                                                    "median": 0.144, 
-                                                    "95%ile": 0.144, 
-                                                    "iteration_count": 1, 
-                                                    "avg": 0.144, 
+                                                    "success": "0.0%",
+                                                    "min": 0.144,
+                                                    "max": 0.144,
+                                                    "median": 0.144,
+                                                    "95%ile": 0.144,
+                                                    "iteration_count": 1,
+                                                    "avg": 0.144,
                                                     "90%ile": 0.144
-                                                }, 
-                                                "display_name": "Novaname", 
-                                                "name": "Novaname", 
-                                                "count_per_iteration": 1, 
+                                                },
+                                                "display_name": "Novaname",
+                                                "name": "Novaname",
+                                                "count_per_iteration": 1,
                                                 "children": []
                                             }
                                         ]
                                     }
-                                }, 
+                                },
                                 "data": [
                                     {
-                                        "timestamp": 1584242238.243732, 
-                                        "error": [], 
-                                        "duration": 0.14420008659362793, 
+                                        "timestamp": 1584242238.243732,
+                                        "error": [],
+                                        "duration": 0.14420008659362793,
                                         "output": {
-                                            "additive": [], 
+                                            "additive": [],
                                             "complete": []
-                                        }, 
-                                        "idle_duration": 0.0, 
+                                        },
+                                        "idle_duration": 0.0,
                                         "atomic_actions": [
                                             {
-                                                "finished_at": 1584242238.38778, 
-                                                "started_at": 1584242238.243803, 
-                                                "name": "Novaname", 
+                                                "finished_at":
+                                                    1584242238.38778,
+                                                "started_at":
+                                                    1584242238.243803,
+                                                "name": "Novaname",
                                                 "children": []
                                             }
                                         ]
                                     }
-                                ], 
-                                "failed_iteration_count": 0, 
-                                "total_iteration_count": 1, 
-                                "created_at": "2020-03-15T03:17:18", 
-                                "updated_at": "2020-03-15T03:17:20", 
-                                "contexts": {}, 
+                                ],
+                                "failed_iteration_count": 0,
+                                "total_iteration_count": 1,
+                                "created_at": "2020-03-15T03:17:18",
+                                "updated_at": "2020-03-15T03:17:20",
+                                "contexts": {},
                                 "contexts_results": [
                                     {
                                         "setup": {
-                                            "finished_at": 1584242238.110536, 
-                                            "started_at": 1584242238.029151, 
-                                            "atomic_actions": [], 
+                                            "finished_at": 1584242238.110536,
+                                            "started_at": 1584242238.029151,
+                                            "atomic_actions": [],
                                             "error": null
-                                        }, 
+                                        },
                                         "cleanup": {
-                                            "finished_at": 1584242238.408476, 
-                                            "started_at": 1584242238.40758, 
-                                            "atomic_actions": [], 
+                                            "finished_at": 1584242238.408476,
+                                            "started_at": 1584242238.40758,
+                                            "atomic_actions": [],
                                             "error": null
-                                        }, 
+                                        },
                                         "plugin_cfg": {
                                             "user_choice_method": "random"
-                                        }, 
+                                        },
                                         "plugin_name": "users@openstack"
                                     }
-                                ], 
-                                "position": 0, 
-                                "pass_sla": true, 
+                                ],
+                                "position": 0,
+                                "pass_sla": true,
                                 "sla_results": {
                                     "sla": [
                                         {
-                                            "criterion": "failure_rate", 
-                                            "detail": "Fake error - FAIL", 
+                                            "criterion": "failure_rate",
+                                            "detail": "Fake error - FAIL",
                                             "success": false
                                         }
                                     ]
-                                }, 
+                                },
                                 "sla": {
                                     "failure_rate": {
                                         "max": 0
@@ -483,181 +492,188 @@ TASK_REPORT_ERROR2 = u"""
 [
     {
         "info": {
-            "generated_at": "2020-03-16T16:07:30", 
-            "rally_version": "2.0.0", 
+            "generated_at": "2020-03-16T16:07:30",
+            "rally_version": "2.0.0",
             "format_version": "1.2"
-        }, 
+        },
         "tasks": [
             {
-                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d", 
-                "title": "", 
-                "description": "", 
-                "status": "finished", 
-                "tags": [], 
-                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708", 
-                "env_name": "teststack_cell02", 
-                "created_at": "2020-03-15T03:17:17", 
-                "updated_at": "2020-03-15T03:17:20", 
-                "pass_sla": true, 
+                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d",
+                "title": "",
+                "description": "",
+                "status": "finished",
+                "tags": [],
+                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708",
+                "env_name": "teststack_cell02",
+                "created_at": "2020-03-15T03:17:17",
+                "updated_at": "2020-03-15T03:17:20",
+                "pass_sla": true,
                 "subtasks": [
                     {
-                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12", 
-                        "title": "GlanceImages.list_images", 
-                        "description": "", 
-                        "status": "finished", 
-                        "created_at": "2020-03-15T03:17:18", 
-                        "updated_at": "2020-03-15T03:17:20", 
-                        "sla": {}, 
+                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12",
+                        "title": "GlanceImages.list_images",
+                        "description": "",
+                        "status": "finished",
+                        "created_at": "2020-03-15T03:17:18",
+                        "updated_at": "2020-03-15T03:17:20",
+                        "sla": {},
                         "workloads": [
                             {
-                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48", 
-                                "description": "List all images.", 
+                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48",
+                                "description": "List all images.",
                                 "runner": {
                                     "constant": {
-                                        "concurrency": 1, 
+                                        "concurrency": 1,
                                         "times": 1
                                     }
-                                }, 
-                                "hooks": [], 
+                                },
+                                "hooks": [],
                                 "scenario": {
                                     "GlanceImages.list_images": {}
-                                }, 
-                                "min_duration": 0.1442, 
-                                "max_duration": 0.1442, 
-                                "start_time": 1584242238.24373, 
-                                "load_duration": 0.1442, 
-                                "full_duration": 0.380141, 
+                                },
+                                "min_duration": 0.1442,
+                                "max_duration": 0.1442,
+                                "start_time": 1584242238.24373,
+                                "load_duration": 0.1442,
+                                "full_duration": 0.380141,
                                 "statistics": {
                                     "durations": {
                                         "total": {
                                             "data": {
-                                                "success": "100.0%", 
-                                                "min": 0.144, 
-                                                "max": 0.144, 
-                                                "median": 0.144, 
-                                                "95%ile": 0.144, 
-                                                "iteration_count": 1, 
-                                                "avg": 0.144, 
+                                                "success": "100.0%",
+                                                "min": 0.144,
+                                                "max": 0.144,
+                                                "median": 0.144,
+                                                "95%ile": 0.144,
+                                                "iteration_count": 1,
+                                                "avg": 0.144,
                                                 "90%ile": 0.144
-                                            }, 
-                                            "display_name": "total", 
-                                            "name": "total", 
-                                            "count_per_iteration": 1, 
+                                            },
+                                            "display_name": "total",
+                                            "name": "total",
+                                            "count_per_iteration": 1,
                                             "children": [
                                                 {
                                                     "data": {
-                                                        "success": "100.0%", 
-                                                        "min": 0.144, 
-                                                        "max": 0.144, 
-                                                        "median": 0.144, 
-                                                        "95%ile": 0.144, 
-                                                        "iteration_count": 1, 
-                                                        "avg": 0.144, 
+                                                        "success": "100.0%",
+                                                        "min": 0.144,
+                                                        "max": 0.144,
+                                                        "median": 0.144,
+                                                        "95%ile": 0.144,
+                                                        "iteration_count": 1,
+                                                        "avg": 0.144,
                                                         "90%ile": 0.144
-                                                    }, 
-                                                    "display_name": "duration", 
-                                                    "name": "duration", 
-                                                    "count_per_iteration": 1, 
+                                                    },
+                                                    "display_name": "duration",
+                                                    "name": "duration",
+                                                    "count_per_iteration": 1,
                                                     "children": []
-                                                }, 
+                                                },
                                                 {
                                                     "data": {
-                                                        "success": "100.0%", 
-                                                        "min": 0.0, 
-                                                        "max": 0.0, 
-                                                        "median": 0.0, 
-                                                        "95%ile": 0.0, 
-                                                        "iteration_count": 1, 
-                                                        "avg": 0.0, 
+                                                        "success": "100.0%",
+                                                        "min": 0.0,
+                                                        "max": 0.0,
+                                                        "median": 0.0,
+                                                        "95%ile": 0.0,
+                                                        "iteration_count": 1,
+                                                        "avg": 0.0,
                                                         "90%ile": 0.0
-                                                    }, 
-                                                    "display_name": "idle_duration", 
-                                                    "name": "idle_duration", 
-                                                    "count_per_iteration": 1, 
+                                                    },
+                                                    "display_name":
+                                                        "idle_duration",
+                                                    "name":
+                                                        "idle_duration",
+                                                    "count_per_iteration": 1,
                                                     "children": []
                                                 }
                                             ]
-                                        }, 
+                                        },
                                         "atomics": [
                                             {
                                                 "data": {
-                                                    "success": "100.0%", 
-                                                    "min": 0.144, 
-                                                    "max": 0.144, 
-                                                    "median": 0.144, 
-                                                    "95%ile": 0.144, 
-                                                    "iteration_count": 1, 
-                                                    "avg": 0.144, 
+                                                    "success": "100.0%",
+                                                    "min": 0.144,
+                                                    "max": 0.144,
+                                                    "median": 0.144,
+                                                    "95%ile": 0.144,
+                                                    "iteration_count": 1,
+                                                    "avg": 0.144,
                                                     "90%ile": 0.144
-                                                }, 
-                                                "display_name": "Novaname", 
-                                                "name": "Novaname", 
-                                                "count_per_iteration": 1, 
+                                                },
+                                                "display_name": "Novaname",
+                                                "name": "Novaname",
+                                                "count_per_iteration": 1,
                                                 "children": []
                                             }
                                         ]
                                     }
-                                }, 
+                                },
                                 "data": [
                                     {
-                                        "timestamp": 1584242238.243732, 
+                                        "timestamp": 1584242238.243732,
                                         "error": [
                                             "-----------",
                                             "Fake error1",
                                             "Fake error2"
-                                        ], 
-                                        "duration": 0.14420008659362793, 
+                                        ],
+                                        "duration": 0.14420008659362793,
                                         "output": {
-                                            "additive": [], 
+                                            "additive": [],
                                             "complete": []
-                                        }, 
-                                        "idle_duration": 0.0, 
+                                        },
+                                        "idle_duration": 0.0,
                                         "atomic_actions": [
                                             {
-                                                "finished_at": 1584242238.38778, 
-                                                "started_at": 1584242238.243803, 
-                                                "name": "Novaname", 
+                                                "finished_at":
+                                                    1584242238.38778,
+                                                "started_at":
+                                                    1584242238.243803,
+                                                "name": "Novaname",
                                                 "children": []
                                             }
                                         ]
                                     }
-                                ], 
-                                "failed_iteration_count": 0, 
-                                "total_iteration_count": 1, 
-                                "created_at": "2020-03-15T03:17:18", 
-                                "updated_at": "2020-03-15T03:17:20", 
-                                "contexts": {}, 
+                                ],
+                                "failed_iteration_count": 0,
+                                "total_iteration_count": 1,
+                                "created_at": "2020-03-15T03:17:18",
+                                "updated_at": "2020-03-15T03:17:20",
+                                "contexts": {},
                                 "contexts_results": [
                                     {
                                         "setup": {
-                                            "finished_at": 1584242238.110536, 
-                                            "started_at": 1584242238.029151, 
-                                            "atomic_actions": [], 
+                                            "finished_at": 1584242238.110536,
+                                            "started_at": 1584242238.029151,
+                                            "atomic_actions": [],
                                             "error": null
-                                        }, 
+                                        },
                                         "cleanup": {
-                                            "finished_at": 1584242238.408476, 
-                                            "started_at": 1584242238.40758, 
-                                            "atomic_actions": [], 
+                                            "finished_at": 1584242238.408476,
+                                            "started_at": 1584242238.40758,
+                                            "atomic_actions": [],
                                             "error": null
-                                        }, 
+                                        },
                                         "plugin_cfg": {
                                             "user_choice_method": "random"
-                                        }, 
+                                        },
                                         "plugin_name": "users@openstack"
                                     }
-                                ], 
-                                "position": 0, 
-                                "pass_sla": true, 
+                                ],
+                                "position": 0,
+                                "pass_sla": true,
                                 "sla_results": {
                                     "sla": [
                                         {
-                                            "criterion": "failure_rate", 
-                                            "detail": "Failure rate criteria 0.00% <= 0.00% <= 0.00% - Passed", 
+                                            "criterion": "failure_rate",
+                                            "detail":
+                                                "Failure rate criteria\
+                                                 0.00% <= 0.00% <= 0.00%\
+                                                  - Passed",
                                             "success": true
                                         }
                                     ]
-                                }, 
+                                },
                                 "sla": {
                                     "failure_rate": {
                                         "max": 0
-- 
GitLab


From b574cc79208bd2794f799e64a049c6393bab8c3b Mon Sep 17 00:00:00 2001
From: ravillar <raul.villar.ramos@cern.ch>
Date: Wed, 25 Mar 2020 11:01:57 +0100
Subject: [PATCH 06/21] OS8753_Send_crash_report

---
 rallytester/rallytester.py            |  2 +-
 rallytester/tests/test_rallytester.py | 25 ++++++++++++++++++++++++-
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index ca4817d..614c859 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -146,7 +146,7 @@ def execute_task(deployment, task, msgTrimmer=None):
     # Check if the task is crashed
     if any('tasks' in j for j in json_result):
         if any(job["tasks"][0]["status"] == "crashed" for job in json_result):
-                        return ("Task status is crashed.")
+                        return json_result
                         # return json here
 
     scenarios = ",".join(
diff --git a/rallytester/tests/test_rallytester.py b/rallytester/tests/test_rallytester.py
index ff645be..9a753f3 100644
--- a/rallytester/tests/test_rallytester.py
+++ b/rallytester/tests/test_rallytester.py
@@ -464,7 +464,30 @@ class TestRallytester(unittest.TestCase):
 
         self.assertEqual(
             (
-                "Task status is crashed."
+                [
+                    {
+                        'info':
+                        {
+                            'generated_at': '2019-03-05T16:50:31',
+                            'rally_version': '0.0.0',
+                            'format_version': '1.1'
+                        },
+                        'tasks':
+                        [
+                            {
+                                'uuid': '5312f2e6-f00d-4a63-9162-40c895cdaea1',
+                                'title': '',
+                                'description': '',
+                                'status': 'crashed',
+                                'tags': [],
+                                'created_at': '2019-03-05T15:14:54',
+                                'updated_at': '2019-03-05T15:14:56',
+                                'pass_sla': True,
+                                'subtasks': []
+                            }
+                        ],
+                    }
+                ]
             ),
             rallytester.execute_task(
                 deployment='fake_deployment',
-- 
GitLab


From 81c90d8c248be32535296df8e69f9a32d24edbb3 Mon Sep 17 00:00:00 2001
From: ravillar <raul.villar.ramos@cern.ch>
Date: Fri, 27 Mar 2020 16:22:36 +0100
Subject: [PATCH 07/21] OS8753_manage_magnum_reports

---
 rallytester/rallytester.py            |  64 ++++-----
 rallytester/tests/fixtures.py         | 197 ++++++++++++++++++++++++++
 rallytester/tests/test_rallytester.py |  45 ++++++
 3 files changed, 272 insertions(+), 34 deletions(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index 614c859..d2ba632 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -153,6 +153,36 @@ def execute_task(deployment, task, msgTrimmer=None):
         [job['tasks'][0]['subtasks'][0]['workloads'][0]
             ['data'][0]['atomic_actions'][0]['name'] for job in json_result])
 
+    # FIXME (lpigueir): cornercase for magnum tasks
+    # result is empty for magnum tasks sometimes and the
+    # real error doesn't appear. Will do this workaround because
+    # `rally task results` is gone/deprecated and we need
+    # to reimplement this part anyway to use `rally task report`
+    # with a different format
+    # 2020 update (Ravillar)
+    if any(job['tasks'][0]['subtasks'][0]['workloads'][0]
+            ['sla_results']['sla'][0]
+            ['criterion'] == 'something_went_wrong'
+            for job in json_result):
+
+        msg = ""
+        tracebacks = ""
+        msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads']
+                [0]['sla_results']['sla'][0]['detail'])
+        log.error('Task failed: {0}'.format(
+                  _msg_enrich(msg.replace('\n', ' '),
+                              msgTrimmer)))
+
+        return (
+            {
+                'task': task,
+                'scenarios': scenarios,
+                'succeed': 0
+            },
+            "{0}\n{1}\nTask descriptor:\n{2}".format(msg, tracebacks,
+                                                     task_descriptor)
+        )
+
     # If there is any error in the reports
     if any(job['tasks'] and
             job['tasks'][0]['subtasks'][0]['workloads'][0]
@@ -181,40 +211,6 @@ def execute_task(deployment, task, msgTrimmer=None):
                                                      task_descriptor)
         )
 
-    # FIXME (lpigueir): cornercase for magnum tasks
-    # result is empty for magnum tasks sometimes and the
-    # real error doesn't appear. Will do this workaround because
-    # `rally task results` is gone/deprecated and we need
-    # to reimplement this part anyway to use `rally task report`
-    # with a different format
-    elif any(
-        not job['tasks'] and
-        'sla' in job and
-        any(sla['criterion'] == 'something_went_wrong' for sla in job['sla'])
-        for job in json_result
-    ):
-        msg = ""
-        tracebacks = ""
-
-        for job in json_result:
-            if 'sla' in job:
-                for sla in job['sla']:
-                    if sla['criterion'] == 'something_went_wrong':
-                        msg += '\n' + sla['detail']
-                        log.error('Task failed: {0}'.format(
-                                  _msg_enrich(msg.replace('\n', ' '),
-                                              msgTrimmer)))
-
-        return (
-            {
-                'task': task,
-                'scenarios': scenarios,
-                'succeed': 0
-            },
-            "{0}\n{1}\nTask descriptor:\n{2}".format(msg, tracebacks,
-                                                     task_descriptor)
-        )
-
     if any('tasks' in j for j in json_result):
         if (any(job['tasks'][0]['subtasks'][0]['workloads']
                 [0]['sla_results']['sla'][0]['success'] is False
diff --git a/rallytester/tests/fixtures.py b/rallytester/tests/fixtures.py
index 04f0626..fd6da73 100644
--- a/rallytester/tests/fixtures.py
+++ b/rallytester/tests/fixtures.py
@@ -688,3 +688,200 @@ TASK_REPORT_ERROR2 = u"""
     }
 ]
 """
+
+TASK_REPORT_MAGNUM = u"""
+[
+    {
+        "info": {
+            "generated_at": "2020-03-16T16:07:30",
+            "rally_version": "2.0.0",
+            "format_version": "1.2"
+        },
+        "tasks": [
+            {
+                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d",
+                "title": "",
+                "description": "",
+                "status": "finished",
+                "tags": [],
+                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708",
+                "env_name": "teststack_cell02",
+                "created_at": "2020-03-15T03:17:17",
+                "updated_at": "2020-03-15T03:17:20",
+                "pass_sla": true,
+                "subtasks": [
+                    {
+                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12",
+                        "title": "GlanceImages.list_images",
+                        "description": "",
+                        "status": "finished",
+                        "created_at": "2020-03-15T03:17:18",
+                        "updated_at": "2020-03-15T03:17:20",
+                        "sla": {},
+                        "workloads": [
+                            {
+                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48",
+                                "description": "List all images.",
+                                "runner": {
+                                    "constant": {
+                                        "concurrency": 1,
+                                        "times": 1
+                                    }
+                                },
+                                "hooks": [],
+                                "scenario": {
+                                    "GlanceImages.list_images": {}
+                                },
+                                "min_duration": 0.1442,
+                                "max_duration": 0.1442,
+                                "start_time": 1584242238.24373,
+                                "load_duration": 0.1442,
+                                "full_duration": 0.380141,
+                                "statistics": {
+                                    "durations": {
+                                        "total": {
+                                            "data": {
+                                                "success": "100.0%",
+                                                "min": 0.144,
+                                                "max": 0.144,
+                                                "median": 0.144,
+                                                "95%ile": 0.144,
+                                                "iteration_count": 1,
+                                                "avg": 0.144,
+                                                "90%ile": 0.144
+                                            },
+                                            "display_name": "total",
+                                            "name": "total",
+                                            "count_per_iteration": 1,
+                                            "children": [
+                                                {
+                                                    "data": {
+                                                        "success": "100.0%",
+                                                        "min": 0.144,
+                                                        "max": 0.144,
+                                                        "median": 0.144,
+                                                        "95%ile": 0.144,
+                                                        "iteration_count": 1,
+                                                        "avg": 0.144,
+                                                        "90%ile": 0.144
+                                                    },
+                                                    "display_name": "duration",
+                                                    "name": "duration",
+                                                    "count_per_iteration": 1,
+                                                    "children": []
+                                                },
+                                                {
+                                                    "data": {
+                                                        "success": "100.0%",
+                                                        "min": 0.0,
+                                                        "max": 0.0,
+                                                        "median": 0.0,
+                                                        "95%ile": 0.0,
+                                                        "iteration_count": 1,
+                                                        "avg": 0.0,
+                                                        "90%ile": 0.0
+                                                    },
+                                                    "display_name":
+                                                        "idle_duration",
+                                                    "name": "idle_duration",
+                                                    "count_per_iteration": 1,
+                                                    "children": []
+                                                }
+                                            ]
+                                        },
+                                        "atomics": [
+                                            {
+                                                "data": {
+                                                    "success": "100.0%",
+                                                    "min": 0.144,
+                                                    "max": 0.144,
+                                                    "median": 0.144,
+                                                    "95%ile": 0.144,
+                                                    "iteration_count": 1,
+                                                    "avg": 0.144,
+                                                    "90%ile": 0.144
+                                                },
+                                                "display_name": "Novaname",
+                                                "name": "Novaname",
+                                                "count_per_iteration": 1,
+                                                "children": []
+                                            }
+                                        ]
+                                    }
+                                },
+                                "data": [
+                                    {
+                                        "timestamp": 1584242238.243732,
+                                        "error": [
+                                            "Fake error_M"
+                                        ],
+                                        "duration": 0.14420008659362793,
+                                        "output": {
+                                            "additive": [],
+                                            "complete": []
+                                        },
+                                        "idle_duration": 0.0,
+                                        "atomic_actions": [
+                                            {
+                                                "finished_at":
+                                                    1584242238.38778,
+                                                "started_at":
+                                                    1584242238.243803,
+                                                "name": "Novaname",
+                                                "children": []
+                                            }
+                                        ]
+                                    }
+                                ],
+                                "failed_iteration_count": 0,
+                                "total_iteration_count": 1,
+                                "created_at": "2020-03-15T03:17:18",
+                                "updated_at": "2020-03-15T03:17:20",
+                                "contexts": {},
+                                "contexts_results": [
+                                    {
+                                        "setup": {
+                                            "finished_at": 1584242238.110536,
+                                            "started_at": 1584242238.029151,
+                                            "atomic_actions": [],
+                                            "error": null
+                                        },
+                                        "cleanup": {
+                                            "finished_at": 1584242238.408476,
+                                            "started_at": 1584242238.40758,
+                                            "atomic_actions": [],
+                                            "error": null
+                                        },
+                                        "plugin_cfg": {
+                                            "user_choice_method": "random"
+                                        },
+                                        "plugin_name": "users@openstack"
+                                    }
+                                ],
+                                "position": 0,
+                                "pass_sla": true,
+                                "sla_results": {
+                                    "sla": [
+                                        {
+                                            "criterion":
+                                                "something_went_wrong",
+                                            "detail":
+                                                "Fake detail example",
+                                            "success": false
+                                        }
+                                    ]
+                                },
+                                "sla": {
+                                    "failure_rate": {
+                                        "max": 0
+                                    }
+                                }
+                            }
+                        ]
+                    }
+                ]
+            }
+        ]
+    }
+]
+"""
diff --git a/rallytester/tests/test_rallytester.py b/rallytester/tests/test_rallytester.py
index 9a753f3..14f319b 100644
--- a/rallytester/tests/test_rallytester.py
+++ b/rallytester/tests/test_rallytester.py
@@ -494,3 +494,48 @@ class TestRallytester(unittest.TestCase):
                 task='fake_task'
             )
         )
+
+    @mock.patch('rallytester.rallytester.subprocess.Popen')
+    @mock.patch(func, return_value=io.StringIO(fixtures.TASK))
+    @mock.patch('os.path.exists', return_value=True)
+    def test_execute_task_magnumfail(self,
+                                     mock_os,
+                                     mock_open,
+                                     mock_sub):
+        mock_pr = mock_sub.return_value
+        type(mock_pr).returncode = mock.PropertyMock(return_value=0)
+
+        mock_pr.communicate.side_effect = [
+            (
+                'Using task: 8d4132ab-3530-4148-9ee3-d5bf6621aa9d',
+                None
+            ),
+            (
+                fixtures.TASK_REPORT_MAGNUM,
+                None
+            )
+        ]
+
+        self.assertEqual(
+            (
+                {
+                    'scenarios': 'Novaname',
+                    'task': 'fake_task',
+                    'succeed': 0
+                },
+                '\nFake detail example\n'
+                '\nTask descriptor:\n\n'
+                '{\n    "NovaServers.list_servers": '
+                '[\n        {\n            "args": '
+                '{\n                "detailed": false'
+                '\n            },\n            "runner": '
+                '{\n                "type": "constant",'
+                '\n                "times": 1,\n       '
+                '         "concurrency": 1\n            }'
+                ',\n            "context": {}\n        }\n    ]\n}\n'
+            ),
+            rallytester.execute_task(
+                deployment='fake_deployment',
+                task='fake_task'
+            )
+        )
-- 
GitLab


From 5f1e0b58be36b9772e3fd02078a2e95d2a700cde Mon Sep 17 00:00:00 2001
From: ravillar <raul.villar.ramos@cern.ch>
Date: Mon, 30 Mar 2020 10:55:30 +0200
Subject: [PATCH 08/21] OS8753_manage_magnum_reports_fix

---
 rallytester/rallytester.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index d2ba632..ede072e 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -167,11 +167,12 @@ def execute_task(deployment, task, msgTrimmer=None):
 
         msg = ""
         tracebacks = ""
-        msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads']
-                [0]['sla_results']['sla'][0]['detail'])
-        log.error('Task failed: {0}'.format(
-                  _msg_enrich(msg.replace('\n', ' '),
-                              msgTrimmer)))
+        for job in json_result:
+            msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads']
+                    [0]['sla_results']['sla'][0]['detail'])
+            log.error('Task failed: {0}'.format(
+                      _msg_enrich(msg.replace('\n', ' '),
+                                  msgTrimmer)))
 
         return (
             {
-- 
GitLab


From 79fc71ad13cc1610d523bd2b486a1cdcf437c0bc Mon Sep 17 00:00:00 2001
From: ravillar <raul.villar.ramos@cern.ch>
Date: Mon, 30 Mar 2020 14:19:15 +0200
Subject: [PATCH 09/21] OS8753_manage_magnum_reports_fix

---
 rallytester/rallytester.py | 70 +++++++++++++++-----------------------
 1 file changed, 28 insertions(+), 42 deletions(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index ede072e..36b2f53 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -144,10 +144,9 @@ def execute_task(deployment, task, msgTrimmer=None):
         )
 
     # Check if the task is crashed
-    if any('tasks' in j for j in json_result):
-        if any(job["tasks"][0]["status"] == "crashed" for job in json_result):
-                        return json_result
-                        # return json here
+    if any(job["tasks"][0]["status"] == "crashed" for job in json_result):
+                    return json_result
+                    # return json here
 
     scenarios = ",".join(
         [job['tasks'][0]['subtasks'][0]['workloads'][0]
@@ -191,16 +190,14 @@ def execute_task(deployment, task, msgTrimmer=None):
         msg = ""
         tracebacks = ""
         for job in json_result:
-            if (job['tasks'] and job['tasks'][0]['subtasks'][0]['workloads'][0]
-                    ['data'][0]['error']):
-                # The real error is in the first result
-                # and the second element of the array
-                msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads']
-                        [0]['data'][0]['error'][1])
-                tracebacks += ('\n' + job['tasks'][0]['subtasks'][0]
-                               ['workloads'][0]['data'][0]['error'][2])
-                log.error('Task failed: {0}'.format(
-                    _msg_enrich(msg.replace('\n', ' '), msgTrimmer)))
+            # The real error is in the first result
+            # and the second element of the array
+            msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads']
+                    [0]['data'][0]['error'][1])
+            tracebacks += ('\n' + job['tasks'][0]['subtasks'][0]
+                           ['workloads'][0]['data'][0]['error'][2])
+            log.error('Task failed: {0}'.format(
+                _msg_enrich(msg.replace('\n', ' '), msgTrimmer)))
 
         return (
             {
@@ -212,34 +209,23 @@ def execute_task(deployment, task, msgTrimmer=None):
                                                      task_descriptor)
         )
 
-    if any('tasks' in j for j in json_result):
-        if (any(job['tasks'][0]['subtasks'][0]['workloads']
-                [0]['sla_results']['sla'][0]['success'] is False
-                for job in json_result)):
-
-            msg = ""
-            tracebacks = ""
-            for job in json_result:
-                if (job['tasks'] and job['tasks'][0]['subtasks']
-                        [0]['workloads'][0]['data'][0]['error']):
-                    # The real error is in the first result
-                    # and the second element of the array
-                    msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads']
-                            [0]['data'][0]['error'][1])
-                    tracebacks += ('\n' + job['tasks'][0]['subtasks']
-                                   [0]['workloads'][0]['data'][0]['error'][2])
-                    log.error('Task failed: {0}'.format(
-                        _msg_enrich(msg.replace('\n', ' '), msgTrimmer)))
-
-            return (
-                {
-                    'task': task,
-                    'scenarios': scenarios,
-                    'succeed': 0
-                },
-                "{0}\n{1}\nTask descriptor:\n{2}".format(msg, tracebacks,
-                                                         task_descriptor)
-            )
+    #Json error2
+    if (any(job['tasks'][0]['subtasks'][0]['workloads']
+            [0]['sla_results']['sla'][0]['success'] is False
+            for job in json_result)):
+
+        msg = ""
+        tracebacks = ""
+
+        return (
+            {
+                'task': task,
+                'scenarios': scenarios,
+                'succeed': 0
+            },
+            "{0}\n{1}\nTask descriptor:\n{2}".format(msg, tracebacks,
+                                                     task_descriptor)
+        )
 
     # In any other case
     log.info('So far, so good, the task has been sucessfully completed')
-- 
GitLab


From 29722cf220956ca43a74514860580c126edfd595 Mon Sep 17 00:00:00 2001
From: ravillar <raul.villar.ramos@cern.ch>
Date: Mon, 30 Mar 2020 14:21:39 +0200
Subject: [PATCH 10/21] OS8753_manage_magnum_reports_fix_p8

---
 rallytester/rallytester.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index 36b2f53..cee746d 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -209,7 +209,7 @@ def execute_task(deployment, task, msgTrimmer=None):
                                                      task_descriptor)
         )
 
-    #Json error2
+    # Json error2
     if (any(job['tasks'][0]['subtasks'][0]['workloads']
             [0]['sla_results']['sla'][0]['success'] is False
             for job in json_result)):
-- 
GitLab


From 89c683ab92564aa5cfcf8acc013ef1c49f7265f2 Mon Sep 17 00:00:00 2001
From: Raul Villar Ramos <raul.villar.ramos@cern.ch>
Date: Wed, 11 Mar 2020 16:47:03 +0100
Subject: [PATCH 11/21] OS8753_Handle_crash_reports

---
 rallytester/rallytester.py            |  9 ++++++++
 rallytester/tests/fixtures.py         | 25 +++++++++++++++++++++
 rallytester/tests/test_rallytester.py | 32 +++++++++++++++++++++++++++
 3 files changed, 66 insertions(+)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index 9bcbf2d..baf97fa 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -143,6 +143,15 @@ def execute_task(deployment, task, msgTrimmer=None):
             .format(msg, output, stderr, task_id, task_descriptor)
         )
 
+    # Check if the task is crashed
+    try:
+        if any(job["tasks"][0]["status"] == "crashed" for job in json_result):
+            return ("Task status is crashed."
+                    "Use: task report --uuid --json "
+                    )
+    except:
+        pass
+
     scenarios = ",".join([job['key']['name'] for job in json_result])
 
     # If there is any error in the results
diff --git a/rallytester/tests/fixtures.py b/rallytester/tests/fixtures.py
index 7601c7b..4de0a8d 100644
--- a/rallytester/tests/fixtures.py
+++ b/rallytester/tests/fixtures.py
@@ -372,3 +372,28 @@ TASK_RESULTS_FAIL2 = u"""
     }
 ]
 """
+
+TASK_RESULTS_CRASH = u"""
+[
+    {
+        "info": {
+            "generated_at": "2019-03-05T16:50:31",
+            "rally_version": "0.0.0",
+            "format_version": "1.1"
+        },
+        "tasks": [
+            {
+                "uuid": "5312f2e6-f00d-4a63-9162-40c895cdaea1",
+                "title": "",
+                "description": "",
+                "status": "crashed",
+                "tags": [],
+                "created_at": "2019-03-05T15:14:54",
+                "updated_at": "2019-03-05T15:14:56",
+                "pass_sla": true,
+                "subtasks": []
+            }
+        ]
+    }
+]
+"""
diff --git a/rallytester/tests/test_rallytester.py b/rallytester/tests/test_rallytester.py
index 89a6b8f..4759d5a 100644
--- a/rallytester/tests/test_rallytester.py
+++ b/rallytester/tests/test_rallytester.py
@@ -440,3 +440,35 @@ class TestRallytester(unittest.TestCase):
                 " Image '{\"name\": \"my_image\"}' not found",
                 rallytester.REASONS_PATTERN)
         )
+
+    @mock.patch('rallytester.rallytester.subprocess.Popen')
+    @mock.patch(func, return_value=io.StringIO(fixtures.TASK_RESULTS_CRASH))
+    @mock.patch('os.path.exists', return_value=True)
+    def test_execute_task_crash(self,
+                                            mock_os,
+                                            mock_open,
+                                            mock_sub):
+        mock_pr = mock_sub.return_value
+        type(mock_pr).returncode = mock.PropertyMock(return_value=0)
+
+        mock_pr.communicate.side_effect = [
+            (
+                'Using task: 5312f2e6-f00d-4a63-9162-40c895cdaea1',
+                None
+            ),
+            (
+                fixtures.TASK_RESULTS_CRASH,
+                None
+            )
+        ]
+
+        self.assertEqual(
+            (
+                    "Task status is crashed."
+                    "Use: task report --uuid --json "
+            ),
+            rallytester.execute_task(
+                deployment='fake_deployment',
+                task='fake_task'
+            )
+        )
-- 
GitLab


From 199d82165601345892bdf70a4546c2cab951ddd5 Mon Sep 17 00:00:00 2001
From: Raul Villar Ramos <raul.villar.ramos@cern.ch>
Date: Wed, 11 Mar 2020 16:57:14 +0100
Subject: [PATCH 12/21] OS8753_Handle_crash_reports_pep8

---
 rallytester/rallytester.py            |  2 +-
 rallytester/tests/test_rallytester.py | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index baf97fa..03ee58b 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -149,7 +149,7 @@ def execute_task(deployment, task, msgTrimmer=None):
             return ("Task status is crashed."
                     "Use: task report --uuid --json "
                     )
-    except:
+    except BaseException:
         pass
 
     scenarios = ",".join([job['key']['name'] for job in json_result])
diff --git a/rallytester/tests/test_rallytester.py b/rallytester/tests/test_rallytester.py
index 4759d5a..f128039 100644
--- a/rallytester/tests/test_rallytester.py
+++ b/rallytester/tests/test_rallytester.py
@@ -445,9 +445,9 @@ class TestRallytester(unittest.TestCase):
     @mock.patch(func, return_value=io.StringIO(fixtures.TASK_RESULTS_CRASH))
     @mock.patch('os.path.exists', return_value=True)
     def test_execute_task_crash(self,
-                                            mock_os,
-                                            mock_open,
-                                            mock_sub):
+                                mock_os,
+                                mock_open,
+                                mock_sub):
         mock_pr = mock_sub.return_value
         type(mock_pr).returncode = mock.PropertyMock(return_value=0)
 
@@ -464,8 +464,8 @@ class TestRallytester(unittest.TestCase):
 
         self.assertEqual(
             (
-                    "Task status is crashed."
-                    "Use: task report --uuid --json "
+                "Task status is crashed."
+                "Use: task report --uuid --json "
             ),
             rallytester.execute_task(
                 deployment='fake_deployment',
-- 
GitLab


From 7ae360fc9a87535aa2ece19d5b8499e536903b97 Mon Sep 17 00:00:00 2001
From: ravillar <raul.villar.ramos@cern.ch>
Date: Mon, 16 Mar 2020 12:44:34 +0100
Subject: [PATCH 13/21] OS8753_handle_crash_report

---
 rallytester/rallytester.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index 03ee58b..db5e315 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -143,14 +143,12 @@ def execute_task(deployment, task, msgTrimmer=None):
             .format(msg, output, stderr, task_id, task_descriptor)
         )
 
-    # Check if the task is crashed
-    try:
+    # Capture crash. Only crash contains 'tasks'
+    if any('tasks' in j for j in json_result):
         if any(job["tasks"][0]["status"] == "crashed" for job in json_result):
-            return ("Task status is crashed."
-                    "Use: task report --uuid --json "
-                    )
-    except BaseException:
-        pass
+                        return ("Task status is crashed."
+                                "Use: task report --uuid --json "
+                                )
 
     scenarios = ",".join([job['key']['name'] for job in json_result])
 
-- 
GitLab


From 6bd19238b8e23626689c718f473c9b114669803b Mon Sep 17 00:00:00 2001
From: ravillar <raul.villar.ramos@cern.ch>
Date: Mon, 23 Mar 2020 16:45:25 +0100
Subject: [PATCH 14/21] OS8753_replace_results_by_reports

---
 rallytester/rallytester.py            |  51 +-
 rallytester/tests/fixtures.py         | 845 +++++++++++++++++---------
 rallytester/tests/test_rallytester.py |  19 +-
 3 files changed, 606 insertions(+), 309 deletions(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index db5e315..c553173 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -110,7 +110,7 @@ def execute_task(deployment, task, msgTrimmer=None):
 
     # Get results executing rally command
     process = subprocess.Popen([  # nosec
-        '/usr/bin/rally', 'task', 'results', task_id],
+        '/usr/bin/rally', 'task', 'report', task_id, '--json'],
         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     output, stderr = process.communicate()
 
@@ -143,26 +143,25 @@ def execute_task(deployment, task, msgTrimmer=None):
             .format(msg, output, stderr, task_id, task_descriptor)
         )
 
-    # Capture crash. Only crash contains 'tasks'
+    # Check if the task is crashed
     if any('tasks' in j for j in json_result):
         if any(job["tasks"][0]["status"] == "crashed" for job in json_result):
-                        return ("Task status is crashed."
-                                "Use: task report --uuid --json "
-                                )
+                        return ("Task status is crashed.")
+                        # return json here
 
-    scenarios = ",".join([job['key']['name'] for job in json_result])
+    scenarios = ",".join([job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['atomic_actions'][0]['name'] for job in json_result])
 
-    # If there is any error in the results
-    if any(job['result'] and
-            job['result'][0]['error'] for job in json_result):
+    # If there is any error in the reports
+    if any(job['tasks'] and 
+            job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'] for job in json_result):
         msg = ""
         tracebacks = ""
         for job in json_result:
-            if job['result'] and job['result'][0]['error']:
+            if job['tasks'] and job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error']:
                 # The real error is in the first result
                 # and the second element of the array
-                msg += '\n' + job['result'][0]['error'][1]
-                tracebacks += '\n' + job['result'][0]['error'][2]
+                msg += '\n' + job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'][1]
+                tracebacks += '\n' + job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'][2]
                 log.error('Task failed: {0}'.format(
                     _msg_enrich(msg.replace('\n', ' '), msgTrimmer)))
 
@@ -183,7 +182,7 @@ def execute_task(deployment, task, msgTrimmer=None):
     # to reimplement this part anyway to use `rally task report`
     # with a different format
     elif any(
-        not job['result'] and
+        not job['tasks'] and
         'sla' in job and
         any(sla['criterion'] == 'something_went_wrong' for sla in job['sla'])
         for job in json_result
@@ -210,6 +209,30 @@ def execute_task(deployment, task, msgTrimmer=None):
                                                      task_descriptor)
         )
 
+    if any('tasks' in j for j in json_result):
+        if any(job['tasks'][0]['subtasks'][0]['workloads'][0]['sla_results']['sla'][0]['success'] == False for job in json_result):
+        
+            msg = ""
+            tracebacks = ""
+            for job in json_result:
+                if job['tasks'] and job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error']:
+                    # The real error is in the first result
+                    # and the second element of the array
+                    msg += '\n' + job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'][1]
+                    tracebacks += '\n' + job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'][2]
+                    log.error('Task failed: {0}'.format(
+                        _msg_enrich(msg.replace('\n', ' '), msgTrimmer)))
+
+            return (
+                {
+                    'task': task,
+                    'scenarios': scenarios,
+                    'succeed': 0
+                },
+                "{0}\n{1}\nTask descriptor:\n{2}".format(msg, tracebacks,
+                                                        task_descriptor)
+            )
+
     # In any other case
     log.info('So far, so good, the task has been sucessfully completed')
     return (
@@ -217,7 +240,7 @@ def execute_task(deployment, task, msgTrimmer=None):
             'task': task,
             'scenarios': scenarios,
             'succeed': 1,
-            'duration': sum([job['result'][0]['duration']
+            'duration': sum([job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['duration']
                              for job in json_result])
         },
         None  # No errors
diff --git a/rallytester/tests/fixtures.py b/rallytester/tests/fixtures.py
index 4de0a8d..95960ae 100644
--- a/rallytester/tests/fixtures.py
+++ b/rallytester/tests/fixtures.py
@@ -74,324 +74,599 @@ TASK = u"""
 }
 """
 
-TASK_RESULTS = u"""
+TASK_REPORT_CRASH = u"""
 [
     {
-        "hooks": [],
-        "created_at": "2019-11-04T13:03:36",
-        "load_duration": 617.198,
-        "result": [
-            {
-                "timestamp": 1554987817.501209,
-                "error": [],
-                "duration": 617.1978030204773,
-                "output": {
-                    "additive": [],
-                    "complete": []
-                },
-                "idle_duration": 10.0,
-                "atomic_actions": {
-                    "cinder_v3.create_volume": 4.72377610206604,
-                    "nova.boot_server": 614.4812738895416,
-                    "nova.delete_server": 7.99204421043396
-                }
-            }
-        ],
-        "key": {
-            "kw": {
-                "runner": {
-                    "type": "constant",
-                    "concurrency": 1,
-                    "times": 1
-                },
-                "hooks": [],
-                "args": {
-                    "volume_size": 1,
-                    "flavor": {
-                        "name": "rally.nano"
-                    },
-                    "image": {
-                        "name": "Rally CirrOS IPv6"
-                    },
-                    "force_delete": false
-                },
-                "sla": {
-                    "failure_rate": {
-                        "max": 0
-                    }
-                },
-                "context": {}
-            },
-            "pos": 0,
-            "name": "Novaname",
-            "description": "Bootdesc"
+        "info": {
+            "generated_at": "2019-03-05T16:50:31",
+            "rally_version": "0.0.0",
+            "format_version": "1.1"
         },
-        "full_duration": 632.436,
-        "sla": [
+        "tasks": [
             {
-                "criterion": "failure_rate",
-                "detail": "Failure rate criteria'\
-                ' 0.00% <= 0.00% <= 0.00% - Passed",
-                "success": true
+                "uuid": "5312f2e6-f00d-4a63-9162-40c895cdaea1",
+                "title": "",
+                "description": "",
+                "status": "crashed",
+                "tags": [],
+                "created_at": "2019-03-05T15:14:54",
+                "updated_at": "2019-03-05T15:14:56",
+                "pass_sla": true,
+                "subtasks": []
             }
         ]
     }
 ]
 """
 
-TASK_RESULTS_FAIL = u"""
-[    {
-        "hooks": [],
-        "created_at": "2019-11-04T13:03:36",
-        "load_duration": 617.198,
-        "result": [
-            {
-                "timestamp": 1554987817.501209,
-                "error": [],
-                "duration": 617.1978030204773,
-                "output": {
-                    "additive": [],
-                    "complete": []
-                },
-                "idle_duration": 10.0,
-                "atomic_actions": {
-                    "cinder_v3.create_volume": 4.72377610206604,
-                    "nova.boot_server": 614.4812738895416,
-                    "nova.delete_server": 7.99204421043396
-                }
-            }
-        ],
-        "key": {
-            "kw": {
-                "runner": {
-                    "type": "constant",
-                    "concurrency": 1,
-                    "times": 1
-                },
-                "hooks": [],
-                "args": {
-                    "volume_size": 1,
-                    "flavor": {
-                        "name": "rally.nano"
-                    },
-                    "image": {
-                        "name": "Rally CirrOS IPv6"
-                    },
-                    "force_delete": false
-                },
-                "sla": {
-                    "failure_rate": {
-                        "max": 0
-                    }
-                },
-                "context": {}
-            },
-            "pos": 0,
-            "name": "Name",
-            "description": "desc"
-        },
-        "full_duration": 632.436,
-        "sla": [
-            {
-                "criterion": "failure_rate",
-                "detail": "Failure rate criteria'\
-                ' 0.00% <= 0.00% <= 0.00% - Passed",
-                "success": true
-            }
-        ]
-    },
+TASK_REPORT = u"""
+[
     {
-        "hooks": [],
-        "created_at": "2019-11-04T13:03:36",
-        "load_duration": 617.198,
-        "result": [
+        "info": {
+            "generated_at": "2020-03-16T16:07:30", 
+            "rally_version": "2.0.0", 
+            "format_version": "1.2"
+        }, 
+        "tasks": [
             {
-                "timestamp": 1554987817.501209,
-                "error": [
-                    "-----------",
-                    "Fake error1",
-                    "Fake error2"
-                ],
-                "duration": 617.1978030204773,
-                "output": {
-                    "additive": [],
-                    "complete": []
-                },
-                "idle_duration": 10.0,
-                "atomic_actions": {
-                    "cinder_v3.create_volume": 4.72377610206604,
-                    "nova.boot_server": 614.4812738895416,
-                    "nova.delete_server": 7.99204421043396
-                }
-            }
-        ],
-        "key": {
-            "kw": {
-                "runner": {
-                    "type": "constant",
-                    "concurrency": 1,
-                    "times": 1
-                },
-                "hooks": [],
-                "args": {
-                    "volume_size": 1,
-                    "flavor": {
-                        "name": "rally.nano"
-                    },
-                    "image": {
-                        "name": "Rally CirrOS IPv6"
-                    },
-                    "force_delete": false
-                },
-                "sla": {
-                    "failure_rate": {
-                        "max": 0
+                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d", 
+                "title": "", 
+                "description": "", 
+                "status": "finished", 
+                "tags": [], 
+                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708", 
+                "env_name": "teststack_cell02", 
+                "created_at": "2020-03-15T03:17:17", 
+                "updated_at": "2020-03-15T03:17:20", 
+                "pass_sla": true, 
+                "subtasks": [
+                    {
+                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12", 
+                        "title": "GlanceImages.list_images", 
+                        "description": "", 
+                        "status": "finished", 
+                        "created_at": "2020-03-15T03:17:18", 
+                        "updated_at": "2020-03-15T03:17:20", 
+                        "sla": {}, 
+                        "workloads": [
+                            {
+                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48", 
+                                "description": "List all images.", 
+                                "runner": {
+                                    "constant": {
+                                        "concurrency": 1, 
+                                        "times": 1
+                                    }
+                                }, 
+                                "hooks": [], 
+                                "scenario": {
+                                    "GlanceImages.list_images": {}
+                                }, 
+                                "min_duration": 0.1442, 
+                                "max_duration": 0.1442, 
+                                "start_time": 1584242238.24373, 
+                                "load_duration": 0.1442, 
+                                "full_duration": 0.380141, 
+                                "statistics": {
+                                    "durations": {
+                                        "total": {
+                                            "data": {
+                                                "success": "100.0%", 
+                                                "min": 0.144, 
+                                                "max": 0.144, 
+                                                "median": 0.144, 
+                                                "95%ile": 0.144, 
+                                                "iteration_count": 1, 
+                                                "avg": 0.144, 
+                                                "90%ile": 0.144
+                                            }, 
+                                            "display_name": "total", 
+                                            "name": "total", 
+                                            "count_per_iteration": 1, 
+                                            "children": [
+                                                {
+                                                    "data": {
+                                                        "success": "100.0%", 
+                                                        "min": 0.144, 
+                                                        "max": 0.144, 
+                                                        "median": 0.144, 
+                                                        "95%ile": 0.144, 
+                                                        "iteration_count": 1, 
+                                                        "avg": 0.144, 
+                                                        "90%ile": 0.144
+                                                    }, 
+                                                    "display_name": "duration", 
+                                                    "name": "duration", 
+                                                    "count_per_iteration": 1, 
+                                                    "children": []
+                                                }, 
+                                                {
+                                                    "data": {
+                                                        "success": "100.0%", 
+                                                        "min": 0.0, 
+                                                        "max": 0.0, 
+                                                        "median": 0.0, 
+                                                        "95%ile": 0.0, 
+                                                        "iteration_count": 1, 
+                                                        "avg": 0.0, 
+                                                        "90%ile": 0.0
+                                                    }, 
+                                                    "display_name": "idle_duration", 
+                                                    "name": "idle_duration", 
+                                                    "count_per_iteration": 1, 
+                                                    "children": []
+                                                }
+                                            ]
+                                        }, 
+                                        "atomics": [
+                                            {
+                                                "data": {
+                                                    "success": "100.0%", 
+                                                    "min": 0.144, 
+                                                    "max": 0.144, 
+                                                    "median": 0.144, 
+                                                    "95%ile": 0.144, 
+                                                    "iteration_count": 1, 
+                                                    "avg": 0.144, 
+                                                    "90%ile": 0.144
+                                                }, 
+                                                "display_name": "Novaname", 
+                                                "name": "Novaname", 
+                                                "count_per_iteration": 1, 
+                                                "children": []
+                                            }
+                                        ]
+                                    }
+                                }, 
+                                "data": [
+                                    {
+                                        "timestamp": 1584242238.243732, 
+                                        "error": [], 
+                                        "duration": 0.14420008659362793, 
+                                        "output": {
+                                            "additive": [], 
+                                            "complete": []
+                                        }, 
+                                        "idle_duration": 0.0, 
+                                        "atomic_actions": [
+                                            {
+                                                "finished_at": 1584242238.38778, 
+                                                "started_at": 1584242238.243803, 
+                                                "name": "Novaname", 
+                                                "children": []
+                                            }
+                                        ]
+                                    }
+                                ], 
+                                "failed_iteration_count": 0, 
+                                "total_iteration_count": 1, 
+                                "created_at": "2020-03-15T03:17:18", 
+                                "updated_at": "2020-03-15T03:17:20", 
+                                "contexts": {}, 
+                                "contexts_results": [
+                                    {
+                                        "setup": {
+                                            "finished_at": 1584242238.110536, 
+                                            "started_at": 1584242238.029151, 
+                                            "atomic_actions": [], 
+                                            "error": null
+                                        }, 
+                                        "cleanup": {
+                                            "finished_at": 1584242238.408476, 
+                                            "started_at": 1584242238.40758, 
+                                            "atomic_actions": [], 
+                                            "error": null
+                                        }, 
+                                        "plugin_cfg": {
+                                            "user_choice_method": "random"
+                                        }, 
+                                        "plugin_name": "users@openstack"
+                                    }
+                                ], 
+                                "position": 0, 
+                                "pass_sla": true, 
+                                "sla_results": {
+                                    "sla": [
+                                        {
+                                            "criterion": "failure_rate", 
+                                            "detail": "Failure rate criteria 0.00% <= 0.00% <= 0.00% - Passed", 
+                                            "success": true
+                                        }
+                                    ]
+                                }, 
+                                "sla": {
+                                    "failure_rate": {
+                                        "max": 0
+                                    }
+                                }
+                            }
+                        ]
                     }
-                },
-                "context": {}
-            },
-            "pos": 0,
-            "name": "name",
-            "description": "des"
-        },
-        "full_duration": 632.436,
-        "sla": [
-            {
-                "criterion": "failure_rate",
-                "detail": "Failure rate criteria'\
-                ' 0.00% <= 0.00% <= 0.00% - Passed",
-                "success": true
+                ]
             }
         ]
     }
 ]
 """
 
-TASK_RESULTS_FAIL2 = u"""
+TASK_REPORT_ERROR1 = u"""
 [
-        {
-        "hooks": [],
-        "created_at": "2019-11-04T13:03:36",
-        "load_duration": 617.198,
-        "result": [
-            {
-                "timestamp": 1554987817.501209,
-                "error": [],
-                "duration": 617.1978030204773,
-                "output": {
-                    "additive": [],
-                    "complete": []
-                },
-                "idle_duration": 10.0,
-                "atomic_actions": {
-                    "cinder_v3.create_volume": 4.72377610206604,
-                    "nova.boot_server": 614.4812738895416,
-                    "nova.delete_server": 7.99204421043396
-                }
-            }
-        ],
-        "key": {
-            "kw": {
-                "runner": {
-                    "type": "constant",
-                    "concurrency": 1,
-                    "times": 1
-                },
-                "hooks": [],
-                "args": {
-                    "volume_size": 1,
-                    "flavor": {
-                        "name": "rally.nano"
-                    },
-                    "image": {
-                        "name": "Rally CirrOS IPv6"
-                    },
-                    "force_delete": false
-                },
-                "sla": {
-                    "failure_rate": {
-                        "max": 0
-                    }
-                },
-                "context": {}
-            },
-            "pos": 0,
-            "name": "NovaServers",
-            "description": "Boot a server"
-        },
-        "full_duration": 632.436
-    },
     {
-        "hooks": [],
-        "created_at": "2019-11-04T13:03:36",
-        "load_duration": 617.198,
-        "result": [],
-        "key": {
-            "kw": {
-                "runner": {
-                    "type": "constant",
-                    "concurrency": 1,
-                    "times": 1
-                },
-                "hooks": [],
-                "args": {
-                    "volume_size": 1,
-                    "flavor": {
-                        "name": "rally.nano"
-                    },
-                    "image": {
-                        "name": "Rally CirrOS IPv6"
-                    },
-                    "force_delete": false
-                },
-                "sla": {
-                    "failure_rate": {
-                        "max": 0
-                    }
-                },
-                "context": {}
-            },
-            "pos": 0,
-            "name": "Novaname",
-            "description": "Bootdesc"
-        },
-        "full_duration": 632.436,
-        "sla": [
-            {
-                "criterion": "failure_rate",
-                "detail": "Failure rate criteria'\
-                ' 0.00% <= 0.00% <= 0.00% - Passed",
-                "success": true
-            },
+        "info": {
+            "generated_at": "2020-03-16T16:07:30", 
+            "rally_version": "2.0.0", 
+            "format_version": "1.2"
+        }, 
+        "tasks": [
             {
-                "criterion": "something_went_wrong",
-                "detail": "Frc",
-                "success": false
+                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d", 
+                "title": "", 
+                "description": "", 
+                "status": "finished", 
+                "tags": [], 
+                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708", 
+                "env_name": "teststack_cell02", 
+                "created_at": "2020-03-15T03:17:17", 
+                "updated_at": "2020-03-15T03:17:20", 
+                "pass_sla": true, 
+                "subtasks": [
+                    {
+                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12", 
+                        "title": "GlanceImages.list_images", 
+                        "description": "", 
+                        "status": "finished", 
+                        "created_at": "2020-03-15T03:17:18", 
+                        "updated_at": "2020-03-15T03:17:20", 
+                        "sla": {}, 
+                        "workloads": [
+                            {
+                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48", 
+                                "description": "List all images.", 
+                                "runner": {
+                                    "constant": {
+                                        "concurrency": 1, 
+                                        "times": 1
+                                    }
+                                }, 
+                                "hooks": [], 
+                                "scenario": {
+                                    "GlanceImages.list_images": {}
+                                }, 
+                                "min_duration": 0.1442, 
+                                "max_duration": 0.1442, 
+                                "start_time": 1584242238.24373, 
+                                "load_duration": 0.1442, 
+                                "full_duration": 0.380141, 
+                                "statistics": {
+                                    "durations": {
+                                        "total": {
+                                            "data": {
+                                                "success": "0.0%", 
+                                                "min": 0.144, 
+                                                "max": 0.144, 
+                                                "median": 0.144, 
+                                                "95%ile": 0.144, 
+                                                "iteration_count": 1, 
+                                                "avg": 0.144, 
+                                                "90%ile": 0.144
+                                            }, 
+                                            "display_name": "total", 
+                                            "name": "total", 
+                                            "count_per_iteration": 1, 
+                                            "children": [
+                                                {
+                                                    "data": {
+                                                        "success": "0.0%", 
+                                                        "min": 0.144, 
+                                                        "max": 0.144, 
+                                                        "median": 0.144, 
+                                                        "95%ile": 0.144, 
+                                                        "iteration_count": 1, 
+                                                        "avg": 0.144, 
+                                                        "90%ile": 0.144
+                                                    }, 
+                                                    "display_name": "duration", 
+                                                    "name": "duration", 
+                                                    "count_per_iteration": 1, 
+                                                    "children": []
+                                                }, 
+                                                {
+                                                    "data": {
+                                                        "success": "0.0%", 
+                                                        "min": 0.0, 
+                                                        "max": 0.0, 
+                                                        "median": 0.0, 
+                                                        "95%ile": 0.0, 
+                                                        "iteration_count": 1, 
+                                                        "avg": 0.0, 
+                                                        "90%ile": 0.0
+                                                    }, 
+                                                    "display_name": "idle_duration", 
+                                                    "name": "idle_duration", 
+                                                    "count_per_iteration": 1, 
+                                                    "children": []
+                                                }
+                                            ]
+                                        }, 
+                                        "atomics": [
+                                            {
+                                                "data": {
+                                                    "success": "0.0%", 
+                                                    "min": 0.144, 
+                                                    "max": 0.144, 
+                                                    "median": 0.144, 
+                                                    "95%ile": 0.144, 
+                                                    "iteration_count": 1, 
+                                                    "avg": 0.144, 
+                                                    "90%ile": 0.144
+                                                }, 
+                                                "display_name": "Novaname", 
+                                                "name": "Novaname", 
+                                                "count_per_iteration": 1, 
+                                                "children": []
+                                            }
+                                        ]
+                                    }
+                                }, 
+                                "data": [
+                                    {
+                                        "timestamp": 1584242238.243732, 
+                                        "error": [], 
+                                        "duration": 0.14420008659362793, 
+                                        "output": {
+                                            "additive": [], 
+                                            "complete": []
+                                        }, 
+                                        "idle_duration": 0.0, 
+                                        "atomic_actions": [
+                                            {
+                                                "finished_at": 1584242238.38778, 
+                                                "started_at": 1584242238.243803, 
+                                                "name": "Novaname", 
+                                                "children": []
+                                            }
+                                        ]
+                                    }
+                                ], 
+                                "failed_iteration_count": 0, 
+                                "total_iteration_count": 1, 
+                                "created_at": "2020-03-15T03:17:18", 
+                                "updated_at": "2020-03-15T03:17:20", 
+                                "contexts": {}, 
+                                "contexts_results": [
+                                    {
+                                        "setup": {
+                                            "finished_at": 1584242238.110536, 
+                                            "started_at": 1584242238.029151, 
+                                            "atomic_actions": [], 
+                                            "error": null
+                                        }, 
+                                        "cleanup": {
+                                            "finished_at": 1584242238.408476, 
+                                            "started_at": 1584242238.40758, 
+                                            "atomic_actions": [], 
+                                            "error": null
+                                        }, 
+                                        "plugin_cfg": {
+                                            "user_choice_method": "random"
+                                        }, 
+                                        "plugin_name": "users@openstack"
+                                    }
+                                ], 
+                                "position": 0, 
+                                "pass_sla": true, 
+                                "sla_results": {
+                                    "sla": [
+                                        {
+                                            "criterion": "failure_rate", 
+                                            "detail": "Fake error - FAIL", 
+                                            "success": false
+                                        }
+                                    ]
+                                }, 
+                                "sla": {
+                                    "failure_rate": {
+                                        "max": 0
+                                    }
+                                }
+                            }
+                        ]
+                    }
+                ]
             }
         ]
     }
 ]
 """
 
-TASK_RESULTS_CRASH = u"""
+TASK_REPORT_ERROR2 = u"""
 [
     {
         "info": {
-            "generated_at": "2019-03-05T16:50:31",
-            "rally_version": "0.0.0",
-            "format_version": "1.1"
-        },
+            "generated_at": "2020-03-16T16:07:30", 
+            "rally_version": "2.0.0", 
+            "format_version": "1.2"
+        }, 
         "tasks": [
             {
-                "uuid": "5312f2e6-f00d-4a63-9162-40c895cdaea1",
-                "title": "",
-                "description": "",
-                "status": "crashed",
-                "tags": [],
-                "created_at": "2019-03-05T15:14:54",
-                "updated_at": "2019-03-05T15:14:56",
-                "pass_sla": true,
-                "subtasks": []
+                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d", 
+                "title": "", 
+                "description": "", 
+                "status": "finished", 
+                "tags": [], 
+                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708", 
+                "env_name": "teststack_cell02", 
+                "created_at": "2020-03-15T03:17:17", 
+                "updated_at": "2020-03-15T03:17:20", 
+                "pass_sla": true, 
+                "subtasks": [
+                    {
+                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12", 
+                        "title": "GlanceImages.list_images", 
+                        "description": "", 
+                        "status": "finished", 
+                        "created_at": "2020-03-15T03:17:18", 
+                        "updated_at": "2020-03-15T03:17:20", 
+                        "sla": {}, 
+                        "workloads": [
+                            {
+                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48", 
+                                "description": "List all images.", 
+                                "runner": {
+                                    "constant": {
+                                        "concurrency": 1, 
+                                        "times": 1
+                                    }
+                                }, 
+                                "hooks": [], 
+                                "scenario": {
+                                    "GlanceImages.list_images": {}
+                                }, 
+                                "min_duration": 0.1442, 
+                                "max_duration": 0.1442, 
+                                "start_time": 1584242238.24373, 
+                                "load_duration": 0.1442, 
+                                "full_duration": 0.380141, 
+                                "statistics": {
+                                    "durations": {
+                                        "total": {
+                                            "data": {
+                                                "success": "100.0%", 
+                                                "min": 0.144, 
+                                                "max": 0.144, 
+                                                "median": 0.144, 
+                                                "95%ile": 0.144, 
+                                                "iteration_count": 1, 
+                                                "avg": 0.144, 
+                                                "90%ile": 0.144
+                                            }, 
+                                            "display_name": "total", 
+                                            "name": "total", 
+                                            "count_per_iteration": 1, 
+                                            "children": [
+                                                {
+                                                    "data": {
+                                                        "success": "100.0%", 
+                                                        "min": 0.144, 
+                                                        "max": 0.144, 
+                                                        "median": 0.144, 
+                                                        "95%ile": 0.144, 
+                                                        "iteration_count": 1, 
+                                                        "avg": 0.144, 
+                                                        "90%ile": 0.144
+                                                    }, 
+                                                    "display_name": "duration", 
+                                                    "name": "duration", 
+                                                    "count_per_iteration": 1, 
+                                                    "children": []
+                                                }, 
+                                                {
+                                                    "data": {
+                                                        "success": "100.0%", 
+                                                        "min": 0.0, 
+                                                        "max": 0.0, 
+                                                        "median": 0.0, 
+                                                        "95%ile": 0.0, 
+                                                        "iteration_count": 1, 
+                                                        "avg": 0.0, 
+                                                        "90%ile": 0.0
+                                                    }, 
+                                                    "display_name": "idle_duration", 
+                                                    "name": "idle_duration", 
+                                                    "count_per_iteration": 1, 
+                                                    "children": []
+                                                }
+                                            ]
+                                        }, 
+                                        "atomics": [
+                                            {
+                                                "data": {
+                                                    "success": "100.0%", 
+                                                    "min": 0.144, 
+                                                    "max": 0.144, 
+                                                    "median": 0.144, 
+                                                    "95%ile": 0.144, 
+                                                    "iteration_count": 1, 
+                                                    "avg": 0.144, 
+                                                    "90%ile": 0.144
+                                                }, 
+                                                "display_name": "Novaname", 
+                                                "name": "Novaname", 
+                                                "count_per_iteration": 1, 
+                                                "children": []
+                                            }
+                                        ]
+                                    }
+                                }, 
+                                "data": [
+                                    {
+                                        "timestamp": 1584242238.243732, 
+                                        "error": [
+                                            "-----------",
+                                            "Fake error1",
+                                            "Fake error2"
+                                        ], 
+                                        "duration": 0.14420008659362793, 
+                                        "output": {
+                                            "additive": [], 
+                                            "complete": []
+                                        }, 
+                                        "idle_duration": 0.0, 
+                                        "atomic_actions": [
+                                            {
+                                                "finished_at": 1584242238.38778, 
+                                                "started_at": 1584242238.243803, 
+                                                "name": "Novaname", 
+                                                "children": []
+                                            }
+                                        ]
+                                    }
+                                ], 
+                                "failed_iteration_count": 0, 
+                                "total_iteration_count": 1, 
+                                "created_at": "2020-03-15T03:17:18", 
+                                "updated_at": "2020-03-15T03:17:20", 
+                                "contexts": {}, 
+                                "contexts_results": [
+                                    {
+                                        "setup": {
+                                            "finished_at": 1584242238.110536, 
+                                            "started_at": 1584242238.029151, 
+                                            "atomic_actions": [], 
+                                            "error": null
+                                        }, 
+                                        "cleanup": {
+                                            "finished_at": 1584242238.408476, 
+                                            "started_at": 1584242238.40758, 
+                                            "atomic_actions": [], 
+                                            "error": null
+                                        }, 
+                                        "plugin_cfg": {
+                                            "user_choice_method": "random"
+                                        }, 
+                                        "plugin_name": "users@openstack"
+                                    }
+                                ], 
+                                "position": 0, 
+                                "pass_sla": true, 
+                                "sla_results": {
+                                    "sla": [
+                                        {
+                                            "criterion": "failure_rate", 
+                                            "detail": "Failure rate criteria 0.00% <= 0.00% <= 0.00% - Passed", 
+                                            "success": true
+                                        }
+                                    ]
+                                }, 
+                                "sla": {
+                                    "failure_rate": {
+                                        "max": 0
+                                    }
+                                }
+                            }
+                        ]
+                    }
+                ]
             }
         ]
     }
diff --git a/rallytester/tests/test_rallytester.py b/rallytester/tests/test_rallytester.py
index f128039..ff645be 100644
--- a/rallytester/tests/test_rallytester.py
+++ b/rallytester/tests/test_rallytester.py
@@ -282,7 +282,7 @@ class TestRallytester(unittest.TestCase):
                 None
             ),
             (
-                fixtures.TASK_RESULTS,
+                fixtures.TASK_REPORT,
                 None
             )
         ]
@@ -293,7 +293,7 @@ class TestRallytester(unittest.TestCase):
                     'task': 'fake_task',
                     'scenarios': 'Novaname',
                     'succeed': 1,
-                    'duration': 617.1978030204773,
+                    'duration': 0.14420008659362793,
                 },
                 None  # No errors
             ),
@@ -319,14 +319,14 @@ class TestRallytester(unittest.TestCase):
                 None
             ),
             (
-                fixtures.TASK_RESULTS_FAIL,
+                fixtures.TASK_REPORT_ERROR2,
                 None
             )
         ]
 
         self.assertEqual(
             (
-                {'task': 'fake_task', 'scenarios': 'Name,name', 'succeed': 0},
+                {'task': 'fake_task', 'scenarios': 'Novaname', 'succeed': 0},
                 '\nFake error1\n\nFake error2\n'
                 'Task descriptor:\n'
                 '\n'
@@ -365,7 +365,7 @@ class TestRallytester(unittest.TestCase):
                 None
             ),
             (
-                fixtures.TASK_RESULTS_FAIL2,
+                fixtures.TASK_REPORT_ERROR1,
                 None
             )
         ]
@@ -374,10 +374,10 @@ class TestRallytester(unittest.TestCase):
             (
                 {
                     'task': 'fake_task',
-                    'scenarios': 'NovaServers,Novaname',
+                    'scenarios': 'Novaname',
                     'succeed': 0,
                 },
-                '\nFrc\n\nTask descriptor:\n\n'
+                '\n\nTask descriptor:\n\n'
                 '{\n'
                 '    "NovaServers.list_servers": [\n'
                 '        {\n'
@@ -442,7 +442,7 @@ class TestRallytester(unittest.TestCase):
         )
 
     @mock.patch('rallytester.rallytester.subprocess.Popen')
-    @mock.patch(func, return_value=io.StringIO(fixtures.TASK_RESULTS_CRASH))
+    @mock.patch(func, return_value=io.StringIO(fixtures.TASK))
     @mock.patch('os.path.exists', return_value=True)
     def test_execute_task_crash(self,
                                 mock_os,
@@ -457,7 +457,7 @@ class TestRallytester(unittest.TestCase):
                 None
             ),
             (
-                fixtures.TASK_RESULTS_CRASH,
+                fixtures.TASK_REPORT_CRASH,
                 None
             )
         ]
@@ -465,7 +465,6 @@ class TestRallytester(unittest.TestCase):
         self.assertEqual(
             (
                 "Task status is crashed."
-                "Use: task report --uuid --json "
             ),
             rallytester.execute_task(
                 deployment='fake_deployment',
-- 
GitLab


From e32d8699018972307d2f7bf6e608575438a08626 Mon Sep 17 00:00:00 2001
From: ravillar <raul.villar.ramos@cern.ch>
Date: Mon, 23 Mar 2020 18:51:26 +0100
Subject: [PATCH 15/21] OS8753_Fix_pep8

---
 rallytester/rallytester.py    |  40 +-
 rallytester/tests/fixtures.py | 664 +++++++++++++++++-----------------
 2 files changed, 366 insertions(+), 338 deletions(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index c553173..ca4817d 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -149,19 +149,25 @@ def execute_task(deployment, task, msgTrimmer=None):
                         return ("Task status is crashed.")
                         # return json here
 
-    scenarios = ",".join([job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['atomic_actions'][0]['name'] for job in json_result])
+    scenarios = ",".join(
+        [job['tasks'][0]['subtasks'][0]['workloads'][0]
+            ['data'][0]['atomic_actions'][0]['name'] for job in json_result])
 
     # If there is any error in the reports
-    if any(job['tasks'] and 
-            job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'] for job in json_result):
+    if any(job['tasks'] and
+            job['tasks'][0]['subtasks'][0]['workloads'][0]
+            ['data'][0]['error'] for job in json_result):
         msg = ""
         tracebacks = ""
         for job in json_result:
-            if job['tasks'] and job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error']:
+            if (job['tasks'] and job['tasks'][0]['subtasks'][0]['workloads'][0]
+                    ['data'][0]['error']):
                 # The real error is in the first result
                 # and the second element of the array
-                msg += '\n' + job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'][1]
-                tracebacks += '\n' + job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'][2]
+                msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads']
+                        [0]['data'][0]['error'][1])
+                tracebacks += ('\n' + job['tasks'][0]['subtasks'][0]
+                               ['workloads'][0]['data'][0]['error'][2])
                 log.error('Task failed: {0}'.format(
                     _msg_enrich(msg.replace('\n', ' '), msgTrimmer)))
 
@@ -210,16 +216,21 @@ def execute_task(deployment, task, msgTrimmer=None):
         )
 
     if any('tasks' in j for j in json_result):
-        if any(job['tasks'][0]['subtasks'][0]['workloads'][0]['sla_results']['sla'][0]['success'] == False for job in json_result):
-        
+        if (any(job['tasks'][0]['subtasks'][0]['workloads']
+                [0]['sla_results']['sla'][0]['success'] is False
+                for job in json_result)):
+
             msg = ""
             tracebacks = ""
             for job in json_result:
-                if job['tasks'] and job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error']:
+                if (job['tasks'] and job['tasks'][0]['subtasks']
+                        [0]['workloads'][0]['data'][0]['error']):
                     # The real error is in the first result
                     # and the second element of the array
-                    msg += '\n' + job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'][1]
-                    tracebacks += '\n' + job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['error'][2]
+                    msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads']
+                            [0]['data'][0]['error'][1])
+                    tracebacks += ('\n' + job['tasks'][0]['subtasks']
+                                   [0]['workloads'][0]['data'][0]['error'][2])
                     log.error('Task failed: {0}'.format(
                         _msg_enrich(msg.replace('\n', ' '), msgTrimmer)))
 
@@ -230,7 +241,7 @@ def execute_task(deployment, task, msgTrimmer=None):
                     'succeed': 0
                 },
                 "{0}\n{1}\nTask descriptor:\n{2}".format(msg, tracebacks,
-                                                        task_descriptor)
+                                                         task_descriptor)
             )
 
     # In any other case
@@ -240,8 +251,9 @@ def execute_task(deployment, task, msgTrimmer=None):
             'task': task,
             'scenarios': scenarios,
             'succeed': 1,
-            'duration': sum([job['tasks'][0]['subtasks'][0]['workloads'][0]['data'][0]['duration']
-                             for job in json_result])
+            'duration': sum([job['tasks'][0]['subtasks'][0]['workloads']
+                            [0]['data'][0]['duration']
+                            for job in json_result])
         },
         None  # No errors
     )
diff --git a/rallytester/tests/fixtures.py b/rallytester/tests/fixtures.py
index 95960ae..04f0626 100644
--- a/rallytester/tests/fixtures.py
+++ b/rallytester/tests/fixtures.py
@@ -103,177 +103,183 @@ TASK_REPORT = u"""
 [
     {
         "info": {
-            "generated_at": "2020-03-16T16:07:30", 
-            "rally_version": "2.0.0", 
+            "generated_at": "2020-03-16T16:07:30",
+            "rally_version": "2.0.0",
             "format_version": "1.2"
-        }, 
+        },
         "tasks": [
             {
-                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d", 
-                "title": "", 
-                "description": "", 
-                "status": "finished", 
-                "tags": [], 
-                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708", 
-                "env_name": "teststack_cell02", 
-                "created_at": "2020-03-15T03:17:17", 
-                "updated_at": "2020-03-15T03:17:20", 
-                "pass_sla": true, 
+                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d",
+                "title": "",
+                "description": "",
+                "status": "finished",
+                "tags": [],
+                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708",
+                "env_name": "teststack_cell02",
+                "created_at": "2020-03-15T03:17:17",
+                "updated_at": "2020-03-15T03:17:20",
+                "pass_sla": true,
                 "subtasks": [
                     {
-                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12", 
-                        "title": "GlanceImages.list_images", 
-                        "description": "", 
-                        "status": "finished", 
-                        "created_at": "2020-03-15T03:17:18", 
-                        "updated_at": "2020-03-15T03:17:20", 
-                        "sla": {}, 
+                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12",
+                        "title": "GlanceImages.list_images",
+                        "description": "",
+                        "status": "finished",
+                        "created_at": "2020-03-15T03:17:18",
+                        "updated_at": "2020-03-15T03:17:20",
+                        "sla": {},
                         "workloads": [
                             {
-                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48", 
-                                "description": "List all images.", 
+                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48",
+                                "description": "List all images.",
                                 "runner": {
                                     "constant": {
-                                        "concurrency": 1, 
+                                        "concurrency": 1,
                                         "times": 1
                                     }
-                                }, 
-                                "hooks": [], 
+                                },
+                                "hooks": [],
                                 "scenario": {
                                     "GlanceImages.list_images": {}
-                                }, 
-                                "min_duration": 0.1442, 
-                                "max_duration": 0.1442, 
-                                "start_time": 1584242238.24373, 
-                                "load_duration": 0.1442, 
-                                "full_duration": 0.380141, 
+                                },
+                                "min_duration": 0.1442,
+                                "max_duration": 0.1442,
+                                "start_time": 1584242238.24373,
+                                "load_duration": 0.1442,
+                                "full_duration": 0.380141,
                                 "statistics": {
                                     "durations": {
                                         "total": {
                                             "data": {
-                                                "success": "100.0%", 
-                                                "min": 0.144, 
-                                                "max": 0.144, 
-                                                "median": 0.144, 
-                                                "95%ile": 0.144, 
-                                                "iteration_count": 1, 
-                                                "avg": 0.144, 
+                                                "success": "100.0%",
+                                                "min": 0.144,
+                                                "max": 0.144,
+                                                "median": 0.144,
+                                                "95%ile": 0.144,
+                                                "iteration_count": 1,
+                                                "avg": 0.144,
                                                 "90%ile": 0.144
-                                            }, 
-                                            "display_name": "total", 
-                                            "name": "total", 
-                                            "count_per_iteration": 1, 
+                                            },
+                                            "display_name": "total",
+                                            "name": "total",
+                                            "count_per_iteration": 1,
                                             "children": [
                                                 {
                                                     "data": {
-                                                        "success": "100.0%", 
-                                                        "min": 0.144, 
-                                                        "max": 0.144, 
-                                                        "median": 0.144, 
-                                                        "95%ile": 0.144, 
-                                                        "iteration_count": 1, 
-                                                        "avg": 0.144, 
+                                                        "success": "100.0%",
+                                                        "min": 0.144,
+                                                        "max": 0.144,
+                                                        "median": 0.144,
+                                                        "95%ile": 0.144,
+                                                        "iteration_count": 1,
+                                                        "avg": 0.144,
                                                         "90%ile": 0.144
-                                                    }, 
-                                                    "display_name": "duration", 
-                                                    "name": "duration", 
-                                                    "count_per_iteration": 1, 
+                                                    },
+                                                    "display_name": "duration",
+                                                    "name": "duration",
+                                                    "count_per_iteration": 1,
                                                     "children": []
-                                                }, 
+                                                },
                                                 {
                                                     "data": {
-                                                        "success": "100.0%", 
-                                                        "min": 0.0, 
-                                                        "max": 0.0, 
-                                                        "median": 0.0, 
-                                                        "95%ile": 0.0, 
-                                                        "iteration_count": 1, 
-                                                        "avg": 0.0, 
+                                                        "success": "100.0%",
+                                                        "min": 0.0,
+                                                        "max": 0.0,
+                                                        "median": 0.0,
+                                                        "95%ile": 0.0,
+                                                        "iteration_count": 1,
+                                                        "avg": 0.0,
                                                         "90%ile": 0.0
-                                                    }, 
-                                                    "display_name": "idle_duration", 
-                                                    "name": "idle_duration", 
-                                                    "count_per_iteration": 1, 
+                                                    },
+                                                    "display_name":
+                                                        "idle_duration",
+                                                    "name": "idle_duration",
+                                                    "count_per_iteration": 1,
                                                     "children": []
                                                 }
                                             ]
-                                        }, 
+                                        },
                                         "atomics": [
                                             {
                                                 "data": {
-                                                    "success": "100.0%", 
-                                                    "min": 0.144, 
-                                                    "max": 0.144, 
-                                                    "median": 0.144, 
-                                                    "95%ile": 0.144, 
-                                                    "iteration_count": 1, 
-                                                    "avg": 0.144, 
+                                                    "success": "100.0%",
+                                                    "min": 0.144,
+                                                    "max": 0.144,
+                                                    "median": 0.144,
+                                                    "95%ile": 0.144,
+                                                    "iteration_count": 1,
+                                                    "avg": 0.144,
                                                     "90%ile": 0.144
-                                                }, 
-                                                "display_name": "Novaname", 
-                                                "name": "Novaname", 
-                                                "count_per_iteration": 1, 
+                                                },
+                                                "display_name": "Novaname",
+                                                "name": "Novaname",
+                                                "count_per_iteration": 1,
                                                 "children": []
                                             }
                                         ]
                                     }
-                                }, 
+                                },
                                 "data": [
                                     {
-                                        "timestamp": 1584242238.243732, 
-                                        "error": [], 
-                                        "duration": 0.14420008659362793, 
+                                        "timestamp": 1584242238.243732,
+                                        "error": [],
+                                        "duration": 0.14420008659362793,
                                         "output": {
-                                            "additive": [], 
+                                            "additive": [],
                                             "complete": []
-                                        }, 
-                                        "idle_duration": 0.0, 
+                                        },
+                                        "idle_duration": 0.0,
                                         "atomic_actions": [
                                             {
-                                                "finished_at": 1584242238.38778, 
-                                                "started_at": 1584242238.243803, 
-                                                "name": "Novaname", 
+                                                "finished_at":
+                                                    1584242238.38778,
+                                                "started_at":
+                                                    1584242238.243803,
+                                                "name": "Novaname",
                                                 "children": []
                                             }
                                         ]
                                     }
-                                ], 
-                                "failed_iteration_count": 0, 
-                                "total_iteration_count": 1, 
-                                "created_at": "2020-03-15T03:17:18", 
-                                "updated_at": "2020-03-15T03:17:20", 
-                                "contexts": {}, 
+                                ],
+                                "failed_iteration_count": 0,
+                                "total_iteration_count": 1,
+                                "created_at": "2020-03-15T03:17:18",
+                                "updated_at": "2020-03-15T03:17:20",
+                                "contexts": {},
                                 "contexts_results": [
                                     {
                                         "setup": {
-                                            "finished_at": 1584242238.110536, 
-                                            "started_at": 1584242238.029151, 
-                                            "atomic_actions": [], 
+                                            "finished_at": 1584242238.110536,
+                                            "started_at": 1584242238.029151,
+                                            "atomic_actions": [],
                                             "error": null
-                                        }, 
+                                        },
                                         "cleanup": {
-                                            "finished_at": 1584242238.408476, 
-                                            "started_at": 1584242238.40758, 
-                                            "atomic_actions": [], 
+                                            "finished_at": 1584242238.408476,
+                                            "started_at": 1584242238.40758,
+                                            "atomic_actions": [],
                                             "error": null
-                                        }, 
+                                        },
                                         "plugin_cfg": {
                                             "user_choice_method": "random"
-                                        }, 
+                                        },
                                         "plugin_name": "users@openstack"
                                     }
-                                ], 
-                                "position": 0, 
-                                "pass_sla": true, 
+                                ],
+                                "position": 0,
+                                "pass_sla": true,
                                 "sla_results": {
                                     "sla": [
                                         {
-                                            "criterion": "failure_rate", 
-                                            "detail": "Failure rate criteria 0.00% <= 0.00% <= 0.00% - Passed", 
+                                            "criterion": "failure_rate",
+                                            "detail":
+                                                "Failure rate criteria\
+                                                 0.00% <= 0.00% <= 0.00%\
+                                                  - Passed",
                                             "success": true
                                         }
                                     ]
-                                }, 
+                                },
                                 "sla": {
                                     "failure_rate": {
                                         "max": 0
@@ -293,177 +299,180 @@ TASK_REPORT_ERROR1 = u"""
 [
     {
         "info": {
-            "generated_at": "2020-03-16T16:07:30", 
-            "rally_version": "2.0.0", 
+            "generated_at": "2020-03-16T16:07:30",
+            "rally_version": "2.0.0",
             "format_version": "1.2"
-        }, 
+        },
         "tasks": [
             {
-                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d", 
-                "title": "", 
-                "description": "", 
-                "status": "finished", 
-                "tags": [], 
-                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708", 
-                "env_name": "teststack_cell02", 
-                "created_at": "2020-03-15T03:17:17", 
-                "updated_at": "2020-03-15T03:17:20", 
-                "pass_sla": true, 
+                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d",
+                "title": "",
+                "description": "",
+                "status": "finished",
+                "tags": [],
+                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708",
+                "env_name": "teststack_cell02",
+                "created_at": "2020-03-15T03:17:17",
+                "updated_at": "2020-03-15T03:17:20",
+                "pass_sla": true,
                 "subtasks": [
                     {
-                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12", 
-                        "title": "GlanceImages.list_images", 
-                        "description": "", 
-                        "status": "finished", 
-                        "created_at": "2020-03-15T03:17:18", 
-                        "updated_at": "2020-03-15T03:17:20", 
-                        "sla": {}, 
+                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12",
+                        "title": "GlanceImages.list_images",
+                        "description": "",
+                        "status": "finished",
+                        "created_at": "2020-03-15T03:17:18",
+                        "updated_at": "2020-03-15T03:17:20",
+                        "sla": {},
                         "workloads": [
                             {
-                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48", 
-                                "description": "List all images.", 
+                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48",
+                                "description": "List all images.",
                                 "runner": {
                                     "constant": {
-                                        "concurrency": 1, 
+                                        "concurrency": 1,
                                         "times": 1
                                     }
-                                }, 
-                                "hooks": [], 
+                                },
+                                "hooks": [],
                                 "scenario": {
                                     "GlanceImages.list_images": {}
-                                }, 
-                                "min_duration": 0.1442, 
-                                "max_duration": 0.1442, 
-                                "start_time": 1584242238.24373, 
-                                "load_duration": 0.1442, 
-                                "full_duration": 0.380141, 
+                                },
+                                "min_duration": 0.1442,
+                                "max_duration": 0.1442,
+                                "start_time": 1584242238.24373,
+                                "load_duration": 0.1442,
+                                "full_duration": 0.380141,
                                 "statistics": {
                                     "durations": {
                                         "total": {
                                             "data": {
-                                                "success": "0.0%", 
-                                                "min": 0.144, 
-                                                "max": 0.144, 
-                                                "median": 0.144, 
-                                                "95%ile": 0.144, 
-                                                "iteration_count": 1, 
-                                                "avg": 0.144, 
+                                                "success": "0.0%",
+                                                "min": 0.144,
+                                                "max": 0.144,
+                                                "median": 0.144,
+                                                "95%ile": 0.144,
+                                                "iteration_count": 1,
+                                                "avg": 0.144,
                                                 "90%ile": 0.144
-                                            }, 
-                                            "display_name": "total", 
-                                            "name": "total", 
-                                            "count_per_iteration": 1, 
+                                            },
+                                            "display_name": "total",
+                                            "name": "total",
+                                            "count_per_iteration": 1,
                                             "children": [
                                                 {
                                                     "data": {
-                                                        "success": "0.0%", 
-                                                        "min": 0.144, 
-                                                        "max": 0.144, 
-                                                        "median": 0.144, 
-                                                        "95%ile": 0.144, 
-                                                        "iteration_count": 1, 
-                                                        "avg": 0.144, 
+                                                        "success": "0.0%",
+                                                        "min": 0.144,
+                                                        "max": 0.144,
+                                                        "median": 0.144,
+                                                        "95%ile": 0.144,
+                                                        "iteration_count": 1,
+                                                        "avg": 0.144,
                                                         "90%ile": 0.144
-                                                    }, 
-                                                    "display_name": "duration", 
-                                                    "name": "duration", 
-                                                    "count_per_iteration": 1, 
+                                                    },
+                                                    "display_name": "duration",
+                                                    "name": "duration",
+                                                    "count_per_iteration": 1,
                                                     "children": []
-                                                }, 
+                                                },
                                                 {
                                                     "data": {
-                                                        "success": "0.0%", 
-                                                        "min": 0.0, 
-                                                        "max": 0.0, 
-                                                        "median": 0.0, 
-                                                        "95%ile": 0.0, 
-                                                        "iteration_count": 1, 
-                                                        "avg": 0.0, 
+                                                        "success": "0.0%",
+                                                        "min": 0.0,
+                                                        "max": 0.0,
+                                                        "median": 0.0,
+                                                        "95%ile": 0.0,
+                                                        "iteration_count": 1,
+                                                        "avg": 0.0,
                                                         "90%ile": 0.0
-                                                    }, 
-                                                    "display_name": "idle_duration", 
-                                                    "name": "idle_duration", 
-                                                    "count_per_iteration": 1, 
+                                                    },
+                                                    "display_name":
+                                                        "idle_duration",
+                                                    "name": "idle_duration",
+                                                    "count_per_iteration": 1,
                                                     "children": []
                                                 }
                                             ]
-                                        }, 
+                                        },
                                         "atomics": [
                                             {
                                                 "data": {
-                                                    "success": "0.0%", 
-                                                    "min": 0.144, 
-                                                    "max": 0.144, 
-                                                    "median": 0.144, 
-                                                    "95%ile": 0.144, 
-                                                    "iteration_count": 1, 
-                                                    "avg": 0.144, 
+                                                    "success": "0.0%",
+                                                    "min": 0.144,
+                                                    "max": 0.144,
+                                                    "median": 0.144,
+                                                    "95%ile": 0.144,
+                                                    "iteration_count": 1,
+                                                    "avg": 0.144,
                                                     "90%ile": 0.144
-                                                }, 
-                                                "display_name": "Novaname", 
-                                                "name": "Novaname", 
-                                                "count_per_iteration": 1, 
+                                                },
+                                                "display_name": "Novaname",
+                                                "name": "Novaname",
+                                                "count_per_iteration": 1,
                                                 "children": []
                                             }
                                         ]
                                     }
-                                }, 
+                                },
                                 "data": [
                                     {
-                                        "timestamp": 1584242238.243732, 
-                                        "error": [], 
-                                        "duration": 0.14420008659362793, 
+                                        "timestamp": 1584242238.243732,
+                                        "error": [],
+                                        "duration": 0.14420008659362793,
                                         "output": {
-                                            "additive": [], 
+                                            "additive": [],
                                             "complete": []
-                                        }, 
-                                        "idle_duration": 0.0, 
+                                        },
+                                        "idle_duration": 0.0,
                                         "atomic_actions": [
                                             {
-                                                "finished_at": 1584242238.38778, 
-                                                "started_at": 1584242238.243803, 
-                                                "name": "Novaname", 
+                                                "finished_at":
+                                                    1584242238.38778,
+                                                "started_at":
+                                                    1584242238.243803,
+                                                "name": "Novaname",
                                                 "children": []
                                             }
                                         ]
                                     }
-                                ], 
-                                "failed_iteration_count": 0, 
-                                "total_iteration_count": 1, 
-                                "created_at": "2020-03-15T03:17:18", 
-                                "updated_at": "2020-03-15T03:17:20", 
-                                "contexts": {}, 
+                                ],
+                                "failed_iteration_count": 0,
+                                "total_iteration_count": 1,
+                                "created_at": "2020-03-15T03:17:18",
+                                "updated_at": "2020-03-15T03:17:20",
+                                "contexts": {},
                                 "contexts_results": [
                                     {
                                         "setup": {
-                                            "finished_at": 1584242238.110536, 
-                                            "started_at": 1584242238.029151, 
-                                            "atomic_actions": [], 
+                                            "finished_at": 1584242238.110536,
+                                            "started_at": 1584242238.029151,
+                                            "atomic_actions": [],
                                             "error": null
-                                        }, 
+                                        },
                                         "cleanup": {
-                                            "finished_at": 1584242238.408476, 
-                                            "started_at": 1584242238.40758, 
-                                            "atomic_actions": [], 
+                                            "finished_at": 1584242238.408476,
+                                            "started_at": 1584242238.40758,
+                                            "atomic_actions": [],
                                             "error": null
-                                        }, 
+                                        },
                                         "plugin_cfg": {
                                             "user_choice_method": "random"
-                                        }, 
+                                        },
                                         "plugin_name": "users@openstack"
                                     }
-                                ], 
-                                "position": 0, 
-                                "pass_sla": true, 
+                                ],
+                                "position": 0,
+                                "pass_sla": true,
                                 "sla_results": {
                                     "sla": [
                                         {
-                                            "criterion": "failure_rate", 
-                                            "detail": "Fake error - FAIL", 
+                                            "criterion": "failure_rate",
+                                            "detail": "Fake error - FAIL",
                                             "success": false
                                         }
                                     ]
-                                }, 
+                                },
                                 "sla": {
                                     "failure_rate": {
                                         "max": 0
@@ -483,181 +492,188 @@ TASK_REPORT_ERROR2 = u"""
 [
     {
         "info": {
-            "generated_at": "2020-03-16T16:07:30", 
-            "rally_version": "2.0.0", 
+            "generated_at": "2020-03-16T16:07:30",
+            "rally_version": "2.0.0",
             "format_version": "1.2"
-        }, 
+        },
         "tasks": [
             {
-                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d", 
-                "title": "", 
-                "description": "", 
-                "status": "finished", 
-                "tags": [], 
-                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708", 
-                "env_name": "teststack_cell02", 
-                "created_at": "2020-03-15T03:17:17", 
-                "updated_at": "2020-03-15T03:17:20", 
-                "pass_sla": true, 
+                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d",
+                "title": "",
+                "description": "",
+                "status": "finished",
+                "tags": [],
+                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708",
+                "env_name": "teststack_cell02",
+                "created_at": "2020-03-15T03:17:17",
+                "updated_at": "2020-03-15T03:17:20",
+                "pass_sla": true,
                 "subtasks": [
                     {
-                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12", 
-                        "title": "GlanceImages.list_images", 
-                        "description": "", 
-                        "status": "finished", 
-                        "created_at": "2020-03-15T03:17:18", 
-                        "updated_at": "2020-03-15T03:17:20", 
-                        "sla": {}, 
+                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12",
+                        "title": "GlanceImages.list_images",
+                        "description": "",
+                        "status": "finished",
+                        "created_at": "2020-03-15T03:17:18",
+                        "updated_at": "2020-03-15T03:17:20",
+                        "sla": {},
                         "workloads": [
                             {
-                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48", 
-                                "description": "List all images.", 
+                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48",
+                                "description": "List all images.",
                                 "runner": {
                                     "constant": {
-                                        "concurrency": 1, 
+                                        "concurrency": 1,
                                         "times": 1
                                     }
-                                }, 
-                                "hooks": [], 
+                                },
+                                "hooks": [],
                                 "scenario": {
                                     "GlanceImages.list_images": {}
-                                }, 
-                                "min_duration": 0.1442, 
-                                "max_duration": 0.1442, 
-                                "start_time": 1584242238.24373, 
-                                "load_duration": 0.1442, 
-                                "full_duration": 0.380141, 
+                                },
+                                "min_duration": 0.1442,
+                                "max_duration": 0.1442,
+                                "start_time": 1584242238.24373,
+                                "load_duration": 0.1442,
+                                "full_duration": 0.380141,
                                 "statistics": {
                                     "durations": {
                                         "total": {
                                             "data": {
-                                                "success": "100.0%", 
-                                                "min": 0.144, 
-                                                "max": 0.144, 
-                                                "median": 0.144, 
-                                                "95%ile": 0.144, 
-                                                "iteration_count": 1, 
-                                                "avg": 0.144, 
+                                                "success": "100.0%",
+                                                "min": 0.144,
+                                                "max": 0.144,
+                                                "median": 0.144,
+                                                "95%ile": 0.144,
+                                                "iteration_count": 1,
+                                                "avg": 0.144,
                                                 "90%ile": 0.144
-                                            }, 
-                                            "display_name": "total", 
-                                            "name": "total", 
-                                            "count_per_iteration": 1, 
+                                            },
+                                            "display_name": "total",
+                                            "name": "total",
+                                            "count_per_iteration": 1,
                                             "children": [
                                                 {
                                                     "data": {
-                                                        "success": "100.0%", 
-                                                        "min": 0.144, 
-                                                        "max": 0.144, 
-                                                        "median": 0.144, 
-                                                        "95%ile": 0.144, 
-                                                        "iteration_count": 1, 
-                                                        "avg": 0.144, 
+                                                        "success": "100.0%",
+                                                        "min": 0.144,
+                                                        "max": 0.144,
+                                                        "median": 0.144,
+                                                        "95%ile": 0.144,
+                                                        "iteration_count": 1,
+                                                        "avg": 0.144,
                                                         "90%ile": 0.144
-                                                    }, 
-                                                    "display_name": "duration", 
-                                                    "name": "duration", 
-                                                    "count_per_iteration": 1, 
+                                                    },
+                                                    "display_name": "duration",
+                                                    "name": "duration",
+                                                    "count_per_iteration": 1,
                                                     "children": []
-                                                }, 
+                                                },
                                                 {
                                                     "data": {
-                                                        "success": "100.0%", 
-                                                        "min": 0.0, 
-                                                        "max": 0.0, 
-                                                        "median": 0.0, 
-                                                        "95%ile": 0.0, 
-                                                        "iteration_count": 1, 
-                                                        "avg": 0.0, 
+                                                        "success": "100.0%",
+                                                        "min": 0.0,
+                                                        "max": 0.0,
+                                                        "median": 0.0,
+                                                        "95%ile": 0.0,
+                                                        "iteration_count": 1,
+                                                        "avg": 0.0,
                                                         "90%ile": 0.0
-                                                    }, 
-                                                    "display_name": "idle_duration", 
-                                                    "name": "idle_duration", 
-                                                    "count_per_iteration": 1, 
+                                                    },
+                                                    "display_name":
+                                                        "idle_duration",
+                                                    "name":
+                                                        "idle_duration",
+                                                    "count_per_iteration": 1,
                                                     "children": []
                                                 }
                                             ]
-                                        }, 
+                                        },
                                         "atomics": [
                                             {
                                                 "data": {
-                                                    "success": "100.0%", 
-                                                    "min": 0.144, 
-                                                    "max": 0.144, 
-                                                    "median": 0.144, 
-                                                    "95%ile": 0.144, 
-                                                    "iteration_count": 1, 
-                                                    "avg": 0.144, 
+                                                    "success": "100.0%",
+                                                    "min": 0.144,
+                                                    "max": 0.144,
+                                                    "median": 0.144,
+                                                    "95%ile": 0.144,
+                                                    "iteration_count": 1,
+                                                    "avg": 0.144,
                                                     "90%ile": 0.144
-                                                }, 
-                                                "display_name": "Novaname", 
-                                                "name": "Novaname", 
-                                                "count_per_iteration": 1, 
+                                                },
+                                                "display_name": "Novaname",
+                                                "name": "Novaname",
+                                                "count_per_iteration": 1,
                                                 "children": []
                                             }
                                         ]
                                     }
-                                }, 
+                                },
                                 "data": [
                                     {
-                                        "timestamp": 1584242238.243732, 
+                                        "timestamp": 1584242238.243732,
                                         "error": [
                                             "-----------",
                                             "Fake error1",
                                             "Fake error2"
-                                        ], 
-                                        "duration": 0.14420008659362793, 
+                                        ],
+                                        "duration": 0.14420008659362793,
                                         "output": {
-                                            "additive": [], 
+                                            "additive": [],
                                             "complete": []
-                                        }, 
-                                        "idle_duration": 0.0, 
+                                        },
+                                        "idle_duration": 0.0,
                                         "atomic_actions": [
                                             {
-                                                "finished_at": 1584242238.38778, 
-                                                "started_at": 1584242238.243803, 
-                                                "name": "Novaname", 
+                                                "finished_at":
+                                                    1584242238.38778,
+                                                "started_at":
+                                                    1584242238.243803,
+                                                "name": "Novaname",
                                                 "children": []
                                             }
                                         ]
                                     }
-                                ], 
-                                "failed_iteration_count": 0, 
-                                "total_iteration_count": 1, 
-                                "created_at": "2020-03-15T03:17:18", 
-                                "updated_at": "2020-03-15T03:17:20", 
-                                "contexts": {}, 
+                                ],
+                                "failed_iteration_count": 0,
+                                "total_iteration_count": 1,
+                                "created_at": "2020-03-15T03:17:18",
+                                "updated_at": "2020-03-15T03:17:20",
+                                "contexts": {},
                                 "contexts_results": [
                                     {
                                         "setup": {
-                                            "finished_at": 1584242238.110536, 
-                                            "started_at": 1584242238.029151, 
-                                            "atomic_actions": [], 
+                                            "finished_at": 1584242238.110536,
+                                            "started_at": 1584242238.029151,
+                                            "atomic_actions": [],
                                             "error": null
-                                        }, 
+                                        },
                                         "cleanup": {
-                                            "finished_at": 1584242238.408476, 
-                                            "started_at": 1584242238.40758, 
-                                            "atomic_actions": [], 
+                                            "finished_at": 1584242238.408476,
+                                            "started_at": 1584242238.40758,
+                                            "atomic_actions": [],
                                             "error": null
-                                        }, 
+                                        },
                                         "plugin_cfg": {
                                             "user_choice_method": "random"
-                                        }, 
+                                        },
                                         "plugin_name": "users@openstack"
                                     }
-                                ], 
-                                "position": 0, 
-                                "pass_sla": true, 
+                                ],
+                                "position": 0,
+                                "pass_sla": true,
                                 "sla_results": {
                                     "sla": [
                                         {
-                                            "criterion": "failure_rate", 
-                                            "detail": "Failure rate criteria 0.00% <= 0.00% <= 0.00% - Passed", 
+                                            "criterion": "failure_rate",
+                                            "detail":
+                                                "Failure rate criteria\
+                                                 0.00% <= 0.00% <= 0.00%\
+                                                  - Passed",
                                             "success": true
                                         }
                                     ]
-                                }, 
+                                },
                                 "sla": {
                                     "failure_rate": {
                                         "max": 0
-- 
GitLab


From 041bb2c3620abf3b1cec43dcc4778bfddb7aeac0 Mon Sep 17 00:00:00 2001
From: ravillar <raul.villar.ramos@cern.ch>
Date: Wed, 25 Mar 2020 11:01:57 +0100
Subject: [PATCH 16/21] OS8753_Send_crash_report

---
 rallytester/rallytester.py            |  2 +-
 rallytester/tests/test_rallytester.py | 25 ++++++++++++++++++++++++-
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index ca4817d..614c859 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -146,7 +146,7 @@ def execute_task(deployment, task, msgTrimmer=None):
     # Check if the task is crashed
     if any('tasks' in j for j in json_result):
         if any(job["tasks"][0]["status"] == "crashed" for job in json_result):
-                        return ("Task status is crashed.")
+                        return json_result
                         # return json here
 
     scenarios = ",".join(
diff --git a/rallytester/tests/test_rallytester.py b/rallytester/tests/test_rallytester.py
index ff645be..9a753f3 100644
--- a/rallytester/tests/test_rallytester.py
+++ b/rallytester/tests/test_rallytester.py
@@ -464,7 +464,30 @@ class TestRallytester(unittest.TestCase):
 
         self.assertEqual(
             (
-                "Task status is crashed."
+                [
+                    {
+                        'info':
+                        {
+                            'generated_at': '2019-03-05T16:50:31',
+                            'rally_version': '0.0.0',
+                            'format_version': '1.1'
+                        },
+                        'tasks':
+                        [
+                            {
+                                'uuid': '5312f2e6-f00d-4a63-9162-40c895cdaea1',
+                                'title': '',
+                                'description': '',
+                                'status': 'crashed',
+                                'tags': [],
+                                'created_at': '2019-03-05T15:14:54',
+                                'updated_at': '2019-03-05T15:14:56',
+                                'pass_sla': True,
+                                'subtasks': []
+                            }
+                        ],
+                    }
+                ]
             ),
             rallytester.execute_task(
                 deployment='fake_deployment',
-- 
GitLab


From 84d09fe19de68264c4b37dcb470913b4e348d347 Mon Sep 17 00:00:00 2001
From: ravillar <raul.villar.ramos@cern.ch>
Date: Fri, 27 Mar 2020 16:22:36 +0100
Subject: [PATCH 17/21] OS8753_manage_magnum_reports

---
 rallytester/rallytester.py            |  64 ++++-----
 rallytester/tests/fixtures.py         | 197 ++++++++++++++++++++++++++
 rallytester/tests/test_rallytester.py |  45 ++++++
 3 files changed, 272 insertions(+), 34 deletions(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index 614c859..d2ba632 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -153,6 +153,36 @@ def execute_task(deployment, task, msgTrimmer=None):
         [job['tasks'][0]['subtasks'][0]['workloads'][0]
             ['data'][0]['atomic_actions'][0]['name'] for job in json_result])
 
+    # FIXME (lpigueir): cornercase for magnum tasks
+    # result is empty for magnum tasks sometimes and the
+    # real error doesn't appear. Will do this workaround because
+    # `rally task results` is gone/deprecated and we need
+    # to reimplement this part anyway to use `rally task report`
+    # with a different format
+    # 2020 update (Ravillar)
+    if any(job['tasks'][0]['subtasks'][0]['workloads'][0]
+            ['sla_results']['sla'][0]
+            ['criterion'] == 'something_went_wrong'
+            for job in json_result):
+
+        msg = ""
+        tracebacks = ""
+        msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads']
+                [0]['sla_results']['sla'][0]['detail'])
+        log.error('Task failed: {0}'.format(
+                  _msg_enrich(msg.replace('\n', ' '),
+                              msgTrimmer)))
+
+        return (
+            {
+                'task': task,
+                'scenarios': scenarios,
+                'succeed': 0
+            },
+            "{0}\n{1}\nTask descriptor:\n{2}".format(msg, tracebacks,
+                                                     task_descriptor)
+        )
+
     # If there is any error in the reports
     if any(job['tasks'] and
             job['tasks'][0]['subtasks'][0]['workloads'][0]
@@ -181,40 +211,6 @@ def execute_task(deployment, task, msgTrimmer=None):
                                                      task_descriptor)
         )
 
-    # FIXME (lpigueir): cornercase for magnum tasks
-    # result is empty for magnum tasks sometimes and the
-    # real error doesn't appear. Will do this workaround because
-    # `rally task results` is gone/deprecated and we need
-    # to reimplement this part anyway to use `rally task report`
-    # with a different format
-    elif any(
-        not job['tasks'] and
-        'sla' in job and
-        any(sla['criterion'] == 'something_went_wrong' for sla in job['sla'])
-        for job in json_result
-    ):
-        msg = ""
-        tracebacks = ""
-
-        for job in json_result:
-            if 'sla' in job:
-                for sla in job['sla']:
-                    if sla['criterion'] == 'something_went_wrong':
-                        msg += '\n' + sla['detail']
-                        log.error('Task failed: {0}'.format(
-                                  _msg_enrich(msg.replace('\n', ' '),
-                                              msgTrimmer)))
-
-        return (
-            {
-                'task': task,
-                'scenarios': scenarios,
-                'succeed': 0
-            },
-            "{0}\n{1}\nTask descriptor:\n{2}".format(msg, tracebacks,
-                                                     task_descriptor)
-        )
-
     if any('tasks' in j for j in json_result):
         if (any(job['tasks'][0]['subtasks'][0]['workloads']
                 [0]['sla_results']['sla'][0]['success'] is False
diff --git a/rallytester/tests/fixtures.py b/rallytester/tests/fixtures.py
index 04f0626..fd6da73 100644
--- a/rallytester/tests/fixtures.py
+++ b/rallytester/tests/fixtures.py
@@ -688,3 +688,200 @@ TASK_REPORT_ERROR2 = u"""
     }
 ]
 """
+
+TASK_REPORT_MAGNUM = u"""
+[
+    {
+        "info": {
+            "generated_at": "2020-03-16T16:07:30",
+            "rally_version": "2.0.0",
+            "format_version": "1.2"
+        },
+        "tasks": [
+            {
+                "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d",
+                "title": "",
+                "description": "",
+                "status": "finished",
+                "tags": [],
+                "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708",
+                "env_name": "teststack_cell02",
+                "created_at": "2020-03-15T03:17:17",
+                "updated_at": "2020-03-15T03:17:20",
+                "pass_sla": true,
+                "subtasks": [
+                    {
+                        "uuid": "258393e8-54b7-4e40-8344-f8f473101d12",
+                        "title": "GlanceImages.list_images",
+                        "description": "",
+                        "status": "finished",
+                        "created_at": "2020-03-15T03:17:18",
+                        "updated_at": "2020-03-15T03:17:20",
+                        "sla": {},
+                        "workloads": [
+                            {
+                                "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48",
+                                "description": "List all images.",
+                                "runner": {
+                                    "constant": {
+                                        "concurrency": 1,
+                                        "times": 1
+                                    }
+                                },
+                                "hooks": [],
+                                "scenario": {
+                                    "GlanceImages.list_images": {}
+                                },
+                                "min_duration": 0.1442,
+                                "max_duration": 0.1442,
+                                "start_time": 1584242238.24373,
+                                "load_duration": 0.1442,
+                                "full_duration": 0.380141,
+                                "statistics": {
+                                    "durations": {
+                                        "total": {
+                                            "data": {
+                                                "success": "100.0%",
+                                                "min": 0.144,
+                                                "max": 0.144,
+                                                "median": 0.144,
+                                                "95%ile": 0.144,
+                                                "iteration_count": 1,
+                                                "avg": 0.144,
+                                                "90%ile": 0.144
+                                            },
+                                            "display_name": "total",
+                                            "name": "total",
+                                            "count_per_iteration": 1,
+                                            "children": [
+                                                {
+                                                    "data": {
+                                                        "success": "100.0%",
+                                                        "min": 0.144,
+                                                        "max": 0.144,
+                                                        "median": 0.144,
+                                                        "95%ile": 0.144,
+                                                        "iteration_count": 1,
+                                                        "avg": 0.144,
+                                                        "90%ile": 0.144
+                                                    },
+                                                    "display_name": "duration",
+                                                    "name": "duration",
+                                                    "count_per_iteration": 1,
+                                                    "children": []
+                                                },
+                                                {
+                                                    "data": {
+                                                        "success": "100.0%",
+                                                        "min": 0.0,
+                                                        "max": 0.0,
+                                                        "median": 0.0,
+                                                        "95%ile": 0.0,
+                                                        "iteration_count": 1,
+                                                        "avg": 0.0,
+                                                        "90%ile": 0.0
+                                                    },
+                                                    "display_name":
+                                                        "idle_duration",
+                                                    "name": "idle_duration",
+                                                    "count_per_iteration": 1,
+                                                    "children": []
+                                                }
+                                            ]
+                                        },
+                                        "atomics": [
+                                            {
+                                                "data": {
+                                                    "success": "100.0%",
+                                                    "min": 0.144,
+                                                    "max": 0.144,
+                                                    "median": 0.144,
+                                                    "95%ile": 0.144,
+                                                    "iteration_count": 1,
+                                                    "avg": 0.144,
+                                                    "90%ile": 0.144
+                                                },
+                                                "display_name": "Novaname",
+                                                "name": "Novaname",
+                                                "count_per_iteration": 1,
+                                                "children": []
+                                            }
+                                        ]
+                                    }
+                                },
+                                "data": [
+                                    {
+                                        "timestamp": 1584242238.243732,
+                                        "error": [
+                                            "Fake error_M"
+                                        ],
+                                        "duration": 0.14420008659362793,
+                                        "output": {
+                                            "additive": [],
+                                            "complete": []
+                                        },
+                                        "idle_duration": 0.0,
+                                        "atomic_actions": [
+                                            {
+                                                "finished_at":
+                                                    1584242238.38778,
+                                                "started_at":
+                                                    1584242238.243803,
+                                                "name": "Novaname",
+                                                "children": []
+                                            }
+                                        ]
+                                    }
+                                ],
+                                "failed_iteration_count": 0,
+                                "total_iteration_count": 1,
+                                "created_at": "2020-03-15T03:17:18",
+                                "updated_at": "2020-03-15T03:17:20",
+                                "contexts": {},
+                                "contexts_results": [
+                                    {
+                                        "setup": {
+                                            "finished_at": 1584242238.110536,
+                                            "started_at": 1584242238.029151,
+                                            "atomic_actions": [],
+                                            "error": null
+                                        },
+                                        "cleanup": {
+                                            "finished_at": 1584242238.408476,
+                                            "started_at": 1584242238.40758,
+                                            "atomic_actions": [],
+                                            "error": null
+                                        },
+                                        "plugin_cfg": {
+                                            "user_choice_method": "random"
+                                        },
+                                        "plugin_name": "users@openstack"
+                                    }
+                                ],
+                                "position": 0,
+                                "pass_sla": true,
+                                "sla_results": {
+                                    "sla": [
+                                        {
+                                            "criterion":
+                                                "something_went_wrong",
+                                            "detail":
+                                                "Fake detail example",
+                                            "success": false
+                                        }
+                                    ]
+                                },
+                                "sla": {
+                                    "failure_rate": {
+                                        "max": 0
+                                    }
+                                }
+                            }
+                        ]
+                    }
+                ]
+            }
+        ]
+    }
+]
+"""
diff --git a/rallytester/tests/test_rallytester.py b/rallytester/tests/test_rallytester.py
index 9a753f3..14f319b 100644
--- a/rallytester/tests/test_rallytester.py
+++ b/rallytester/tests/test_rallytester.py
@@ -494,3 +494,48 @@ class TestRallytester(unittest.TestCase):
                 task='fake_task'
             )
         )
+
+    @mock.patch('rallytester.rallytester.subprocess.Popen')
+    @mock.patch(func, return_value=io.StringIO(fixtures.TASK))
+    @mock.patch('os.path.exists', return_value=True)
+    def test_execute_task_magnumfail(self,
+                                     mock_os,
+                                     mock_open,
+                                     mock_sub):
+        mock_pr = mock_sub.return_value
+        type(mock_pr).returncode = mock.PropertyMock(return_value=0)
+
+        mock_pr.communicate.side_effect = [
+            (
+                'Using task: 8d4132ab-3530-4148-9ee3-d5bf6621aa9d',
+                None
+            ),
+            (
+                fixtures.TASK_REPORT_MAGNUM,
+                None
+            )
+        ]
+
+        self.assertEqual(
+            (
+                {
+                    'scenarios': 'Novaname',
+                    'task': 'fake_task',
+                    'succeed': 0
+                },
+                '\nFake detail example\n'
+                '\nTask descriptor:\n\n'
+                '{\n    "NovaServers.list_servers": '
+                '[\n        {\n            "args": '
+                '{\n                "detailed": false'
+                '\n            },\n            "runner": '
+                '{\n                "type": "constant",'
+                '\n                "times": 1,\n       '
+                '         "concurrency": 1\n            }'
+                ',\n            "context": {}\n        }\n    ]\n}\n'
+            ),
+            rallytester.execute_task(
+                deployment='fake_deployment',
+                task='fake_task'
+            )
+        )
-- 
GitLab


From f296dcee2f91e1020e3b5d59ee070a19b36bda21 Mon Sep 17 00:00:00 2001
From: ravillar <raul.villar.ramos@cern.ch>
Date: Mon, 30 Mar 2020 10:55:30 +0200
Subject: [PATCH 18/21] OS8753_manage_magnum_reports_fix

---
 rallytester/rallytester.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index d2ba632..ede072e 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -167,11 +167,12 @@ def execute_task(deployment, task, msgTrimmer=None):
 
         msg = ""
         tracebacks = ""
-        msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads']
-                [0]['sla_results']['sla'][0]['detail'])
-        log.error('Task failed: {0}'.format(
-                  _msg_enrich(msg.replace('\n', ' '),
-                              msgTrimmer)))
+        for job in json_result:
+            msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads']
+                    [0]['sla_results']['sla'][0]['detail'])
+            log.error('Task failed: {0}'.format(
+                      _msg_enrich(msg.replace('\n', ' '),
+                                  msgTrimmer)))
 
         return (
             {
-- 
GitLab


From aeafa6b67fdc3dc108b02b64f5d2b22686565f0c Mon Sep 17 00:00:00 2001
From: ravillar <raul.villar.ramos@cern.ch>
Date: Mon, 30 Mar 2020 14:19:15 +0200
Subject: [PATCH 19/21] OS8753_manage_magnum_reports_fix

---
 rallytester/rallytester.py | 70 +++++++++++++++-----------------------
 1 file changed, 28 insertions(+), 42 deletions(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index ede072e..36b2f53 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -144,10 +144,9 @@ def execute_task(deployment, task, msgTrimmer=None):
         )
 
     # Check if the task is crashed
-    if any('tasks' in j for j in json_result):
-        if any(job["tasks"][0]["status"] == "crashed" for job in json_result):
-                        return json_result
-                        # return json here
+    if any(job["tasks"][0]["status"] == "crashed" for job in json_result):
+                    return json_result
+                    # return json here
 
     scenarios = ",".join(
         [job['tasks'][0]['subtasks'][0]['workloads'][0]
@@ -191,16 +190,14 @@ def execute_task(deployment, task, msgTrimmer=None):
         msg = ""
         tracebacks = ""
         for job in json_result:
-            if (job['tasks'] and job['tasks'][0]['subtasks'][0]['workloads'][0]
-                    ['data'][0]['error']):
-                # The real error is in the first result
-                # and the second element of the array
-                msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads']
-                        [0]['data'][0]['error'][1])
-                tracebacks += ('\n' + job['tasks'][0]['subtasks'][0]
-                               ['workloads'][0]['data'][0]['error'][2])
-                log.error('Task failed: {0}'.format(
-                    _msg_enrich(msg.replace('\n', ' '), msgTrimmer)))
+            # The real error is in the first result
+            # and the second element of the array
+            msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads']
+                    [0]['data'][0]['error'][1])
+            tracebacks += ('\n' + job['tasks'][0]['subtasks'][0]
+                           ['workloads'][0]['data'][0]['error'][2])
+            log.error('Task failed: {0}'.format(
+                _msg_enrich(msg.replace('\n', ' '), msgTrimmer)))
 
         return (
             {
@@ -212,34 +209,23 @@ def execute_task(deployment, task, msgTrimmer=None):
                                                      task_descriptor)
         )
 
-    if any('tasks' in j for j in json_result):
-        if (any(job['tasks'][0]['subtasks'][0]['workloads']
-                [0]['sla_results']['sla'][0]['success'] is False
-                for job in json_result)):
-
-            msg = ""
-            tracebacks = ""
-            for job in json_result:
-                if (job['tasks'] and job['tasks'][0]['subtasks']
-                        [0]['workloads'][0]['data'][0]['error']):
-                    # The real error is in the first result
-                    # and the second element of the array
-                    msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads']
-                            [0]['data'][0]['error'][1])
-                    tracebacks += ('\n' + job['tasks'][0]['subtasks']
-                                   [0]['workloads'][0]['data'][0]['error'][2])
-                    log.error('Task failed: {0}'.format(
-                        _msg_enrich(msg.replace('\n', ' '), msgTrimmer)))
-
-            return (
-                {
-                    'task': task,
-                    'scenarios': scenarios,
-                    'succeed': 0
-                },
-                "{0}\n{1}\nTask descriptor:\n{2}".format(msg, tracebacks,
-                                                         task_descriptor)
-            )
+    #Json error2
+    if (any(job['tasks'][0]['subtasks'][0]['workloads']
+            [0]['sla_results']['sla'][0]['success'] is False
+            for job in json_result)):
+
+        msg = ""
+        tracebacks = ""
+
+        return (
+            {
+                'task': task,
+                'scenarios': scenarios,
+                'succeed': 0
+            },
+            "{0}\n{1}\nTask descriptor:\n{2}".format(msg, tracebacks,
+                                                     task_descriptor)
+        )
 
     # In any other case
     log.info('So far, so good, the task has been sucessfully completed')
-- 
GitLab


From d79589556a57d9c14ee87eb09c4e065671001c6b Mon Sep 17 00:00:00 2001
From: ravillar <raul.villar.ramos@cern.ch>
Date: Mon, 30 Mar 2020 14:21:39 +0200
Subject: [PATCH 20/21] OS8753_manage_magnum_reports_fix_p8

---
 rallytester/rallytester.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index 36b2f53..cee746d 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -209,7 +209,7 @@ def execute_task(deployment, task, msgTrimmer=None):
                                                      task_descriptor)
         )
 
-    #Json error2
+    # Json error2
     if (any(job['tasks'][0]['subtasks'][0]['workloads']
             [0]['sla_results']['sla'][0]['success'] is False
             for job in json_result)):
-- 
GitLab


From f5aae593a470be48eb64af5f3204071e2df04a31 Mon Sep 17 00:00:00 2001
From: Raul Villar Ramos <raul.villar.ramos@cern.ch>
Date: Wed, 1 Apr 2020 14:38:58 +0000
Subject: [PATCH 21/21] Remove unnecessary text

---
 rallytester/rallytester.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py
index cee746d..b18663b 100644
--- a/rallytester/rallytester.py
+++ b/rallytester/rallytester.py
@@ -152,13 +152,7 @@ def execute_task(deployment, task, msgTrimmer=None):
         [job['tasks'][0]['subtasks'][0]['workloads'][0]
             ['data'][0]['atomic_actions'][0]['name'] for job in json_result])
 
-    # FIXME (lpigueir): cornercase for magnum tasks
-    # result is empty for magnum tasks sometimes and the
-    # real error doesn't appear. Will do this workaround because
-    # `rally task results` is gone/deprecated and we need
-    # to reimplement this part anyway to use `rally task report`
-    # with a different format
-    # 2020 update (Ravillar)
+    # Magnum
     if any(job['tasks'][0]['subtasks'][0]['workloads'][0]
             ['sla_results']['sla'][0]
             ['criterion'] == 'something_went_wrong'
-- 
GitLab