diff --git a/rallytester/rallytester.py b/rallytester/rallytester.py index 9bcbf2d8306c8407a732aed1cab084c788e515ce..b18663b43324e8c608f46e68d33404917bf25326 100644 --- a/rallytester/rallytester.py +++ b/rallytester/rallytester.py @@ -110,7 +110,7 @@ def execute_task(deployment, task, msgTrimmer=None): # Get results executing rally command process = subprocess.Popen([ # nosec - '/usr/bin/rally', 'task', 'results', task_id], + '/usr/bin/rally', 'task', 'report', task_id, '--json'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, stderr = process.communicate() @@ -143,21 +143,29 @@ def execute_task(deployment, task, msgTrimmer=None): .format(msg, output, stderr, task_id, task_descriptor) ) - scenarios = ",".join([job['key']['name'] for job in json_result]) + # Check if the task is crashed + if any(job["tasks"][0]["status"] == "crashed" for job in json_result): + return json_result + # return json here + + scenarios = ",".join( + [job['tasks'][0]['subtasks'][0]['workloads'][0] + ['data'][0]['atomic_actions'][0]['name'] for job in json_result]) + + # Magnum + if any(job['tasks'][0]['subtasks'][0]['workloads'][0] + ['sla_results']['sla'][0] + ['criterion'] == 'something_went_wrong' + for job in json_result): - # If there is any error in the results - if any(job['result'] and - job['result'][0]['error'] for job in json_result): msg = "" tracebacks = "" for job in json_result: - if job['result'] and job['result'][0]['error']: - # The real error is in the first result - # and the second element of the array - msg += '\n' + job['result'][0]['error'][1] - tracebacks += '\n' + job['result'][0]['error'][2] - log.error('Task failed: {0}'.format( - _msg_enrich(msg.replace('\n', ' '), msgTrimmer))) + msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads'] + [0]['sla_results']['sla'][0]['detail']) + log.error('Task failed: {0}'.format( + _msg_enrich(msg.replace('\n', ' '), + msgTrimmer))) return ( { @@ -169,29 +177,39 @@ def execute_task(deployment, task, msgTrimmer=None): task_descriptor) ) - # FIXME (lpigueir): cornercase for magnum tasks - # result is empty for magnum tasks sometimes and the - # real error doesn't appear. Will do this workaround because - # `rally task results` is gone/deprecated and we need - # to reimplement this part anyway to use `rally task report` - # with a different format - elif any( - not job['result'] and - 'sla' in job and - any(sla['criterion'] == 'something_went_wrong' for sla in job['sla']) - for job in json_result - ): + # If there is any error in the reports + if any(job['tasks'] and + job['tasks'][0]['subtasks'][0]['workloads'][0] + ['data'][0]['error'] for job in json_result): msg = "" tracebacks = "" - for job in json_result: - if 'sla' in job: - for sla in job['sla']: - if sla['criterion'] == 'something_went_wrong': - msg += '\n' + sla['detail'] - log.error('Task failed: {0}'.format( - _msg_enrich(msg.replace('\n', ' '), - msgTrimmer))) + # The real error is in the first result + # and the second element of the array + msg += ('\n' + job['tasks'][0]['subtasks'][0]['workloads'] + [0]['data'][0]['error'][1]) + tracebacks += ('\n' + job['tasks'][0]['subtasks'][0] + ['workloads'][0]['data'][0]['error'][2]) + log.error('Task failed: {0}'.format( + _msg_enrich(msg.replace('\n', ' '), msgTrimmer))) + + return ( + { + 'task': task, + 'scenarios': scenarios, + 'succeed': 0 + }, + "{0}\n{1}\nTask descriptor:\n{2}".format(msg, tracebacks, + task_descriptor) + ) + + # Json error2 + if (any(job['tasks'][0]['subtasks'][0]['workloads'] + [0]['sla_results']['sla'][0]['success'] is False + for job in json_result)): + + msg = "" + tracebacks = "" return ( { @@ -210,8 +228,9 @@ def execute_task(deployment, task, msgTrimmer=None): 'task': task, 'scenarios': scenarios, 'succeed': 1, - 'duration': sum([job['result'][0]['duration'] - for job in json_result]) + 'duration': sum([job['tasks'][0]['subtasks'][0]['workloads'] + [0]['data'][0]['duration'] + for job in json_result]) }, None # No errors ) diff --git a/rallytester/tests/fixtures.py b/rallytester/tests/fixtures.py index 7601c7ba9b4c585c46b4e05e4625d58d11c1c3fd..fd6da735860b4c09b908d19722d2ae7e26352e7a 100644 --- a/rallytester/tests/fixtures.py +++ b/rallytester/tests/fixtures.py @@ -74,299 +74,812 @@ TASK = u""" } """ -TASK_RESULTS = u""" +TASK_REPORT_CRASH = u""" [ { - "hooks": [], - "created_at": "2019-11-04T13:03:36", - "load_duration": 617.198, - "result": [ - { - "timestamp": 1554987817.501209, - "error": [], - "duration": 617.1978030204773, - "output": { - "additive": [], - "complete": [] - }, - "idle_duration": 10.0, - "atomic_actions": { - "cinder_v3.create_volume": 4.72377610206604, - "nova.boot_server": 614.4812738895416, - "nova.delete_server": 7.99204421043396 - } - } - ], - "key": { - "kw": { - "runner": { - "type": "constant", - "concurrency": 1, - "times": 1 - }, - "hooks": [], - "args": { - "volume_size": 1, - "flavor": { - "name": "rally.nano" - }, - "image": { - "name": "Rally CirrOS IPv6" - }, - "force_delete": false - }, - "sla": { - "failure_rate": { - "max": 0 - } - }, - "context": {} - }, - "pos": 0, - "name": "Novaname", - "description": "Bootdesc" + "info": { + "generated_at": "2019-03-05T16:50:31", + "rally_version": "0.0.0", + "format_version": "1.1" }, - "full_duration": 632.436, - "sla": [ + "tasks": [ { - "criterion": "failure_rate", - "detail": "Failure rate criteria'\ - ' 0.00% <= 0.00% <= 0.00% - Passed", - "success": true + "uuid": "5312f2e6-f00d-4a63-9162-40c895cdaea1", + "title": "", + "description": "", + "status": "crashed", + "tags": [], + "created_at": "2019-03-05T15:14:54", + "updated_at": "2019-03-05T15:14:56", + "pass_sla": true, + "subtasks": [] } ] } ] """ -TASK_RESULTS_FAIL = u""" -[ { - "hooks": [], - "created_at": "2019-11-04T13:03:36", - "load_duration": 617.198, - "result": [ - { - "timestamp": 1554987817.501209, - "error": [], - "duration": 617.1978030204773, - "output": { - "additive": [], - "complete": [] - }, - "idle_duration": 10.0, - "atomic_actions": { - "cinder_v3.create_volume": 4.72377610206604, - "nova.boot_server": 614.4812738895416, - "nova.delete_server": 7.99204421043396 - } - } - ], - "key": { - "kw": { - "runner": { - "type": "constant", - "concurrency": 1, - "times": 1 - }, - "hooks": [], - "args": { - "volume_size": 1, - "flavor": { - "name": "rally.nano" - }, - "image": { - "name": "Rally CirrOS IPv6" - }, - "force_delete": false - }, - "sla": { - "failure_rate": { - "max": 0 - } - }, - "context": {} - }, - "pos": 0, - "name": "Name", - "description": "desc" +TASK_REPORT = u""" +[ + { + "info": { + "generated_at": "2020-03-16T16:07:30", + "rally_version": "2.0.0", + "format_version": "1.2" }, - "full_duration": 632.436, - "sla": [ + "tasks": [ { - "criterion": "failure_rate", - "detail": "Failure rate criteria'\ - ' 0.00% <= 0.00% <= 0.00% - Passed", - "success": true + "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d", + "title": "", + "description": "", + "status": "finished", + "tags": [], + "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708", + "env_name": "teststack_cell02", + "created_at": "2020-03-15T03:17:17", + "updated_at": "2020-03-15T03:17:20", + "pass_sla": true, + "subtasks": [ + { + "uuid": "258393e8-54b7-4e40-8344-f8f473101d12", + "title": "GlanceImages.list_images", + "description": "", + "status": "finished", + "created_at": "2020-03-15T03:17:18", + "updated_at": "2020-03-15T03:17:20", + "sla": {}, + "workloads": [ + { + "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48", + "description": "List all images.", + "runner": { + "constant": { + "concurrency": 1, + "times": 1 + } + }, + "hooks": [], + "scenario": { + "GlanceImages.list_images": {} + }, + "min_duration": 0.1442, + "max_duration": 0.1442, + "start_time": 1584242238.24373, + "load_duration": 0.1442, + "full_duration": 0.380141, + "statistics": { + "durations": { + "total": { + "data": { + "success": "100.0%", + "min": 0.144, + "max": 0.144, + "median": 0.144, + "95%ile": 0.144, + "iteration_count": 1, + "avg": 0.144, + "90%ile": 0.144 + }, + "display_name": "total", + "name": "total", + "count_per_iteration": 1, + "children": [ + { + "data": { + "success": "100.0%", + "min": 0.144, + "max": 0.144, + "median": 0.144, + "95%ile": 0.144, + "iteration_count": 1, + "avg": 0.144, + "90%ile": 0.144 + }, + "display_name": "duration", + "name": "duration", + "count_per_iteration": 1, + "children": [] + }, + { + "data": { + "success": "100.0%", + "min": 0.0, + "max": 0.0, + "median": 0.0, + "95%ile": 0.0, + "iteration_count": 1, + "avg": 0.0, + "90%ile": 0.0 + }, + "display_name": + "idle_duration", + "name": "idle_duration", + "count_per_iteration": 1, + "children": [] + } + ] + }, + "atomics": [ + { + "data": { + "success": "100.0%", + "min": 0.144, + "max": 0.144, + "median": 0.144, + "95%ile": 0.144, + "iteration_count": 1, + "avg": 0.144, + "90%ile": 0.144 + }, + "display_name": "Novaname", + "name": "Novaname", + "count_per_iteration": 1, + "children": [] + } + ] + } + }, + "data": [ + { + "timestamp": 1584242238.243732, + "error": [], + "duration": 0.14420008659362793, + "output": { + "additive": [], + "complete": [] + }, + "idle_duration": 0.0, + "atomic_actions": [ + { + "finished_at": + 1584242238.38778, + "started_at": + 1584242238.243803, + "name": "Novaname", + "children": [] + } + ] + } + ], + "failed_iteration_count": 0, + "total_iteration_count": 1, + "created_at": "2020-03-15T03:17:18", + "updated_at": "2020-03-15T03:17:20", + "contexts": {}, + "contexts_results": [ + { + "setup": { + "finished_at": 1584242238.110536, + "started_at": 1584242238.029151, + "atomic_actions": [], + "error": null + }, + "cleanup": { + "finished_at": 1584242238.408476, + "started_at": 1584242238.40758, + "atomic_actions": [], + "error": null + }, + "plugin_cfg": { + "user_choice_method": "random" + }, + "plugin_name": "users@openstack" + } + ], + "position": 0, + "pass_sla": true, + "sla_results": { + "sla": [ + { + "criterion": "failure_rate", + "detail": + "Failure rate criteria\ + 0.00% <= 0.00% <= 0.00%\ + - Passed", + "success": true + } + ] + }, + "sla": { + "failure_rate": { + "max": 0 + } + } + } + ] + } + ] } ] - }, + } +] +""" + +TASK_REPORT_ERROR1 = u""" +[ { - "hooks": [], - "created_at": "2019-11-04T13:03:36", - "load_duration": 617.198, - "result": [ - { - "timestamp": 1554987817.501209, - "error": [ - "-----------", - "Fake error1", - "Fake error2" - ], - "duration": 617.1978030204773, - "output": { - "additive": [], - "complete": [] - }, - "idle_duration": 10.0, - "atomic_actions": { - "cinder_v3.create_volume": 4.72377610206604, - "nova.boot_server": 614.4812738895416, - "nova.delete_server": 7.99204421043396 - } - } - ], - "key": { - "kw": { - "runner": { - "type": "constant", - "concurrency": 1, - "times": 1 - }, - "hooks": [], - "args": { - "volume_size": 1, - "flavor": { - "name": "rally.nano" - }, - "image": { - "name": "Rally CirrOS IPv6" - }, - "force_delete": false - }, - "sla": { - "failure_rate": { - "max": 0 - } - }, - "context": {} - }, - "pos": 0, - "name": "name", - "description": "des" + "info": { + "generated_at": "2020-03-16T16:07:30", + "rally_version": "2.0.0", + "format_version": "1.2" }, - "full_duration": 632.436, - "sla": [ + "tasks": [ { - "criterion": "failure_rate", - "detail": "Failure rate criteria'\ - ' 0.00% <= 0.00% <= 0.00% - Passed", - "success": true + "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d", + "title": "", + "description": "", + "status": "finished", + "tags": [], + "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708", + "env_name": "teststack_cell02", + "created_at": "2020-03-15T03:17:17", + "updated_at": "2020-03-15T03:17:20", + "pass_sla": true, + "subtasks": [ + { + "uuid": "258393e8-54b7-4e40-8344-f8f473101d12", + "title": "GlanceImages.list_images", + "description": "", + "status": "finished", + "created_at": "2020-03-15T03:17:18", + "updated_at": "2020-03-15T03:17:20", + "sla": {}, + "workloads": [ + { + "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48", + "description": "List all images.", + "runner": { + "constant": { + "concurrency": 1, + "times": 1 + } + }, + "hooks": [], + "scenario": { + "GlanceImages.list_images": {} + }, + "min_duration": 0.1442, + "max_duration": 0.1442, + "start_time": 1584242238.24373, + "load_duration": 0.1442, + "full_duration": 0.380141, + "statistics": { + "durations": { + "total": { + "data": { + "success": "0.0%", + "min": 0.144, + "max": 0.144, + "median": 0.144, + "95%ile": 0.144, + "iteration_count": 1, + "avg": 0.144, + "90%ile": 0.144 + }, + "display_name": "total", + "name": "total", + "count_per_iteration": 1, + "children": [ + { + "data": { + "success": "0.0%", + "min": 0.144, + "max": 0.144, + "median": 0.144, + "95%ile": 0.144, + "iteration_count": 1, + "avg": 0.144, + "90%ile": 0.144 + }, + "display_name": "duration", + "name": "duration", + "count_per_iteration": 1, + "children": [] + }, + { + "data": { + "success": "0.0%", + "min": 0.0, + "max": 0.0, + "median": 0.0, + "95%ile": 0.0, + "iteration_count": 1, + "avg": 0.0, + "90%ile": 0.0 + }, + "display_name": + "idle_duration", + "name": "idle_duration", + "count_per_iteration": 1, + "children": [] + } + ] + }, + "atomics": [ + { + "data": { + "success": "0.0%", + "min": 0.144, + "max": 0.144, + "median": 0.144, + "95%ile": 0.144, + "iteration_count": 1, + "avg": 0.144, + "90%ile": 0.144 + }, + "display_name": "Novaname", + "name": "Novaname", + "count_per_iteration": 1, + "children": [] + } + ] + } + }, + "data": [ + { + "timestamp": 1584242238.243732, + "error": [], + "duration": 0.14420008659362793, + "output": { + "additive": [], + "complete": [] + }, + "idle_duration": 0.0, + "atomic_actions": [ + { + "finished_at": + 1584242238.38778, + "started_at": + 1584242238.243803, + "name": "Novaname", + "children": [] + } + ] + } + ], + "failed_iteration_count": 0, + "total_iteration_count": 1, + "created_at": "2020-03-15T03:17:18", + "updated_at": "2020-03-15T03:17:20", + "contexts": {}, + "contexts_results": [ + { + "setup": { + "finished_at": 1584242238.110536, + "started_at": 1584242238.029151, + "atomic_actions": [], + "error": null + }, + "cleanup": { + "finished_at": 1584242238.408476, + "started_at": 1584242238.40758, + "atomic_actions": [], + "error": null + }, + "plugin_cfg": { + "user_choice_method": "random" + }, + "plugin_name": "users@openstack" + } + ], + "position": 0, + "pass_sla": true, + "sla_results": { + "sla": [ + { + "criterion": "failure_rate", + "detail": "Fake error - FAIL", + "success": false + } + ] + }, + "sla": { + "failure_rate": { + "max": 0 + } + } + } + ] + } + ] } ] } ] """ -TASK_RESULTS_FAIL2 = u""" +TASK_REPORT_ERROR2 = u""" [ - { - "hooks": [], - "created_at": "2019-11-04T13:03:36", - "load_duration": 617.198, - "result": [ + { + "info": { + "generated_at": "2020-03-16T16:07:30", + "rally_version": "2.0.0", + "format_version": "1.2" + }, + "tasks": [ { - "timestamp": 1554987817.501209, - "error": [], - "duration": 617.1978030204773, - "output": { - "additive": [], - "complete": [] - }, - "idle_duration": 10.0, - "atomic_actions": { - "cinder_v3.create_volume": 4.72377610206604, - "nova.boot_server": 614.4812738895416, - "nova.delete_server": 7.99204421043396 - } - } - ], - "key": { - "kw": { - "runner": { - "type": "constant", - "concurrency": 1, - "times": 1 - }, - "hooks": [], - "args": { - "volume_size": 1, - "flavor": { - "name": "rally.nano" - }, - "image": { - "name": "Rally CirrOS IPv6" - }, - "force_delete": false - }, - "sla": { - "failure_rate": { - "max": 0 + "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d", + "title": "", + "description": "", + "status": "finished", + "tags": [], + "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708", + "env_name": "teststack_cell02", + "created_at": "2020-03-15T03:17:17", + "updated_at": "2020-03-15T03:17:20", + "pass_sla": true, + "subtasks": [ + { + "uuid": "258393e8-54b7-4e40-8344-f8f473101d12", + "title": "GlanceImages.list_images", + "description": "", + "status": "finished", + "created_at": "2020-03-15T03:17:18", + "updated_at": "2020-03-15T03:17:20", + "sla": {}, + "workloads": [ + { + "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48", + "description": "List all images.", + "runner": { + "constant": { + "concurrency": 1, + "times": 1 + } + }, + "hooks": [], + "scenario": { + "GlanceImages.list_images": {} + }, + "min_duration": 0.1442, + "max_duration": 0.1442, + "start_time": 1584242238.24373, + "load_duration": 0.1442, + "full_duration": 0.380141, + "statistics": { + "durations": { + "total": { + "data": { + "success": "100.0%", + "min": 0.144, + "max": 0.144, + "median": 0.144, + "95%ile": 0.144, + "iteration_count": 1, + "avg": 0.144, + "90%ile": 0.144 + }, + "display_name": "total", + "name": "total", + "count_per_iteration": 1, + "children": [ + { + "data": { + "success": "100.0%", + "min": 0.144, + "max": 0.144, + "median": 0.144, + "95%ile": 0.144, + "iteration_count": 1, + "avg": 0.144, + "90%ile": 0.144 + }, + "display_name": "duration", + "name": "duration", + "count_per_iteration": 1, + "children": [] + }, + { + "data": { + "success": "100.0%", + "min": 0.0, + "max": 0.0, + "median": 0.0, + "95%ile": 0.0, + "iteration_count": 1, + "avg": 0.0, + "90%ile": 0.0 + }, + "display_name": + "idle_duration", + "name": + "idle_duration", + "count_per_iteration": 1, + "children": [] + } + ] + }, + "atomics": [ + { + "data": { + "success": "100.0%", + "min": 0.144, + "max": 0.144, + "median": 0.144, + "95%ile": 0.144, + "iteration_count": 1, + "avg": 0.144, + "90%ile": 0.144 + }, + "display_name": "Novaname", + "name": "Novaname", + "count_per_iteration": 1, + "children": [] + } + ] + } + }, + "data": [ + { + "timestamp": 1584242238.243732, + "error": [ + "-----------", + "Fake error1", + "Fake error2" + ], + "duration": 0.14420008659362793, + "output": { + "additive": [], + "complete": [] + }, + "idle_duration": 0.0, + "atomic_actions": [ + { + "finished_at": + 1584242238.38778, + "started_at": + 1584242238.243803, + "name": "Novaname", + "children": [] + } + ] + } + ], + "failed_iteration_count": 0, + "total_iteration_count": 1, + "created_at": "2020-03-15T03:17:18", + "updated_at": "2020-03-15T03:17:20", + "contexts": {}, + "contexts_results": [ + { + "setup": { + "finished_at": 1584242238.110536, + "started_at": 1584242238.029151, + "atomic_actions": [], + "error": null + }, + "cleanup": { + "finished_at": 1584242238.408476, + "started_at": 1584242238.40758, + "atomic_actions": [], + "error": null + }, + "plugin_cfg": { + "user_choice_method": "random" + }, + "plugin_name": "users@openstack" + } + ], + "position": 0, + "pass_sla": true, + "sla_results": { + "sla": [ + { + "criterion": "failure_rate", + "detail": + "Failure rate criteria\ + 0.00% <= 0.00% <= 0.00%\ + - Passed", + "success": true + } + ] + }, + "sla": { + "failure_rate": { + "max": 0 + } + } + } + ] } - }, - "context": {} - }, - "pos": 0, - "name": "NovaServers", - "description": "Boot a server" - }, - "full_duration": 632.436 - }, + ] + } + ] + } +] +""" + +TASK_REPORT_MAGNUM = u""" +[ { - "hooks": [], - "created_at": "2019-11-04T13:03:36", - "load_duration": 617.198, - "result": [], - "key": { - "kw": { - "runner": { - "type": "constant", - "concurrency": 1, - "times": 1 - }, - "hooks": [], - "args": { - "volume_size": 1, - "flavor": { - "name": "rally.nano" - }, - "image": { - "name": "Rally CirrOS IPv6" - }, - "force_delete": false - }, - "sla": { - "failure_rate": { - "max": 0 - } - }, - "context": {} - }, - "pos": 0, - "name": "Novaname", - "description": "Bootdesc" + "info": { + "generated_at": "2020-03-16T16:07:30", + "rally_version": "2.0.0", + "format_version": "1.2" }, - "full_duration": 632.436, - "sla": [ + "tasks": [ { - "criterion": "failure_rate", - "detail": "Failure rate criteria'\ - ' 0.00% <= 0.00% <= 0.00% - Passed", - "success": true - }, - { - "criterion": "something_went_wrong", - "detail": "Frc", - "success": false + "uuid": "8d4132ab-3530-4148-9ee3-d5bf6621aa9d", + "title": "", + "description": "", + "status": "finished", + "tags": [], + "env_uuid": "62eb0285-247a-4b22-aecf-b1c93d757708", + "env_name": "teststack_cell02", + "created_at": "2020-03-15T03:17:17", + "updated_at": "2020-03-15T03:17:20", + "pass_sla": true, + "subtasks": [ + { + "uuid": "258393e8-54b7-4e40-8344-f8f473101d12", + "title": "GlanceImages.list_images", + "description": "", + "status": "finished", + "created_at": "2020-03-15T03:17:18", + "updated_at": "2020-03-15T03:17:20", + "sla": {}, + "workloads": [ + { + "uuid": "609a426a-9ea9-4738-aee7-ea8c3bda6d48", + "description": "List all images.", + "runner": { + "constant": { + "concurrency": 1, + "times": 1 + } + }, + "hooks": [], + "scenario": { + "GlanceImages.list_images": {} + }, + "min_duration": 0.1442, + "max_duration": 0.1442, + "start_time": 1584242238.24373, + "load_duration": 0.1442, + "full_duration": 0.380141, + "statistics": { + "durations": { + "total": { + "data": { + "success": "100.0%", + "min": 0.144, + "max": 0.144, + "median": 0.144, + "95%ile": 0.144, + "iteration_count": 1, + "avg": 0.144, + "90%ile": 0.144 + }, + "display_name": "total", + "name": "total", + "count_per_iteration": 1, + "children": [ + { + "data": { + "success": "100.0%", + "min": 0.144, + "max": 0.144, + "median": 0.144, + "95%ile": 0.144, + "iteration_count": 1, + "avg": 0.144, + "90%ile": 0.144 + }, + "display_name": "duration", + "name": "duration", + "count_per_iteration": 1, + "children": [] + }, + { + "data": { + "success": "100.0%", + "min": 0.0, + "max": 0.0, + "median": 0.0, + "95%ile": 0.0, + "iteration_count": 1, + "avg": 0.0, + "90%ile": 0.0 + }, + "display_name": + "idle_duration", + "name": "idle_duration", + "count_per_iteration": 1, + "children": [] + } + ] + }, + "atomics": [ + { + "data": { + "success": "100.0%", + "min": 0.144, + "max": 0.144, + "median": 0.144, + "95%ile": 0.144, + "iteration_count": 1, + "avg": 0.144, + "90%ile": 0.144 + }, + "display_name": "Novaname", + "name": "Novaname", + "count_per_iteration": 1, + "children": [] + } + ] + } + }, + "data": [ + { + "timestamp": 1584242238.243732, + "error": [ + "Fake error_M" + ], + "duration": 0.14420008659362793, + "output": { + "additive": [], + "complete": [] + }, + "idle_duration": 0.0, + "atomic_actions": [ + { + "finished_at": + 1584242238.38778, + "started_at": + 1584242238.243803, + "name": "Novaname", + "children": [] + } + ] + } + ], + "failed_iteration_count": 0, + "total_iteration_count": 1, + "created_at": "2020-03-15T03:17:18", + "updated_at": "2020-03-15T03:17:20", + "contexts": {}, + "contexts_results": [ + { + "setup": { + "finished_at": 1584242238.110536, + "started_at": 1584242238.029151, + "atomic_actions": [], + "error": null + }, + "cleanup": { + "finished_at": 1584242238.408476, + "started_at": 1584242238.40758, + "atomic_actions": [], + "error": null + }, + "plugin_cfg": { + "user_choice_method": "random" + }, + "plugin_name": "users@openstack" + } + ], + "position": 0, + "pass_sla": true, + "sla_results": { + "sla": [ + { + "criterion": + "something_went_wrong", + "detail": + "Fake detail example", + "success": false + } + ] + }, + "sla": { + "failure_rate": { + "max": 0 + } + } + } + ] + } + ] } ] } diff --git a/rallytester/tests/test_rallytester.py b/rallytester/tests/test_rallytester.py index 89a6b8fbf7b4beb4d3bd38af94ea8420a742a598..14f319bfbd1c4825115430e6398bc8e202724772 100644 --- a/rallytester/tests/test_rallytester.py +++ b/rallytester/tests/test_rallytester.py @@ -282,7 +282,7 @@ class TestRallytester(unittest.TestCase): None ), ( - fixtures.TASK_RESULTS, + fixtures.TASK_REPORT, None ) ] @@ -293,7 +293,7 @@ class TestRallytester(unittest.TestCase): 'task': 'fake_task', 'scenarios': 'Novaname', 'succeed': 1, - 'duration': 617.1978030204773, + 'duration': 0.14420008659362793, }, None # No errors ), @@ -319,14 +319,14 @@ class TestRallytester(unittest.TestCase): None ), ( - fixtures.TASK_RESULTS_FAIL, + fixtures.TASK_REPORT_ERROR2, None ) ] self.assertEqual( ( - {'task': 'fake_task', 'scenarios': 'Name,name', 'succeed': 0}, + {'task': 'fake_task', 'scenarios': 'Novaname', 'succeed': 0}, '\nFake error1\n\nFake error2\n' 'Task descriptor:\n' '\n' @@ -365,7 +365,7 @@ class TestRallytester(unittest.TestCase): None ), ( - fixtures.TASK_RESULTS_FAIL2, + fixtures.TASK_REPORT_ERROR1, None ) ] @@ -374,10 +374,10 @@ class TestRallytester(unittest.TestCase): ( { 'task': 'fake_task', - 'scenarios': 'NovaServers,Novaname', + 'scenarios': 'Novaname', 'succeed': 0, }, - '\nFrc\n\nTask descriptor:\n\n' + '\n\nTask descriptor:\n\n' '{\n' ' "NovaServers.list_servers": [\n' ' {\n' @@ -440,3 +440,102 @@ class TestRallytester(unittest.TestCase): " Image '{\"name\": \"my_image\"}' not found", rallytester.REASONS_PATTERN) ) + + @mock.patch('rallytester.rallytester.subprocess.Popen') + @mock.patch(func, return_value=io.StringIO(fixtures.TASK)) + @mock.patch('os.path.exists', return_value=True) + def test_execute_task_crash(self, + mock_os, + mock_open, + mock_sub): + mock_pr = mock_sub.return_value + type(mock_pr).returncode = mock.PropertyMock(return_value=0) + + mock_pr.communicate.side_effect = [ + ( + 'Using task: 5312f2e6-f00d-4a63-9162-40c895cdaea1', + None + ), + ( + fixtures.TASK_REPORT_CRASH, + None + ) + ] + + self.assertEqual( + ( + [ + { + 'info': + { + 'generated_at': '2019-03-05T16:50:31', + 'rally_version': '0.0.0', + 'format_version': '1.1' + }, + 'tasks': + [ + { + 'uuid': '5312f2e6-f00d-4a63-9162-40c895cdaea1', + 'title': '', + 'description': '', + 'status': 'crashed', + 'tags': [], + 'created_at': '2019-03-05T15:14:54', + 'updated_at': '2019-03-05T15:14:56', + 'pass_sla': True, + 'subtasks': [] + } + ], + } + ] + ), + rallytester.execute_task( + deployment='fake_deployment', + task='fake_task' + ) + ) + + @mock.patch('rallytester.rallytester.subprocess.Popen') + @mock.patch(func, return_value=io.StringIO(fixtures.TASK)) + @mock.patch('os.path.exists', return_value=True) + def test_execute_task_magnumfail(self, + mock_os, + mock_open, + mock_sub): + mock_pr = mock_sub.return_value + type(mock_pr).returncode = mock.PropertyMock(return_value=0) + + mock_pr.communicate.side_effect = [ + ( + 'Using task: 8d4132ab-3530-4148-9ee3-d5bf6621aa9d', + None + ), + ( + fixtures.TASK_REPORT_MAGNUM, + None + ) + ] + + self.assertEqual( + ( + { + 'scenarios': 'Novaname', + 'task': 'fake_task', + 'succeed': 0 + }, + '\nFake detail example\n' + '\nTask descriptor:\n\n' + '{\n "NovaServers.list_servers": ' + '[\n {\n "args": ' + '{\n "detailed": false' + '\n },\n "runner": ' + '{\n "type": "constant",' + '\n "times": 1,\n ' + ' "concurrency": 1\n }' + ',\n "context": {}\n }\n ]\n}\n' + ), + rallytester.execute_task( + deployment='fake_deployment', + task='fake_task' + ) + )