Commit 5f3879ee authored by Domenico Giordano's avatar Domenico Giordano
Browse files

reduce size of dataset

parent 229c115a
......@@ -56,11 +56,11 @@
#output dir in user HDFS area
outbasepath="test_rally_errors_ipynb"
#output base file name
outbasefile="rally_errors.parquet"
#input file path with data to process with spark
inbasepath="/project/monitoring/archive/openstack/logs/generic/rallytester/2021/0*/01"
inbasepath="/project/monitoring/archive/openstack/logs/generic/rallytester/2021/0[2-3]/01"
#schema file
schemafile='rally_schema.json'
```
......
......@@ -127,7 +127,7 @@ def run_tests():
# output dir in user HDFS area
outbasepath = "test_rally_errors_py"
# input file path with data to process with spark
inbasepath = "/project/monitoring/archive/openstack/logs/generic/rallytester/2021/0*/0[1-2]" # noqa: E501
inbasepath = "/project/monitoring/archive/openstack/logs/generic/rallytester/2021/02/0[1-2]" # noqa: E501
# schema file
schemafile = 'rally_schema.json'
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment