Commit 2da98fe2 authored by Domenico Giordano's avatar Domenico Giordano
Browse files

change destination hdfs folder

parent 5f3879ee
......@@ -52,11 +52,11 @@
``` python
#Configuration PARAMS
#Define configuration parameters
#output dir in user HDFS area
outbasepath="test_rally_errors_ipynb"
outbasepath="/project/it_cloud_data_analytics/CI/test_rally_errors_ipynb"
#output base file name
outbasefile="rally_errors.parquet"
#input file path with data to process with spark
inbasepath="/project/monitoring/archive/openstack/logs/generic/rallytester/2021/0[2-3]/01"
#schema file
......
......@@ -125,7 +125,7 @@ def run_tests():
# Define configuration parameters
# ################################
# output dir in user HDFS area
outbasepath = "test_rally_errors_py"
outbasepath = "/project/it_cloud_data_analytics/CI/test_rally_errors_py"
# input file path with data to process with spark
inbasepath = "/project/monitoring/archive/openstack/logs/generic/rallytester/2021/02/0[1-2]" # noqa: E501
# schema file
......
......@@ -47,7 +47,7 @@ hdfs dfs -ls /project/monitoring || fail 'test access to hdfs'
echo -e "\n@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
echo -e "\ntest python pyspark module\n"
echo -e "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n"
outbasepath="test_rally_errors_py"
outbasepath="/project/it_cloud_data_analytics/CI/test_rally_errors_py"
hdfs dfs -test -d $outbasepath; [[ "$?" -eq "0" ]] && echo "have to remove folder $outbasepath" && hdfs dfs -rm -r $outbasepath
python3 ${WORK_DIR}/test_spark_connector.py
......@@ -56,6 +56,6 @@ python3 ${WORK_DIR}/test_spark_connector.py
#echo -e "\ntest jupyter notebook\n"
jupyter nbconvert --version
#echo -e "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n"
outbasepath="test_rally_errors_ipynb"
outbasepath="/project/it_cloud_data_analytics/CI/test_rally_errors_ipynb"
hdfs dfs -test -d $outbasepath; [[ "$?" -eq "0" ]] && echo "have to remove folder $outbasepath" && hdfs dfs -rm -r $outbasepath
jupyter nbconvert --ExecutePreprocessor.timeout=600 --to notebook --execute ${WORK_DIR}/test_notebook_spark_connector.ipynb
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment