Commit 03f2d049 authored by Domenico Giordano's avatar Domenico Giordano
Browse files

support yaml. remove not needed code

parent 55c629ac
......@@ -9,6 +9,7 @@ echo "ad_config values after jinja rendering $ad_config"
cat > prepare_json_config_from_template.py << 'EOF'
import json
import sys
import yaml
ad_config_from_dag = json.loads(sys.argv[1])
ad_config_ref_file = ad_config_from_dag["template_config_file"]
......@@ -16,8 +17,8 @@ ad_config_ref_file = ad_config_from_dag["template_config_file"]
print("ad_config_ref_file %s" % ad_config_ref_file)
print("ad_config_from_dag dict %s" % ad_config_from_dag)
with open(ad_config_ref_file,"r") as f:
ad_config_ref_dict = json.loads(f.read())
with open(ad_config_ref_file, "r", encoding="utf-8") as f:
ad_config_ref_dict = yaml.safe_load(f.read())
f.close()
print(ad_config_ref_dict)
......@@ -62,32 +63,11 @@ klist -c {{var.value.KRB5CCNAME}};
local_cache_folder=`python -c "import json; f=open(\"ad_config_file.json\"); print(json.loads(f.read())[\"local_cache_folder\"])"`
echo "local_cache_folder is $local_cache_folder"
mkdir -p $local_cache_folder;
[[ "$?" -ne 0 ]] && exit 1;
#mkdir -p $local_cache_folder;
#[[ "$?" -ne 0 ]] && exit 1;
source set_spark_analytix.sh;
local_cache_folder=`python -c "import json; f=open(\"ad_config_file.json\"); print(json.loads(f.read())[\"local_cache_folder\"])"`
echo -e "\nlocal_cache_folder is $local_cache_folder"
mkdir -p $local_cache_folder;
[[ "$?" -ne 0 ]] && exit 1;
hdfs_out_folder=`python -c "import json; f=open(\"ad_config_file.json\"); print(json.loads(f.read())[\"hdfs_out_folder\"])"`
echo -e "\nhdfs_out_folder is $hdfs_out_folder"
hdfs dfs -mkdir -p $hdfs_out_folder;
[[ "$?" -ne 0 ]] && exit 1;
hdfs_cache_folder=`python -c "import json; f=open(\"ad_config_file.json\"); print(json.loads(f.read())[\"hdfs_cache_folder\"])"`
echo -e "\nhdfs_cache_folder is $hdfs_cache_folder"
hdfs dfs -mkdir -p $hdfs_cache_folder;
[[ "$?" -ne 0 ]] && exit 1;
normalization_out_folder=`python -c "import json; f=open(\"ad_config_file.json\"); print(json.loads(f.read())[\"normalization_out_folder\"])"`
echo -e "\nnormalization_out_folder is $normalization_out_folder"
hdfs dfs -mkdir -p $normalization_out_folder;
[[ "$?" -ne 0 ]] && exit 1;
whereis data_mining
{{params.ad_task}};
EOF
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment