Commit 229c115a authored by Domenico Giordano's avatar Domenico Giordano
Browse files

check empty df

parent 914b0b11
...@@ -97,7 +97,11 @@ def getSparkDF(inpath, outpath, spark, schemafile): ...@@ -97,7 +97,11 @@ def getSparkDF(inpath, outpath, spark, schemafile):
output_df = regex_df output_df = regex_df
# Let's now write to the output dir # Let's now write to the output dir
cluster_utils.write_spark_df(output_df, outpath) df_empty = (len(output_df.head(1))==0)
print("Is the dataframe empty? %s" % df_empty)
if not df_empty:
print("Saving dataframe")
cluster_utils.write_spark_df(output_df,outpath)
return output_df return output_df
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment