removed bug with mismatching datatypes

src/spark/get_data.py

@@ -136,10 +136,11 @@ class DataProcessor(ABC):
             # resample to seconds
             seconds = [t1 + step * x for x in range(int((t2 - t1) / step) + 1)]
             time = [t for t in seconds if t in ts]
-            print(seconds, file=sys.stderr)
             # only keep ipoc timestamps
             return time
-
+        
+        # map timestamps to int because otherwise data_range filters all entries becuase int is not in list of strings
+        timestamps = [int(t) for t in timestamps]
         # udf to generate upsampled timestamps
         date_range_udf = udf(lambda x, y: date_range(x, y, timestamps, step), ArrayType(LongType()))
         # get upper limit of timestamps for final timestamp in dataframe (handles edge case for last timestamp)
@@ -155,7 +156,6 @@ class DataProcessor(ABC):
         # join dataframes on timestamp index (rounded to nearest second)
         self.data = self.data.dropDuplicates([_dfindex])
         # Remove old timestamps if they did not align with ipoc timestamps
-
         self.data = self.data.where(f.col(_dfindex).isin(timestamps))
 
     @staticmethod

[Kyle Bringmans]

• this fixed the bug which caused all rows to be thrown away
• works on data from 1 month
• testing for 1 year (2017)

[Kyle Bringmans]

• Get core dump for resample of 1 year

• 1 month gives BroadcastTimeout -> rerunning with timeout turned off

[Kyle Bringmans]

Retrieved 9 months of data (in 1 month segments)