diff --git a/property_pull.py b/property_pull.py index 33ce127a8bda4e5c425a5eac978c8daf69cbbe16..cd29379b369d0a4426e0e6fc46c79d983a39431a 100644 --- a/property_pull.py +++ b/property_pull.py @@ -18,8 +18,6 @@ import numpy as np import pandas as pd _dest_dir = './pulledData' -_files = False -_db_write = True # mysqlDB variables mydb = mysql.connector.connect( @@ -52,8 +50,8 @@ sums_cols = udf(lambda arr: 0 if arr == [] else sum(arr), IntegerType()) ''' -def write_file(_device, _start_time, _end_time, _property, data): - if _files: +def write_file(_device, _start_time, _end_time, _property, data, _files_write): + if _files_write: # Writing output # with open(_dest_dir + '/' + _device + '_' + _start_time.replace(' ', '_') + 'to' + _end_time.replace(' ', '_') + '_' + _property + '.dat', 'w') as outfile: @@ -75,7 +73,7 @@ def uncompress_pandas(_data_array, _index_array, _size, _nrows): # noinspection SqlResolve -def pull_histogram_pandas(_fill_name, _mode, _device, _start_time, _end_time): +def pull_histogram_pandas(_fill_name, _mode, _device, _start_time, _end_time, _files_write=False, _db_write=True): _property = 'LoggingHistogram' data = {} @@ -150,11 +148,13 @@ def pull_histogram_pandas(_fill_name, _mode, _device, _start_time, _end_time): db_cursor = mydb.cursor() db_cursor.execute('UPDATE Data SET histogram = %s where id = %s', (json.dumps(data), data_id,)) - write_file(_device, _start_time, _end_time, _property, data) + write_file(_device, _start_time, _end_time, _property, data, _files_write) + + return data # noinspection SqlResolve -def pull_integral_pandas(_fill_name, _mode, _device, _start_time, _end_time): +def pull_integral_pandas(_fill_name, _mode, _device, _start_time, _end_time, _files_write=False, _db_write=True): _property = "LoggingIntegral" data = {} @@ -233,11 +233,13 @@ def pull_integral_pandas(_fill_name, _mode, _device, _start_time, _end_time): db_cursor = mydb.cursor() db_cursor.execute('UPDATE Data SET integral = %s where id = %s', (json.dumps(data), data_id,)) - write_file(_device, _start_time, _end_time, _property, data) + write_file(_device, _start_time, _end_time, _property, data, _files_write) + + return data # noinspection SqlResolve -def pull_raw_dist_pandas(_fill_name, _mode, _device, _start_time, _end_time): +def pull_raw_dist_pandas(_fill_name, _mode, _device, _start_time, _end_time, _files_write=False, _db_write=True): _property = "LoggingRawDist" data = {} @@ -305,11 +307,13 @@ def pull_raw_dist_pandas(_fill_name, _mode, _device, _start_time, _end_time): db_cursor = mydb.cursor() db_cursor.execute('UPDATE Data SET raw_dist = %s where id = %s', (json.dumps(data), data_id,)) - write_file(_device, _start_time, _end_time, _property, data) + write_file(_device, _start_time, _end_time, _property, data, _files_write) + + return data # noinspection SqlResolve -def pull_integral_dist_pandas(_fill_name, _mode, _device, _start_time, _end_time): +def pull_integral_dist_pandas(_fill_name, _mode, _device, _start_time, _end_time, _files_write=False, _db_write=True): _property = "LoggingIntegralDist" data = {} @@ -381,11 +385,13 @@ def pull_integral_dist_pandas(_fill_name, _mode, _device, _start_time, _end_time db_cursor = mydb.cursor() db_cursor.execute('UPDATE Data SET integral_dist = %s where id = %s', (json.dumps(data), data_id,)) - write_file(_device, _start_time, _end_time, _property, data) + write_file(_device, _start_time, _end_time, _property, data, _files_write) + + return data # noinspection SqlResolve -def pull_turnloss_pandas(_fill_name, _mode, _device, _start_time, _end_time): +def pull_turnloss_pandas(_fill_name, _mode, _device, _start_time, _end_time, _files_write=False, _db_write=True): _property = "LoggingTurnLoss" data = {} @@ -463,25 +469,24 @@ def pull_turnloss_pandas(_fill_name, _mode, _device, _start_time, _end_time): db_cursor = mydb.cursor() db_cursor.execute('UPDATE Data SET turnloss = %s where id = %s', (json.dumps(data), data_id,)) - write_file(_device, _start_time, _end_time, _property, data) + write_file(_device, _start_time, _end_time, _property, data, _files_write) + return data -def property_pull(_fill_name, _mode, _device, _start, _end, _files_input=False, _db_write_input=True): - global _files - _files = _files_input - global _db_write - _db_write = _db_write_input +def property_pull(_fill_name, _mode, _device, _start, _end, _files_write=False, _db_write=True): gcursor = mydb.cursor() gcursor.execute('select count(*) from Mode where name=%s', (_mode,)) mode_exists = gcursor.fetchone()[0] if mode_exists == 0: - print('Mode ' + _mode + ' ') + print('Mode ' + _mode + ' unknown ') gcursor.execute('SELECT id FROM Device where name=%s limit 1', (_device,)) device_id = gcursor.fetchone()[0] + if device_id is None: + print('Device ' + _device + ' unknown') gcursor.execute("select count(*) from Measure where device_id=%s " "and fill_mode_id in ( select id from FillMode " @@ -514,11 +519,11 @@ def property_pull(_fill_name, _mode, _device, _start, _end, _files_input=False, 'VALUES ( %s, %s, %s )', (fill_mode_id, data_id, device_id)) - pull_histogram_pandas(_fill_name, _mode, _device, _start, _end) - pull_integral_pandas(_fill_name, _mode, _device, _start, _end) - pull_raw_dist_pandas(_fill_name, _mode, _device, _start, _end) - pull_integral_dist_pandas(_fill_name, _mode, _device, _start, _end) - pull_turnloss_pandas(_fill_name, _mode, _device, _start, _end) + pull_histogram_pandas(_fill_name, _mode, _device, _start, _end, _files_write, _db_write) + pull_integral_pandas(_fill_name, _mode, _device, _start, _end, _files_write, _db_write) + pull_raw_dist_pandas(_fill_name, _mode, _device, _start, _end, _files_write, _db_write) + pull_integral_dist_pandas(_fill_name, _mode, _device, _start, _end, _files_write, _db_write) + pull_turnloss_pandas(_fill_name, _mode, _device, _start, _end, _files_write, _db_write) mydb.commit() print('Pulled done, data pushed on DB for fill '