From 4faa1a6d81530cf61f0aebd4fbe789098202ae0f Mon Sep 17 00:00:00 2001
From: Quentin Codelupi <quentin.codelupi@cern.ch>
Date: Wed, 24 Jul 2019 17:44:36 +0200
Subject: [PATCH] [fix] files output option + cases of collect empty
---
property-pull.py | 157 ++++++++++++++++++++++++++++++++---------------
1 file changed, 109 insertions(+), 48 deletions(-)
diff --git a/property-pull.py b/property-pull.py
index ea23c8b..a488b95 100644
--- a/property-pull.py
+++ b/property-pull.py
@@ -100,8 +100,14 @@ def pull_histogram(_start_time, _end_time, _device):
data['rows'] = array_dataset_hg.count()
data['startTime'] = _start_time
data['endTime'] = _end_time
- data['treshold_hg'] = df.select('threshold_hg').collect()[0]
- data['treshold_lg'] = df.select('threshold_lg').collect()[0]
+
+ data['treshold_hg'] = df.select('threshold_hg').collect()
+ if len(data['treshold_hg']) > 0:
+ data['treshold_hg'] = data['treshold_hg'][0]
+
+ data['treshold_lg'] = df.select('threshold_lg').collect()
+ if len(data['treshold_lg']) > 0:
+ data['treshold_lg'] = data['treshold_lg'][0]
# Timestamp #
time_stamp = df.select("__record_timestamp__")
@@ -134,12 +140,12 @@ def pull_histogram(_start_time, _end_time, _device):
db_cursor = mydb.cursor()
db_cursor.execute('UPDATE data SET histogram = %s where id = %s', (json.dumps(data), data_id,))
- # Writing output
- with open(_dest_dir + '/' + _device + '_' + _start_time.replace(' ', '_') + 'to'
- + _end_time.replace(' ', '_') + '_' + _property + '.dat', 'w') as outfile:
- json.dump(data, outfile)
-
if _files:
+ # Writing output
+ with open(_dest_dir + '/' + _device + '_' + _start_time.replace(' ', '_') + 'to'
+ + _end_time.replace(' ', '_') + '_' + _property + '.dat', 'w') as outfile:
+ json.dump(data, outfile)
+
print(_dest_dir + '/' + _device + '_' + _start_time.replace(' ', '_') + 'to'
+ _end_time.replace(' ', '_') + '_' + _property + '.dat' + " Pulled")
@@ -179,11 +185,26 @@ def pull_integral(_start_time, _end_time, _device):
data['rows'] = df.count()
data['startTime'] = _start_time
data['endTime'] = _end_time
- data['zsthr_hg'] = df.select('zsthr_hg').collect()[0]
- data['zsthrave_hg'] = df.select('zsthrave_hg').collect()[0]
- data['zsthr_lg'] = df.select('zsthr_lg').collect()[0]
- data['zsthrave_lg'] = df.select('zsthrave_lg').collect()[0]
- data['baselinesub'] = df.select('baselinesub').collect()[0]
+
+ data['zsthr_hg'] = df.select('zsthr_hg').collect()
+ if len(data['zsthr_hg']) > 0:
+ data['zsthr_hg'] = data['zsthr_hg'][0]
+
+ data['zsthrave_hg'] = df.select('zsthrave_hg').collect()
+ if len(data['zsthrave_hg']) > 0:
+ data['zsthrave_hg'] = data['zsthrave_hg'][0]
+
+ data['zsthr_lg'] = df.select('zsthr_lg').collect()
+ if len(data['zsthr_lg']) > 0:
+ data['zsthr_lg'] = data['zsthr_lg'][0]
+
+ data['zsthrave_lg'] = df.select('zsthrave_lg').collect()
+ if len(data['zsthrave_lg']) > 0:
+ data['zsthrave_lg'] = data['zsthrave_lg'][0]
+
+ data['baselinesub'] = df.select('baselinesub').collect()
+ if len(data['baselinesub']) > 0:
+ data['baselinesub'] = data['baselinesub'][0]
# Intime Data #
result_intime_hg = intime_array_hg.collect()
@@ -222,12 +243,12 @@ def pull_integral(_start_time, _end_time, _device):
db_cursor = mydb.cursor()
db_cursor.execute('UPDATE data SET integral = %s where id = %s', (json.dumps(data), data_id,))
- # Writing output #
- with open(_dest_dir + '/' + _device + '_' + _start_time.replace(' ', '_') + 'to'
- + _end_time.replace(' ', '_') + '_' + _property + '.dat', 'w') as outfile:
- json.dump(data, outfile)
-
if _files:
+ # Writing output #
+ with open(_dest_dir + '/' + _device + '_' + _start_time.replace(' ', '_') + 'to'
+ + _end_time.replace(' ', '_') + '_' + _property + '.dat', 'w') as outfile:
+ json.dump(data, outfile)
+
print(_dest_dir + '/' + _device + '_' + _start_time.replace(' ', '_') + 'to'
+ _end_time.replace(' ', '_') + '_' + _property + '.dat' + " Pulled")
@@ -261,10 +282,22 @@ def pull_raw_dist(_start_time, _end_time, _device):
data['rows'] = df.count()
data['startTime'] = _start_time
data['endTime'] = _end_time
- data['lsbcut_hg'] = df.select('lsbcut_hg').collect()[0]
- data['offset_hg'] = df.select('offset_hg').collect()[0]
- data['lsbcut_lg'] = df.select('lsbcut_lg').collect()[0]
- data['offset_lg'] = df.select('offset_lg').collect()[0]
+
+ data['lsbcut_hg'] = df.select('lsbcut_hg').collect()
+ if len(data['lsbcut_hg']) > 0:
+ data['lsbcut_hg'] = data['lsbcut_hg'][0]
+
+ data['offset_hg'] = df.select('offset_hg').collect()
+ if len(data['offset_hg']) > 0:
+ data['offset_hg'] = data['offset_hg'][0]
+
+ data['lsbcut_lg'] = df.select('lsbcut_lg').collect()
+ if len(data['lsbcut_lg']) > 0:
+ data['lsbcut_lg'] = data['lsbcut_lg'][0]
+
+ data['offset_lg'] = df.select('offset_lg').collect()
+ if len(data['offset_lg']) > 0:
+ data['offset_lg'] = data['offset_lg'][0]
# Array Data #
result_hg = array_dataset_hg.collect()
@@ -290,12 +323,12 @@ def pull_raw_dist(_start_time, _end_time, _device):
db_cursor = mydb.cursor()
db_cursor.execute('UPDATE data SET raw_dist = %s where id = %s', (json.dumps(data), data_id,))
- # Writing output #
- with open(_dest_dir + '/' + _device + '_' + _start_time.replace(' ', '_') + 'to'
- + _end_time.replace(' ', '_') + '_' + _property + '.dat', 'w') as outfile:
- json.dump(data, outfile)
-
if _files:
+ # Writing output #
+ with open(_dest_dir + '/' + _device + '_' + _start_time.replace(' ', '_') + 'to'
+ + _end_time.replace(' ', '_') + '_' + _property + '.dat', 'w') as outfile:
+ json.dump(data, outfile)
+
print(_dest_dir + '/' + _device + '_' + _start_time.replace(' ', '_') + 'to'
+ _end_time.replace(' ', '_') + '_' + _property + '.dat' + " Pulled")
@@ -327,11 +360,26 @@ def pull_integral_dist(_start_time, _end_time, _device):
data['rows'] = df.count()
data['startTime'] = _start_time
data['endTime'] = _end_time
- data['lsbcut_hg'] = df.select('lsbcut_hg').collect()[0]
- data['offset_hg'] = df.select('offset_hg').collect()[0]
- data['lsbcut_lg'] = df.select('lsbcut_lg').collect()[0]
- data['offset_lg'] = df.select('offset_lg').collect()[0]
- data['nturns'] = df.select('nturns').collect()[0]
+
+ data['lsbcut_hg'] = df.select('lsbcut_hg').collect()
+ if len(data['lsbcut_hg']) > 0:
+ data['lsbcut_hg'] = data['lsbcut_hg'][0]
+
+ data['offset_hg'] = df.select('offset_hg').collect()
+ if len(data['offset_hg']) > 0:
+ data['offset_hg'] = data['offset_hg'][0]
+
+ data['lsbcut_lg'] = df.select('lsbcut_lg').collect()
+ if len(data['lsbcut_lg']) > 0:
+ data['lsbcut_lg'] = data['lsbcut_lg'][0]
+
+ data['offset_lg'] = df.select('offset_lg').collect()
+ if len(data['offset_lg']) > 0:
+ data['offset_lg'] = data['offset_lg'][0]
+
+ data['nturns'] = df.select('nturns').collect()
+ if len(data['nturns']) > 0:
+ data['nturns'] = data['nturns'][0]
# Array Data #
result_hg = array_dataset_hg.collect()
@@ -358,12 +406,12 @@ def pull_integral_dist(_start_time, _end_time, _device):
db_cursor = mydb.cursor()
db_cursor.execute('UPDATE data SET integral_dist = %s where id = %s', (json.dumps(data), data_id,))
- # Writing output #
- with open(_dest_dir + '/' + _device + '_' + _start_time.replace(' ', '_') + 'to'
- + _end_time.replace(' ', '_') + '_' + _property + '.dat', 'w') as outfile:
- json.dump(data, outfile)
-
if _files:
+ # Writing output #
+ with open(_dest_dir + '/' + _device + '_' + _start_time.replace(' ', '_') + 'to'
+ + _end_time.replace(' ', '_') + '_' + _property + '.dat', 'w') as outfile:
+ json.dump(data, outfile)
+
print(_dest_dir + '/' + _device + '_' + _start_time.replace(' ', '_') + 'to'
+ _end_time.replace(' ', '_') + '_' + _property + '.dat' + " Pulled")
@@ -408,10 +456,21 @@ def pull_turnloss(_start_time, _end_time, _device):
# DATA #
data['startTime'] = _start_time
data['endTime'] = _end_time
- data['nturns'] = df.select('nturns').collect()[0]
- data['zsthr_hg'] = df.select('zsthr_hg').collect()[0]
- data['zsthr_lg'] = df.select('zsthr_lg').collect()[0]
- data['baselinesub'] = df.select('baselinesub').collect()[0]
+ data['nturns'] = df.select('nturns').collect()
+ if len(data['nturns']) > 0:
+ data['nturns'] = data['nturns'][0]
+
+ data['zsthr_hg'] = df.select('zsthr_hg').collect()
+ if len(data['zsthr_hg']) > 0:
+ data['zsthr_hg'] = data['zsthr_hg'][0]
+
+ data['zsthr_lg'] = df.select('zsthr_lg').collect()
+ if len(data['zsthr_lg']) > 0:
+ data['zsthr_lg'] = data['zsthr_lg'][0]
+
+ data['baselinesub'] = df.select('baselinesub').collect()
+ if len(data['baselinesub']) > 0:
+ data['baselinesub'] = data['baselinesub'][0]
# Timestamp #
time_stamp = df.select("__record_timestamp__")
@@ -445,12 +504,12 @@ def pull_turnloss(_start_time, _end_time, _device):
db_cursor = mydb.cursor()
db_cursor.execute('UPDATE data SET turnloss = %s where id = %s', (json.dumps(data), data_id,))
- # Writing output #
- with open(_dest_dir + '/' + _device + '_' + _start_time.replace(' ', '_') + 'to'
- + _end_time.replace(' ', '_') + '_' + _property + '.dat', 'w') as outfile:
- json.dump(data, outfile)
-
if _files:
+ # Writing output #
+ with open(_dest_dir + '/' + _device + '_' + _start_time.replace(' ', '_') + 'to'
+ + _end_time.replace(' ', '_') + '_' + _property + '.dat', 'w') as outfile:
+ json.dump(data, outfile)
+
print(_dest_dir + '/' + _device + '_' + _start_time.replace(' ', '_') + 'to'
+ _end_time.replace(' ', '_') + '_' + _property + '.dat' + " Pulled")
@@ -479,7 +538,7 @@ if __name__ == '__main__':
" <start> : the start time of the pull you want to do. \n"
" format: 'yyyy-mm-dd hh:mm:ss.ms' or 'yyyy-mm-dd_hh:mm:ss.ms'")
sys.exit(0)
- elif len(sys.argv) < 5:
+ elif len(sys.argv) < 6:
print("usage: python property-pull.py <fillName> <device> <mode> <start> <end> [-f] \n"
"where : \n"
" -f also generate .dat files \n"
@@ -491,8 +550,10 @@ if __name__ == '__main__':
_mode = sys.argv[3]
_start = sys.argv[4]
_end = sys.argv[5]
- if len(sys.argv) > 5 and sys.argv[6] == "-f":
- _files = True
+ if len(sys.argv) > 6:
+ _files = sys.argv[6] == "-f"
+
+ print(_files)
print('Pulling data for device ' + _device + ' mode ' + _mode + ' from ' + _start + ' to ' + _end)
--
GitLab