Commit 18d93070 authored by Antonin Dvorak's avatar Antonin Dvorak
Browse files

add upperbound param to sqlite3 wrapper

parent fc7e9a56
......@@ -369,19 +369,26 @@ class BaseOutlierAnalyser(ABC):
# connect to the db
conn_score = sqlite3.connect(score_folder + '/scores.db', timeout=120)
modify_db(conn_score, '''CREATE TABLE IF NOT EXISTS scores
modify_db(
conn = conn_score,
query = '''CREATE TABLE IF NOT EXISTS scores
(hostgroup text, hostname text, algorithm text,
score real, end_window int, noramlization_id text,
PRIMARY KEY (hostgroup, hostname, algorithm,
end_window, noramlization_id))''')
end_window, noramlization_id))''',
upperbound = 10)
# add row by row
num_rows = len(df)
for i in range(num_rows):
# Try inserting the row
row = df.iloc[i]
modify_db(conn_score, '''INSERT OR IGNORE INTO scores
VALUES (?, ?, ?, ?, ?, ?)''', row)
modify_db(
conn = conn_score,
query = '''INSERT OR IGNORE INTO scores
VALUES (?, ?, ?, ?, ?, ?)''',
upperbound = 10,
args = row)
conn_score.close()
......
......@@ -231,19 +231,26 @@ def save_scores_local_sqlite(analyser,
+ '/scores_' + algorithm_name + '.db', timeout=120)
# ensure the table is there
modify_db(conn_score, '''CREATE TABLE IF NOT EXISTS scores
modify_db(
conn = conn_score,
query = '''CREATE TABLE IF NOT EXISTS scores
(hostgroup text, hostname text, algorithm text,
score real, end_window int, noramlization_id text,
PRIMARY KEY (hostgroup, hostname, algorithm, end_window,
noramlization_id))''')
noramlization_id))''',
upperbound = 10)
# add row by row
num_rows = len(df)
for i in range(num_rows):
# Try inserting the row
row = df.iloc[i]
modify_db(conn_score, '''INSERT OR IGNORE INTO scores
VALUES (?, ?, ?, ?, ?, ?)''', row)
modify_db(
conn = conn_score,
query = '''INSERT OR IGNORE INTO scores
VALUES (?, ?, ?, ?, ?, ?)''',
upperbound = 10,
args = row)
conn_score.close()
......@@ -640,20 +647,26 @@ def analysis(module_name, class_name, alias_name, hyperparameters,
# connect to the db
conn = sqlite3.connect(folder_training_time + '/time.db', timeout=120)
modify_db(conn, '''CREATE TABLE IF NOT EXISTS time
(date_start text, date_end_excluded text,
long_algo_description text,
training_time real, measurement_time text,
PRIMARY KEY (date_start, date_end_excluded,
long_algo_description,
measurement_time))''')
modify_db(conn, '''INSERT INTO time
VALUES (?, ?, ?, ?, datetime('now', 'localtime'))''',
[data_dict_train["date_start"],
data_dict_train["date_end_excluded"],
algo_name,
training_time])
modify_db(
conn = conn,
query = '''CREATE TABLE IF NOT EXISTS time
(date_start text, date_end_excluded text,
long_algo_description text,
training_time real, measurement_time text,
PRIMARY KEY (date_start, date_end_excluded,
long_algo_description,
measurement_time))''',
upperbound = 10)
modify_db(
conn = conn,
query = '''INSERT INTO time
VALUES (?, ?, ?, ?, datetime('now', 'localtime'))''',
upperbound = 10,
args = [data_dict_train["date_start"],
data_dict_train["date_end_excluded"],
algo_name,
training_time])
conn.close()
# with open(file_path_config_train) as json_file:
......
......@@ -360,11 +360,14 @@ def score_benchmark(folder_scores, hostgroup,
conn_score = sqlite3.connect(labels_folder + '/week_metrics.db',
timeout=120)
# ensure the table is there
modify_db(conn_score, '''CREATE TABLE IF NOT EXISTS auc
modify_db(
conn = conn_score,
query = '''CREATE TABLE IF NOT EXISTS auc
(hostgroup text, algorithm text, family text,
auc_score real, week_index int, end_week int,
PRIMARY KEY
(hostgroup, algorithm, end_week))''')
(hostgroup, algorithm, end_week))''',
upperbound = 10)
# FOR EVERY WEEK -------------------------------------------------------
for w in sorted(weeks_available):
......@@ -384,9 +387,12 @@ def score_benchmark(folder_scores, hostgroup,
# ax=dict_ax_rocs[algo_name], alpha=0.2)
print("AUC: ", roc_auc)
aucs_weeks.append(roc_auc)
modify_db(conn_score, '''INSERT OR IGNORE INTO auc
VALUES (?, ?, ?, ?, ?, ?)''',
(hostgroup, algo_name, family, roc_auc, int(w), end_week))
modify_db(
conn = conn_score,
query = '''INSERT OR IGNORE INTO auc
VALUES (?, ?, ?, ?, ?, ?)''',
upperbound = 10,
args = (hostgroup, algo_name, family, roc_auc, int(w), end_week))
conn_score.close()
# # CUMULATIVE QUANTITIES
......
......@@ -3,6 +3,7 @@ import time
def modify_db(conn,
query,
upperbound=10,
*args):
"""Wrapper for modifications (INSERT, UPDATE, DELETE or REPLACE) of Sqlite3 database.
Executes given query and commits it - in case of lock it retries the commit
......@@ -12,14 +13,15 @@ def modify_db(conn,
existing sqlite3 connection to use
query: sql
query to run
upperbound: int
number of retries to execute query / commit to db
args: array
additional arguments for execute query, optional
"""
upperbound = 11
c = conn.cursor()
# retry db execute
for x in range(1, upperbound):
for x in range(1, upperbound + 1):
try:
if args:
c.execute(query, *args)
......@@ -33,11 +35,11 @@ def modify_db(conn,
print("Sqlite3 execute successful, breaking the retry cycle.")
break
# max attempts achieved, quit
if x == upperbound - 1:
if x == upperbound:
raise Exception("Sqlite3 - achieved max retries to execute with no success, giving up...")
# retry db commit
for x in range(1, upperbound):
for x in range(1, upperbound + 1):
try:
conn.commit()
except Exception as e:
......@@ -48,5 +50,5 @@ def modify_db(conn,
print("Sqlite3 commit successful, breaking the retry cycle.")
break
# max attempts achieved, quit
if x == upperbound - 1:
if x == upperbound:
raise Exception("Sqlite3 - achieved max retries to commit with no success, giving up...")
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment