add upperbound param to sqlite3 wrapper

adcern/analyser.py

@@ -369,19 +369,26 @@ class BaseOutlierAnalyser(ABC):
         # connect to the db
         conn_score = sqlite3.connect(score_folder + '/scores.db', timeout=120)
 
-        modify_db(conn_score, '''CREATE TABLE IF NOT EXISTS scores
+        modify_db(
+                conn = conn_score, 
+                query = '''CREATE TABLE IF NOT EXISTS scores
                     (hostgroup text, hostname text, algorithm text,
                     score real, end_window int, noramlization_id text,
                     PRIMARY KEY (hostgroup, hostname, algorithm,
-                    end_window, noramlization_id))''')
+                    end_window, noramlization_id))''',
+                upperbound = 10)
 
         # add row by row
         num_rows = len(df)
         for i in range(num_rows):
             # Try inserting the row
             row = df.iloc[i]
-            modify_db(conn_score, '''INSERT OR IGNORE INTO scores
-                        VALUES (?, ?, ?, ?, ?, ?)''', row)
+            modify_db(
+                    conn = conn_score,
+                    query = '''INSERT OR IGNORE INTO scores
+                        VALUES (?, ?, ?, ?, ?, ?)''',
+                    upperbound = 10,
+                    args = row)
 
         conn_score.close()
 

adcern/cmd/data_mining.py

@@ -231,19 +231,26 @@ def save_scores_local_sqlite(analyser,
                         + '/scores_' + algorithm_name + '.db', timeout=120)
 
     # ensure the table is there
-    modify_db(conn_score, '''CREATE TABLE IF NOT EXISTS scores
+    modify_db(
+            conn = conn_score, 
+            query = '''CREATE TABLE IF NOT EXISTS scores
                 (hostgroup text, hostname text, algorithm text,
                 score real, end_window int, noramlization_id text,
                 PRIMARY KEY (hostgroup, hostname, algorithm, end_window,
-                noramlization_id))''')
+                noramlization_id))''',
+            upperbound = 10)
 
     # add row by row
     num_rows = len(df)
     for i in range(num_rows):
         # Try inserting the row
         row = df.iloc[i]
-        modify_db(conn_score, '''INSERT OR IGNORE INTO scores
-                        VALUES (?, ?, ?, ?, ?, ?)''', row)
+        modify_db(
+                conn = conn_score,
+                query = '''INSERT OR IGNORE INTO scores
+                    VALUES (?, ?, ?, ?, ?, ?)''', 
+                upperbound = 10,                 
+                args = row)
 
     conn_score.close()
 
@@ -640,20 +647,26 @@ def analysis(module_name, class_name, alias_name, hyperparameters,
         # connect to the db
         conn = sqlite3.connect(folder_training_time + '/time.db', timeout=120)
 
-        modify_db(conn, '''CREATE TABLE IF NOT EXISTS time
-                (date_start text, date_end_excluded text,
-                long_algo_description text,
-                training_time real, measurement_time text,
-                PRIMARY KEY (date_start, date_end_excluded,
-                            long_algo_description,
-                            measurement_time))''')
-
-        modify_db(conn, '''INSERT INTO time
-                        VALUES (?, ?, ?, ?, datetime('now', 'localtime'))''',
-                [data_dict_train["date_start"],
-                data_dict_train["date_end_excluded"],
-                algo_name,
-                training_time])
+        modify_db(
+                conn = conn, 
+                query = '''CREATE TABLE IF NOT EXISTS time
+                    (date_start text, date_end_excluded text,
+                    long_algo_description text,
+                    training_time real, measurement_time text,
+                    PRIMARY KEY (date_start, date_end_excluded,
+                                long_algo_description,
+                                measurement_time))''',
+                upperbound = 10)
+
+        modify_db(
+                conn = conn, 
+                query = '''INSERT INTO time
+                    VALUES (?, ?, ?, ?, datetime('now', 'localtime'))''',
+                upperbound = 10,
+                args = [data_dict_train["date_start"],
+                    data_dict_train["date_end_excluded"],
+                    algo_name,
+                    training_time])
 
         conn.close()
     # with open(file_path_config_train) as json_file:

adcern/cmd/elaborate_scores.py

@@ -360,11 +360,14 @@ def score_benchmark(folder_scores, hostgroup,
     conn_score = sqlite3.connect(labels_folder + '/week_metrics.db',
                                  timeout=120)
     # ensure the table is there
-    modify_db(conn_score, '''CREATE TABLE IF NOT EXISTS auc
+    modify_db(
+            conn = conn_score, 
+            query = '''CREATE TABLE IF NOT EXISTS auc
                 (hostgroup text, algorithm text, family text,
                 auc_score real, week_index int, end_week int,
                 PRIMARY KEY
-                (hostgroup, algorithm, end_week))''')
+                (hostgroup, algorithm, end_week))''',
+            upperbound = 10)
 
     # FOR EVERY WEEK -------------------------------------------------------
     for w in sorted(weeks_available):
@@ -384,9 +387,12 @@ def score_benchmark(folder_scores, hostgroup,
         #             ax=dict_ax_rocs[algo_name], alpha=0.2)
         print("AUC: ", roc_auc)
         aucs_weeks.append(roc_auc)
-        modify_db(conn_score, '''INSERT OR IGNORE INTO auc
-                     VALUES (?, ?, ?, ?, ?, ?)''',
-                  (hostgroup, algo_name, family, roc_auc, int(w), end_week))
+        modify_db(
+                conn = conn_score,
+                query = '''INSERT OR IGNORE INTO auc
+                    VALUES (?, ?, ?, ?, ?, ?)''',
+                upperbound = 10,
+                args = (hostgroup, algo_name, family, roc_auc, int(w), end_week))
     conn_score.close()
 
     # # CUMULATIVE QUANTITIES

adcern/sqlite3_backend.py

@@ -3,6 +3,7 @@ import time
 
 def modify_db(conn,
               query,
+              upperbound=10,
               *args):
     """Wrapper for modifications (INSERT, UPDATE, DELETE or REPLACE) of Sqlite3 database.
     Executes given query and commits it - in case of lock it retries the commit
@@ -12,14 +13,15 @@ def modify_db(conn,
         existing sqlite3 connection to use
     query: sql
         query to run
+    upperbound: int
+        number of retries to execute query / commit to db
     args: array
         additional arguments for execute query, optional
     """
-    upperbound = 11
-
+    
     c = conn.cursor()
     # retry db execute
-    for x in range(1, upperbound):
+    for x in range(1, upperbound + 1):
         try:
             if args:
                 c.execute(query, *args)
@@ -33,11 +35,11 @@ def modify_db(conn,
             print("Sqlite3 execute successful, breaking the retry cycle.")
             break
         # max attempts achieved, quit
-        if x == upperbound - 1:
+        if x == upperbound:
             raise Exception("Sqlite3 - achieved max retries to execute with no success, giving up...")
 
     # retry db commit
-    for x in range(1, upperbound):
+    for x in range(1, upperbound + 1):
         try:
             conn.commit()
         except Exception as e:
@@ -48,5 +50,5 @@ def modify_db(conn,
             print("Sqlite3 commit successful, breaking the retry cycle.")
             break
         # max attempts achieved, quit            
-        if x == upperbound - 1:
+        if x == upperbound:
             raise Exception("Sqlite3 - achieved max retries to commit with no success, giving up...")