Merge branch 'qa-v0.4_antonin' of...

Merge branch 'qa-v0.4_antonin' of ssh://gitlab.cern.ch:7999/cloud-infrastructure/data-analytics into qa-v0.4_antonin

adcern/cmd/data_mining.py

@@ -638,24 +638,23 @@ def analysis(module_name, class_name, alias_name, hyperparameters,
         with open(file_path_config_train) as json_file:
             data_dict_train = json.load(json_file)
         # connect to the db
-        conn = sqlite3.connect(folder_training_time + '/time.db', timeout = 120)
-        c = conn.cursor()
-        # ensure the table is there
-        c.execute('''CREATE TABLE IF NOT EXISTS time
-                        (date_start text, date_end_excluded text,
-                        long_algo_description text,
-                        training_time real, measurement_time text,
-                        PRIMARY KEY (date_start, date_end_excluded,
-                                    long_algo_description,
-                                    measurement_time))''')
-        conn.commit()
-        c.execute('''INSERT INTO time
+        conn = sqlite3.connect(folder_training_time + '/time.db', timeout=120)
+
+        modify_db(conn, '''CREATE TABLE IF NOT EXISTS time
+                (date_start text, date_end_excluded text,
+                long_algo_description text,
+                training_time real, measurement_time text,
+                PRIMARY KEY (date_start, date_end_excluded,
+                            long_algo_description,
+                            measurement_time))''')
+
+        modify_db(conn, '''INSERT INTO time
                         VALUES (?, ?, ?, ?, datetime('now', 'localtime'))''',
-                  [data_dict_train["date_start"],
-                   data_dict_train["date_end_excluded"],
-                   algo_name,
-                   training_time])
-        conn.commit()
+                [data_dict_train["date_start"],
+                data_dict_train["date_end_excluded"],
+                algo_name,
+                training_time])
+
         conn.close()
     # with open(file_path_config_train) as json_file:
     #     data_dict_train = json.load(json_file)

adcern/sqlite3_backend.py

-import sqlite3 
+import sqlite3
+import time
 
 def modify_db(conn,
               query,
@@ -16,21 +17,30 @@ def modify_db(conn,
     args: array
         additional arguments for execute query, optional
     """
-       
-    with conn.cursor() as c:  
-        if args:
-            c.execute(query,args)
-        else:
-            c.execute(query)     
+
+    with conn.cursor() as c:
+        for x in range(0, 10):
+            try:
+                if args:
+                    c.execute(query, args)
+                else:
+                    c.execute(query)
+            except:
+                print("Sqlite3 execute unsuccessful, retrying....")
+                time.sleep(1)
+                pass
+            else:
+                print("Sqlite3 execute successful, breaking the retry cycle.")
+                break
 
     # retry commit - db might be locked by different process
     for x in range(0, 10):
         try:
             conn.commit()
         except:
-            print("Commit to sqlite unsuccessful, retrying....")
+            print("Sqlite3 commit unsuccessful, retrying....")
             time.sleep(1)
             pass
-        finally:
+        else:
+            print("Sqlite3 commit successful, breaking the retry cycle.")
             break
-    
\ No newline at end of file