From b4818b9390e4cf9b5affff48fa5af63323a4b62c Mon Sep 17 00:00:00 2001
From: Domenico Giordano <domenico.giordano@cern.ch>
Date: Fri, 30 Apr 2021 09:51:40 +0200
Subject: [PATCH] Merge branch 'qa-v0.4_antonin' of
ssh://gitlab.cern.ch:7999/cloud-infrastructure/data-analytics into
qa-v0.4_antonin
---
adcern/cmd/data_mining.py | 33 ++++++++++++++++-----------------
adcern/sqlite3_backend.py | 30 ++++++++++++++++++++----------
2 files changed, 36 insertions(+), 27 deletions(-)
diff --git a/adcern/cmd/data_mining.py b/adcern/cmd/data_mining.py
index acc0c88b..3e3af0fb 100755
--- a/adcern/cmd/data_mining.py
+++ b/adcern/cmd/data_mining.py
@@ -638,24 +638,23 @@ def analysis(module_name, class_name, alias_name, hyperparameters,
with open(file_path_config_train) as json_file:
data_dict_train = json.load(json_file)
# connect to the db
- conn = sqlite3.connect(folder_training_time + '/time.db', timeout = 120)
- c = conn.cursor()
- # ensure the table is there
- c.execute('''CREATE TABLE IF NOT EXISTS time
- (date_start text, date_end_excluded text,
- long_algo_description text,
- training_time real, measurement_time text,
- PRIMARY KEY (date_start, date_end_excluded,
- long_algo_description,
- measurement_time))''')
- conn.commit()
- c.execute('''INSERT INTO time
+ conn = sqlite3.connect(folder_training_time + '/time.db', timeout=120)
+
+ modify_db(conn, '''CREATE TABLE IF NOT EXISTS time
+ (date_start text, date_end_excluded text,
+ long_algo_description text,
+ training_time real, measurement_time text,
+ PRIMARY KEY (date_start, date_end_excluded,
+ long_algo_description,
+ measurement_time))''')
+
+ modify_db(conn, '''INSERT INTO time
VALUES (?, ?, ?, ?, datetime('now', 'localtime'))''',
- [data_dict_train["date_start"],
- data_dict_train["date_end_excluded"],
- algo_name,
- training_time])
- conn.commit()
+ [data_dict_train["date_start"],
+ data_dict_train["date_end_excluded"],
+ algo_name,
+ training_time])
+
conn.close()
# with open(file_path_config_train) as json_file:
# data_dict_train = json.load(json_file)
diff --git a/adcern/sqlite3_backend.py b/adcern/sqlite3_backend.py
index c6fb78a2..fd967e93 100644
--- a/adcern/sqlite3_backend.py
+++ b/adcern/sqlite3_backend.py
@@ -1,4 +1,5 @@
-import sqlite3
+import sqlite3
+import time
def modify_db(conn,
query,
@@ -16,21 +17,30 @@ def modify_db(conn,
args: array
additional arguments for execute query, optional
"""
-
- with conn.cursor() as c:
- if args:
- c.execute(query,args)
- else:
- c.execute(query)
+
+ with conn.cursor() as c:
+ for x in range(0, 10):
+ try:
+ if args:
+ c.execute(query, args)
+ else:
+ c.execute(query)
+ except:
+ print("Sqlite3 execute unsuccessful, retrying....")
+ time.sleep(1)
+ pass
+ else:
+ print("Sqlite3 execute successful, breaking the retry cycle.")
+ break
# retry commit - db might be locked by different process
for x in range(0, 10):
try:
conn.commit()
except:
- print("Commit to sqlite unsuccessful, retrying....")
+ print("Sqlite3 commit unsuccessful, retrying....")
time.sleep(1)
pass
- finally:
+ else:
+ print("Sqlite3 commit successful, breaking the retry cycle.")
break
-
\ No newline at end of file
--
GitLab