Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
cloud-infrastructure
data-analytics
Commits
18d93070
Commit
18d93070
authored
May 02, 2021
by
Antonin Dvorak
Browse files
add upperbound param to sqlite3 wrapper
parent
fc7e9a56
Changes
4
Hide whitespace changes
Inline
Side-by-side
adcern/analyser.py
View file @
18d93070
...
...
@@ -369,19 +369,26 @@ class BaseOutlierAnalyser(ABC):
# connect to the db
conn_score
=
sqlite3
.
connect
(
score_folder
+
'/scores.db'
,
timeout
=
120
)
modify_db
(
conn_score
,
'''CREATE TABLE IF NOT EXISTS scores
modify_db
(
conn
=
conn_score
,
query
=
'''CREATE TABLE IF NOT EXISTS scores
(hostgroup text, hostname text, algorithm text,
score real, end_window int, noramlization_id text,
PRIMARY KEY (hostgroup, hostname, algorithm,
end_window, noramlization_id))'''
)
end_window, noramlization_id))'''
,
upperbound
=
10
)
# add row by row
num_rows
=
len
(
df
)
for
i
in
range
(
num_rows
):
# Try inserting the row
row
=
df
.
iloc
[
i
]
modify_db
(
conn_score
,
'''INSERT OR IGNORE INTO scores
VALUES (?, ?, ?, ?, ?, ?)'''
,
row
)
modify_db
(
conn
=
conn_score
,
query
=
'''INSERT OR IGNORE INTO scores
VALUES (?, ?, ?, ?, ?, ?)'''
,
upperbound
=
10
,
args
=
row
)
conn_score
.
close
()
...
...
adcern/cmd/data_mining.py
View file @
18d93070
...
...
@@ -231,19 +231,26 @@ def save_scores_local_sqlite(analyser,
+
'/scores_'
+
algorithm_name
+
'.db'
,
timeout
=
120
)
# ensure the table is there
modify_db
(
conn_score
,
'''CREATE TABLE IF NOT EXISTS scores
modify_db
(
conn
=
conn_score
,
query
=
'''CREATE TABLE IF NOT EXISTS scores
(hostgroup text, hostname text, algorithm text,
score real, end_window int, noramlization_id text,
PRIMARY KEY (hostgroup, hostname, algorithm, end_window,
noramlization_id))'''
)
noramlization_id))'''
,
upperbound
=
10
)
# add row by row
num_rows
=
len
(
df
)
for
i
in
range
(
num_rows
):
# Try inserting the row
row
=
df
.
iloc
[
i
]
modify_db
(
conn_score
,
'''INSERT OR IGNORE INTO scores
VALUES (?, ?, ?, ?, ?, ?)'''
,
row
)
modify_db
(
conn
=
conn_score
,
query
=
'''INSERT OR IGNORE INTO scores
VALUES (?, ?, ?, ?, ?, ?)'''
,
upperbound
=
10
,
args
=
row
)
conn_score
.
close
()
...
...
@@ -640,20 +647,26 @@ def analysis(module_name, class_name, alias_name, hyperparameters,
# connect to the db
conn
=
sqlite3
.
connect
(
folder_training_time
+
'/time.db'
,
timeout
=
120
)
modify_db
(
conn
,
'''CREATE TABLE IF NOT EXISTS time
(date_start text, date_end_excluded text,
long_algo_description text,
training_time real, measurement_time text,
PRIMARY KEY (date_start, date_end_excluded,
long_algo_description,
measurement_time))'''
)
modify_db
(
conn
,
'''INSERT INTO time
VALUES (?, ?, ?, ?, datetime('now', 'localtime'))'''
,
[
data_dict_train
[
"date_start"
],
data_dict_train
[
"date_end_excluded"
],
algo_name
,
training_time
])
modify_db
(
conn
=
conn
,
query
=
'''CREATE TABLE IF NOT EXISTS time
(date_start text, date_end_excluded text,
long_algo_description text,
training_time real, measurement_time text,
PRIMARY KEY (date_start, date_end_excluded,
long_algo_description,
measurement_time))'''
,
upperbound
=
10
)
modify_db
(
conn
=
conn
,
query
=
'''INSERT INTO time
VALUES (?, ?, ?, ?, datetime('now', 'localtime'))'''
,
upperbound
=
10
,
args
=
[
data_dict_train
[
"date_start"
],
data_dict_train
[
"date_end_excluded"
],
algo_name
,
training_time
])
conn
.
close
()
# with open(file_path_config_train) as json_file:
...
...
adcern/cmd/elaborate_scores.py
View file @
18d93070
...
...
@@ -360,11 +360,14 @@ def score_benchmark(folder_scores, hostgroup,
conn_score
=
sqlite3
.
connect
(
labels_folder
+
'/week_metrics.db'
,
timeout
=
120
)
# ensure the table is there
modify_db
(
conn_score
,
'''CREATE TABLE IF NOT EXISTS auc
modify_db
(
conn
=
conn_score
,
query
=
'''CREATE TABLE IF NOT EXISTS auc
(hostgroup text, algorithm text, family text,
auc_score real, week_index int, end_week int,
PRIMARY KEY
(hostgroup, algorithm, end_week))'''
)
(hostgroup, algorithm, end_week))'''
,
upperbound
=
10
)
# FOR EVERY WEEK -------------------------------------------------------
for
w
in
sorted
(
weeks_available
):
...
...
@@ -384,9 +387,12 @@ def score_benchmark(folder_scores, hostgroup,
# ax=dict_ax_rocs[algo_name], alpha=0.2)
print
(
"AUC: "
,
roc_auc
)
aucs_weeks
.
append
(
roc_auc
)
modify_db
(
conn_score
,
'''INSERT OR IGNORE INTO auc
VALUES (?, ?, ?, ?, ?, ?)'''
,
(
hostgroup
,
algo_name
,
family
,
roc_auc
,
int
(
w
),
end_week
))
modify_db
(
conn
=
conn_score
,
query
=
'''INSERT OR IGNORE INTO auc
VALUES (?, ?, ?, ?, ?, ?)'''
,
upperbound
=
10
,
args
=
(
hostgroup
,
algo_name
,
family
,
roc_auc
,
int
(
w
),
end_week
))
conn_score
.
close
()
# # CUMULATIVE QUANTITIES
...
...
adcern/sqlite3_backend.py
View file @
18d93070
...
...
@@ -3,6 +3,7 @@ import time
def
modify_db
(
conn
,
query
,
upperbound
=
10
,
*
args
):
"""Wrapper for modifications (INSERT, UPDATE, DELETE or REPLACE) of Sqlite3 database.
Executes given query and commits it - in case of lock it retries the commit
...
...
@@ -12,14 +13,15 @@ def modify_db(conn,
existing sqlite3 connection to use
query: sql
query to run
upperbound: int
number of retries to execute query / commit to db
args: array
additional arguments for execute query, optional
"""
upperbound
=
11
c
=
conn
.
cursor
()
# retry db execute
for
x
in
range
(
1
,
upperbound
):
for
x
in
range
(
1
,
upperbound
+
1
):
try
:
if
args
:
c
.
execute
(
query
,
*
args
)
...
...
@@ -33,11 +35,11 @@ def modify_db(conn,
print
(
"Sqlite3 execute successful, breaking the retry cycle."
)
break
# max attempts achieved, quit
if
x
==
upperbound
-
1
:
if
x
==
upperbound
:
raise
Exception
(
"Sqlite3 - achieved max retries to execute with no success, giving up..."
)
# retry db commit
for
x
in
range
(
1
,
upperbound
):
for
x
in
range
(
1
,
upperbound
+
1
):
try
:
conn
.
commit
()
except
Exception
as
e
:
...
...
@@ -48,5 +50,5 @@ def modify_db(conn,
print
(
"Sqlite3 commit successful, breaking the retry cycle."
)
break
# max attempts achieved, quit
if
x
==
upperbound
-
1
:
if
x
==
upperbound
:
raise
Exception
(
"Sqlite3 - achieved max retries to commit with no success, giving up..."
)
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment