From 47103cbda774cf2d598483560dab5d63a789b4c9 Mon Sep 17 00:00:00 2001
From: Ross Hunter <ross.john.hunter@cern.ch>
Date: Tue, 28 Nov 2023 11:08:19 +0000
Subject: [PATCH 1/6] Allow xrdcp force for overwrite
---
handlers/utils/publish.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/handlers/utils/publish.py b/handlers/utils/publish.py
index 283d017e..002afc2b 100644
--- a/handlers/utils/publish.py
+++ b/handlers/utils/publish.py
@@ -17,7 +17,7 @@ def get_gitlab_server():
return gitlab.Gitlab(GITLAB_URL, private_token=os.getenv("GITLAB_TOKEN"))
-def upload_eos_www(source, destination, baseurl=None):
+def upload_eos_www(source, destination, baseurl=None, force=False):
"""Copy a file to EOS via xrdcp."""
baseurl = baseurl or os.getenv("LHCBPR_WWW_EOS")
if not baseurl:
@@ -27,7 +27,11 @@ def upload_eos_www(source, destination, baseurl=None):
full_destination = os.path.join(baseurl, destination)
log.info(f"Uploading {source} to {full_destination}")
try:
- cmd = ["xrdcp", source, full_destination]
+ cmd = ["xrdcp"]
+ if force:
+ # use case is to overwrite files already present
+ cmd.append("-f")
+ cmd += [source, full_destination]
log.debug("Calling {}".format(" ".join(map(repr, cmd))))
subprocess.check_call(cmd)
except subprocess.CalledProcessError as e:
--
GitLab
From 9f612f1fab6d9cc75ccf55ca5f4b419c244c4709 Mon Sep 17 00:00:00 2001
From: Ross Hunter <ross.john.hunter@cern.ch>
Date: Tue, 28 Nov 2023 11:38:31 +0000
Subject: [PATCH 2/6] Only copy once and reduce log output
---
handlers/BandwidthTestHandler.py | 20 ++++----------------
1 file changed, 4 insertions(+), 16 deletions(-)
diff --git a/handlers/BandwidthTestHandler.py b/handlers/BandwidthTestHandler.py
index 7f49d3e8..293cd548 100644
--- a/handlers/BandwidthTestHandler.py
+++ b/handlers/BandwidthTestHandler.py
@@ -509,28 +509,16 @@ class BandwidthTestHandler(BaseHandler):
process,
)
- # Temporary test of the system: copy the full stream MDF to eos
+ # Copy the full stream MDF and tck.json to eos for use in sprucing tests
# Only do so if test was completely successful.
if options == "Moore_hlt2_bandwidth" and slot == "lhcb-master" and status:
from datetime import datetime
for fsuffix in ["__full.mdf", ".tck.json"]:
- before_copy = datetime.now()
full_stream_out = os.path.join(
directory, 'tmp', 'MDF',
f"hlt2_bw_testing__production{fsuffix}")
- log.info(
- f"Trying to copy {full_stream_out} to eos. File size: {os.stat(full_stream_out).st_size / 1e6} MB."
- )
- publish.upload_eos_www(
- full_stream_out,
- os.path.basename(full_stream_out),
- baseurl=os.path.join(
- os.getenv("LHCBPR_WWW_EOS"), "UpgradeRateTest"))
- log.info(f"Copied {full_stream_out} to eos. This took " +
- str(datetime.now() - before_copy))
- log.info("Trying to copy to new directory...")
- before_second_copy = datetime.now()
+ before_copy = datetime.now()
publish.upload_eos_www(
full_stream_out,
os.path.join("current_hlt2_output",
@@ -538,8 +526,8 @@ class BandwidthTestHandler(BaseHandler):
baseurl=os.path.join(
os.getenv("LHCBPR_WWW_EOS"), "UpgradeRateTest"))
log.info(
- f"Copied {full_stream_out} to eos new directory. This took "
- + str(datetime.now() - before_second_copy))
+ f"Copied {full_stream_out} to eos. This took "
+ + str(datetime.now() - before_copy))
# Post GitLab MR feedback; a bit more complex as comparing to a reference build.
if (slot in [
--
GitLab
From e62af5545e7e5b59bb226c2c84d82c96a32c239c Mon Sep 17 00:00:00 2001
From: Ross Hunter <ross.john.hunter@cern.ch>
Date: Tue, 28 Nov 2023 11:39:05 +0000
Subject: [PATCH 3/6] Force overwrite
---
handlers/BandwidthTestHandler.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/handlers/BandwidthTestHandler.py b/handlers/BandwidthTestHandler.py
index 293cd548..cb9c6a2d 100644
--- a/handlers/BandwidthTestHandler.py
+++ b/handlers/BandwidthTestHandler.py
@@ -524,7 +524,7 @@ class BandwidthTestHandler(BaseHandler):
os.path.join("current_hlt2_output",
os.path.basename(full_stream_out)),
baseurl=os.path.join(
- os.getenv("LHCBPR_WWW_EOS"), "UpgradeRateTest"))
+ os.getenv("LHCBPR_WWW_EOS"), "UpgradeRateTest"), force=True)
log.info(
f"Copied {full_stream_out} to eos. This took "
+ str(datetime.now() - before_copy))
--
GitLab
From 4fe5a08ed3dca2c21801ab35950d711973d5451b Mon Sep 17 00:00:00 2001
From: Ross Hunter <ross.john.hunter@cern.ch>
Date: Tue, 28 Nov 2023 14:18:04 +0000
Subject: [PATCH 4/6] Fix formatting
---
handlers/BandwidthTestHandler.py | 8 ++++----
handlers/utils/publish.py | 8 ++++++--
2 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/handlers/BandwidthTestHandler.py b/handlers/BandwidthTestHandler.py
index cb9c6a2d..bf886687 100644
--- a/handlers/BandwidthTestHandler.py
+++ b/handlers/BandwidthTestHandler.py
@@ -524,10 +524,10 @@ class BandwidthTestHandler(BaseHandler):
os.path.join("current_hlt2_output",
os.path.basename(full_stream_out)),
baseurl=os.path.join(
- os.getenv("LHCBPR_WWW_EOS"), "UpgradeRateTest"), force=True)
- log.info(
- f"Copied {full_stream_out} to eos. This took "
- + str(datetime.now() - before_copy))
+ os.getenv("LHCBPR_WWW_EOS"), "UpgradeRateTest"),
+ force=True)
+ log.info(f"Copied {full_stream_out} to eos. This took " +
+ str(datetime.now() - before_copy))
# Post GitLab MR feedback; a bit more complex as comparing to a reference build.
if (slot in [
diff --git a/handlers/utils/publish.py b/handlers/utils/publish.py
index 002afc2b..0dc8802d 100644
--- a/handlers/utils/publish.py
+++ b/handlers/utils/publish.py
@@ -21,7 +21,8 @@ def upload_eos_www(source, destination, baseurl=None, force=False):
"""Copy a file to EOS via xrdcp."""
baseurl = baseurl or os.getenv("LHCBPR_WWW_EOS")
if not baseurl:
- log.error(f"LHCBPR_WWW_EOS env is not set. Will not upload {source} to EOS")
+ log.error(
+ f"LHCBPR_WWW_EOS env is not set. Will not upload {source} to EOS")
else:
# don't use urljoin here as it can't cope with root://
full_destination = os.path.join(baseurl, destination)
@@ -51,7 +52,10 @@ def post_mattermost(message, webhook=None):
log.error(f"Failed to post to Mattermost: {e}")
-def post_gitlab_feedback(trigger_source, message, add_labels=[], remove_labels=[]):
+def post_gitlab_feedback(trigger_source,
+ message,
+ add_labels=[],
+ remove_labels=[]):
"""Post feedback to GitLab for a ci-test trigger.
Args:
--
GitLab
From 1e0c69b07c9647dba95377c09a2f606fde5aa6e5 Mon Sep 17 00:00:00 2001
From: Ross Hunter <ross.john.hunter@cern.ch>
Date: Tue, 28 Nov 2023 14:25:19 +0000
Subject: [PATCH 5/6] reduce diff - revert unrelated formatting changes
---
handlers/utils/publish.py | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/handlers/utils/publish.py b/handlers/utils/publish.py
index 0dc8802d..b46054b8 100644
--- a/handlers/utils/publish.py
+++ b/handlers/utils/publish.py
@@ -52,10 +52,7 @@ def post_mattermost(message, webhook=None):
log.error(f"Failed to post to Mattermost: {e}")
-def post_gitlab_feedback(trigger_source,
- message,
- add_labels=[],
- remove_labels=[]):
+def post_gitlab_feedback(trigger_source, message, add_labels=[], remove_labels=[]):
"""Post feedback to GitLab for a ci-test trigger.
Args:
--
GitLab
From 25b2a20c9eea064ef25726fff012e299d91ab206 Mon Sep 17 00:00:00 2001
From: Ross Hunter <ross.john.hunter@cern.ch>
Date: Tue, 28 Nov 2023 14:26:05 +0000
Subject: [PATCH 6/6] Revert "reduce diff - revert unrelated formatting
changes"
This reverts commit 1e0c69b07c9647dba95377c09a2f606fde5aa6e5.
---
handlers/utils/publish.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/handlers/utils/publish.py b/handlers/utils/publish.py
index b46054b8..0dc8802d 100644
--- a/handlers/utils/publish.py
+++ b/handlers/utils/publish.py
@@ -52,7 +52,10 @@ def post_mattermost(message, webhook=None):
log.error(f"Failed to post to Mattermost: {e}")
-def post_gitlab_feedback(trigger_source, message, add_labels=[], remove_labels=[]):
+def post_gitlab_feedback(trigger_source,
+ message,
+ add_labels=[],
+ remove_labels=[]):
"""Post feedback to GitLab for a ci-test trigger.
Args:
--
GitLab