handlers/BandwidthTestHandler.py

@@ -509,37 +509,25 @@ class BandwidthTestHandler(BaseHandler):
             process,
         )
 
-        # Temporary test of the system: copy the full stream MDF to eos
+        # Copy the full stream MDF and tck.json to eos for use in sprucing tests
         # Only do so if test was completely successful.
         if options == "Moore_hlt2_bandwidth" and slot == "lhcb-master" and status:
             from datetime import datetime
             for fsuffix in ["__full.mdf", ".tck.json"]:
 
-                before_copy = datetime.now()
                 full_stream_out = os.path.join(
                     directory, 'tmp', 'MDF',
                     f"hlt2_bw_testing__production{fsuffix}")
-                log.info(
-                    f"Trying to copy {full_stream_out} to eos. File size: {os.stat(full_stream_out).st_size / 1e6} MB."
-                )
-                publish.upload_eos_www(
-                    full_stream_out,
-                    os.path.basename(full_stream_out),
-                    baseurl=os.path.join(
-                        os.getenv("LHCBPR_WWW_EOS"), "UpgradeRateTest"))
-                log.info(f"Copied {full_stream_out} to eos. This took " +
-                         str(datetime.now() - before_copy))
-                log.info("Trying to copy to new directory...")
-                before_second_copy = datetime.now()
+                before_copy = datetime.now()
                 publish.upload_eos_www(
                     full_stream_out,
                     os.path.join("current_hlt2_output",
                                  os.path.basename(full_stream_out)),
                     baseurl=os.path.join(
-                        os.getenv("LHCBPR_WWW_EOS"), "UpgradeRateTest"))
-                log.info(
-                    f"Copied {full_stream_out} to eos new directory. This took "
-                    + str(datetime.now() - before_second_copy))
+                        os.getenv("LHCBPR_WWW_EOS"), "UpgradeRateTest"),
+                    force=True)
+                log.info(f"Copied {full_stream_out} to eos. This took " +
+                         str(datetime.now() - before_copy))
 
         # Post GitLab MR feedback; a bit more complex as comparing to a reference build.
         if (slot in [

handlers/utils/publish.py

@@ -17,17 +17,22 @@ def get_gitlab_server():
     return gitlab.Gitlab(GITLAB_URL, private_token=os.getenv("GITLAB_TOKEN"))
 
 
-def upload_eos_www(source, destination, baseurl=None):
+def upload_eos_www(source, destination, baseurl=None, force=False):
     """Copy a file to EOS via xrdcp."""
     baseurl = baseurl or os.getenv("LHCBPR_WWW_EOS")
     if not baseurl:
-        log.error(f"LHCBPR_WWW_EOS env is not set. Will not upload {source} to EOS")
+        log.error(
+            f"LHCBPR_WWW_EOS env is not set. Will not upload {source} to EOS")
     else:
         # don't use urljoin here as it can't cope with root://
         full_destination = os.path.join(baseurl, destination)
         log.info(f"Uploading {source} to {full_destination}")
         try:
-            cmd = ["xrdcp", source, full_destination]
+            cmd = ["xrdcp"]
+            if force:
+                # use case is to overwrite files already present
+                cmd.append("-f")
+            cmd += [source, full_destination]
             log.debug("Calling {}".format(" ".join(map(repr, cmd))))
             subprocess.check_call(cmd)
         except subprocess.CalledProcessError as e:
@@ -47,7 +52,10 @@ def post_mattermost(message, webhook=None):
             log.error(f"Failed to post to Mattermost: {e}")
 
 
-def post_gitlab_feedback(trigger_source, message, add_labels=[], remove_labels=[]):
+def post_gitlab_feedback(trigger_source,
+                         message,
+                         add_labels=[],
+                         remove_labels=[]):
     """Post feedback to GitLab for a ci-test trigger.
 
     Args: