diff --git a/handlers/BandwidthTestHandler.py b/handlers/BandwidthTestHandler.py index 7f49d3e8bee86d5fc2031fdfabcc3be39c1311a5..bf88668706117fef92a48873f0cc3c4707ee7bbe 100644 --- a/handlers/BandwidthTestHandler.py +++ b/handlers/BandwidthTestHandler.py @@ -509,37 +509,25 @@ class BandwidthTestHandler(BaseHandler): process, ) - # Temporary test of the system: copy the full stream MDF to eos + # Copy the full stream MDF and tck.json to eos for use in sprucing tests # Only do so if test was completely successful. if options == "Moore_hlt2_bandwidth" and slot == "lhcb-master" and status: from datetime import datetime for fsuffix in ["__full.mdf", ".tck.json"]: - before_copy = datetime.now() full_stream_out = os.path.join( directory, 'tmp', 'MDF', f"hlt2_bw_testing__production{fsuffix}") - log.info( - f"Trying to copy {full_stream_out} to eos. File size: {os.stat(full_stream_out).st_size / 1e6} MB." - ) - publish.upload_eos_www( - full_stream_out, - os.path.basename(full_stream_out), - baseurl=os.path.join( - os.getenv("LHCBPR_WWW_EOS"), "UpgradeRateTest")) - log.info(f"Copied {full_stream_out} to eos. This took " + - str(datetime.now() - before_copy)) - log.info("Trying to copy to new directory...") - before_second_copy = datetime.now() + before_copy = datetime.now() publish.upload_eos_www( full_stream_out, os.path.join("current_hlt2_output", os.path.basename(full_stream_out)), baseurl=os.path.join( - os.getenv("LHCBPR_WWW_EOS"), "UpgradeRateTest")) - log.info( - f"Copied {full_stream_out} to eos new directory. This took " - + str(datetime.now() - before_second_copy)) + os.getenv("LHCBPR_WWW_EOS"), "UpgradeRateTest"), + force=True) + log.info(f"Copied {full_stream_out} to eos. This took " + + str(datetime.now() - before_copy)) # Post GitLab MR feedback; a bit more complex as comparing to a reference build. if (slot in [ diff --git a/handlers/utils/publish.py b/handlers/utils/publish.py index 283d017ecad87a055bdcbaf8473f40ebccbf169b..0dc8802d6aa74f55be31bfd5676e5993b6ef9f60 100644 --- a/handlers/utils/publish.py +++ b/handlers/utils/publish.py @@ -17,17 +17,22 @@ def get_gitlab_server(): return gitlab.Gitlab(GITLAB_URL, private_token=os.getenv("GITLAB_TOKEN")) -def upload_eos_www(source, destination, baseurl=None): +def upload_eos_www(source, destination, baseurl=None, force=False): """Copy a file to EOS via xrdcp.""" baseurl = baseurl or os.getenv("LHCBPR_WWW_EOS") if not baseurl: - log.error(f"LHCBPR_WWW_EOS env is not set. Will not upload {source} to EOS") + log.error( + f"LHCBPR_WWW_EOS env is not set. Will not upload {source} to EOS") else: # don't use urljoin here as it can't cope with root:// full_destination = os.path.join(baseurl, destination) log.info(f"Uploading {source} to {full_destination}") try: - cmd = ["xrdcp", source, full_destination] + cmd = ["xrdcp"] + if force: + # use case is to overwrite files already present + cmd.append("-f") + cmd += [source, full_destination] log.debug("Calling {}".format(" ".join(map(repr, cmd)))) subprocess.check_call(cmd) except subprocess.CalledProcessError as e: @@ -47,7 +52,10 @@ def post_mattermost(message, webhook=None): log.error(f"Failed to post to Mattermost: {e}") -def post_gitlab_feedback(trigger_source, message, add_labels=[], remove_labels=[]): +def post_gitlab_feedback(trigger_source, + message, + add_labels=[], + remove_labels=[]): """Post feedback to GitLab for a ci-test trigger. Args: