Skip to content
Snippets Groups Projects

[RTADPA BW Tests] Introducing an Hlt1-bandwidth test via Moore_in_Allen

Merged Luke Grazette requested to merge lugrazet-BW-initialhlt1test into master
All threads resolved!
1 file
+ 0
66
Compare changes
  • Side-by-side
  • Inline
@@ -160,69 +160,3 @@ def download_mdf_inputs_locally(urls,
if max_size and size >= max_size:
break
return local_fns
def download_digi_inputs_locally(urls, dest_dir, max_size=None):
"""Download raw inputs locally partially or fully.
Downloads files identified by `urls`. If specified, download until
`max_size` cumulative size is reached. The last file will be cut
off.
Args:
urls: (xrootd) URLs to download.
dest_dir: Path to destination directory.
max_size (int): Limit of the total downloaded file size.
Returns:
list: Paths of the local files.
Raises:
OSError: If an error occurred while downloading a file.
"""
size = 0
local_fns = []
local_files = {} # url: (local_fn, size)
for url in urls:
if url not in local_files:
# prepend hash of full URL in case basenames are not unique
basename = '{}_{}'.format(
hashlib.md5(url.encode()).hexdigest(), os.path.basename(url))
assert basename.endswith('.digi') or basename.endswith('.xdigi')
dest = os.path.join(dest_dir, basename)
src_size = xrd_size(url)
size_to_copy = (min(src_size,
int(max_size) - size)
if max_size else src_size)
# check if we don't have the file already locally
try:
dest_size = os.path.getsize(dest)
except OSError:
dest_size = 0
# if the local file is missing or smaller, download again
if dest_size < size_to_copy:
avail_space = get_fs_avail_space(dest_dir) + dest_size
if avail_space < size_to_copy:
raise RuntimeError(
'Destination {!r} will not have enough available '
'space (need {:.1f}GB has {:.1f}GB).'.format(
dest_dir, size_to_copy / 1024**3,
avail_space / 1024**3))
if size_to_copy < src_size:
xrdcp_part(url, dest, size_to_copy)
else:
xrdcp(url, dest)
dest_size = os.path.getsize(dest)
try:
os.chmod(dest, 0o666)
except OSError:
pass
local_files[url] = (dest, dest_size)
else:
dest, dest_size = local_files[url]
local_fns.append(dest)
size += dest_size
if max_size and size >= max_size:
break
return local_fns
Loading