Skip to content
Snippets Groups Projects
Commit ac67611e authored by Ben Couturier's avatar Ben Couturier
Browse files

Added possibility to list LFNs

parent 12cbec02
No related branches found
No related tags found
No related merge requests found
Pipeline #5490530 passed
......@@ -24,9 +24,10 @@ __all__ = [
"cache_ap_info",
"auth",
"authw",
"ApdReturnType",
]
from .analysis_data import AnalysisData, get_analysis_data
from .analysis_data import AnalysisData, ApdReturnType, get_analysis_data
from .ap_info import (
SampleCollection,
cache_ap_info,
......
......@@ -20,6 +20,7 @@ import copy
import itertools
import logging
import os
from enum import Enum
from pathlib import Path
from apd.ap_info import (
......@@ -41,6 +42,12 @@ APD_METADATA_LIFETIME_DEFAULT = 600
APD_DATA_CACHE_DIR = "APD_DATA_CACHE_DIR"
class ApdReturnType(Enum):
PFN = 0
LFN = 1
SAMPLE = 2
def _load_and_setup_cache(
cache_dir, working_group, analysis, ap_date=None, api_url="https://lbap.app.cern.ch"
):
......@@ -335,7 +342,7 @@ class AnalysisData:
def __call__(
self,
*,
return_pfns=True,
return_type=ApdReturnType.PFN,
check_data=True,
use_local_cache=True,
showmax=10,
......@@ -390,12 +397,17 @@ class AnalysisData:
logger.debug("Error loading data: %s", error_txt)
raise ValueError("Error loading data: " + error_txt)
if return_pfns:
if use_local_cache:
return self._transform_pfns(samples.PFNs())
return samples.PFNs()
if return_type == ApdReturnType.SAMPLE:
return samples
if return_type == ApdReturnType.LFN:
print("Returning lfns")
return samples.LFNs()
return samples
# by default we return the PFns
if use_local_cache:
return self._transform_pfns(samples.PFNs())
return samples.PFNs()
def _transform_pfns(self, pfns):
"""Method to return PFNs, useful as it can be overriden in inheriting classes"""
......
......@@ -371,6 +371,13 @@ class SampleCollection:
pfns.append(auth(pfnlist[0]))
return pfns
def LFNs(self):
"""Collects the LFNs"""
lfns = []
for sample in self.info:
lfns += sample["lfns"].keys()
return lfns
def byte_count(self):
"""Collects the number of files from all the samples"""
count = 0
......
......@@ -11,7 +11,7 @@
import pytest
from apd import AnalysisData
from apd import AnalysisData, ApdReturnType
@pytest.mark.parametrize("with_tokens", [[False], [True]])
......@@ -189,6 +189,7 @@ def test_name_case_sensitivity(apd_cache):
def test_unknown_tag_value(apd_cache):
"""CHeck that we throw a ValueError when a value does not exist for a given tag."""
datasets = AnalysisData("b2oc", "b02dkpi")
with pytest.raises(ValueError, match="No sample for tag datatype=2032.*"):
......@@ -198,3 +199,19 @@ def test_unknown_tag_value(apd_cache):
eventtype="11164047",
mc=True,
)
def test_lfn(apd_cache):
"""Check that the method to return the LFNs is functional."""
datasets = AnalysisData("b2oc", "b02dkpi")
lfns = datasets(
datatype="2011",
eventtype="11164047",
polarity="magdown",
return_type=ApdReturnType.LFN,
)
assert (
len(lfns) == 5
and all("00128098_0000" in x for x in lfns)
and all(x.startswith("/lhcb") for x in lfns)
)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment