Skip to content
Snippets Groups Projects

Sweeping !1602 from master to devel: fix (DataIntegrity): detect missing files better

Merged Sweeping !1602 from master to devel: fix (DataIntegrity): detect missing files better
Merged LHCbDIRAC Bot requested to merge cherry-pick-6d1db832f-devel into devel
1 file
+ 4
2
Compare changes
  • Side-by-side
  • Inline
@@ -11,10 +11,13 @@
"""This is the Data Integrity Client which allows the simple reporting of
problematic file and replicas to the IntegrityDB and their status correctly
updated in the FileCatalog."""
import errno
import re
from DIRAC import S_OK, gLogger
from DIRAC.Core.Utilities.Adler import compareAdler
from DIRAC.Core.Utilities.DErrno import cmpError
from DIRAC.Core.Utilities.ReturnValues import returnSingleResult
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.Resources.Storage.StorageElement import StorageElement
@@ -329,7 +332,6 @@ class DataIntegrityClient(DIRACDataIntegrityClient):
"""Check obtain the physical file metadata and check the files are
available."""
gLogger.info(f"Checking the integrity of {len(lfns)} physical files at {se}")
res = StorageElement(se).getFileMetadata(lfns)
if not res["OK"]:
gLogger.error("Failed to get metadata for lfns.", res["Message"])
@@ -338,7 +340,7 @@ class DataIntegrityClient(DIRACDataIntegrityClient):
# If the replicas are completely missing
missingReplicas = []
for lfn, reason in res["Value"]["Failed"].items():
if re.search("File does not exist", reason):
if re.search("File does not exist", reason) or cmpError(reason, errno.ENOENT):
missingReplicas.append((lfn, "deprecatedUrl", se, "PFNMissing"))
if missingReplicas:
self.reportProblematicReplicas(missingReplicas, se, "PFNMissing")
Loading