diff --git a/DataQuality/DQDefects/python/db.py b/DataQuality/DQDefects/python/db.py
index be88bac2b335f7115d8b31acf5f70f034655b895..813b7d5db6cd34fb2bf11874b0af7898dc61efa8 100644
--- a/DataQuality/DQDefects/python/db.py
+++ b/DataQuality/DQDefects/python/db.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration
"""
Authors: Peter Waller <peter.waller@cern.ch> and "Peter Onyisi" <peter.onyisi@cern.ch>
@@ -19,23 +19,21 @@ from logging import getLogger; log = getLogger("DQDefects.db")
from contextlib import contextmanager
from DQUtils import fetch_iovs, IOVSet
+from DQUtils.sugar.iovtype import IOVType
from DQUtils.channel_mapping import list_to_channelselection
+from DQUtils.sugar import RunLumi
from DQDefects import DEFAULT_CONNECTION_STRING
from .exceptions import DefectExistsError, DefectUnknownError
from .folders import DefectsDBFoldersMixin
from .ids import DefectsDBIDsNamesMixin, choose_new_defect_id
-from .tags import DefectsDBTagsMixin
+from .tags import DefectsDBTagsMixin, tagtype
from .virtual_mixin import DefectsDBVirtualDefectsMixin
from .virtual_calculator import calculate_virtual_defects
+from typing import Union, Tuple, Optional, Iterable, Collection
import six
-if six.PY2:
- def _encode (s, enc): return s.encode(enc)
-else:
- def _encode (s, enc): return s
-
class DefectsDB(DefectsDBVirtualDefectsMixin,
DefectsDBTagsMixin,
@@ -60,8 +58,8 @@ class DefectsDB(DefectsDBVirtualDefectsMixin,
Public interface is nominally defined in this class (DefectsDB).
"""
- def __init__(self, connection_string=DEFAULT_CONNECTION_STRING,
- read_only=True, create=False, tag="HEAD"):
+ def __init__(self, connection_string: str = DEFAULT_CONNECTION_STRING,
+ read_only: bool = True, create: bool = False, tag: Union[str, Tuple] = "HEAD") -> None:
"""
Create a new DefectsDB instance.
@@ -85,8 +83,6 @@ class DefectsDB(DefectsDBVirtualDefectsMixin,
self.connection_string = connection_string
self._read_only = read_only
self._create = create
- import collections
- tagtype = collections.namedtuple('tagtype', ['defects', 'logic'])
if isinstance(tag, six.string_types):
self._tag = tagtype(tag, tag) if tag else tagtype("HEAD", "HEAD")
else:
@@ -97,8 +93,8 @@ class DefectsDB(DefectsDBVirtualDefectsMixin,
if len(tag) != 2:
raise TypeError('tag argument must be a 2-element sequence')
self._tag = tag
- self._tag = tagtype(_encode(self._tag[0],'ascii'),
- _encode(self._tag[1],'ascii'))
+ self._tag = tagtype(self._tag[0],
+ self._tag[1])
# COOL has no way of emptying a storage buffer. Creating a new storage
# buffer flushes the old one. Therefore, if an exception happens
@@ -114,7 +110,7 @@ class DefectsDB(DefectsDBVirtualDefectsMixin,
self.defects_folder
self.defect_logic_folder
- def __del__(self):
+ def __del__(self) -> None:
"""
Ideally we would use inheritance to call destructors, but this isn't
possible in the general case with the way we (ab)use mixins, so we just
@@ -122,7 +118,7 @@ class DefectsDB(DefectsDBVirtualDefectsMixin,
"""
self._clear_connections()
- def create_defect(self, name, description):
+ def create_defect(self, name: str, description: str) -> None:
"""
Create a new type of defect; tries to figure out system ID from the
defect name. See also: `create_defect_with_id`, `new_system_defect`.
@@ -135,9 +131,9 @@ class DefectsDB(DefectsDBVirtualDefectsMixin,
"""
sysid = choose_new_defect_id(self.defect_id_map, name)
log.info("Creating new defect %s: system ID %08x", name, sysid)
- return self._create_defect_with_id(sysid, name, description)
+ self._create_defect_with_id(sysid, name, description)
- def _create_defect_with_id(self, did, name, description):
+ def _create_defect_with_id(self, did: int, name: str, description: str) -> None:
"""
Create a new type of defect, specifying the defect ID.
See also: `create_defect`, `new_system_defect`.
@@ -151,21 +147,21 @@ class DefectsDB(DefectsDBVirtualDefectsMixin,
if did in self.defect_ids: raise DefectExistsError(did)
try:
oldname = self.normalize_defect_names(name)
- already_exists = True
+ raise DefectExistsError(f'Defect {oldname} already exists')
except DefectUnknownError:
- already_exists = False
- if already_exists:
- raise DefectExistsError('Defect %s already exists' % oldname)
+ pass
self.defects_folder.createChannel(did,
- _encode(name,'ascii'),
- _encode(description,'utf-8'))
+ name.encode('ascii'),
+ description.encode('utf-8'))
self._new_defect(did, name)
- def retrieve(self, since=None, until=None, channels=None, nonpresent=False,
- primary_only=False, ignore=None,
- with_primary_dependencies=False, intersect=False,
- with_time=False, evaluate_full=True):
+ def retrieve(self, since: Optional[Union[int, Tuple[int,int], RunLumi]] = None,
+ until: Optional[Union[int, Tuple[int,int], RunLumi]] = None,
+ channels: Optional[Iterable[Union[str,int]]] = None, nonpresent: bool = False,
+ primary_only: bool = False, ignore: Collection[str] = None,
+ with_primary_dependencies: bool = False, intersect: bool = False,
+ with_time: bool = False, evaluate_full: bool = True) -> IOVSet:
"""
Retrieve defects from the database.
@@ -203,7 +199,7 @@ class DefectsDB(DefectsDBVirtualDefectsMixin,
virtual_channels = None
else:
virtual_channels = self.virtual_defect_ids
- primary_channels = sorted(self.defect_ids)
+ primary_channels = set(self.defect_ids)
query_channels = None # (all)
primary_output_names = [self.defect_id_map[pid] for pid in primary_channels]
@@ -230,6 +226,8 @@ class DefectsDB(DefectsDBVirtualDefectsMixin,
for logic in ordered_logics:
logic.set_evaluation(evaluate_full)
+ else:
+ ordered_logics = []
# Figure out if the set of channels will produce too many ranges for COOL
if query_channels is not None:
@@ -301,8 +299,8 @@ class DefectsDB(DefectsDBVirtualDefectsMixin,
self.defects_folder.flushStorageBuffer()
return thunk()
- def insert(self, defect_id, since, until, comment, added_by,
- present=True, recoverable=False):
+ def insert(self, defect_id: Union[str, int], since: int, until: int, comment: str, added_by: str,
+ present: bool = True, recoverable: bool = False) -> None:
"""
Insert a new defect into the database.
@@ -319,7 +317,7 @@ class DefectsDB(DefectsDBVirtualDefectsMixin,
return self._insert(defect_id, since, until, comment, added_by,
present, recoverable, self.defects_tag)
- def _insert_iov(self, iov, tag):
+ def _insert_iov(self, iov: IOVType, tag: str) -> None:
"""
Helper function for inserting IOV objects, since record order doesn't
match function argument order
@@ -327,8 +325,9 @@ class DefectsDB(DefectsDBVirtualDefectsMixin,
return self._insert(iov.channel, iov.since, iov.until, iov.comment,
iov.user, iov.present, iov.recoverable, tag)
- def _insert(self, defect_id, since, until, comment, added_by,
- present=True, recoverable=False, tag='HEAD'):
+ def _insert(self, defect_id: Union[str, int], since: Union[int, Tuple[int, int], RunLumi],
+ until: Union[int, Tuple[int, int], RunLumi], comment: str, added_by: str,
+ present: bool = True, recoverable: bool = False, tag: str = 'HEAD') -> None:
"""
Implementation of insert, allows tag specification for internal
functions
@@ -342,10 +341,10 @@ class DefectsDB(DefectsDBVirtualDefectsMixin,
p["present"] = present
p["recoverable"] = recoverable
- p["user"] = _encode(added_by, 'utf-8')
- p["comment"] = _encode(comment, 'utf-8')
+ p["user"] = added_by.encode('utf-8')
+ p["comment"] = comment.encode('utf-8')
defect_id = self.defect_chan_as_id(defect_id, True)
- store(since, until, p, defect_id, tag,
+ store(since, until, p, defect_id, tag.encode('ascii'),
(True if tag != 'HEAD' else False))
diff --git a/DataQuality/DQDefects/python/folders.py b/DataQuality/DQDefects/python/folders.py
index 5e8e9b56aaca94d66bf9a8ed401e01ac19462e5b..cc61f1629f71b143bfb673d0203405f2ae483512 100644
--- a/DataQuality/DQDefects/python/folders.py
+++ b/DataQuality/DQDefects/python/folders.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration
from logging import getLogger; log = getLogger("DQDefects.folders")
@@ -8,6 +8,9 @@ from DQDefects import (DEFECTS_FOLDER, DEFECT_FOLDER_DESC, DEFECT_LOGIC_FOLDER,
DEFECT_LOGIC_FOLDER_DESC, PARENT_FOLDERSET)
from PyCool import cool
+
+from typing import Tuple
+
ST = cool.StorageType
FV = cool.FolderVersioning
@@ -16,21 +19,21 @@ class DefectsDBFoldersMixin(object):
This mixin contains code for managing the defect folders
"""
- def __init__(self):
+ def __init__(self) -> None:
self._defects_folder = None
self._defect_logic_folder = None
self._parent_folderset = None
self._connections = []
super(DefectsDBFoldersMixin, self).__init__()
- def _clear_connections(self):
+ def _clear_connections(self) -> None:
log.debug("DefectsDB connections cleared")
for connection in self._connections:
connection.closeDatabase()
self._connections = []
@property
- def parent_folderset(self):
+ def parent_folderset(self) -> cool.IFolderSet:
"""
Lazy-loads parent folderset (useful for heirarchical tag manipulation)
"""
@@ -40,7 +43,7 @@ class DefectsDBFoldersMixin(object):
self._parent_folderset = db.getFolderSet(PARENT_FOLDERSET)
return self._parent_folderset
- def _load_folder(self, folder, create_function):
+ def _load_folder(self, folder, create_function) -> Tuple[cool.IFolder, cool.Record]:
"""
Internal function used to load a COOL folder
"""
@@ -61,14 +64,14 @@ class DefectsDBFoldersMixin(object):
return folder, payload
@property
- def defects_folder(self):
+ def defects_folder(self) -> cool.IFolder:
"""
Returns the folder containing the defects, loading it if necessary
"""
if self._defects_folder is None: self._load_defects_folder()
return self._defects_folder
- def _load_defects_folder(self, read_only=True, create=False):
+ def _load_defects_folder(self) -> None:
"""
Internal function for populating the self._defects_folder variable
"""
@@ -76,7 +79,7 @@ class DefectsDBFoldersMixin(object):
res = self._load_folder(DEFECTS_FOLDER, self._create_defects_folder)
self._defects_folder, self._defect_payload = res
- def _create_defects_folder(self, db):
+ def _create_defects_folder(self, db: cool.IDatabase) -> cool.IFolder:
"""
Creates the COOL database/folder for defects, if they don't exist.
Internal - use create=True in the constructor to create the COOL folder.
@@ -94,14 +97,14 @@ class DefectsDBFoldersMixin(object):
DEFECT_FOLDER_DESC, True)
@property
- def defect_logic_folder(self):
+ def defect_logic_folder(self) -> cool.IFolder:
"""
Returns the folder containing the virtual defect logic, loading it if necessary
"""
if self._defect_logic_folder is None: self._load_defect_logic_folder()
return self._defect_logic_folder
- def _load_defect_logic_folder(self):
+ def _load_defect_logic_folder(self) -> None:
"""
Internal function for populating the self._defect_logic_folder variable
"""
@@ -109,7 +112,7 @@ class DefectsDBFoldersMixin(object):
self._create_defect_logic_folder)
self._defect_logic_folder, self._defect_logic_payload = res
- def _create_defect_logic_folder(self, db):
+ def _create_defect_logic_folder(self, db: cool.IDatabase) -> cool.IFolder:
"""
Creates the COOL database/folder for virtual defect logic, if they don't exist.
Internal - use create=True in the constructor to create the COOL folder.
diff --git a/DataQuality/DQDefects/python/ids.py b/DataQuality/DQDefects/python/ids.py
index 6befeb8ddecc93be0a2381e0fe6ad2a63de2c53f..c24417e44c0abcbb03058228936f20e9e3326032 100644
--- a/DataQuality/DQDefects/python/ids.py
+++ b/DataQuality/DQDefects/python/ids.py
@@ -1,8 +1,8 @@
-# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration
from logging import getLogger; log = getLogger("DQDefects.defect_ids")
-from ctypes import Union, Structure, c_uint
+from ctypes import Structure, c_uint, Union as cUnion
from DQUtils.channel_mapping import get_channel_ids_names
@@ -10,10 +10,8 @@ from .exceptions import (DefectUnknownError,
InvalidLogicTagError)
import six
-if six.PY2:
- def _decode (s): return s.decode('utf-8')
-else:
- def _decode (s): return s
+from typing import Set, Iterable, Tuple, Union, List
+from collections.abc import Mapping, MutableMapping
class DefectIDBitfield(Structure):
@@ -26,7 +24,7 @@ class DefectIDBitfield(Structure):
return "(is_virtual=%s, defect=%i)" % args
-class DefectID(Union):
+class DefectID(cUnion):
_fields_ = [("as_int", c_uint),
("as_bitfield", DefectIDBitfield)]
@@ -47,7 +45,8 @@ class DefectID(Union):
args = self.as_int, self.as_bitfield
return '<DefectID value=0x%08x fields=%r>' % args
-def choose_new_defect_id(existing_map, defect_name, virtual=False):
+def choose_new_defect_id(existing_map: Mapping[Union[str, int], Union[str,int]],
+ defect_name: str, virtual: bool = False) -> int:
"""
Function to create a new defect ID.
@@ -77,14 +76,18 @@ class DefectsDBIDsNamesMixin(object):
Contains the logic for storing knowledge of which defects exist, and their
channel names and IDs
"""
- def __init__(self):
- self._defect_id_map = self._defect_ids = self._defect_names = None
- self._virtual_defect_id_map = None
- self._virtual_defect_ids = None
- self._virtual_defect_names = None
+ def __init__(self) -> None:
+ self._initialized = False
+ self._virtual_initialized = False
+ self._defect_id_map: MutableMapping[Union[str,int], Union[str,int]] = {}
+ self._defect_ids: Set[int] = set()
+ self._defect_names: Set[str] = set()
+ self._virtual_defect_id_map: MutableMapping[Union[str,int], Union[str,int]] = {}
+ self._virtual_defect_ids: Set[int] = set()
+ self._virtual_defect_names: Set[str] = set()
super(DefectsDBIDsNamesMixin, self).__init__()
- def _populate_defect_ids(self):
+ def _populate_defect_ids(self) -> None:
"""
Called the first time any of defect_{ids,names,id_map,etc} is called,
and populates internal variables to store the channel ids/names for the
@@ -95,13 +98,13 @@ class DefectsDBIDsNamesMixin(object):
self._defect_names = set(names)
self._defect_id_map = mapping
- def _populate_virtual_defect_ids(self):
+ def _populate_virtual_defect_ids(self) -> None:
"""
Called the first time any of virtual_defect_{ids,names,id_map,etc} is called,
and populates internal variables to store the channel ids/names for the
life of the DefectsDB instance.
"""
- ids, names, mapping = get_channel_ids_names(self.defect_logic_folder)
+ _, _, mapping = get_channel_ids_names(self.defect_logic_folder)
try:
self._virtual_defect_names = set(self.virtual_defect_logics.keys())
self._virtual_defect_ids = set(mapping[name] for name in self._virtual_defect_names)
@@ -115,7 +118,7 @@ class DefectsDBIDsNamesMixin(object):
self._virtual_defect_ids = set()
self._virtual_defect_map = {}
- def _new_defect(self, did, dname):
+ def _new_defect(self, did: int, dname: str) -> None:
"""
Internal function used to keep defect IDs/names uptodate.
"""
@@ -124,7 +127,7 @@ class DefectsDBIDsNamesMixin(object):
self.defect_id_map[did] = dname
self.defect_id_map[dname] = did
- def _new_virtual_defect(self, did, dname):
+ def _new_virtual_defect(self, did: int, dname: str) -> None:
"""
Internal function used to keep defect IDs/names uptodate.
"""
@@ -136,55 +139,68 @@ class DefectsDBIDsNamesMixin(object):
self._virtual_defect_logics = None
@property
- def defect_ids(self):
+ def defect_ids(self) -> Set[int]:
"""
Gives the set of defect IDs that exist in COOL
"""
- if self._defect_ids is None: self._populate_defect_ids()
+ if not self._initialized:
+ self._populate_defect_ids()
+ self._initialized = True
return self._defect_ids
@property
- def defect_names(self):
+ def defect_names(self) -> Set[str]:
"""
Gives the set of defect names that exist in COOL
"""
- if self._defect_names is None: self._populate_defect_ids()
+ if not self._initialized:
+ self._populate_defect_ids()
+ self._initialized = True
+ assert self._defect_names is not None, self._initialized
return self._defect_names
@property
- def defect_id_map(self):
+ def defect_id_map(self) -> MutableMapping[Union[str,int], Union[str,int]]:
"""
Gives the dictionary relating COOL channel ids to defect names and vice
versa, retrieving them from the database if necessary.
"""
- if self._defect_id_map is None: self._populate_defect_ids()
+ if not self._initialized:
+ self._populate_defect_ids()
+ self._initialized = True
return self._defect_id_map
@property
- def virtual_defect_ids(self):
+ def virtual_defect_ids(self) -> Set[int]:
"""
Returns the set of existing virtual defect IDs
"""
- if self._virtual_defect_ids is None: self._populate_virtual_defect_ids()
+ if not self._virtual_initialized:
+ self._populate_virtual_defect_ids()
+ self._virtual_initialized = True
return self._virtual_defect_ids
@property
- def virtual_defect_names(self):
+ def virtual_defect_names(self) -> Set[str]:
"""
Returns the set of existing virtual defect names
"""
- if self._virtual_defect_names is None: self._populate_virtual_defect_ids()
+ if not self._virtual_initialized:
+ self._populate_virtual_defect_ids()
+ self._virtual_initialized = True
return self._virtual_defect_names
@property
- def virtual_defect_id_map(self):
+ def virtual_defect_id_map(self) -> MutableMapping[Union[str,int], Union[str,int]]:
"""
Returns a dict() mapping virtual defect names to IDs and vice-versa.
"""
- if self._virtual_defect_id_map is None: self._populate_virtual_defect_ids()
+ if not self._virtual_initialized:
+ self._populate_virtual_defect_ids()
+ self._virtual_initialized = True
return self._virtual_defect_id_map
- def defect_chan_as_id(self, channel, primary_only=False):
+ def defect_chan_as_id(self, channel: Union[str, int], primary_only: bool = False) -> int:
"""
Returns the defect ID for a virtual defect.
Accepts a `channel` as an integer/string and returns it as an integer.
@@ -209,44 +225,44 @@ class DefectsDBIDsNamesMixin(object):
raise RuntimeError("Invalid `channel` type, got %s, expected integer"
" or string" % type(channel))
- def defect_names_as_ids(self, channels):
+ def defect_names_as_ids(self, channels: Iterable[Union[str, int]]) -> List[int]:
"""
Returns a list of channels as IDs
"""
return [self.defect_chan_as_id(chan) for chan in channels]
- def get_channels(self):
+ def get_channels(self) -> Tuple[Set[int], Set[str], Mapping[Union[str, int], Union[str, int]]]:
"""
Return channel IDs, names, and dict relating the two
"""
return self.defect_ids, self.defect_names, self.defect_id_map
- def get_virtual_channels(self):
+ def get_virtual_channels(self) -> Tuple[Set[int], Set[str], Mapping[Union[str, int], Union[str, int]]]:
"""
Return channel IDs, names, and dict relating the two
"""
return self.virtual_defect_ids, self.virtual_defect_names, self.virtual_defect_id_map
- def get_channel_descriptions(self, channels):
+ def get_channel_descriptions(self, channels: Iterable[Union[str, int]]) -> MutableMapping[Union[str, int], str]:
"""
For the list of channel IDs "channels", return dict mapping ID to
description
"""
get_desc = self.defects_folder.channelDescription
- return dict((channel, _decode (get_desc(self.defect_chan_as_id(channel))))
- for channel in channels)
+ return {channel: get_desc(self.defect_chan_as_id(channel))
+ for channel in channels}
- def get_virtual_channel_descriptions(self, channels):
+ def get_virtual_channel_descriptions(self, channels: Iterable[Union[str, int]]) -> MutableMapping[Union[str, int], str]:
"""
For the list of channel IDs "channels", return dict mapping ID to
descriptiondefect_id
"""
get_desc = self.defect_logic_folder.channelDescription
- return dict((channel, _decode(get_desc(self.defect_chan_as_id(channel))))
- for channel in channels)
+ return {channel: get_desc(self.defect_chan_as_id(channel))
+ for channel in channels}
@property
- def all_defect_descriptions(self):
+ def all_defect_descriptions(self) -> Mapping[Union[str, int], str]:
"""
A dictionary of all (virtual and primary) defect descriptions
"""
@@ -255,7 +271,7 @@ class DefectsDBIDsNamesMixin(object):
result.update(gvcd(self.virtual_defect_names))
return result
- def set_channel_description(self, channel, description):
+ def set_channel_description(self, channel: Union[str, int], description: str) -> None:
"""
Set a defect description
"""
@@ -264,9 +280,9 @@ class DefectsDBIDsNamesMixin(object):
folder = self.defect_logic_folder
else:
folder = self.defects_folder
- folder.setChannelDescription(chan_id, description)
+ folder.setChannelDescription(chan_id, description.encode('utf-8'))
- def defect_is_virtual(self, defect_id):
+ def defect_is_virtual(self, defect_id: Union[str, int]) -> bool:
"""
Returns True if the `defect_id` represents a virtual defect, False if it
is not and raises if it doesn't exist
@@ -289,7 +305,7 @@ class DefectsDBIDsNamesMixin(object):
raise DefectUnknownError(defect_id)
- def normalize_defect_names(self, defect_id):
+ def normalize_defect_names(self, defect_id: Union[str, Iterable[str]]) -> Union[str, List[str]]:
"""
Returns correct name(s) of defects, given name(s) that possibly differ
from the correct ones by case. Raises if an input name doesn't map to
diff --git a/DataQuality/DQDefects/python/tags.py b/DataQuality/DQDefects/python/tags.py
index 1cb562cf0f0da0fae57d13285e15a57f1e7e31a6..89519567414131d5e43b0ff73c91477da2363d73 100644
--- a/DataQuality/DQDefects/python/tags.py
+++ b/DataQuality/DQDefects/python/tags.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration
from logging import getLogger; log = getLogger("DQDefects.tags")
@@ -12,34 +12,33 @@ from .exceptions import InvalidTagError, InvalidDefectTagError, InvalidLogicTagE
from .virtual_mixin import NONHEAD_MODIFICATION_MSG
-import six
-if six.PY2:
- def _encode (s, enc): return s.encode(enc)
-else:
- def _encode (s, enc): return s
+from typing import List, Optional, Iterable
+
+import collections
+tagtype = collections.namedtuple('tagtype', ['defects', 'logic'])
class DefectsDBTagsMixin(object):
- def __init__(self):
+ def __init__(self) -> None:
super(DefectsDBTagsMixin, self).__init__()
self.check_tag_validity()
@property
- def logics_tags(self):
+ def logics_tags(self) -> List[str]:
"""
Returns a list of existing logic tags
"""
- return [t for t in self.defect_logic_folder.listTags()]
+ return [str(t) for t in self.defect_logic_folder.listTags()]
@property
- def defects_tags(self):
+ def defects_tags(self) -> List[str]:
"""
Returns a list of existing defect tags
"""
- return [t for t in self.defects_folder.listTags()]
+ return [str(t) for t in self.defects_folder.listTags()]
@property
- def next_logics_tag(self):
+ def next_logics_tag(self) -> str:
"""
Gives the next available DEFECTLOGICS tag
"""
@@ -50,17 +49,16 @@ class DefectsDBTagsMixin(object):
new_id = 0
return DEFECT_LOGIC_TAG_FORMAT % new_id
- def _tag_head_and_lock(self, folder, name, description):
+ def _tag_head_and_lock(self, folder: cool.IFolder, name: str, description: str) -> None:
"""
Give the current HEAD of `folder` a new tag and lock it.
"""
LOCKED = cool.HvsTagLock.LOCKED
- name = _encode(name,'ascii')
- description = _encode(description,'utf-8')
- folder.cloneTagAsUserTag('HEAD', name, description)
- folder.setTagLockStatus(name, LOCKED)
+ aname = name.encode('ascii')
+ folder.cloneTagAsUserTag('HEAD', aname, description.encode('utf-8'))
+ folder.setTagLockStatus(aname, LOCKED)
- def new_hierarchical_tag(self, defects_tag, logics_tag):
+ def new_hierarchical_tag(self, defects_tag: str, logics_tag: str) -> str:
"""
Create a new heirarchical tag relating the defects and logics
@@ -69,28 +67,30 @@ class DefectsDBTagsMixin(object):
and has description
"(v%i) blah"
"""
- defects_tag = _encode(defects_tag,'ascii')
- logics_tag = _encode(logics_tag,'ascii')
logic_revision = int(logics_tag.split("-")[-1])
defect_part = "-".join(defects_tag.split("-")[1:])
hierarchical_tag = "DetStatus-v%02i-%s" % (logic_revision, defect_part)
+ logicspart = "(%i) " % logic_revision
- defect_descr = self.defects_folder.tagDescription(defects_tag)
+ adefects_tag = defects_tag.encode('ascii')
+ alogics_tag = logics_tag.encode('ascii')
+ ahierarchical_tag = hierarchical_tag.encode('ascii')
+ logicspart = logicspart.encode('ascii')
+ defect_descr = self.defects_folder.tagDescription(adefects_tag).encode('utf-8')
- logicspart = "(%i) " % logic_revision
# Protection here against making descriptions too long
description = logicspart + defect_descr[:255 - len(logicspart)]
parent_folder = self.parent_folderset
- self.defects_folder.createTagRelation(hierarchical_tag, defects_tag)
- self.defect_logic_folder.createTagRelation(hierarchical_tag, logics_tag)
- parent_folder.setTagDescription(hierarchical_tag, description)
+ self.defects_folder.createTagRelation(ahierarchical_tag, adefects_tag)
+ self.defect_logic_folder.createTagRelation(ahierarchical_tag, alogics_tag)
+ parent_folder.setTagDescription(ahierarchical_tag, description)
log.info("New hierarchical tag %s", hierarchical_tag)
return hierarchical_tag
@property
- def defects_tag_valid(self):
+ def defects_tag_valid(self) -> bool:
try:
self.defects_tag
except InvalidTagError:
@@ -99,7 +99,7 @@ class DefectsDBTagsMixin(object):
return True
@property
- def logics_tag_valid(self):
+ def logics_tag_valid(self) -> bool:
try:
self.logics_tag
except InvalidTagError:
@@ -107,7 +107,7 @@ class DefectsDBTagsMixin(object):
else:
return True
- def check_tag_validity(self):
+ def check_tag_validity(self) -> None:
"""
Ensure that the tags that this DefectsDB instance was constructed with
are functional
@@ -116,7 +116,7 @@ class DefectsDBTagsMixin(object):
raise InvalidTagError("Tag doesn't resolve: {0}".format(self._tag))
@property
- def defects_tag(self):
+ def defects_tag(self) -> str:
"""
Return the tag used for retrieving defects on this DefectsDB instance
"""
@@ -130,7 +130,7 @@ class DefectsDBTagsMixin(object):
return self._tag.defects
@property
- def logics_tag(self):
+ def logics_tag(self) -> str:
"""
Return the tag used for retrieving virtual defect clauses
"""
@@ -144,27 +144,26 @@ class DefectsDBTagsMixin(object):
return self._tag.logic
@property
- def tag(self):
+ def tag(self) -> tagtype:
"""
Readonly property stating the tag that the database was instantiated on
"""
return self._tag
@property
- def tags(self):
+ def tags(self) -> List[str]:
"""
The list of tags which are on the database
"""
- return [t.decode() for t in self.parent_folderset.listTags()]
+ return [str(t) for t in self.parent_folderset.listTags()]
- def new_logics_tag(self, description=""):
+ def new_logics_tag(self, description: str = "") -> str:
"""
Create a new tag for the logic folder and lock it.
Parameters:
`description` : What changed in this tag? (optional, default "")
"""
- description = _encode(description,'utf-8')
assert self.logics_tag == "HEAD", NONHEAD_MODIFICATION_MSG
new_tag_name = self.next_logics_tag
@@ -173,7 +172,7 @@ class DefectsDBTagsMixin(object):
log.info("Tagged DEFECTLOGICS HEAD with %s", new_tag_name)
return new_tag_name
- def new_defects_tag(self, name, description, iovranges=None):
+ def new_defects_tag(self, name: str, description: str, iovranges: Optional[Iterable] = None) -> str:
"""
Clones the current DEFECTS tag (specified in the constructor) to a new one
If iovranges != None, does the slower thing of copying IOVs one by one
@@ -182,17 +181,17 @@ class DefectsDBTagsMixin(object):
`name` : Name of the new tag
`description` : Description of the contents of this tag
"""
- name = _encode(name,'ascii')
- description = _encode(description,'utf-8')
if name.startswith("DetStatus"):
raise RuntimeError("Only specify the last part of the defect tag")
+ adescription = description.encode('utf-8')
log.info("Creating new tag %s", name)
- name = "DetStatusDEFECTS-%s" % name
+ name = f"DetStatusDEFECTS-{name}"
+ aname = name.encode('ascii')
if iovranges is None:
- self.defects_folder.cloneTagAsUserTag(self.defects_tag, name, description)
+ self.defects_folder.cloneTagAsUserTag(self.defects_tag.encode('ascii'), aname, adescription)
return name
# Fetch all primary defects relevant to `iovranges`
@@ -208,6 +207,6 @@ class DefectsDBTagsMixin(object):
# If there are no IOVs to copy, there is no new tag to describe
if to_copy:
- self.defects_folder.setTagDescription(name, description)
+ self.defects_folder.setTagDescription(aname, adescription)
return name
diff --git a/DataQuality/DQDefects/python/tests/__init__.py b/DataQuality/DQDefects/python/tests/__init__.py
index 533078dfa0d44abe3212f427eb4695f1015bcfa9..0e1d9513fea7b84b8b01698837076699c9b55b4b 100644
--- a/DataQuality/DQDefects/python/tests/__init__.py
+++ b/DataQuality/DQDefects/python/tests/__init__.py
@@ -1,10 +1,11 @@
-# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration
# -*- coding: utf-8 -*-
# flake8: noqa
# unicode characters embedded below confuse flake8
from os import unlink, environ
from os.path import exists
from nose import with_setup
+from nose.tools import raises
from logging import getLogger; log = getLogger("DQDefects.tests")
@@ -188,6 +189,22 @@ def test_defect_insertion_retrieval_unicode():
assert iov.comment == TEST_COMMENT
assert ddb.all_defect_descriptions[TEST_DEFECT_NAME] == TEST_DEFECT_DESCRIPTION
+@raises(UnicodeEncodeError)
+@with_setup(create_database, teardown_database)
+def test_defect_failure_nonascii_name():
+ """
+ Check that we raise an error if the defect name is not ASCII
+ """
+ if six.PY3:
+ import ROOT
+ if ROOT.gROOT.GetVersionInt() < 62000:
+ # Passing str objects using multibyte encodings is broken
+ # with pyroot up to 6.18. Should be fixed in 6.20?
+ return
+ ddb = DefectsDB(TEST_DATABASE, read_only=False)
+
+ ddb.create_defect(u"DQD_TÉST_DÉFÉCT_0", "Test")
+
@with_setup(create_database, teardown_database)
def test_defect_empty_retrieval():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
@@ -227,6 +244,28 @@ def test_virtual_defect_creation():
assert len(ddb.virtual_defect_ids) == 1
assert "DQD_TEST_VIRTUAL_DEFECT" in ddb.virtual_defect_names
+@with_setup(create_database, teardown_database)
+def test_defect_list_update_on_creation():
+ """
+ Check that the defect name list is updated when a defect is created
+ """
+ ddb = DefectsDB(TEST_DATABASE, read_only=False)
+ name = create_defect_type(ddb, 0)
+
+ assert ddb.defect_names == set([name])
+
+@with_setup(create_database, teardown_database)
+def test_virtual_defect_list_update_on_creation():
+ """
+ Check that the virtual defect name list is updated when a defect is created
+ """
+ ddb = DefectsDB(TEST_DATABASE, read_only=False)
+ name = create_defect_type(ddb, 0)
+ ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT",
+ "Comment", name)
+
+ assert ddb.virtual_defect_names == set(["DQD_TEST_VIRTUAL_DEFECT"])
+
@with_setup(create_database, teardown_database)
def test_virtual_defect_creation_unicode():
@@ -249,6 +288,24 @@ def test_virtual_defect_creation_unicode():
assert len(ddb.virtual_defect_ids) == 1
assert "DQD_TEST_VIRTUAL_DEFECT" in ddb.virtual_defect_names
+@raises(UnicodeEncodeError)
+@with_setup(create_database, teardown_database)
+def test_virtual_defect_failure_nonascii_name():
+ """
+ Check that we raise an error if the virtual defect name is not ASCII
+ """
+ if six.PY3:
+ import ROOT
+ if ROOT.gROOT.GetVersionInt() < 62000:
+ # Passing str objects using multibyte encodings is broken
+ # with pyroot up to 6.18. Should be fixed in 6.20?
+ return
+ ddb = DefectsDB(TEST_DATABASE, read_only=False)
+
+ ddb.create_defect(u"DQD_TEST_DEFECT_0", "Test")
+ ddb.new_virtual_defect(u"DQD_TÉST_VIRTUAL_DÉFÉCT",
+ u"Comment", u"DQD_TEST_DEFECT_0")
+
@with_setup(create_database, teardown_database)
def test_nonpresent():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
@@ -630,6 +687,8 @@ def test_independent_tags():
ltag2 = ddb.new_logics_tag()
+ assert ltag1 != ltag2, f'{ltag1} and {ltag2} should be different'
+
iovs = DefectsDB(TEST_DATABASE, tag=("HEAD", ltag1)).retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
assert iov_ranges(iovs) == [(100, 200)], str(iov_ranges(iovs))
@@ -687,7 +746,7 @@ def test_virtual_defect_consistency():
"""
Checking that virtual flags don't depend on non-existent flags
"""
- DefectsDB('oracle://ATLAS_COOLPROD;schema=ATLAS_COOLOFL_GLOBAL;dbname=CONDBR2')._virtual_defect_consistency_check()
+ DefectsDB()._virtual_defect_consistency_check()
@with_setup(create_database, teardown_database)
def test_iov_tag_defects():
@@ -706,7 +765,7 @@ def test_iov_tag_defects():
ddb.new_defects_tag("dqd-test", "New iov tag",
iovranges=[(0, 51),
((0,210), (0,306))])
- ddb2 = DefectsDB(TEST_DATABASE, tag='DetStatusDEFECTS-dqd-test')
+ ddb2 = DefectsDB(TEST_DATABASE, tag=('DetStatusDEFECTS-dqd-test','HEAD'))
iovs = ddb2.retrieve(nonpresent=True)
assert len(iovs) == 2
assert ((iovs[0].channel, iovs[0].since, iovs[0].until,
@@ -788,7 +847,45 @@ def test_reject_duplicate_names():
except DefectExistsError:
wasthrown = True
assert wasthrown, 'Creation of duplicate defects should throw, even with different case'
-
+
+ wasthrown = False
+ try:
+ ddb.create_defect('A_test_defect_2', 'A test defect 2')
+ ddb.rename_defect('A_test_defect_2', 'A_TEST_DEFECT')
+ except DefectExistsError:
+ wasthrown = True
+ assert wasthrown, 'Rename of defect to existing defect name should throw'
+
+ ddb.new_virtual_defect('TEST_VIRTUAL_DEFECT', 'Comment', 'A_TEST_DEFECT')
+ wasthrown = False
+ try:
+ ddb.new_virtual_defect('Test_Virtual_Defect', 'Comment', 'A_TEST_DEFECT')
+ except DefectExistsError:
+ wasthrown = True
+ assert wasthrown, 'Creation of duplicate virtual defects should throw'
+
+ ddb.new_virtual_defect('Test_Virtual_Defect_2', 'Comment', 'A_TEST_DEFECT')
+ wasthrown = False
+ try:
+ ddb.rename_defect('Test_Virtual_Defect_2', 'TEST_VIRTUAL_DEFECT')
+ except DefectExistsError:
+ wasthrown = True
+ assert wasthrown, 'Rename of virtual defect to existing virtual defect name should throw'
+
+ wasthrown = False
+ try:
+ ddb.rename_defect('Test_Virtual_Defect_2', 'A_TEST_DEFECT')
+ except DefectExistsError:
+ wasthrown = True
+ assert wasthrown, 'Rename of virtual defect to existing defect name should throw'
+
+ wasthrown = False
+ try:
+ ddb.rename_defect('A_TEST_DEFECT', 'TEST_VIRTUAL_DEFECT')
+ except DefectExistsError:
+ wasthrown = True
+ assert wasthrown, 'Rename of defect to existing virtual defect name should throw'
+
@with_setup(create_database, teardown_database)
def test_get_intolerable_defects():
'''
@@ -845,3 +942,57 @@ def test_noncontiguous_defectid_creation():
assert m == {0: 'DQD_TEST_DEFECT_0', 1: 'DQD_TEST_DEFECT_1', 2: 'TEST_DEFECT_DQD_2', 3: 'DQD_TEST_DEFECT_3', 'DQD_TEST_DEFECT_3': 3, 'DQD_TEST_DEFECT_0': 0, 'DQD_TEST_DEFECT_1': 1, 'TEST_DEFECT_DQD_2': 2}, 'Primary defect problem'
ids, names, m = ddb.get_virtual_channels()
assert m == {2147483648: 'DQD_TEST_VIRTUAL_DEFECT', 2147483649: 'DQD_TEST_VIRTUAL_DEFECT_2', 'DQD_TEST_VIRTUAL_DEFECT': 2147483648, 'DQD_TEST_VIRTUAL_DEFECT_2': 2147483649}, 'Virtual defect problem'
+
+@with_setup(create_database, teardown_database)
+def test_return_types():
+ """
+ Test that return types are appropriate
+ """
+ ddb = DefectsDB(TEST_DATABASE, read_only=False)
+
+ # Create a defect
+ create_defect_type(ddb, 0)
+ did = ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT",
+ "Comment", "DQD_TEST_DEFECT_0")
+ assert type(did) == int
+
+ ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "", "")
+
+ dtag = ddb.new_defects_tag("dqd-test", "New iov tag")
+ assert type(dtag) == str, f'{type(dtag)} instead of str'
+ ltag = ddb.new_logics_tag()
+ assert type(ltag) == str
+ htag = ddb.new_hierarchical_tag(dtag, ltag)
+ assert type(htag) == str
+
+ assert type(ddb.defects_tags) == list, f'{type(ddb.defects_tags)} instead of list'
+ assert type(ddb.defects_tags[0]) == str, f'{type(ddb.defects_tags[0])} instead of str'
+ assert type(ddb.logics_tags) == list, f'{type(ddb.logics_tags)} instead of list'
+ assert type(ddb.logics_tags[0]) == str, f'{type(ddb.logics_tags[0])} instead of str'
+ assert type(ddb.tags) == list, f'{type(ddb.tags)} instead of list'
+ assert type(ddb.tags[0]) == str, f'{type(ddb.tags[0])} instead of str'
+
+ ids, names, _ = ddb.get_channels()
+ ids = set(ids); names = set(names) # Duplicate so we don't stomp on the underlying data
+ assert type(ids.pop()) == int
+ assert type(names.pop()) == str
+ ids, names, _ = ddb.get_virtual_channels()
+ ids = set(ids); names = set(names)
+ assert type(ids.pop()) == int
+ assert type(names.pop()) == str
+
+ iov = ddb.retrieve()
+ assert type(iov[0].channel) == str
+ assert type(iov[0].comment) == str
+ assert type(iov[0].user) == str
+ assert type(iov[0].present) == bool
+ assert type(iov[0].recoverable) == bool
+
+ assert type(ddb.all_defect_descriptions["DQD_TEST_DEFECT_0"]) == str, str(ddb.all_defect_descriptions)
+ assert type(ddb.all_defect_descriptions["DQD_TEST_VIRTUAL_DEFECT"]) == str, str(ddb.all_defect_descriptions)
+
+ ddb2 = DefectsDB(TEST_DATABASE, read_only=False, tag=htag)
+ assert type(ddb2.defects_tag) == str
+ assert type(ddb2.logics_tag) == str
+ assert type(ddb2.tag.defects) == str
+ assert type(ddb2.tag.logic) == str
\ No newline at end of file
diff --git a/DataQuality/DQDefects/python/virtual_calculator.py b/DataQuality/DQDefects/python/virtual_calculator.py
index 8e451f87f93a1161e2e2024e237d31e14bfc7be3..25cb345429682b3e2d89efbce7ff2642ceb86f18 100644
--- a/DataQuality/DQDefects/python/virtual_calculator.py
+++ b/DataQuality/DQDefects/python/virtual_calculator.py
@@ -1,6 +1,8 @@
# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
-from logging import getLogger; log = getLogger("DQDefects.virtual_defect_calculator")
+from logging import getLogger
+
+from .virtual_logic import DefectLogic; log = getLogger("DQDefects.virtual_defect_calculator")
from collections import defaultdict
@@ -11,8 +13,15 @@ from DQDefects import DEFECT_IOV
import six
+from collections.abc import Container
+from typing import Tuple, Union, Optional, Iterable, Generator, Mapping
+
-def generate_virtual_defects(by_channel, logics, since_cr, until_cr, ignore):
+def generate_virtual_defects(by_channel: Mapping[str, IOVSet], logics: Iterable[DefectLogic],
+ since_cr: Union[int, Tuple[int, int], RunLumi],
+ until_cr: Union[int, Tuple[int, int], RunLumi],
+ ignore: Optional[Container[str]]
+ ) -> Generator[Tuple[RunLumi, RunLumi, Mapping[str, DEFECT_IOV]], None, None]:
"""
An iterator which emits (since, until, {channel name : iov})
@@ -27,7 +36,7 @@ def generate_virtual_defects(by_channel, logics, since_cr, until_cr, ignore):
for since, until, current_iovs, changes in process_iovs_changed(*iovsets):
# Update things that changed since last iteration in the states dict
for change in changes:
- if ignore and channels[change] in ignore:
+ if ignore is not None and channels[change] in ignore:
continue
states[channels[change]] = current_iovs[change]
@@ -43,7 +52,7 @@ def generate_virtual_defects(by_channel, logics, since_cr, until_cr, ignore):
yield since, until, states
-def bad_iov(since, until):
+def bad_iov(since: RunLumi, until: RunLumi) -> bool:
"""
Skip some commonly emitted nonsensical IoVs.
"""
@@ -54,9 +63,11 @@ def bad_iov(since, until):
(since.lumi == 0 and until.lumi == 1 and since.run == until.run)
)
-def calculate_virtual_defects(primary_iovs, evaluation_order,
- virtual_output_channels, primary_output_channels,
- since, until, ignore):
+def calculate_virtual_defects(primary_iovs: IOVSet, evaluation_order: Iterable[DefectLogic],
+ virtual_output_channels: Iterable[str], primary_output_channels: Iterable[str],
+ since: Optional[Union[int, Tuple[int, int], RunLumi]],
+ until: Optional[Union[int, Tuple[int, int], RunLumi]],
+ ignore: Optional[Container[str]]) -> IOVSet:
"""
Returns a list of IoVs for a given query in the normal COOL order
(sorted by channelID, then by since)
@@ -73,7 +84,7 @@ def calculate_virtual_defects(primary_iovs, evaluation_order,
# Copy desired primary channels to the result
for primary_channel, primary_iovs in six.iteritems(primary_by_channel):
if primary_channel in primary_output_channels:
- if ignore and primary_channel in ignore:
+ if ignore is not None and primary_channel in ignore:
continue
result[primary_channel] = primary_iovs
@@ -96,7 +107,7 @@ def calculate_virtual_defects(primary_iovs, evaluation_order,
# Sort them by traditional COOL sort ordering (by channelId first,
# then by since. `iovs` are already ordered by since.)
result_list = IOVSet()
- for channel, iovs in sorted(six.iteritems(result)):
+ for _, iovs in sorted(six.iteritems(result)):
result_list.extend(iovs.solidify(DEFECT_IOV))
return result_list
diff --git a/DataQuality/DQDefects/python/virtual_logic.py b/DataQuality/DQDefects/python/virtual_logic.py
index 01770d6930158f1f86c06839c8b89e2a7bea060a..dca4d13431987ff0c4979740c5831382af65961a 100644
--- a/DataQuality/DQDefects/python/virtual_logic.py
+++ b/DataQuality/DQDefects/python/virtual_logic.py
@@ -1,29 +1,32 @@
-# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration
+from __future__ import annotations
from DQDefects import DEFECT_IOV
from functools import reduce
+from typing import Mapping
+
class DefectLogic(object):
- def __init__(self, defect_iov):
+ def __init__(self, defect_iov) -> None:
self.name = defect_iov.channel
self.clauses = defect_iov.clauses.split()
self.operations = set()
self.inverting = set()
self._operations_parse()
self.realclauses = set(_[1] for _ in self.operations)
- self.primary_defects = None
+ self.primary_defects = set()
self.evaluate = self.bad_evaluator
- def bad_evaluator(self):
+ def bad_evaluator(self, states) -> DEFECT_IOV:
assert False, "bad_evaluation should never be called"
- def set_evaluation(self, full):
+ def set_evaluation(self, full: bool) -> None:
if full:
self.evaluate = self.evaluate_full
else:
self.evaluate = self.evaluate_partial
- def _operations_parse(self):
+ def _operations_parse(self) -> None:
from operator import truth, not_
for name in self.clauses:
if name[0] == '!':
@@ -32,7 +35,7 @@ class DefectLogic(object):
else:
self.operations.add((truth, name))
- def _populate(self, defect_logics):
+ def _populate(self, defect_logics: Mapping[str, DefectLogic]) -> None:
"""
Populate the dependencies with logic objects, and the primary defects.
"""
@@ -43,7 +46,7 @@ class DefectLogic(object):
self.primary_defects = set(c for _, c in self.operations
if c not in defect_logics)
- def evaluate_full(self, states):
+ def evaluate_full(self, states: Mapping[str, DEFECT_IOV]) -> DEFECT_IOV:
"""
Evaluate this defect logic for a point in time.
@@ -65,7 +68,7 @@ class DefectLogic(object):
+['!'+_ for _ in uninverted]))
)
- def evaluate_partial(self, states):
+ def evaluate_partial(self, states: Mapping[str, DEFECT_IOV]) -> DEFECT_IOV:
"""
Evaluate this defect logic for a point in time.
diff --git a/DataQuality/DQDefects/python/virtual_mixin.py b/DataQuality/DQDefects/python/virtual_mixin.py
index 4270c90afa82029412c9263a328d4d0cf93c8874..cb63c78f67b2411170f459f7dbab522f2331294d 100644
--- a/DataQuality/DQDefects/python/virtual_mixin.py
+++ b/DataQuality/DQDefects/python/virtual_mixin.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration
from logging import getLogger; log = getLogger("DQDefects.virtual_defect_mixin")
from DQUtils import fetch_iovs
@@ -7,10 +7,9 @@ from .exceptions import DefectUnknownError, DefectExistsError
from .ids import choose_new_defect_id
from .virtual_logic import DefectLogic
import six
-if six.PY2:
- def _encode (s, enc): return s.encode(enc)
-else:
- def _encode (s, enc): return s
+
+from typing import Optional, Iterable, Mapping, Set, Sequence, List
+
NONHEAD_MODIFICATION_MSG = ("Operations which modify virtual defects can only "
"be done on the HEAD tag.")
@@ -43,84 +42,85 @@ class DefectsDBVirtualDefectsMixin(object):
A DefectsDB mixin for managing virtual defects
"""
- def __init__(self):
+ def __init__(self) -> None:
self._virtual_defect_logics = None
super(DefectsDBVirtualDefectsMixin, self).__init__()
- def validate_clauses(self, clauses):
+ def validate_clauses(self, clauses: str) -> None:
all_defects = self.defect_names | self.virtual_defect_names
for clause in clauses.split():
if clause[0] in ('!', '-'):
clause = clause[1:]
assert clause in all_defects, (clause + " is not a known defect")
- def update_virtual_defect(self, defect_name, clauses, comment=None):
+ def update_virtual_defect(self, defect_name: str, clauses: str, comment: Optional[str] = None) -> None:
# HEAD protection is here, to allow other functions to
# change other tags in a known manner with _update_virtual_defect
assert self.logics_tag == "HEAD", NONHEAD_MODIFICATION_MSG
assert not self._read_only, "Insertion on read-only database"
- clauses = _encode(clauses,'ascii')
- comment = _encode(comment,'utf-8') if comment is not None else comment
self._update_virtual_defect(defect_name, clauses, comment)
- def _update_virtual_defect(self, defect_name, clauses, comment=None, tag=None):
- if not tag: tag = 'HEAD'
+ def _update_virtual_defect(self, defect_name: str, clauses: str,
+ comment: Optional[str] = None, tag: Optional[str] = None) -> None:
defect_id = self.defect_chan_as_id(defect_name)
assert self.defect_is_virtual(defect_id), ("Tried to update nonvirtual"
" defect with update_virtual_defect()")
self.validate_clauses(clauses)
-
- self._defect_logic_payload["clauses"] = clauses
+
+ tag = tag if tag is not None else 'HEAD'
+ ucomment = comment.encode('utf-8') if comment is not None else None
+
+ self._defect_logic_payload["clauses"] = clauses.encode('ascii')
store = self.defect_logic_folder.storeObject
- store(0, 2**63-1, self._defect_logic_payload, defect_id, tag,
+ store(0, 2**63-1, self._defect_logic_payload, defect_id, tag.encode('ascii'),
(True if tag != 'HEAD' else False))
- if comment:
- self.defect_logic_folder.setChannelDescription(defect_id, comment)
+ if comment is not None:
+ self.defect_logic_folder.setChannelDescription(defect_id, ucomment)
- def rename_defect(self, defect_name, new_defect_name):
+ def rename_defect(self, defect_name: str, new_defect_name: str) -> None:
"""
Rename a defect (primary or virtual). Will keep data and channel ID.
Will fix up all virtual defect dependencies in all tags.
"""
assert not self._read_only, "Channel rename on read-only database"
- defect_name = _encode(defect_name,'ascii')
- new_defect_name = _encode(new_defect_name,'ascii')
try:
oldname = self.normalize_defect_names(new_defect_name)
- already_exists = True
- except DefectUnknownError:
- already_exists = False
- if already_exists:
raise DefectExistsError('Defect %s already exists' % oldname)
+ except DefectUnknownError:
+ pass
defect_id = self.defect_chan_as_id(defect_name)
+ anew_defect_name = new_defect_name.encode('ascii')
if self.defect_is_virtual(defect_id):
- self.defect_logic_folder.setChannelName(defect_id, new_defect_name)
- self._virtual_defect_names = None
- self._virtual_defect_id_map = None
+ self.defect_logic_folder.setChannelName(defect_id, anew_defect_name)
+ self._virtual_defect_names = set()
+ self._virtual_defect_id_map = {}
self._virtual_defect_logics = None
+ self._virtual_initialized = False
else:
- self.defects_folder.setChannelName(defect_id, new_defect_name)
- self._defect_names = None
- self._defect_id_map = None
+ self.defects_folder.setChannelName(defect_id, anew_defect_name)
+ self._defect_names = set()
+ self._defect_id_map = {}
+ self._initialized = False
import contextlib
@contextlib.contextmanager
def logics_unlocking(tag):
+ atag = tag.encode('ascii')
log.info('Unlocking tag %s', tag)
folder = self.defect_logic_folder
- orig_status = folder.tagLockStatus(tag)
+ orig_status = folder.tagLockStatus(atag)
if orig_status != 0:
- folder.setTagLockStatus(tag, 0)
+ folder.setTagLockStatus(atag, 0)
yield
if orig_status != 0:
- folder.setTagLockStatus(tag, orig_status)
+ folder.setTagLockStatus(atag, orig_status)
log.info('Done with tag %s', tag)
def logic_substitute(clause, oldname, newname):
@@ -146,16 +146,13 @@ class DefectsDBVirtualDefectsMixin(object):
self._update_virtual_defect(vd, newclauses, tag=tag)
self._virtual_defect_logics = None
- def new_virtual_defect(self, defect_name, comment, clauses):
+ def new_virtual_defect(self, defect_name: str, comment: Optional[str], clauses: str) -> int:
"""
Create a new virtual defect
"""
assert self.logics_tag == "HEAD", NONHEAD_MODIFICATION_MSG
assert not self._read_only, "Insertion on read-only database"
from DQUtils.channel_mapping import get_channel_ids_names
- clauses = _encode(clauses,'ascii')
- defect_name = _encode(defect_name,'ascii')
- comment = _encode(comment,'utf-8') if comment is not None else comment
# Force load of defects_folder to populate _defect_payload
store = self.defect_logic_folder.storeObject
@@ -163,30 +160,28 @@ class DefectsDBVirtualDefectsMixin(object):
self.validate_clauses(clauses)
- p["clauses"] = clauses
+ p["clauses"] = clauses.encode('ascii')
# need to get true mapping of folder, not just what is visible
# from this tag
- ids, names, mapping = get_channel_ids_names(self.defect_logic_folder)
+ ids, _, mapping = get_channel_ids_names(self.defect_logic_folder)
defect_id = choose_new_defect_id(mapping, defect_name, True)
log.debug("Creating new defect %s: 0x%08x", defect_name, defect_id)
try:
oldname = self.normalize_defect_names(defect_name)
- already_exists = True
- except DefectUnknownError:
- already_exists = False
- if already_exists:
raise DefectExistsError('Defect %s already exists' % oldname)
+ except DefectUnknownError:
+ pass
if (defect_id in ids):
raise DefectExistsError(defect_name)
- self.defect_logic_folder.createChannel(defect_id, defect_name, comment)
+ self.defect_logic_folder.createChannel(defect_id, defect_name.encode('ascii'), comment.encode('utf-8'))
store(0, 2**63-1, p, defect_id)
self._new_virtual_defect(defect_id, defect_name)
return defect_id
- def _resolve_evaluation_order(self, defects=None):
+ def _resolve_evaluation_order(self, defects: Optional[Iterable[str]] = None) -> List[DefectLogic]:
"""
Returns a list of DefectLogic objects that need to be evaluated,
in the correct order for them to be consistent.
@@ -211,7 +206,7 @@ class DefectsDBVirtualDefectsMixin(object):
resolved.remove(MasterNode)
return resolved
- def resolve_primary_defects(self, defect_logics):
+ def resolve_primary_defects(self, defect_logics: Iterable[DefectLogic]) -> Set[str]:
"""
Determine which primary flags are used for
a given list of input `virtual_defect_names`.
@@ -223,7 +218,7 @@ class DefectsDBVirtualDefectsMixin(object):
return primary_defects
@property
- def virtual_defect_logics(self):
+ def virtual_defect_logics(self) -> Mapping[str, DefectLogic]:
"""
Returns all virtual defects in the form {"defect_name" : DefectLogic()}
for the tag DefectDB was constructed on.
@@ -235,23 +230,23 @@ class DefectsDBVirtualDefectsMixin(object):
return self._virtual_defect_logics
- def _get_virtual_defect_logics(self, tag):
- if tag != "HEAD" and not self.defect_logic_folder.existsUserTag(tag):
+ def _get_virtual_defect_logics(self, tag: str) -> Mapping[str, DefectLogic]:
+ if tag != "HEAD" and not self.defect_logic_folder.existsUserTag(tag.encode('ascii')):
# The tag doesn't exist, so there is nothing to fetch.
return {}
- logics = fetch_iovs(self.defect_logic_folder, tag=tag,
+ logics = fetch_iovs(self.defect_logic_folder, tag=tag.encode('ascii'),
named_channels=True)
-
- logics = dict((l.channel, DefectLogic(l)) for l in logics)
+
+ logics = {l.channel: DefectLogic(l) for l in logics}
- for defect_name, defect_logic in six.iteritems (logics):
+ for _, defect_logic in six.iteritems (logics):
defect_logic._populate(logics)
return logics
- def get_intolerable_defects(self, old_primary_only=True,
- exclude=['TIGHT', 'IDTIGHT', 'PHYS_.*']):
+ def get_intolerable_defects(self, old_primary_only: bool = True,
+ exclude: Sequence[str] = ['TIGHT', 'IDTIGHT', 'PHYS_.*']):
"""
Returns primary defects that are depended on by a virtual defect
if old_primary_only == True, only return those depended on by a
@@ -268,7 +263,7 @@ class DefectsDBVirtualDefectsMixin(object):
vdl = dict(l for l in vdl.items() if not rex.match(l[0]))
return self.resolve_primary_defects(vdl.values())
- def _virtual_defect_consistency_check(self):
+ def _virtual_defect_consistency_check(self) -> bool:
"""
When called, uses an assertion to check that there are no missing
defects. This is a database consistency check which should never be
diff --git a/DataQuality/DQDefects/share/DQDefects.ref b/DataQuality/DQDefects/share/DQDefects.ref
index 0a6d4f2f829be72554f797afbf29a6f8c27b79b3..2b56d25d11c5069db9c9c15114a372e269053fd7 100644
--- a/DataQuality/DQDefects/share/DQDefects.ref
+++ b/DataQuality/DQDefects/share/DQDefects.ref
@@ -5,9 +5,13 @@ DQDefects.tests.test_defect_creation ... ok
Testing virtual defects involving inversion ... ok
DQDefects.tests.test_defect_insertion_retrieval ... ok
DQDefects.tests.test_defect_insertion_retrieval_unicode ... ok
+Check that we raise an error if the defect name is not ASCII ... ok
DQDefects.tests.test_defect_empty_retrieval ... ok
DQDefects.tests.test_virtual_defect_creation ... ok
+Check that the defect name list is updated when a defect is created ... ok
+Check that the virtual defect name list is updated when a defect is created ... ok
DQDefects.tests.test_virtual_defect_creation_unicode ... ok
+Check that we raise an error if the virtual defect name is not ASCII ... ok
DQDefects.tests.test_nonpresent ... ok
Testing many virtual channels with sparsely placed IoVs ... ok
Testing virtual defect basics ... ok
@@ -29,8 +33,9 @@ DQDefects.tests.test_reject_duplicate_names ... ok
Check that the intolerable defect listing function works ... ok
Checking that listing a tag works even if the tag is empty ... ok
Test that defects are inserted correctly into non-contiguous ranges ... ok
+Test that return types are appropriate ... ok
----------------------------------------------------------------------
-Ran 31 tests in 26.579s
+Ran 36 tests in 26.579s
OK