diff --git a/CI/CMakeLists.txt b/CI/CMakeLists.txt
deleted file mode 100644
index e7116b2f2f5bb620cceedc2a5300945c3209b03d..0000000000000000000000000000000000000000
--- a/CI/CMakeLists.txt
+++ /dev/null
@@ -1,12 +0,0 @@
-
-# Declare the package's name to the build. This is necessary for it
-# to show up nicely in the build results.
-atlas_subdir( CI )
-
-# Declare tests for the "package":
-add_test (NAME DomainMapTests COMMAND python -m test.test_domain_map WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
-add_test (NAME WatchListTests COMMAND python -m test.test_watch_list WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
-set_property (TEST DomainMapTests WatchListTests APPEND PROPERTY LABELS CI)
-
-# install executables
-atlas_install_scripts( sweep_MR.py )
diff --git a/CI/domain_map.py b/CI/domain_map.py
deleted file mode 100644
index fd5cc81edafd5dbc954833f31e04df39c397cdd8..0000000000000000000000000000000000000000
--- a/CI/domain_map.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# dictionary for mapping package names to software domains
-# keys ... must be strings for the name of the software domain
-# values ... must be sets containing valid regular expression strings
-DOMAIN_MAP = {}
-
-DOMAIN_MAP['Analysis'] = set(['^PhysicsAnalysis/'])
-DOMAIN_MAP['BTagging'] = set(['JetTagging', 'ParticleJetTools', 'FlavourTag'])
-DOMAIN_MAP['Build'] = set(['^Build$','^Projects/'])
-DOMAIN_MAP['Calorimeter'] = set(['^Calorimeter/'])
-DOMAIN_MAP['CI'] = set(['^CI'])
-DOMAIN_MAP['Core'] = set(['^Control/'])
-DOMAIN_MAP['Database'] = set(['^Database/'])
-DOMAIN_MAP['Digitization'] = set(['Digitization','TileSimAlgs','PileUpComps','PileUpTools','RunDependentSim'])
-DOMAIN_MAP['DQ'] = set(['^DataQuality/','^LumiBlock/'])
-DOMAIN_MAP['EDM'] = set(['^Event/'])
-DOMAIN_MAP['Egamma'] = set(['^egamma'])
-DOMAIN_MAP['EventDisplay'] = set(['^graphics/'])
-DOMAIN_MAP['Externals'] = set(['^External/'])
-DOMAIN_MAP['ForwardDetectors'] = set(['^ForwardDetectors/'])
-DOMAIN_MAP['Generators'] = set(['^Generators/'])
-DOMAIN_MAP['Geometry'] = set(['^AtlasGeometryCommon/','^DetectorDescription/'])
-DOMAIN_MAP['InnerDetector'] = set(['^InnerDetector/'])
-DOMAIN_MAP['JetEtmiss'] = set(['Jet','(?<!spectro)(?<!geo)MET','MissingEt','EventShape'])
-DOMAIN_MAP['LAr'] = set(['^LArCalorimeter/'])
-DOMAIN_MAP['Magnets'] = set(['^MagneticField/'])
-DOMAIN_MAP['MuonSpectrometer'] = set(['^MuonSpectrometer/'])
-DOMAIN_MAP['Other'] = set(['^Commission/','^LCG_Interfaces/','^PackDist/','^ReleaseTests/'])
-DOMAIN_MAP['Overlay'] = set(['Overlay'])
-DOMAIN_MAP['Reconstruction'] = set(['^Reconstruction/'])
-DOMAIN_MAP['Simulation'] = set(['^Simulation/','G4','InDetSimUtils','SubDetectorEnvelopes','SimEvent'])
-DOMAIN_MAP['Tau'] = set(['Tau'])
-DOMAIN_MAP['Test'] = set(['^AtlasTest/','^TestPolicy/'])
-DOMAIN_MAP['TestBeam'] = set(['^TestBeam/'])
-DOMAIN_MAP['Tile'] = set(['^TileCalorimeter/'])
-DOMAIN_MAP['Tools'] = set(['^Tools/'])
-DOMAIN_MAP['Tracking'] = set(['^Tracking/'])
-DOMAIN_MAP['Trigger'] = set(['^Trigger/','^HLT/','Trig','^DetectorDescription/.?Reg','^DetectorDescription/RoiDescriptor','RegionSelector'])
-DOMAIN_MAP['TriggerMenu'] = set(['^Trigger/TriggerCommon/TriggerMenu'])
-
diff --git a/CI/get_srl_domain_information.py b/CI/get_srl_domain_information.py
deleted file mode 100644
index 852aff382c8dbddf8a6b791de6cd18ad85cf2a3b..0000000000000000000000000000000000000000
--- a/CI/get_srl_domain_information.py
+++ /dev/null
@@ -1,93 +0,0 @@
-import argparse, glob, logging, os, pickle, re, sys
-
-def get_srl_package_mapping(directory):
- domains = {}
- logging.debug("get SRL domain <--> package mapping from directory '%s'" % directory)
- srl_files = glob.glob(os.path.join(directory,"atlas-srl-*"))
- for f in srl_files:
- logging.debug("reading file '%s'" % f)
- m = re.match("atlas-srl-(\w+)",os.path.basename(f))
- domain = m.group(1)
- logging.info("extracted domain '%s' from filename '%s'" % (domain,os.path.basename(f)))
- fh = open(f)
- pkgs = [package.strip() for package in fh.readlines() if package.strip()]
- fh.close()
- logging.debug("found packages %s in file '%s'" % (str(pkgs),f))
- if not domain in domains:
- domains[domain] = set(pkgs)
- else:
- domains[domain].update(pkgs)
-
- for domain,packages in domains.items():
- logging.debug("domain '%s': packages = %s" % (domain,str(packages)))
-
- return domains
-
-def get_srl_group_members(group_def_file):
- experts = {}
- logging.debug("read SRL group definitions from '%s'" % group_def_file)
- fh = open(group_def_file)
- groups = [l.strip() for l in fh.readlines()]
- fh.close()
- for g in groups:
- m = re.match("atlas-srl-(\w+).*=([\w ,]+)",g)
- domain = m.group(1)
- expert_list = m.group(2).split(',')
- members = set([e.strip() for e in expert_list])
- logging.info("group '%s' consists of members %s" % (domain,members))
- if not domain in experts:
- experts[domain] = members
- else:
- experts[domain].update(members)
-
- for domain,expert_list in experts.items():
- logging.debug("domain '%s': experts = %s" % (domain,str(expert_list)))
-
- return experts
-
-def merge_srl_packages_experts(pkg_map,expert_map):
- domains = {}
- for domain,pkgs in pkg_map.items():
- if domain in expert_map:
- domains[domain] = {"packages": pkgs, "experts": expert_map[domain]}
- else:
- logging.error("found package mapping for domain '%s' but no experts" % domain)
-
- missing = set(expert_map.keys()) - set(domains.keys())
- if len(missing) > 0:
- logging.error("following domains have experts but no packages: %s" % str(missing))
-
- return domains
-
-def main():
- parser = argparse.ArgumentParser(description="ATLAS SRL mapper",
- formatter_class=argparse.ArgumentDefaultsHelpFormatter)
- parser.add_argument("indir",type=str,help="directory with ATLAS SRL definitions")
- parser.add_argument("outfile",type=str,help="output pickle file with dictionary")
- parser.add_argument("-v","--verbose",default="INFO",choices=["DEBUG","INFO","WARNING","ERROR","CRITICAL"],help="verbosity level")
-
- # get command line arguments
- args = parser.parse_args()
- args.indir = os.path.abspath(args.indir)
-
- # configure log output
- logging.basicConfig(format='%(asctime)s %(levelname)-10s %(message)s',
- datefmt='%H:%M:%S',
- level=logging.getLevelName(args.verbose))
-
- # delegate
- packages = get_srl_package_mapping(os.path.join(args.indir,"done"))
- experts = get_srl_group_members(os.path.join(args.indir,"atlas-srl_group_defs.txt"))
- domains = merge_srl_packages_experts(packages,experts)
-
- if os.path.isfile(args.outfile):
- var = ""
- while not (var == 'y' or var == 'n'):
- var = raw_input("The output file '%s' already exists. Do you want to overwrite it? y|[n] " % args.outfile)
- if var == 'n':
- sys.exit(0)
-
- pickle.dump(domains,open(args.outfile,'w'))
-
-if __name__ == "__main__":
- main()
diff --git a/CI/gitlab_mr_helpers.py b/CI/gitlab_mr_helpers.py
deleted file mode 100644
index edf627f84c60cb9ea854e1937e4386a11555d9f1..0000000000000000000000000000000000000000
--- a/CI/gitlab_mr_helpers.py
+++ /dev/null
@@ -1,80 +0,0 @@
-import logging, os, re
-from domain_map import DOMAIN_MAP
-
-DOMAINS = DOMAIN_MAP.keys()
-
-def print_collection(c):
- return " - " + "\n - ".join(c)
-
-def map_filename_to_package(fname):
- """
- fname ... full path of filename
-
- note: only works if this file resides in <ATHENA_ROOT>/CI
-
- return: package path for given full file path
- """
- # get Athena root directory (specific to current layout) which is one level up
- athena_root = os.path.dirname(os.path.abspath(os.path.join(os.path.realpath(__file__),'../')))
- logging.debug("found Athena root directory '%s'",athena_root)
-
- # start from directory name
- pkg_name = os.path.dirname(fname)
-
- # recursively move up in directory hierarchy and look for CMakeLists.txt
- while pkg_name and pkg_name != '/':
- if os.path.isfile(os.path.join(athena_root,pkg_name,'CMakeLists.txt')):
- break
- pkg_name = os.path.dirname(pkg_name)
-
- logging.debug("mapped file '%s' to package '%s'" % (fname,pkg_name))
- return pkg_name
-
-def map_package_domain(pkg):
- """
- map the package name to a domain
-
- pkg ... package path
-
- return: domain(s) of package
- """
- domains = set()
- for domain,pattern_list in DOMAIN_MAP.items():
- for pattern in pattern_list:
- if re.search(pattern,pkg,re.I):
- domains.add(domain)
- break
-
- return domains
-
-def list_changed_packages(mr):
- """
- mr ... Gitlab merge request object
-
- return: sorted set of package names affected by this merge request
- """
- changed_files = set([c[p] for c in mr.changes()['changes'] for p in ['old_path','new_path']])
- logging.debug("changed files:\n" + print_collection(changed_files))
-
- return sorted(set([map_filename_to_package(f) for f in changed_files]))
-
-def list_affected_domains(packages):
- """
- packages ... set of Athena packages
-
- return: sorted set of domains responsible for the given packages
- and a set of all packages which could not be mapped
- """
- domains = set()
- unmapped_pkgs = set()
- for pkg in packages:
- this_domain = map_package_domain(pkg)
- logging.debug("mapped package '%s' to domain '%s'" % (pkg,this_domain))
- if not this_domain:
- unmapped_pkgs.add(pkg)
- else:
- domains.update(this_domain)
- logging.debug("affected domains:\n" + print_collection(domains))
- logging.debug("unmapped packages:\n" + print_collection(unmapped_pkgs))
-
- return domains,unmapped_pkgs
diff --git a/CI/handle_new_mr.py b/CI/handle_new_mr.py
deleted file mode 100644
index 4e1d032065c47b346e1f1e0c8c7dfb1afe48fb04..0000000000000000000000000000000000000000
--- a/CI/handle_new_mr.py
+++ /dev/null
@@ -1,174 +0,0 @@
-import argparse, gitlab, logging, re, sys
-from gitlab.exceptions import GitlabGetError
-from gitlab_mr_helpers import print_collection, list_changed_packages, list_affected_domains
-from watch_list import WATCH_LIST
-
-def comment_affected_packages(packages):
- """
- prepare initial comment to merge request listing affected packages
-
- packages ... list of affected packages
-
- return: comment text
- """
- n_packages = len(packages)
- if n_packages == 0:
- comment = "This merge request affects no known packages. Consult an expert!"
- elif n_packages == 1:
- comment = "This merge request affects 1 package: \n- " + list(packages)[0]
- elif n_packages <= 20:
- comment = "This merge request affects %d packages: \n- " % n_packages
- comment += " \n- ".join(sorted(packages))
- else:
- comment = "This merge request affects %d packages. Since this is a long list, I will not print it here." % n_packages
-
- return comment
-
-def warn_unmapped_packages(packages):
- """
- prepare warning for packages which could not be mapped to domains
-
- packages ... set of packages which could not be mapped
-
- return: comment text
- """
- if not packages:
- return ""
-
- comment = " \n \nCould not map the following packages to a domain: \n \n"
- for p in packages:
- comment += "- %s \n" % p
- return comment
-
-def add_watchers(packages):
- """
- prepare comment to add watchers for the modified packages
-
- packages ... set of affected packages
-
- return: comment text
- """
- watchers = set()
- for pkg in packages:
- for pattern,users in WATCH_LIST.items():
- if re.search(pattern,pkg):
- logging.debug("package '%s' matches watch list pattern '%s'" % (pkg,pattern))
- logging.debug("subscribing users:\n" + print_collection(users))
- watchers.update(users)
-
- logging.debug("list of watchers for this MR:\n" + print_collection(watchers))
-
- if watchers:
- watchers = ['@' + w for w in watchers]
- comment = " \n \nAdding "
- comment += " ,".join(watchers)
- comment += " as watcher%s" % ('' if len(watchers) == 1 else 's')
- else:
- comment = ""
-
- return comment
-
-def add_labels(mr,domains):
- """
- add review and domain labels to merge request
-
- mr ... Gitlab merge request object
- domains ... set of affected domains
-
- return: updated list of labels
- """
- # labels assigned during MR creation
- labels = set(mr.labels)
- logging.debug("labels assigned during MR creation:\n" + print_collection(labels))
- # assign labels for all affected domains
- for d in domains:
- labels.add(d)
- # remove possible complete review labels
- for label in list(labels):
- if re.match("review-",label):
- labels.discard(label)
- # add review-pending flag
- labels.add("review-pending-level-1")
- # add label for target branch
- labels.add(mr.target_branch)
- logging.debug("updated labels:\n" + print_collection(labels))
- mr.labels = list(labels)
- mr.save()
-
- return labels
-
-def handle_new_merge_request(args):
- """
- handle new merge request in Gitlab
-
- This involves the following steps:
- - get a list of packages affected by these changes
- - post a comment about the affected packages
- - determine domain experts and mention those in a comment
- - assign labels for sign-off process by domain experts
- """
- # get GitLab API handler
- gl = gitlab.Gitlab(args.url,args.token)
- try:
- # get Gitlab project object
- project = gl.projects.get(args.project_name)
- logging.debug("retrieved Gitlab project handle")
- # get Gitlab merge request object
- mr = project.mergerequests.get(args.mr_id)
- logging.debug("retrieved Gitlab merge request handle")
- except GitlabGetError as e:
- logging.critical("error communication with Gitlab API '%s'" % (e.error_message))
- sys.exit(1)
-
- handled_mr_states = ["opened","reopened","merged"]
- if not mr.state in handled_mr_states:
- logging.debug("ignore merge request in '%s' state" % mr.state)
- sys.exit(0)
-
- # get list of affected packages
- affected_packages = filter(None,list_changed_packages(mr))
- # get list of domains
- affected_domains,unmapped_pkgs = list_affected_domains(affected_packages)
- # add CI as fallback check if no responsible domain was found
- if not affected_domains:
- logging.debug("no responsible domain found -> please check that this is expected")
- affected_domains = set(['no-domain'])
- # assemble atlasbot comment
- comment = ""
- # add comment listing affected packages
- comment += comment_affected_packages(affected_packages)
- # warn about unmapped packages
- comment += warn_unmapped_packages(unmapped_pkgs)
- # add watchers for affected packages
- comment += add_watchers(affected_packages)
-
- # publish comment
- logging.debug("add comment to merge request:\n\n" + comment + "\n")
- mr.notes.create({'body':comment})
-
- # add labels
- add_labels(mr,affected_domains)
-
-def main():
- parser = argparse.ArgumentParser(description="GitLab merge request handler",formatter_class=argparse.ArgumentDefaultsHelpFormatter)
- parser.add_argument("-m","--merge-request-id",dest="mr_id",required=True,type=int,help="(unique) ID of merge request (not the project specific IID)")
- parser.add_argument("-p","--project-name",dest="project_name",required=True,help="GitLab project with namespace (e.g. user/my-project)")
- parser.add_argument("-t","--token",required=True,help="private GitLab user token")
- parser.add_argument("-u","--url",default="https://gitlab.cern.ch",help="URL of GitLab instance")
- parser.add_argument("-v","--verbose",default="INFO",choices=["DEBUG","INFO","WARNING","ERROR","CRITICAL"],help="verbosity level")
-
- # get command line arguments
- args = parser.parse_args()
-
- # configure log output
- logging.basicConfig(format='%(asctime)s %(levelname)-10s %(message)s',
- datefmt='%H:%M:%S',
- level=logging.getLevelName(args.verbose))
-
- logging.debug("parsed arguments:\n" + repr(args))
-
- # delegate
- handle_new_merge_request(args)
-
-if __name__ == "__main__":
- main()
diff --git a/CI/post_ci_result.py b/CI/post_ci_result.py
deleted file mode 100644
index 910fa9710a24624544c387edb450bedc84df97d5..0000000000000000000000000000000000000000
--- a/CI/post_ci_result.py
+++ /dev/null
@@ -1,93 +0,0 @@
-import argparse, gitlab, logging, sys
-from gitlab.exceptions import GitlabGetError, GitlabCreateError
-
-def compose_result_text(status):
- """
- generate comment line describing global result of CI job
-
- status ... Jenkins exit code as string
- """
- text = ":question: **CI Result UNSET**"
- if status == "SUCCESS":
- text = ":white_check_mark: **CI Result SUCCESS**"
- elif status == "FAILURE":
- text = ":negative_squared_cross_mark: **CI Result FAILURE**"
- elif status == "ABORT":
- text = ":point_up: **CI Result ABORTED**"
-
- return text
-
-def compose_stage_text(stage,result):
- """
- generate comment line describing result of CI job stage
-
- result ... Jenkin exit code as string
- """
- if result == "SUCCESS":
- text = ":white_check_mark: "
- elif result == "FAILURE":
- text = ":o: "
- # everything else means that this stage was not run
- else:
- text = ":white_circle: "
-
- return text + stage
-
-def main():
- parser = argparse.ArgumentParser(description="GitLab merge request commentator",formatter_class=argparse.ArgumentDefaultsHelpFormatter)
- parser.add_argument("--externals",required=True,type=str,help="result of external building step")
- parser.add_argument("--cmake",required=True,type=str,help="result of cmake configuration step")
- parser.add_argument("--make",required=True,type=str,help="result of build step")
- parser.add_argument("--test",required=True,type=str,help="result of test step")
- parser.add_argument("--status",required=True,choices=["SUCCESS","FAILURE","ABORT"],help="final result of CI job")
- parser.add_argument("-i","--ci-job-id",dest="ci_id",required=True,type=int,help="ID of CI job")
- parser.add_argument("-m","--merge-request-id",dest="mr_id",required=True,type=int,help="(unique) ID of merge request (not the project specific IID)")
- parser.add_argument("-n","--nicos-project-relname",dest="nicos_name",required=True,type=str,help="NICOS project release name for this build")
- parser.add_argument("-p","--project-name",dest="project_name",required=True,help="GitLab project with namespace (e.g. user/my-project)")
- parser.add_argument("-t","--token",required=True,help="private GitLab user token")
- parser.add_argument("-u","--url",default="https://gitlab.cern.ch",help="URL of GitLab instance")
- parser.add_argument("-v","--verbose",default="INFO",choices=["DEBUG","INFO","WARNING","ERROR","CRITICAL"],help="verbosity level")
-
- # get command line arguments
- args = parser.parse_args()
-
- # configure log output
- logging.basicConfig(format='%(asctime)s %(levelname)-10s %(message)s',
- datefmt='%H:%M:%S',
- level=logging.getLevelName(args.verbose))
-
- logging.debug("parsed arguments:\n" + repr(args))
-
- # get GitLab API handler
- gl = gitlab.Gitlab(args.url,args.token)
- try:
- # get Gitlab project object
- project = gl.projects.get(args.project_name)
- logging.debug("retrieved Gitlab project handle")
- # get Gitlab merge request object
- mr = project.mergerequests.get(args.mr_id)
- logging.debug("retrieved Gitlab merge request handle")
- except GitlabGetError as e:
- logging.critical("error communication with Gitlab API '%s'" % (e.error_message))
- sys.exit(1)
-
- # prepare individual parts of the CI comment
- comment = compose_result_text(args.status)
- comment += " \n"
- comment += compose_stage_text("externals",args.externals) + " \n"
- comment += compose_stage_text("cmake",args.cmake) + " \n"
- comment += compose_stage_text("make",args.make) + " \n"
- comment += compose_stage_text("test",args.test) + " \n"
- comment += " \n"
- comment += "Full details available at [NICOS {0}](http://atlas-nightlies-browser.cern.ch/~platinum/nightlies/info?tp=g&nightly=MR-CI-builds&rel={0}&ar=*) \n".format(args.nicos_name)
- comment += "For experts only: Jenkins output [[CI-MERGE-REQUEST {0}]](http://aibuild080.cern.ch:8080/job/CI-MERGE-REQUEST/{0}/) (for remote access see instructions in Jenkins section [here](https://atlassoftwaredocs.web.cern.ch/MRtutorial/tools/))".format(args.ci_id)
-
- logging.debug(comment)
- try:
- mr.notes.create({'body':comment})
- except GitlabCreateError as e:
- logging.critical("error creating MR comment '%s'" % (e.error_message))
- sys.exit(1)
-
-if __name__ == "__main__":
- main()
diff --git a/CI/run_unit_tests_for_mr.py b/CI/run_unit_tests_for_mr.py
deleted file mode 100644
index 551e1ef318422bc2c0f9322ee5b28436e666d933..0000000000000000000000000000000000000000
--- a/CI/run_unit_tests_for_mr.py
+++ /dev/null
@@ -1,71 +0,0 @@
-import argparse, gitlab, logging, os, subprocess, sys
-from gitlab.exceptions import GitlabGetError
-from gitlab_mr_helpers import print_collection, map_filename_to_package
-
-def run_unit_tests(args):
- # get GitLab API handler
- gl = gitlab.Gitlab(args.url,args.token)
- try:
- # get Gitlab project object
- project = gl.projects.get(args.project_name)
- logging.debug("retrieved Gitlab project handle")
- # get Gitlab merge request object
- mr = project.mergerequests.get(args.mr_id)
- logging.debug("retrieved Gitlab merge request handle")
- except GitlabGetError as e:
- logging.critical("error communication with Gitlab API '%s'" % (e.error_message))
- sys.exit(1)
-
- changes = mr.changes()['changes']
- changed_files = set([c[p] for c in changes for p in ['old_path','new_path']])
- logging.debug("changed files:\n" + print_collection(changed_files))
- affected_packages = sorted(set([map_filename_to_package(f) for f in changed_files]))
-
- # construct list of patterns for matching test labels
- pattern_list = []
- for package_path in affected_packages:
- # ignore empty package paths which would trigger all tests
- # (empty package paths may appear due to failed mapping)
- if not package_path:
- continue
-
- # label is package name and not full package path
- pattern_list.append("^" + os.path.basename(package_path) + "$")
-
- # only run tests if we found some patterns for test labels
- if pattern_list:
- # assemble ctest command
- ctest_cmd = "ctest --output-on-failure -O ctest.log "
- ctest_cmd += "-L \"" + "|".join(pattern_list) + "\""
-
- # execute
- logging.debug("ctest command = '%s'" % ctest_cmd)
- status = subprocess.call(ctest_cmd,shell=True)
- return status
- # no tests -> return success
- else:
- return 0
-
-def main():
- parser = argparse.ArgumentParser(description="ATLAS unit test runner",
- formatter_class=argparse.ArgumentDefaultsHelpFormatter)
- parser.add_argument("-m","--merge-request-id",dest="mr_id",required=True,type=int,help="(unique) ID of merge request (not the project specific IID)")
- parser.add_argument("-p","--project-name",dest="project_name",required=True,help="GitLab project with namespace (e.g. user/my-project)")
- parser.add_argument("-t","--token",required=True,help="private GitLab user token")
- parser.add_argument("-u","--url",default="https://gitlab.cern.ch",help="URL of GitLab instance")
- parser.add_argument("-v","--verbose",default="INFO",choices=["DEBUG","INFO","WARNING","ERROR","CRITICAL"],help="verbosity level")
-
- # get command line arguments
- args = parser.parse_args()
-
- # configure log output
- logging.basicConfig(format='%(asctime)s %(levelname)-10s %(message)s',
- datefmt='%H:%M:%S',
- level=logging.getLevelName(args.verbose))
-
- # delegate
- status = run_unit_tests(args)
- sys.exit(status)
-
-if __name__ == "__main__":
- main()
diff --git a/CI/sweep_MR.py b/CI/sweep_MR.py
deleted file mode 100644
index 63aeefb4f039ddcf8b6ff1621b16916c1988fc94..0000000000000000000000000000000000000000
--- a/CI/sweep_MR.py
+++ /dev/null
@@ -1,287 +0,0 @@
-import argparse, gitlab, logging, os, re, subprocess, sys, yaml
-from gitlab.exceptions import GitlabGetError, GitlabCreateError, GitlabCherryPickError
-from gitlab_mr_helpers import list_changed_packages
-
-def execute_command_with_retry(cmd,max_attempts=3):
- logging.debug("running command '%s' with max attempts %d",cmd,max_attempts)
- attempt = 0
- while attempt < max_attempts:
- attempt += 1
- logging.debug('running attempt %d',attempt)
- process = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)
- out,err = process.communicate()
- status = process.returncode
- out = out.strip()
- err = err.strip()
- logging.debug('command returned %d',status)
- if out:
- logging.debug('stdout:\n%s',out)
- if err:
- logging.debug('stderr:\n%s',err)
-
- # break loop if execution was successfull
- if status == 0:
- break
-
- return status,out,err
-
-def get_list_of_merge_commits(branch,since):
- logging.info("looking for merge commits on '%s' since '%s'",branch,since)
- git_cmd = 'git log --merges --first-parent --oneline --since="{0}" {1}'.format(since,branch)
- status,out,_ = execute_command_with_retry(git_cmd)
-
- # bail out in case of errors
- if status != 0:
- logging.critical("failed to retrieve merge commits")
- return None
-
- # extract hashes of merge commits
- hash_list = set()
- for line in out.split('\n'):
- # skip empty lines
- if not line:
- continue
- match = re.match(r"[0-9a-f]{6,}",line)
- if not match:
- logging.critical("could not extract merge commit hash from '%s'",line)
- continue
- hash_list.add(match.group())
- logging.info("found %d merge commits",len(hash_list))
- logging.debug("hashes : %s",repr(hash_list))
-
- return hash_list
-
-def get_sweep_target_branch_rules(src_branch):
- git_cmd = "git show {0}:CI/config.yaml".format(src_branch)
- status,out,_ = execute_command_with_retry(git_cmd)
-
- # bail out in case of errors
- if status != 0:
- logging.critical("failed to retrieve CI configuration")
- return None
-
- try:
- CI_config = yaml.load(out)
- except:
- logging.critical("failed to interpret the following text as YAML:\n%s",out)
- return None
-
- if not 'sweep-targets' in CI_config or not CI_config['sweep-targets']:
- logging.info("no sweep targets for branch '%s' configured",src_branch)
- return None
-
- target_branch_rules = CI_config['sweep-targets']
- logging.info("read %d sweeping rules for MRs to '%s",len(target_branch_rules),src_branch)
- logging.debug("sweeping rules: %r",target_branch_rules)
-
- return target_branch_rules
-
-def cherry_pick_mr(merge_commit,source_branch,target_branch_rules,project):
- # keep track of successful and failed cherry-picks
- good_branches = set()
- failed_branches = set()
-
- # get merge commit object
- try:
- commit = project.commits.get(merge_commit)
- except GitlabGetError as e:
- logging.critical("failed to get merge commit '%s' with\n%s",merge_commit,e.error_message)
- return
-
- # get merge request IID from commit message
- _,out,_ = execute_command_with_retry("git show {0}".format(merge_commit))
- match = re.search("See merge request !(\d+)",out)
- if match:
- MR_IID = int(match.group(1))
- logging.debug("merge commit '%s' corresponds to MR IID %d",merge_commit,MR_IID)
-
- # retrieve gitlab API object
- try:
- mr_handle = project.mergerequests.list(iid=MR_IID)[0]
- except:
- logging.critical("failed to retrieve Gitlab merge request handle")
- return
- else:
- logging.debug("retrieved Gitlab merge request handle")
- else:
- logging.critical("failed to determine MR IID")
- return
-
- # save original author so that we can add as watcher
- original_mr_author = mr_handle.author.username
-
- # handle sweep labels
- labels = set(mr_handle.labels)
- if "sweep:done" in labels:
- logging.info("merge commit '%s' was already swept -> skipping",merge_commit)
- return
- if "sweep:ignore" in labels:
- logging.info("merge commit '%s' is marked as ignore -> skipping",merge_commit)
- return
-
- labels.add("sweep:done")
- mr_handle.labels = list(labels)
- mr_handle.save()
-
- # get list of affected packages for this MR
- affected_packages = list_changed_packages(mr_handle)
- logging.debug("MR %d affects the following packages: %r",MR_IID,affected_packages)
-
- # determine set of target branches from rules and affected packages
- target_branches = set()
- for rule,branches in target_branch_rules.items():
- # get pattern expression for affected packages
- pkg_pattern = re.compile(rule)
- # only add target branches if ALL affected packages match the given pattern
- matches = [pkg_pattern.match(pkg_name) for pkg_name in affected_packages]
- if all(matches):
- logging.debug("add branches for rule '%s'",rule)
- target_branches.update(branches)
- else:
- logging.debug("skip branches for rule '%s'",rule)
-
- logging.info("MR %d is swept to %d branches",MR_IID,len(target_branches))
- logging.debug("sweep target branches: %r",target_branches)
-
- # get initial MR commit title and description
- _,mr_title,_ = execute_command_with_retry('git show {0} --pretty=format:"%s"'.format(merge_commit))
- _,mr_desc,_ = execute_command_with_retry('git show {0} --pretty=format:"%b"'.format(merge_commit))
-
- # perform cherry-pick to all target branches
- for tbranch in target_branches:
- failed = False
-
- # create remote branch for containing the cherry-pick commit
- cherry_pick_branch = "cherry-pick-{0}-{1}".format(merge_commit,tbranch)
- try:
- project.branches.create({'branch_name': cherry_pick_branch, 'ref': tbranch})
- except GitlabCreateError as e:
- logging.critical("failed to create remote branch '%s' with\n%s",cherry_pick_branch,e.error_message)
- failed = True
- else:
- # perform cherry-pick
- try:
- commit.cherry_pick(cherry_pick_branch)
- except GitlabCherryPickError as e:
- logging.critical("failed to cherry pick merge commit '%s' with\n%s",merge_commit,e.error_message)
- failed = True
-
- # only create MR if cherry-pick succeeded
- if failed:
- logging.critical("failed to cherry-pick '%s' into '%s'",merge_commit,tbranch)
- failed_branches.add(tbranch)
- else:
- logging.info("cherry-picked '%s' into '%s'",merge_commit,tbranch)
-
- # create merge request
- mr_data = {}
- mr_data['source_branch'] = cherry_pick_branch
- mr_data['target_branch'] = tbranch
- mr_data['title'] = mr_title
- mr_data['description'] = mr_desc
- mr_data['labels'] = "sweep:from {0}".format(os.path.basename(source_branch))
- mr_data['remove_source_branch'] = True
- try:
- new_mr = project.mergerequests.create(mr_data)
- except GitlabCreateError as e:
- logging.critical("failed to create merge request for '%s' into '%s' with\n%s",cherry_pick_branch,tbranch,e.error_message)
- failed_branches.add(tbranch)
- else:
- good_branches.add(tbranch)
- # adding original author as watcher
- notification_text = "Adding original author @{0:s} as watcher.".format(original_mr_author)
- new_mr.notes.create({'body': notification_text})
-
- # compile comment about sweep results
- comment = "**Sweep summary** \n"
- if good_branches:
- comment += "successful: \n* " + "\n* ".join(sorted(good_branches)) + " \n \n"
- if failed_branches:
- comment += "failed: \n* " + "\n* ".join(sorted(failed_branches))
- # add label to original MR indicating cherry-pick problem
- mr_handle.labels = list(set(mr_handle.labels) | {"sweep:failed"})
- mr_handle.save()
-
- # add sweep summary to MR in Gitlab
- try:
- mr_handle.notes.create({'body': comment})
- except GitlabCreateError as e:
- logging.critical("failed to add comment with sweep summary with\n{0:s}".format(e.error_message))
-
-def main():
- parser = argparse.ArgumentParser(description="GitLab merge request commentator",formatter_class=argparse.ArgumentDefaultsHelpFormatter)
- parser.add_argument("-b","--branch",required=True,help="remote branch whose merge commits should be swept (e.g. origin/master)")
- parser.add_argument("-p","--project-name",dest="project_name",required=True,help="GitLab project with namespace (e.g. user/my-project)")
- parser.add_argument("-s","--since",default="1 day",help="time interval for sweeping MR (e.g. 1 week)")
- parser.add_argument("-t","--token",required=True,help="private GitLab user token")
- parser.add_argument("-u","--url",default="https://gitlab.cern.ch",help="URL of GitLab instance")
- parser.add_argument("-v","--verbose",default="INFO",choices=["DEBUG","INFO","WARNING","ERROR","CRITICAL"],help="verbosity level")
- parser.add_argument("--repository-root",dest="root",default=os.path.dirname(os.path.abspath(os.path.join(os.path.realpath(__file__),'../'))),help="path to root directory of git repository")
-
- # get command line arguments
- args = parser.parse_args()
-
- # configure log output
- logging.basicConfig(format='%(asctime)s %(levelname)-10s %(message)s',
- datefmt='%H:%M:%S',
- level=logging.getLevelName(args.verbose))
-
- logging.debug("parsed arguments:\n" + repr(args))
-
- # we only support porting merge commits from remote branches since we expect
- # them to be created through the Gitlab web interface
- # -> branch must contain the name of the remote repository (e.g. upstream/master)
- # -> infer it
- tokens = args.branch.split('/')
- if len(tokens) < 2:
- logging.critical("expect branch to specify a remote branch (e.g. 'upstream/master')")
- logging.critical("received branch '%s' which does not look like a remote branch",args.branch)
- logging.critical("--> aborting")
- sys.exit(1)
-
- # set name of remote repository
- args.remote_name = tokens[0]
-
- # get GitLab API handler
- gl = gitlab.Gitlab(args.url,args.token)
- try:
- # get Gitlab project object
- project = gl.projects.get(args.project_name)
- logging.debug("retrieved Gitlab project handle")
- except GitlabGetError as e:
- logging.critical("error communication with Gitlab API '%s'" % (e.error_message))
- sys.exit(1)
-
- # get top-level directory of git repository (specific to current directory structure)
- workdir = os.path.abspath(args.root)
-
- logging.info("changing to root directory of git repository '%s'",workdir)
- current_dir = os.getcwd()
- os.chdir(workdir)
-
- # fetch latest changes
- status,_,_ = execute_command_with_retry("git fetch --prune {0}".format(args.remote_name))
- if status != 0:
- logging.critical("failed to fetch from '%s'",args.remote_name)
- return None
-
- # get list of branches MRs should be forwarded to
- target_branch_rules = get_sweep_target_branch_rules(args.branch)
- if not target_branch_rules:
- logging.info("no sweeping rules for branch '%s' found",args.branch)
- sys.exit(0)
-
- # get list of MRs in relevant period
- MR_list = get_list_of_merge_commits(args.branch,args.since)
- if not MR_list:
- logging.info("no MRs to '%s' found during last %s",args.branch,args.since)
- sys.exit(0)
-
- # do the actual cherry-picking
- for mr in MR_list:
- cherry_pick_mr(mr,args.branch,target_branch_rules,project)
- # change back to initial directory
- os.chdir(current_dir)
-
-if __name__ == "__main__":
- main()
diff --git a/CI/test/__init__.py b/CI/test/__init__.py
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/CI/test/test_domain_map.py b/CI/test/test_domain_map.py
deleted file mode 100644
index 3fcb2fac37a9c50a5f64951f66e527cb6fdb6540..0000000000000000000000000000000000000000
--- a/CI/test/test_domain_map.py
+++ /dev/null
@@ -1,25 +0,0 @@
-import re, unittest
-from domain_map import DOMAIN_MAP
-
-class TestDomainMapDictionary(unittest.TestCase):
- def test_DomainMapIsDict(self):
- self.assertIsInstance(DOMAIN_MAP,dict)
-
- def test_KeysAreStrings(self):
- for domain_name in DOMAIN_MAP.keys():
- self.assertIsInstance(domain_name,str)
-
- def test_ValuesAreSets(self):
- for package_patterns in DOMAIN_MAP.values():
- self.assertIsInstance(package_patterns,set)
-
- def test_PatternsAreValidRegex(self):
- for package_patterns in DOMAIN_MAP.values():
- for pattern in package_patterns:
- try:
- _ = re.compile(pattern)
- except:
- self.fail("compiling '%s' as regular expression failed" % pattern)
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/CI/test/test_watch_list.py b/CI/test/test_watch_list.py
deleted file mode 100644
index f8221f89dfac8ece2457ea6eea902d6144dca389..0000000000000000000000000000000000000000
--- a/CI/test/test_watch_list.py
+++ /dev/null
@@ -1,25 +0,0 @@
-import re, unittest
-from watch_list import WATCH_LIST
-
-class TestWatchListDictionary(unittest.TestCase):
- def test_WatchListIsDict(self):
- self.assertIsInstance(WATCH_LIST,dict)
-
- def test_ValuesAreSets(self):
- for watchers in WATCH_LIST.values():
- self.assertIsInstance(watchers,set)
-
- def test_UsernamesAreStrings(self):
- for watchers in WATCH_LIST.values():
- for username in watchers:
- self.assertIsInstance(username,str)
-
- def test_PatternsAreValidRegex(self):
- for package_pattern in WATCH_LIST.keys():
- try:
- _ = re.compile(package_pattern)
- except:
- self.fail("compiling '%s' as regular expression failed" % package_pattern)
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/CI/watch_list.py b/CI/watch_list.py
deleted file mode 100644
index 31e4593ae31065fa131d6734e960c993e6c40741..0000000000000000000000000000000000000000
--- a/CI/watch_list.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# dictionary defining a watch list for MR notifications
-# keys ... must be valid regular expression
-# values ... must be sets containing Gitlab usernames as strings
-WATCH_LIST = {}
-
-WATCH_LIST['^CI$'] = set(['cgumpert'])
-WATCH_LIST['Simulation'] = set(['ritsch','jchapman','vpascuzz'])
-WATCH_LIST['Digitization'] = set(['jchapman'])
-WATCH_LIST['Overlay'] = set(['jchapman','ahaas','tkharlam','tadej'])
-WATCH_LIST['TrkiPatFitter'] = set(['pop'])
-WATCH_LIST['MooPerformance'] = set(['pop'])
-WATCH_LIST['JetCalibTools'] = set(['jbossios'])
-WATCH_LIST['AFP'] = set(['ggach'])
-WATCH_LIST['MuonEfficiencyCorrections'] = set(['nkoehler','jojungge'])
-WATCH_LIST['MuonTPTools'] = set(['nkoehler','jojungge'])
-WATCH_LIST['MuonPerformanceAlgs'] = set(['nkoehler','jojungge'])
-WATCH_LIST['MuonPerformanceHistUtils'] = set(['nkoehler','jojungge'])
-WATCH_LIST['(PixelMonitoring)|(PixelConditionsServices)|(PixelRawData)'] = set(['kzoch','ibragimo'])
-WATCH_LIST['LongLivedParticleDPDMaker'] = set(['cohm','leejr','kdipetri','ctreado','will'])
-WATCH_LIST['TrkV0Fitter'] = set(['bouhova'])
-WATCH_LIST['TrkVertexAnalysisUtils'] = set(['bouhova'])
-WATCH_LIST['InDetV0Finder'] = set(['bouhova'])
-WATCH_LIST['InDetBeamSpot'] = set(['csuster', 'amorley'])