From 76257216ef9a9d39b4be8c1242eff3d42f3562eb Mon Sep 17 00:00:00 2001
From: Atlas-Software Librarian <Atlas-Software.Librarian@cern.ch>
Date: Thu, 3 Sep 2015 22:27:30 +0200
Subject: [PATCH] import and fix merge_files.py from GaudiPolicy - adding
previously missing file (AtlasCommonPolicy-00-00-55)
* temporary fix for https://its.cern.ch/jira/browse/ATEAM-151
* import and fix merge_files.py from GaudiPolicy
* tagging AtlasCommonPolicy-00-00-55
2015-06-17 Rolf Seuster
* tagging AtlasCommonPolicy-00-00-54
* proper dependencies when splitting off debug symbols
2015-02-20 Emil Obreshkov <obreshkov@cern.ch>
* cmt/requirements: moving setting of JOBOPTSEARCHPATH, PYTHONPATH, DATAPATH, ROOTMAPSEARCHPATH, XMLPATH from ExternalPolicy here
so that DetCommon project can have these paths set
* Tagged as AtlasCommonPolicy-00-00-53
---
AtlasCommonPolicy/cmt/requirements | 41 +++++
AtlasCommonPolicy/scripts/merge_files.py | 209 +++++++++++++++++++++++
2 files changed, 250 insertions(+)
create mode 100755 AtlasCommonPolicy/scripts/merge_files.py
diff --git a/AtlasCommonPolicy/cmt/requirements b/AtlasCommonPolicy/cmt/requirements
index 4f8fcefa661..4ff67a1e798 100755
--- a/AtlasCommonPolicy/cmt/requirements
+++ b/AtlasCommonPolicy/cmt/requirements
@@ -113,6 +113,7 @@ pattern with_optdebug_library \
private ; \
document optdebug_library <name>_optdebug_library library=<name> ; \
macro_append <name>_optdebug_library_dependencies " <name>" ; \
+ macro_append <name>_library_dependencies " <name>_optdebug_library " ; \
action <name>_NICOS_Fix_debuginfo ' if test ! -z "$(NICOS_PROJECT_HOME)" -a ! -z "$(NICOS_PROJECT_RELNAME)" -a \
! -z "$(NICOS_COPY_HOME)" -a ! -z "$(NICOS_PROJECT_RELNAME_COPY)" -a \
-x /usr/lib/rpm/debugedit ; then \
@@ -160,3 +161,43 @@ tag_exclude NICOS NICOSrel_nightly
# requires that the new path is shorter than the old path. ccache requires that the replaced and original path do not depend on the nightly it runs.
macro_append AtlasCommonPolicy_cppflags "" \
NICOSrel_nightly ' -fdebug-prefix-map=$(NICOS_PROJECT_HOME)=$(NICOS_COPY_HOME) '
+
+#--------------------------------------------
+## Filling the JOBOPTSEARCHPATH list with list of currently visible installation areas
+#--------------------------------------------
+
+#path_remove JOBOPTSEARCHPATH "InstallArea"
+#path_prepend JOBOPTSEARCHPATH `${ATLASPOLICYROOT}/cmt/jo.sh`
+
+cmtpath_pattern \
+ path JOBOPTSEARCHPATH "" ; \
+ path_remove PYTHONPATH "$(cmt_installarea_prefix)" ; \
+ path_remove DATAPATH "$(cmt_installarea_prefix)" ; \
+ path_remove ROOTMAPSEARCHPATH "$(cmt_installarea_prefix)" ; \
+ path_remove XMLPATH "$(cmt_installarea_prefix)"
+
+cmtpath_pattern \
+ path_prepend JOBOPTSEARCHPATH "" \
+ <project>_with_installarea&Unix "<path>/$(cmt_installarea_prefix)/jobOptions" ; \
+ path_prepend PYTHONPATH "" \
+ <project>_with_installarea&Unix "<path>/$(cmt_installarea_prefix)/python" ; \
+ path_prepend PYTHONPATH "" \
+ <project>_with_installarea&Unix "<path>/$(cmt_installarea_prefix)/${CMTCONFIG}/lib/python$(Python_version)" ; \
+ path_prepend DATAPATH "" \
+ <project>_with_installarea&Unix "<path>/$(cmt_installarea_prefix)/share" ; \
+ path_prepend DATAPATH "" \
+ <project>_with_installarea&Unix "<path>/$(cmt_installarea_prefix)/${CMTCONFIG}/share" ; \
+ path_prepend ROOTMAPSEARCHPATH "" \
+ <project>_with_installarea&Unix "<path>/$(cmt_installarea_prefix)/rootmap" ; \
+ path_prepend XMLPATH "" \
+ <project>_with_installarea&Unix "<path>/$(cmt_installarea_prefix)/XML"
+
+cmtpath_pattern \
+ path_prepend JOBOPTSEARCHPATH "."
+
+# temporary fix for https://its.cern.ch/jira/browse/ATEAM-151
+# import and fix merge_files.py from GaudiPolicy
+macro merge_componentslist_cmd "python $(AtlasCommonPolicy_root)/scripts/merge_files.py"
+macro merge_rootmap_cmd "python $(AtlasCommonPolicy_root)/scripts/merge_files.py"
+macro merge_genconfDb_cmd "python $(AtlasCommonPolicy_root)/scripts/merge_files.py"
+macro merge_componentslist_cmd "python $(AtlasCommonPolicy_root)/scripts/merge_files.py"
diff --git a/AtlasCommonPolicy/scripts/merge_files.py b/AtlasCommonPolicy/scripts/merge_files.py
new file mode 100755
index 00000000000..ead7e78b063
--- /dev/null
+++ b/AtlasCommonPolicy/scripts/merge_files.py
@@ -0,0 +1,209 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
+# @file: GaudiPolicy/cmt/fragments/merge_files.py
+# @purpose: merge_files <fragment> file into a 'per-project' <merged> file
+# @author: Sebastien Binet <binet@cern.ch>
+
+import os
+import sys
+from datetime import datetime
+import fcntl
+
+def mergeFiles( fragFileNames, mergedFileName, commentChar, doMerge, ignoreMissing ):
+
+ startMark = "%s --Beg " % commentChar
+ timeMark = "%s --Date inserted: %s" % (commentChar, datetime.now())
+ endMark = "%s --End " % commentChar
+ nameOffset = len(startMark)
+
+ basenames = map(os.path.basename, fragFileNames)
+
+ isNewFile = not os.path.exists(mergedFileName)
+
+ # create an empty file if it does not exist
+ # "append mode" ensures that, in case of two processes trying to
+ # create the file, they do not truncate each other file
+ if isNewFile:
+ # check if the destination directory exists
+ path_to_file = os.path.split(mergedFileName)[0]
+ if path_to_file and not os.path.isdir(path_to_file):
+ # if doesn't exist, create it
+ os.makedirs(path_to_file)
+ open(mergedFileName,'a')
+
+ lockFile = open( mergedFileName + '.lock', 'a' )
+
+ # locking file, gaining exclusive access to it
+ # code from locker.py, only posix relevant part - we don't support NT - did we ever ??
+ # Lock with a simple call to lockf() - this blocks until the lock is aquired
+ try:
+ fcntl.lockf( lockFile, fcntl.LOCK_EX )
+ except IOError, exc_value:
+ print "Problem when trying to lock {0}, IOError {1}".format(mergedFile, exc_value[0])
+ raise
+
+ mergedFile = open( mergedFileName, 'r' )
+
+ try:
+
+ newLines = [ ]
+ skipBlock = ""
+ for line in mergedFile.readlines():
+ if line.startswith(startMark) and line[nameOffset:].strip() in basenames:
+ skipBlock = endMark + line[nameOffset:].strip()
+ # remove all the empty lines occurring before the start mark
+ while (len(newLines) > 0) and (newLines[-1].strip() == ''):
+ newLines.pop()
+ if not skipBlock:
+ newLines.append(line)
+ if line.startswith(skipBlock):
+ skipBlock = ""
+ if skipBlock:
+ print "WARNING: missing end mark ('%s')" % skipBlock
+
+ if doMerge:
+ for f in fragFileNames:
+ if ignoreMissing and not os.path.exists(f):
+ print "WARNING: '%s' does not exist, I'm ignoring it" % f
+ continue
+ # I do not want to add 2 empty lines at the beginning of a file
+ if newLines:
+ newLines.append('\n\n')
+ bf = os.path.basename(f)
+ newLines.append(startMark + bf + '\n')
+ newLines.append(timeMark + '\n')
+ fileData = open(f, 'r').read()
+ newLines.append(fileData)
+ if fileData and fileData[-1] != '\n':
+ newLines.append('\n')
+ newLines.append(endMark + bf + '\n')
+
+ #mergedFile.seek(0)
+ #mergedFile.truncate(0)
+ #mergedFile.writelines(newLines)
+
+ newFile = open( mergedFileName + ".new" , 'w' )
+ newFile.writelines(newLines)
+ newFile.close()
+ os.rename(mergedFileName + ".new",mergedFileName)
+
+ finally:
+ # unlock file
+ fcntl.lockf( lockFile, fcntl.LOCK_UN )
+
+ return 0
+
+if __name__ == "__main__":
+
+ from optparse import OptionParser
+ parser = OptionParser(usage="usage: %prog [options]")
+ parser.add_option(
+ "-i",
+ "--input-file",
+ action = "append",
+ dest = "fragFileNames",
+ default = [],
+ help = "The path and name of the file one wants to merge into the 'master' one"
+ )
+ parser.add_option(
+ "-m",
+ "--merged-file",
+ dest = "mergedFileName",
+ default = None,
+ help = "The path and name of the 'master' file which will hold the content of all the other fragment files"
+ )
+ parser.add_option(
+ "-c",
+ "--comment-char",
+ dest = "commentChar",
+ default = "#",
+ help = "The type of the commenting character for the type of files at hand (this is an attempt at handling the largest possible use cases)"
+ )
+ parser.add_option(
+ "--do-merge",
+ dest = "doMerge",
+ action = "store_true",
+ default = True,
+ help = "Switch to actually carry on with the merging procedure"
+ )
+ parser.add_option(
+ "--un-merge",
+ dest = "unMerge",
+ action = "store_true",
+ default = False,
+ help = "Switch to remove our fragment file from the 'master' file"
+ )
+ parser.add_option(
+ "--stamp-dir",
+ dest = "stampDir",
+ action = "store",
+ default = None,
+ help = "Create the stamp file in the specified directory. If not specified"
+ +" the directory of the source file is used."
+ )
+ parser.add_option(
+ "--no-stamp",
+ action = "store_true",
+ help = "Do no create stamp files."
+ )
+ parser.add_option(
+ "--ignore-missing",
+ dest = "ignoreMissing",
+ action = "store_true",
+ help = "Print a warning if a fragment file is missing, but do not fail."
+ )
+
+ (options, args) = parser.parse_args()
+
+ # ensure consistency...
+ options.doMerge = not options.unMerge
+
+ # allow command line syntax as
+ # merge_files.py [options] <fragment file1> [<fragment file2>...] <merged file>
+ if args:
+ options.mergedFileName = args[-1]
+ options.fragFileNames += args[:-1]
+
+ sc = 1
+ if not options.fragFileNames or \
+ not options.mergedFileName :
+ str(parser.print_help() or "")
+ print "*** ERROR ***",sys.argv
+ sys.exit(sc)
+ pass
+
+ if options.stampDir is None:
+ stampFileName = lambda x: x + ".stamp"
+ else:
+ stampFileName = lambda x: os.path.join(options.stampDir,
+ os.path.basename(x)
+ + ".stamp")
+ # Configure Python logging
+ import logging
+ logging.basicConfig(level = logging.INFO)
+
+ if "GAUDI_BUILD_LOCK" in os.environ:
+ globalLock = locker.LockFile(os.environ["GAUDI_BUILD_LOCK"], temporary = True)
+ else:
+ globalLock = None
+
+ if True: #try:
+ sc = mergeFiles( options.fragFileNames, options.mergedFileName,
+ options.commentChar,
+ doMerge = options.doMerge,
+ ignoreMissing = options.ignoreMissing)
+ if not options.no_stamp:
+ for stamp in map(stampFileName, options.fragFileNames):
+ open(stamp, 'w')
+ #except IOError, err:
+ # print "ERROR:",err
+ #except Exception, err:
+ # print "ERROR:",err
+ #except:
+ # print "ERROR: unknown error !!"
+
+ del globalLock
+
+ sys.exit( sc )
--
GitLab