ignore --ccache-skip if it is in CFLAGS (PyCmt-00-00-36)

Tools/PyCmt/bin/make_pkgbuild.py

+#!/usr/bin/env python
+
+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+# @purpose: driver for the pkgbuild-based builders
+
+from __future__ import with_statement
+
+### imports -------------------------------------------------------------------
+import sys
+import os
+import optparse
+
+import PyCmt.pkgbuild as pkgbuild
+
+msg = pkgbuild.msg
+
+### functions -----------------------------------------------------------------
+def main():
+    parser = optparse.OptionParser(usage="usage: %prog pkgbuild.py")
+    p = parser.add_option
+    p("-f", "--file",
+      default = None,
+      dest = "fname",
+      help = "path to the pkgbuild.py file containing build instructions")
+    p("-p", "--pkg-root",
+      default = None,
+      dest = "pkg_root",
+      help = "directory containing the cmt/requirements file (so we can "+\
+             "query CMT for build environment and options)")
+
+    (opts, args) = parser.parse_args()
+
+    if opts.fname is None or opts.pkg_root is None:
+        parser.print_help()
+        return 1
+
+    import os.path as osp
+    def _norm_path(p):
+        p = osp.expanduser(osp.expandvars(p))
+        p = osp.realpath(osp.normpath(p))
+        return p
+    opts.fname = _norm_path(opts.fname)
+    opts.pkg_root = _norm_path(opts.pkg_root)
+
+    # may raise
+    os.access(opts.fname, os.R_OK)
+    os.access(opts.pkg_root, os.R_OK)
+    assert osp.isdir(opts.pkg_root)
+    assert osp.isfile(opts.fname)
+
+    env = os.environ.copy()
+    env['pkg_root'] = opts.pkg_root
+    return pkgbuild.build_package(opts.fname, env=env, msg=msg)
+
+if __name__ == "__main__":
+    import sys
+    sys.exit(main())
+    
+        
+    
+    

Tools/PyCmt/cmt/requirements

+package PyCmt
+
+author Sebastien Binet <binet@cern.ch>
+
+use AtlasPolicy AtlasPolicy-*
+use AtlasPython	AtlasPython-*	External -no_auto_imports
+
+branches python bin
+
+##################################
+# for pkgbuild-enabled packages...
+cmtpath_pattern_reverse \
+ macro_append includes ' $(ppcmd)"<path>/$(cmt_installarea_prefix)/$(CMTCONFIG)/include" '
+##################################
+
+apply_pattern declare_scripts files="\
+ -s=$(PyCmt_root)/bin \
+ make_pkgbuild.py \
+"
+
+private
+
+apply_pattern declare_python_modules files="*.py pyshell pkgbuild"
+
+pattern make_pkgbuild \
+ private ; \
+ macro_append constituents " make_pkgbuild_<name>" ; \
+ action make_pkgbuild_<name> "$(PyCmt_root)/bin/make_pkgbuild.py --pkg-root=$(<package>_root) --file=<file>" ; \
+ end_private
+
+end_private

Tools/PyCmt/python/Decorators.py

+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
+# @author: Sebastien Binet <binet@cern.ch>
+# @date:   March 2008
+# @purpose: a set of decorators. Most of them (if not all) have been stolen
+#           from here:
+#           http://www.phyast.pitt.edu/~micheles/python/documentation.html
+#
+from __future__ import with_statement
+
+__version__ = "$Revision$"
+__author__  = "Sebastien Binet <binet@cern.ch>"
+
+__all__ = [
+    'memoize',
+    'forking',
+    'async',
+    ]
+
+import sys
+import os
+import itertools
+from decorator import *
+
+@decorator
+def memoize(func, *args):
+    """This decorator implements the memoize pattern, i.e. it caches the result
+    of a function in a dictionary, so that the next time the function is called
+    with the same input parameters the result is retrieved from the cache and
+    not recomputed.
+    """
+    try:
+        mem_dict = getattr(func, "_mem_dict")
+    except AttributeError:
+        # look-up failed so we have to build the cache holder
+        mem_dict = {}
+        setattr(func, "_mem_dict", mem_dict)
+    try:
+        return mem_dict[args]
+    except KeyError:
+        # look-up failed so we have to build the result the first time around
+        # then we cache
+        mem_dict[args] = result = func(*args)
+        return result
+
+# FIXME: does not work... func is an instance of FunctionMaker which cannot
+#        be pickled...
+import __builtin__
+@decorator
+def mp_forking(func, *args, **kwargs):
+    import multiprocessing as mp
+    ## pool = mp.Pool (processes=1)
+    ## return pool.apply (func, *args, **kwargs)
+
+    # create a local queue to fetch the results back
+    def wrapping(func):
+        q = mp.Queue()
+        def wrap_fct(*args, **kwargs):
+            try:
+                res = func(*args, **kwargs)
+            # catch *everything* and 're-raise'
+            except BaseException,err:
+                #import traceback; traceback.print_exc()
+                res = err
+            q.put(res)
+        wrap_fct.q = q
+        return wrap_fct
+
+    func = wrapping(func)
+    proc = mp.Process(target=func, args=args, kwargs=kwargs)
+    proc.start()
+    res = func.q.get()
+    proc.join()
+    proc.terminate()
+    if isinstance(res, BaseException):
+        #import traceback; traceback.print_exc()
+        raise res
+        #reraise_exception(exc,exc_info)
+    return res
+
+def reraise_exception(new_exc, exc_info=None):
+    if exc_info is None:
+        exc_info = sys.exc_info()
+    _exc_class, _exc, tb = exc_info
+    raise new_exc.__class__, new_exc, tb
+    
+@decorator
+def forking(func, *args, **kwargs):
+    """
+    This decorator implements the forking patterns, i.e. it runs the function
+    in a forked process.
+    see:
+     http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/511474
+    """
+    import os
+    try:
+        import cPickle as pickle
+    except ImportError:
+        import pickle
+        
+    # create a pipe which will be shared between parent and child
+    pread, pwrite = os.pipe()
+
+    # do fork
+    pid = os.fork()
+
+    ## parent ##
+    if pid > 0:
+        os.close(pwrite)
+        with os.fdopen(pread, 'rb') as f:
+            status, result = pickle.load(f)
+        os.waitpid(pid, 0)
+        if status == 0:
+            return result
+        else:
+            remote_exc = result[0]
+            reraise_exception(remote_exc)
+            
+    ## child ##
+    else:
+        os.close(pread)
+        try:
+            result = func(*args, **kwargs)
+            status = 0
+        except (Exception, KeyboardInterrupt), exc:
+            import traceback
+            exc_string = traceback.format_exc(limit=10)
+            for l in exc_string.splitlines():
+                print "[%d]"%os.getpid(),l.rstrip()
+            result = exc, exc_string
+            status = 1
+        with os.fdopen(pwrite, 'wb') as f:
+            try:
+                pickle.dump((status,result), f, pickle.HIGHEST_PROTOCOL)
+            except pickle.PicklingError, exc:
+                pickle.dump((2,exc), f, pickle.HIGHEST_PROTOCOL)
+        os._exit(0)
+    pass # forking
+
+            
+### a decorator converting blocking functions into asynchronous functions
+#   stolen from http://pypi.python.org/pypi/decorator/3.0.0
+def _async_on_success(result): # default implementation
+    "Called on the result of the function"
+    return result
+
+def _async_on_failure(exc_info): # default implementation
+    "Called if the function fails"
+    _exc_class, _exc, tb = exc_info
+    raise _exc_class, _exc, tb
+    pass
+
+def _async_on_closing(): # default implementation
+    "Called at the end, both in case of success and failure"
+    pass
+
+class Async(object):
+    """
+    A decorator converting blocking functions into asynchronous
+    functions, by using threads or processes. Examples:
+
+    async_with_threads =  Async(threading.Thread)
+    async_with_processes =  Async(multiprocessing.Process)
+    """
+
+    def __init__(self, threadfactory):
+        self.threadfactory = threadfactory
+
+    def __call__(self, func,
+                 on_success=_async_on_success,
+                 on_failure=_async_on_failure,
+                 on_closing=_async_on_closing):
+        # every decorated function has its own independent thread counter
+        func.counter = itertools.count(1)
+        func.on_success = on_success
+        func.on_failure = on_failure
+        func.on_closing = on_closing
+        return decorator(self.call, func)
+
+    def call(self, func, *args, **kw):
+        def func_wrapper():
+            try:
+                result = func(*args, **kw)
+            except:
+                func.on_failure(sys.exc_info())
+            else:
+                return func.on_success(result)
+            finally:
+                func.on_closing()
+        name = '%s-%s' % (func.__name__, func.counter.next())
+        thread = self.threadfactory(None, func_wrapper, name)
+        thread.start()
+        return thread
+
+# default async decorator: using processes
+def async(async_type='mp'):
+    if async_type in ("mp", "multiprocessing"):
+        from multiprocessing import Process
+        factory = Process
+    elif async_type in ("th", "threading"):
+        from threading import Thread
+        factory = Thread
+    else:
+        raise ValueError ("async_type must be either 'multiprocessing' "
+                          "or 'threading' (got: %s)"%async_type)
+    async_obj = Async (factory)
+    return async_obj
+        
+    

Tools/PyCmt/python/Logging.py

+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
+## @author: Sebastien Binet
+## @file :  Logging.py
+## @purpose: try to import Logging from AthenaCommon.
+##           falls back on stdlib's one
+
+__version__ = "$Revision$"
+__author__  = "Sebastien Binet"
+
+__all__ = ['msg', 'logging']
+
+try:
+    import AthenaCommon.Logging as L
+    msg = L.log
+    logging = L.logging
+except ImportError:
+    import logging
+    logging.VERBOSE = logging.DEBUG - 1
+    logging.ALL     = logging.DEBUG - 2
+    logging.addLevelName( logging.VERBOSE, 'VERBOSE' )
+    logging.addLevelName( logging.ALL, 'ALL' )
+    logging.basicConfig(level=logging.INFO,
+                        format="Py:%(name)-14s%(levelname)8s %(message)s")
+    cls = logging.getLoggerClass()
+    def verbose(self, msg, *args, **kwargs):
+        if self.manager.disable >= logging.VERBOSE:
+            return
+        if logging.VERBOSE >= self.getEffectiveLevel():
+            apply(self._log, (logging.VERBOSE, msg, args), kwargs)
+    cls.verbose = verbose
+    del verbose
+    
+    log = msg = logging.getLogger("Athena")
+    
+            

Tools/PyCmt/python/__init__.py

+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+

Tools/PyCmt/python/bwdcompat.py

+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
+import subprocess
+
+### monkey-patch subprocess (fwd compat w/ py-3.x) ----------------------------
+def getstatusoutput(cmd, *popenargs, **kwargs):
+    if isinstance(cmd, basestring):
+        cmd = cmd.split()
+    if 'stdout' in kwargs:
+        raise ValueError('stdout argument not allowed, it will be overridden.')
+    kwargs['stdout'] = subprocess.PIPE
+    p = subprocess.Popen(cmd, *popenargs, **kwargs)
+    sc = p.returncode
+    if sc is None:
+        sc = 0
+    fd = p.stdout
+    out= ''.join(list(fd.readlines()))
+    if out[-1:] == '\n': out = out[:-1]
+    return sc,out
+subprocess.getstatusoutput = getstatusoutput
+del getstatusoutput
+
+def getstatus(cmd, *popenargs, **kwargs):
+    return subprocess.getstatusoutput(cmd, *popenargs, **kwargs)[0]
+subprocess.getstatus = getstatus
+del getstatus
+
+def getoutput(cmd, *popenargs, **kwargs):
+    return subprocess.getstatusoutput(cmd, *popenargs, **kwargs)[1]
+subprocess.getoutput = getoutput
+del getoutput
+### ---------------------------------------------------------------------------

Tools/PyCmt/python/decorator.py

+##########################     LICENCE     ###############################
+##
+##   Copyright (c) 2005-2011, Michele Simionato
+##   All rights reserved.
+##
+##   Redistributions of source code must retain the above copyright 
+##   notice, this list of conditions and the following disclaimer.
+##   Redistributions in bytecode form must reproduce the above copyright
+##   notice, this list of conditions and the following disclaimer in
+##   the documentation and/or other materials provided with the
+##   distribution. 
+
+##   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+##   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+##   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+##   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+##   HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+##   INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+##   BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+##   OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+##   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+##   TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+##   USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+##   DAMAGE.
+
+"""
+Decorator module, see http://pypi.python.org/pypi/decorator
+for the documentation.
+"""
+
+__version__ = '3.3.0'
+
+__all__ = ["decorator", "FunctionMaker", "partial"]
+
+import sys, re, inspect
+
+try:
+    from functools import partial
+except ImportError: # for Python version < 2.5
+    class partial(object):
+        "A simple replacement of functools.partial"
+        def __init__(self, func, *args, **kw):
+            self.func = func
+            self.args = args                
+            self.keywords = kw
+        def __call__(self, *otherargs, **otherkw):
+            kw = self.keywords.copy()
+            kw.update(otherkw)
+            return self.func(*(self.args + otherargs), **kw)
+
+if sys.version >= '3':
+    from inspect import getfullargspec
+else:
+    class getfullargspec(object):
+        "A quick and dirty replacement for getfullargspec for Python 2.X"
+        def __init__(self, f):
+            self.args, self.varargs, self.varkw, self.defaults = \
+                inspect.getargspec(f)
+            self.kwonlyargs = []
+            self.kwonlydefaults = None
+            self.annotations = getattr(f, '__annotations__', {})
+        def __iter__(self):
+            yield self.args
+            yield self.varargs
+            yield self.varkw
+            yield self.defaults
+
+DEF = re.compile('\s*def\s*([_\w][_\w\d]*)\s*\(')
+
+# basic functionality
+class FunctionMaker(object):
+    """
+    An object with the ability to create functions with a given signature.
+    It has attributes name, doc, module, signature, defaults, dict and
+    methods update and make.
+    """
+    def __init__(self, func=None, name=None, signature=None,
+                 defaults=None, doc=None, module=None, funcdict=None):
+        self.shortsignature = signature
+        if func:
+            # func can be a class or a callable, but not an instance method
+            self.name = func.__name__
+            if self.name == '<lambda>': # small hack for lambda functions
+                self.name = '_lambda_' 
+            self.doc = func.__doc__
+            self.module = func.__module__
+            if inspect.isfunction(func):
+                argspec = getfullargspec(func)
+                for a in ('args', 'varargs', 'varkw', 'defaults', 'kwonlyargs',
+                          'kwonlydefaults', 'annotations'):
+                    setattr(self, a, getattr(argspec, a))
+                for i, arg in enumerate(self.args):
+                    setattr(self, 'arg%d' % i, arg)
+                self.signature = inspect.formatargspec(
+                    formatvalue=lambda val: "", *argspec)[1:-1]
+                allargs = list(self.args)
+                if self.varargs:
+                    allargs.append('*' + self.varargs)
+                if self.varkw:
+                    allargs.append('**' + self.varkw)
+                try:
+                    self.shortsignature = ', '.join(allargs)
+                except TypeError: # exotic signature, valid only in Python 2.X
+                    self.shortsignature = self.signature
+                self.dict = func.__dict__.copy()
+        # func=None happens when decorating a caller
+        if name:
+            self.name = name
+        if signature is not None:
+            self.signature = signature
+        if defaults:
+            self.defaults = defaults
+        if doc:
+            self.doc = doc
+        if module:
+            self.module = module
+        if funcdict:
+            self.dict = funcdict
+        # check existence required attributes
+        assert hasattr(self, 'name')
+        if not hasattr(self, 'signature'):
+            raise TypeError('You are decorating a non function: %s' % func)
+
+    def update(self, func, **kw):
+        "Update the signature of func with the data in self"
+        func.__name__ = self.name
+        func.__doc__ = getattr(self, 'doc', None)
+        func.__dict__ = getattr(self, 'dict', {})
+        func.func_defaults = getattr(self, 'defaults', ())
+        callermodule = sys._getframe(3).f_globals.get('__name__', '?')
+        func.__module__ = getattr(self, 'module', callermodule)
+        func.__dict__.update(kw)
+
+    def make(self, src_templ, evaldict=None, addsource=False, **attrs):
+        "Make a new function from a given template and update the signature"
+        src = src_templ % vars(self) # expand name and signature
+        evaldict = evaldict or {}
+        mo = DEF.match(src)
+        if mo is None:
+            raise SyntaxError('not a valid function template\n%s' % src)
+        name = mo.group(1) # extract the function name
+        names = set([name] + [arg.strip(' *') for arg in 
+                             self.shortsignature.split(',')])
+        for n in names:
+            if n in ('_func_', '_call_'):
+                raise NameError('%s is overridden in\n%s' % (n, src))
+        if not src.endswith('\n'): # add a newline just for safety
+            src += '\n' # this is needed in old versions of Python
+        try:
+            code = compile(src, '<string>', 'single')
+            # print >> sys.stderr, 'Compiling %s' % src
+            exec code in evaldict
+        except:
+            print >> sys.stderr, 'Error in generated code:'
+            print >> sys.stderr, src
+            raise
+        func = evaldict[name]
+        if addsource:
+            attrs['__source__'] = src
+        self.update(func, **attrs)
+        return func
+
+    @classmethod
+    def create(cls, obj, body, evaldict, defaults=None,
+               doc=None, module=None, addsource=True,**attrs):
+        """
+        Create a function from the strings name, signature and body.
+        evaldict is the evaluation dictionary. If addsource is true an attribute
+        __source__ is added to the result. The attributes attrs are added,
+        if any.
+        """
+        if isinstance(obj, str): # "name(signature)"
+            name, rest = obj.strip().split('(', 1)
+            signature = rest[:-1] #strip a right parens            
+            func = None
+        else: # a function
+            name = None
+            signature = None
+            func = obj
+        self = cls(func, name, signature, defaults, doc, module)
+        ibody = '\n'.join('    ' + line for line in body.splitlines())
+        return self.make('def %(name)s(%(signature)s):\n' + ibody, 
+                        evaldict, addsource, **attrs)
+  
+def decorator(caller, func=None):
+    """
+    decorator(caller) converts a caller function into a decorator;
+    decorator(caller, func) decorates a function using a caller.
+    """
+    if func is not None: # returns a decorated function
+        evaldict = func.func_globals.copy()
+        evaldict['_call_'] = caller
+        evaldict['_func_'] = func
+        return FunctionMaker.create(
+            func, "return _call_(_func_, %(shortsignature)s)",
+            evaldict, undecorated=func)
+    else: # returns a decorator
+        if isinstance(caller, partial):
+            return partial(decorator, caller)
+        # otherwise assume caller is a function
+        first = inspect.getargspec(caller)[0][0] # first arg
+        evaldict = caller.func_globals.copy()
+        evaldict['_call_'] = caller
+        evaldict['decorator'] = decorator
+        return FunctionMaker.create(
+            '%s(%s)' % (caller.__name__, first), 
+            'return decorator(_call_, %s)' % first,
+            evaldict, undecorated=caller,
+            doc=caller.__doc__, module=caller.__module__)

Tools/PyCmt/python/pkgbuild/__init__.py

+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
+# -*- python -*-
+# @file PyCmt/python/pkgbuild/__init__.py
+# @purpose a simple build system
+# @author Sebastien Binet <binet@cern.ch>
+
+from __future__ import with_statement
+
+__doc__ = "A simple build system"
+__author__ = "Sebastien Binet <binet@cern.ch>"
+__version__= "$Revision$"
+