Commit c349d7b2 authored by Atlas-Software Librarian's avatar Atlas-Software Librarian Committed by Graeme Stewart
Browse files

ignore --ccache-skip if it is in CFLAGS (PyCmt-00-00-36)

parent ce721aa6
#!/usr/bin/env python
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
# @purpose: driver for the pkgbuild-based builders
from __future__ import with_statement
### imports -------------------------------------------------------------------
import sys
import os
import optparse
import PyCmt.pkgbuild as pkgbuild
msg = pkgbuild.msg
### functions -----------------------------------------------------------------
def main():
parser = optparse.OptionParser(usage="usage: %prog pkgbuild.py")
p = parser.add_option
p("-f", "--file",
default = None,
dest = "fname",
help = "path to the pkgbuild.py file containing build instructions")
p("-p", "--pkg-root",
default = None,
dest = "pkg_root",
help = "directory containing the cmt/requirements file (so we can "+\
"query CMT for build environment and options)")
(opts, args) = parser.parse_args()
if opts.fname is None or opts.pkg_root is None:
parser.print_help()
return 1
import os.path as osp
def _norm_path(p):
p = osp.expanduser(osp.expandvars(p))
p = osp.realpath(osp.normpath(p))
return p
opts.fname = _norm_path(opts.fname)
opts.pkg_root = _norm_path(opts.pkg_root)
# may raise
os.access(opts.fname, os.R_OK)
os.access(opts.pkg_root, os.R_OK)
assert osp.isdir(opts.pkg_root)
assert osp.isfile(opts.fname)
env = os.environ.copy()
env['pkg_root'] = opts.pkg_root
return pkgbuild.build_package(opts.fname, env=env, msg=msg)
if __name__ == "__main__":
import sys
sys.exit(main())
package PyCmt
author Sebastien Binet <binet@cern.ch>
use AtlasPolicy AtlasPolicy-*
use AtlasPython AtlasPython-* External -no_auto_imports
branches python bin
##################################
# for pkgbuild-enabled packages...
cmtpath_pattern_reverse \
macro_append includes ' $(ppcmd)"<path>/$(cmt_installarea_prefix)/$(CMTCONFIG)/include" '
##################################
apply_pattern declare_scripts files="\
-s=$(PyCmt_root)/bin \
make_pkgbuild.py \
"
private
apply_pattern declare_python_modules files="*.py pyshell pkgbuild"
pattern make_pkgbuild \
private ; \
macro_append constituents " make_pkgbuild_<name>" ; \
action make_pkgbuild_<name> "$(PyCmt_root)/bin/make_pkgbuild.py --pkg-root=$(<package>_root) --file=<file>" ; \
end_private
end_private
This diff is collapsed.
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
# @author: Sebastien Binet <binet@cern.ch>
# @date: March 2008
# @purpose: a set of decorators. Most of them (if not all) have been stolen
# from here:
# http://www.phyast.pitt.edu/~micheles/python/documentation.html
#
from __future__ import with_statement
__version__ = "$Revision$"
__author__ = "Sebastien Binet <binet@cern.ch>"
__all__ = [
'memoize',
'forking',
'async',
]
import sys
import os
import itertools
from decorator import *
@decorator
def memoize(func, *args):
"""This decorator implements the memoize pattern, i.e. it caches the result
of a function in a dictionary, so that the next time the function is called
with the same input parameters the result is retrieved from the cache and
not recomputed.
"""
try:
mem_dict = getattr(func, "_mem_dict")
except AttributeError:
# look-up failed so we have to build the cache holder
mem_dict = {}
setattr(func, "_mem_dict", mem_dict)
try:
return mem_dict[args]
except KeyError:
# look-up failed so we have to build the result the first time around
# then we cache
mem_dict[args] = result = func(*args)
return result
# FIXME: does not work... func is an instance of FunctionMaker which cannot
# be pickled...
import __builtin__
@decorator
def mp_forking(func, *args, **kwargs):
import multiprocessing as mp
## pool = mp.Pool (processes=1)
## return pool.apply (func, *args, **kwargs)
# create a local queue to fetch the results back
def wrapping(func):
q = mp.Queue()
def wrap_fct(*args, **kwargs):
try:
res = func(*args, **kwargs)
# catch *everything* and 're-raise'
except BaseException,err:
#import traceback; traceback.print_exc()
res = err
q.put(res)
wrap_fct.q = q
return wrap_fct
func = wrapping(func)
proc = mp.Process(target=func, args=args, kwargs=kwargs)
proc.start()
res = func.q.get()
proc.join()
proc.terminate()
if isinstance(res, BaseException):
#import traceback; traceback.print_exc()
raise res
#reraise_exception(exc,exc_info)
return res
def reraise_exception(new_exc, exc_info=None):
if exc_info is None:
exc_info = sys.exc_info()
_exc_class, _exc, tb = exc_info
raise new_exc.__class__, new_exc, tb
@decorator
def forking(func, *args, **kwargs):
"""
This decorator implements the forking patterns, i.e. it runs the function
in a forked process.
see:
http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/511474
"""
import os
try:
import cPickle as pickle
except ImportError:
import pickle
# create a pipe which will be shared between parent and child
pread, pwrite = os.pipe()
# do fork
pid = os.fork()
## parent ##
if pid > 0:
os.close(pwrite)
with os.fdopen(pread, 'rb') as f:
status, result = pickle.load(f)
os.waitpid(pid, 0)
if status == 0:
return result
else:
remote_exc = result[0]
reraise_exception(remote_exc)
## child ##
else:
os.close(pread)
try:
result = func(*args, **kwargs)
status = 0
except (Exception, KeyboardInterrupt), exc:
import traceback
exc_string = traceback.format_exc(limit=10)
for l in exc_string.splitlines():
print "[%d]"%os.getpid(),l.rstrip()
result = exc, exc_string
status = 1
with os.fdopen(pwrite, 'wb') as f:
try:
pickle.dump((status,result), f, pickle.HIGHEST_PROTOCOL)
except pickle.PicklingError, exc:
pickle.dump((2,exc), f, pickle.HIGHEST_PROTOCOL)
os._exit(0)
pass # forking
### a decorator converting blocking functions into asynchronous functions
# stolen from http://pypi.python.org/pypi/decorator/3.0.0
def _async_on_success(result): # default implementation
"Called on the result of the function"
return result
def _async_on_failure(exc_info): # default implementation
"Called if the function fails"
_exc_class, _exc, tb = exc_info
raise _exc_class, _exc, tb
pass
def _async_on_closing(): # default implementation
"Called at the end, both in case of success and failure"
pass
class Async(object):
"""
A decorator converting blocking functions into asynchronous
functions, by using threads or processes. Examples:
async_with_threads = Async(threading.Thread)
async_with_processes = Async(multiprocessing.Process)
"""
def __init__(self, threadfactory):
self.threadfactory = threadfactory
def __call__(self, func,
on_success=_async_on_success,
on_failure=_async_on_failure,
on_closing=_async_on_closing):
# every decorated function has its own independent thread counter
func.counter = itertools.count(1)
func.on_success = on_success
func.on_failure = on_failure
func.on_closing = on_closing
return decorator(self.call, func)
def call(self, func, *args, **kw):
def func_wrapper():
try:
result = func(*args, **kw)
except:
func.on_failure(sys.exc_info())
else:
return func.on_success(result)
finally:
func.on_closing()
name = '%s-%s' % (func.__name__, func.counter.next())
thread = self.threadfactory(None, func_wrapper, name)
thread.start()
return thread
# default async decorator: using processes
def async(async_type='mp'):
if async_type in ("mp", "multiprocessing"):
from multiprocessing import Process
factory = Process
elif async_type in ("th", "threading"):
from threading import Thread
factory = Thread
else:
raise ValueError ("async_type must be either 'multiprocessing' "
"or 'threading' (got: %s)"%async_type)
async_obj = Async (factory)
return async_obj
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
## @author: Sebastien Binet
## @file : Logging.py
## @purpose: try to import Logging from AthenaCommon.
## falls back on stdlib's one
__version__ = "$Revision$"
__author__ = "Sebastien Binet"
__all__ = ['msg', 'logging']
try:
import AthenaCommon.Logging as L
msg = L.log
logging = L.logging
except ImportError:
import logging
logging.VERBOSE = logging.DEBUG - 1
logging.ALL = logging.DEBUG - 2
logging.addLevelName( logging.VERBOSE, 'VERBOSE' )
logging.addLevelName( logging.ALL, 'ALL' )
logging.basicConfig(level=logging.INFO,
format="Py:%(name)-14s%(levelname)8s %(message)s")
cls = logging.getLoggerClass()
def verbose(self, msg, *args, **kwargs):
if self.manager.disable >= logging.VERBOSE:
return
if logging.VERBOSE >= self.getEffectiveLevel():
apply(self._log, (logging.VERBOSE, msg, args), kwargs)
cls.verbose = verbose
del verbose
log = msg = logging.getLogger("Athena")
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
import subprocess
### monkey-patch subprocess (fwd compat w/ py-3.x) ----------------------------
def getstatusoutput(cmd, *popenargs, **kwargs):
if isinstance(cmd, basestring):
cmd = cmd.split()
if 'stdout' in kwargs:
raise ValueError('stdout argument not allowed, it will be overridden.')
kwargs['stdout'] = subprocess.PIPE
p = subprocess.Popen(cmd, *popenargs, **kwargs)
sc = p.returncode
if sc is None:
sc = 0
fd = p.stdout
out= ''.join(list(fd.readlines()))
if out[-1:] == '\n': out = out[:-1]
return sc,out
subprocess.getstatusoutput = getstatusoutput
del getstatusoutput
def getstatus(cmd, *popenargs, **kwargs):
return subprocess.getstatusoutput(cmd, *popenargs, **kwargs)[0]
subprocess.getstatus = getstatus
del getstatus
def getoutput(cmd, *popenargs, **kwargs):
return subprocess.getstatusoutput(cmd, *popenargs, **kwargs)[1]
subprocess.getoutput = getoutput
del getoutput
### ---------------------------------------------------------------------------
########################## LICENCE ###############################
##
## Copyright (c) 2005-2011, Michele Simionato
## All rights reserved.
##
## Redistributions of source code must retain the above copyright
## notice, this list of conditions and the following disclaimer.
## Redistributions in bytecode form must reproduce the above copyright
## notice, this list of conditions and the following disclaimer in
## the documentation and/or other materials provided with the
## distribution.
## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
## "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
## LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
## A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
## HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
## INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
## BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
## OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
## ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
## TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
## USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
## DAMAGE.
"""
Decorator module, see http://pypi.python.org/pypi/decorator
for the documentation.
"""
__version__ = '3.3.0'
__all__ = ["decorator", "FunctionMaker", "partial"]
import sys, re, inspect
try:
from functools import partial
except ImportError: # for Python version < 2.5
class partial(object):
"A simple replacement of functools.partial"
def __init__(self, func, *args, **kw):
self.func = func
self.args = args
self.keywords = kw
def __call__(self, *otherargs, **otherkw):
kw = self.keywords.copy()
kw.update(otherkw)
return self.func(*(self.args + otherargs), **kw)
if sys.version >= '3':
from inspect import getfullargspec
else:
class getfullargspec(object):
"A quick and dirty replacement for getfullargspec for Python 2.X"
def __init__(self, f):
self.args, self.varargs, self.varkw, self.defaults = \
inspect.getargspec(f)
self.kwonlyargs = []
self.kwonlydefaults = None
self.annotations = getattr(f, '__annotations__', {})
def __iter__(self):
yield self.args
yield self.varargs
yield self.varkw
yield self.defaults
DEF = re.compile('\s*def\s*([_\w][_\w\d]*)\s*\(')
# basic functionality
class FunctionMaker(object):
"""
An object with the ability to create functions with a given signature.
It has attributes name, doc, module, signature, defaults, dict and
methods update and make.
"""
def __init__(self, func=None, name=None, signature=None,
defaults=None, doc=None, module=None, funcdict=None):
self.shortsignature = signature
if func:
# func can be a class or a callable, but not an instance method
self.name = func.__name__
if self.name == '<lambda>': # small hack for lambda functions
self.name = '_lambda_'
self.doc = func.__doc__
self.module = func.__module__
if inspect.isfunction(func):
argspec = getfullargspec(func)
for a in ('args', 'varargs', 'varkw', 'defaults', 'kwonlyargs',
'kwonlydefaults', 'annotations'):
setattr(self, a, getattr(argspec, a))
for i, arg in enumerate(self.args):
setattr(self, 'arg%d' % i, arg)
self.signature = inspect.formatargspec(
formatvalue=lambda val: "", *argspec)[1:-1]
allargs = list(self.args)
if self.varargs:
allargs.append('*' + self.varargs)
if self.varkw:
allargs.append('**' + self.varkw)
try:
self.shortsignature = ', '.join(allargs)
except TypeError: # exotic signature, valid only in Python 2.X
self.shortsignature = self.signature
self.dict = func.__dict__.copy()
# func=None happens when decorating a caller
if name:
self.name = name
if signature is not None:
self.signature = signature
if defaults:
self.defaults = defaults
if doc:
self.doc = doc
if module:
self.module = module
if funcdict:
self.dict = funcdict
# check existence required attributes
assert hasattr(self, 'name')
if not hasattr(self, 'signature'):
raise TypeError('You are decorating a non function: %s' % func)
def update(self, func, **kw):
"Update the signature of func with the data in self"
func.__name__ = self.name
func.__doc__ = getattr(self, 'doc', None)
func.__dict__ = getattr(self, 'dict', {})
func.func_defaults = getattr(self, 'defaults', ())
callermodule = sys._getframe(3).f_globals.get('__name__', '?')
func.__module__ = getattr(self, 'module', callermodule)
func.__dict__.update(kw)
def make(self, src_templ, evaldict=None, addsource=False, **attrs):
"Make a new function from a given template and update the signature"
src = src_templ % vars(self) # expand name and signature
evaldict = evaldict or {}
mo = DEF.match(src)
if mo is None:
raise SyntaxError('not a valid function template\n%s' % src)
name = mo.group(1) # extract the function name
names = set([name] + [arg.strip(' *') for arg in
self.shortsignature.split(',')])
for n in names:
if n in ('_func_', '_call_'):
raise NameError('%s is overridden in\n%s' % (n, src))
if not src.endswith('\n'): # add a newline just for safety
src += '\n' # this is needed in old versions of Python
try:
code = compile(src, '<string>', 'single')
# print >> sys.stderr, 'Compiling %s' % src
exec code in evaldict
except:
print >> sys.stderr, 'Error in generated code:'
print >> sys.stderr, src
raise
func = evaldict[name]
if addsource:
attrs['__source__'] = src
self.update(func, **attrs)
return func
@classmethod
def create(cls, obj, body, evaldict, defaults=None,
doc=None, module=None, addsource=True,**attrs):
"""
Create a function from the strings name, signature and body.
evaldict is the evaluation dictionary. If addsource is true an attribute
__source__ is added to the result. The attributes attrs are added,
if any.
"""
if isinstance(obj, str): # "name(signature)"
name, rest = obj.strip().split('(', 1)
signature = rest[:-1] #strip a right parens
func = None
else: # a function
name = None
signature = None
func = obj
self = cls(func, name, signature, defaults, doc, module)
ibody = '\n'.join(' ' + line for line in body.splitlines())
return self.make('def %(name)s(%(signature)s):\n' + ibody,
evaldict, addsource, **attrs)
def decorator(caller, func=None):
"""
decorator(caller) converts a caller function into a decorator;
decorator(caller, func) decorates a function using a caller.
"""
if func is not None: # returns a decorated function
evaldict = func.func_globals.copy()
evaldict['_call_'] = caller
evaldict['_func_'] = func
return FunctionMaker.create(
func, "return _call_(_func_, %(shortsignature)s)",
evaldict, undecorated=func)
else: # returns a decorator
if isinstance(caller, partial):
return partial(decorator, caller)
# otherwise assume caller is a function
first = inspect.getargspec(caller)[0][0] # first arg
evaldict = caller.func_globals.copy()
evaldict['_call_'] = caller
evaldict['decorator'] = decorator
return FunctionMaker.create(
'%s(%s)' % (caller.__name__, first),
'return decorator(_call_, %s)' % first,
evaldict, undecorated=caller,
doc=caller.__doc__, module=caller.__module__)
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
# -*- python -*-
# @file PyCmt/python/pkgbuild/__init__.py
# @purpose a simple build system
# @author Sebastien Binet <binet@cern.ch>
from __future__ import with_statement
__doc__ = "A simple build system"
__author__ = "Sebastien Binet <binet@cern.ch>"
__version__= "$Revision$"