Merge branch 'flake8_atlas' into 'master'

Import flake8_atlas as new package into atlasexternals

See merge request !483

External/flake8_atlas/CMakeLists.txt

+# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+
+# The name of the package:
+atlas_subdir( flake8_atlas )
+
+# Since this package doesn't depend on any other ATLAS packages,
+# in release recompilation mode don't even go further than this.
+if( ATLAS_RELEASE_MODE )
+   return()
+endif()
+
+# External(s) needed::
+find_package( PythonInterp REQUIRED )
+find_package( pip REQUIRED )
+
+# Installation directory:
+set( _buildDir    ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/flake8_atlasBuild )
+set( _stamp "${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/flake8_atlas.stamp" )
+
+# Build the package with pip:
+add_custom_command( OUTPUT "${_stamp}"
+   COMMAND ${CMAKE_COMMAND} -E touch ${_stamp}
+   COMMAND ${CMAKE_COMMAND} -E env --unset=SHELL PYTHONUSERBASE=${_buildDir}
+   ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/atlas_build_run.sh
+   "${PIP_EXECUTABLE} install --disable-pip-version-check --no-warn-script-location --no-warn-conflicts"
+   "--no-cache-dir --user ${CMAKE_CURRENT_SOURCE_DIR}"
+   COMMAND ${CMAKE_COMMAND} -E copy_directory ${_buildDir} ${CMAKE_BINARY_DIR}/${ATLAS_PLATFORM}
+   WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}"
+   COMMENT "Building flake8_atlas...")
+
+# Add target and make the package target depend on it:
+add_custom_target( flake8_atlas_plugins ALL
+   DEPENDS "${_stamp}" )
+
+add_dependencies( Package_flake8_atlas flake8_atlas_plugins )
+
+# Install the package:
+install( DIRECTORY ${_buildDir}/
+   DESTINATION .
+   USE_SOURCE_PERMISSIONS OPTIONAL )
+
+# Define the package tests as ctest:
+atlas_add_test( test
+   SCRIPT python -m unittest discover -s ${CMAKE_CURRENT_SOURCE_DIR} )

External/flake8_atlas/README.md

+# ATLAS plugins for flake8
+
+This module contains ATLAS-specific `flake8_atlas` plugins. It is built as part of the
+athena externals. To verify if the plugin is found after setting up a release
+(e.g. via `asetup`), run:
+```
+> flake8 --version
+3.7.7 (flake8_atlas: 1.0.0, mccabe: 0.6.1, pycodestyle: 2.5.0, pyflakes: 2.1.1) CPython 2.7.15 on Linux
+```
+
+Plugins with codes `>=ATL900` are disabled by default. To enable them use
+```
+flake8 --enable-extension=ATL901
+```
+In addition to ATLAS specific plugins most of the "python23" plugins from
+the [hacking](https://github.com/openstack-dev/hacking) plugin by OpenStack
+were imported (and some modified).
+
+
+## Local installation
+To install the `flake8_atlas` plugins locally and use them in your preferred IDE run:
+```
+pip install git+https://:@gitlab.cern.ch:8443/atlas/atlasexternals.git#subdirectory=External/flake8_atlas
+```
+Replace the repository URL with your preferred method of authentication (see the "Clone" button on
+[atlas/atlasexternals](https://gitlab.cern.ch/atlas/atlasexternals)).
+Run `flake8 --version` to verify if the plugin was successfully installed.
+To check if an update exists, compare your local version of `flake8_atlas` with the 
+[version in the repository](https://gitlab.cern.ch/atlas/atlasexternals/blob/master/External/flake8_atlas/flake8_atlas/__init__.py).
+
+
+## Addition of new plugins
+New plugins should be added following the existing examples:
+*  Add an entry_point to `setup.py`.
+*  Add the plugin code to one of the existing python files.
+*  Build the package (only needed when adding/changing entry points).
+*  Before releasing update the version number in `flake8_atlas/__init__.py`
+
+To test a full local installation, use one the following:
+*  from a git branch: `pip install git+[GITURL]@[BRANCH]#subdirectory=External/flake8_atlas`
+*  from a local source: `pip install path/to/External/flake8_atlas`
+
+### Other Documentation
+*  http://flake8.pycqa.org/en/latest/plugin-development
+*  AST-based plugins: https://greentreesnakes.readthedocs.io/en/latest/

External/flake8_atlas/flake8_atlas/__init__.py

+# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+
+# Version of the package:
+__version__ = '1.0.0'

External/flake8_atlas/flake8_atlas/checks.py

+# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+"""ATLAS-specific flake8 plugins
+
+Documentation: http://flake8.pycqa.org/en/latest/plugin-development
+"""
+
+from flake8_atlas import utils
+import ast
+import re
+
+# Inspired by: https://github.com/openstack-dev/hacking/blob/master/hacking/checks/other.py
+RE_LOGGING = re.compile(r".*\.(?:error|warn|warning|info|debug)"
+                        r"\([^,]*(%)[^,]*[,)]")
+@utils.flake8_atlas
+def delayed_string_interpolation(logical_line):
+    r"""String interpolation should be delayed at logging calls.
+    Fail: log.debug('Example: %s' % 'bad')
+    Pass: log.debug('Example: %s', 'good')
+    """
+    msg = "ATL100: use lazy string formatting in logging calls (',' instead of '%')"
+
+    m = RE_LOGGING.match(logical_line)
+    if m is not None:
+       col = m.start(1)
+       yield (col, msg)
+
+
+######################################################################
+# ATL9xy: Specialized plugins (disabled by default)
+######################################################################
+
+@utils.flake8_atlas
+@utils.off_by_default
+class OutputLevel(object):
+   """Check if an explicit OutputLevel is set
+   Fail: myalg.OutputLevel = DEBUG
+   """
+   msg = ('ATL900: Do not assign an explicit OutputLevel', 'ATL900')
+
+   def __init__(self, tree):
+      self.tree = tree
+
+   def run(self):
+      # Walk AST and find assignment to OutputLevel
+      for node in ast.walk(self.tree):
+         # Find: c.OutputLevel = DEBUG
+         if isinstance(node, ast.Assign):
+            for t in node.targets:
+               if isinstance(t,ast.Attribute) and t.attr=='OutputLevel':
+                  yield (node.lineno, node.col_offset) + self.msg
+
+         # Find: setattr(c,'OutputLevel',DEBUG)
+         if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id=='setattr':
+            a = node.args[1]
+            if isinstance(a, ast.Str) and a.s=='OutputLevel':
+               yield (node.lineno, node.col_offset) + self.msg
+
+
+RE_PRINT = re.compile(r"\bprint\b")
+@utils.flake8_atlas
+@utils.off_by_default
+def print_for_logging(logical_line):
+    """Check for occurences of plain 'print'"""
+
+    for match in RE_PRINT.finditer(logical_line):
+        yield match.start(0), (
+            "ATL901: use 'AthenaCommon.Logging' instead of 'print'")

External/flake8_atlas/flake8_atlas/python23.py

+# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+#
+# This is a modified version of the 'hacking' flake8 plugin:
+#   https://github.com/openstack-dev/hacking
+#
+# The unmodified original checks are kept with the "hacking_" prefix.
+# For modified versions (including bugfixes) the prefix was removed.
+#
+
+# Original licence:
+# -----------------
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+
+from flake8_atlas import utils
+import re
+import tokenize
+import ast
+
+def import_normalize(line):
+    # convert "from x import y" to "import x.y"
+    # handle "from x import y as z" to "import x.y as z"
+    split_line = line.split()
+    if ("import" in line and line.startswith("from ") and "," not in line and
+            split_line[2] == "import" and split_line[3] != "*" and
+            split_line[1] != "__future__" and
+            (len(split_line) == 4 or
+             (len(split_line) == 6 and split_line[4] == "as"))):
+        return "import %s.%s" % (split_line[1], split_line[3])
+    else:
+       return line
+
+
+#def hacking_python3x_except_compatible(logical_line, noqa):
+#Deleted as this is covered by pycodestyle E722
+
+RE_OCTAL = re.compile(r"0+([1-9]\d*)")
+
+@utils.flake8_atlas
+def hacking_python3x_octal_literals(logical_line, tokens, noqa):
+    r"""Check for octal literals in Python 3.x compatible form.
+
+    As of Python 3.x, the construct "0755" has been removed.
+    Use "0o755" instead".
+
+    Okay:   f(0o755)
+    Okay:   f(0755)
+    Okay:   f(755)
+    Okay:   f(0)
+    Okay:   f(000)
+    Okay:   MiB = 1.0415
+    Fail:   f(0755)
+    Okay:   f(0755)  # noqa
+    """
+    if noqa:
+        return
+
+    for token_type, text, _, _, _ in tokens:
+        if token_type == tokenize.NUMBER:
+            match = RE_OCTAL.match(text)
+            if match:
+                yield 0, ("ATL231: Python 3.x incompatible octal %s should be "
+                          "written as 0o%s " %
+                          (match.group(0)[1:], match.group(1)))
+
+
+@utils.flake8_atlas
+class incompatible_print_statement(object):
+    r"""Check if a Py3 incompatible print statement is used.
+
+    Check for the use of print statements. But only flag those that are
+    indeed Py3 incompatible. If print_function has been imported, by definition
+    there are no print statements in the code, and this check will never fire.
+
+    Okay: print msg       # but caught by ATL233
+    Okay: print(msg)
+    Fail: print("a","b")  # unless 'from __future__ import print_function'
+    Fail: print
+    """
+
+    def __init__(self, tree):
+        self.tree = tree
+        self.code = 'ATL233'
+        self.msg = ('%s: Python 3.x incompatible use of print statement' % self.code, self.code)
+
+    def run(self):
+        for node in ast.walk(self.tree):
+            if isinstance(node, ast.Print):   # Py2 print statement
+                # no arguments
+                if len(node.values)==0:
+                    yield (node.lineno, node.col_offset) + self.msg
+                # tuple as argument
+                if len(node.values)>0 and isinstance(node.values[0], ast.Tuple) and len(node.values[0].elts)>1:
+                    yield (node.lineno, node.col_offset) + self.msg
+
+
+RE_PRINT = re.compile(r"\bprint\b\s*[^\(]")
+@utils.flake8_atlas
+def print_statement(logical_line):
+    r"""Check if a Py3 incompatible print statement is used.
+
+    Check if a print statement without brackets is used. This check
+    complements ATL232.
+
+    Okay:   print(msg)
+    Fail:   print msg
+    """
+    msg = 'ATL233: Python 3.x incompatible use of non function-like print statement'
+    for match in RE_PRINT.finditer(logical_line):
+        yield match.start(0), msg
+
+
+@utils.flake8_atlas
+def hacking_no_assert_equals(logical_line, tokens, noqa):
+    r"""assert(Not)Equals() is deprecated, use assert(Not)Equal instead.
+
+    Okay:   self.assertEqual(0, 0)
+    Okay:   self.assertNotEqual(0, 1)
+    ATL234: self.assertEquals(0, 0)
+    ATL234: self.assertNotEquals(0, 1)
+    Okay:   self.assertEquals(0, 0)  # noqa
+    Okay:   self.assertNotEquals(0, 1)  # noqa
+    """
+
+    if noqa:
+        return
+    for token_type, text, start_index, _, _ in tokens:
+
+        if token_type == tokenize.NAME:
+            if text == "assertEquals" or text == "assertNotEquals":
+                yield (start_index[1],
+                       "ATL234: %s is deprecated, use %s" % (text, text[:-1]))
+
+
+@utils.flake8_atlas
+def hacking_no_assert_underscore(logical_line, tokens, noqa):
+    r"""assert_() is deprecated, use assertTrue instead.
+
+    Okay: self.assertTrue(foo)
+    ATL235: self.assert_(foo)
+    Okay: self.assert_(foo)  # noqa
+    """
+    if noqa:
+        return
+    for token_type, text, start_index, _, _ in tokens:
+
+        if token_type == tokenize.NAME and text == "assert_":
+            yield (start_index[1],
+                   "ATL235: assert_ is deprecated, use assertTrue")
+
+
+@utils.flake8_atlas
+def hacking_python3x_metaclass(logical_line, noqa):
+    r"""Check for metaclass to be Python 3.x compatible.
+
+    Okay:   @six.add_metaclass(Meta)\nclass Foo(object):\n    pass
+    Okay:   @six.with_metaclass(Meta)\nclass Foo(object):\n    pass
+    Okay:   class Foo(object):\n    '''docstring\n\n    __metaclass__ = Meta\n'''
+    ATL236: class Foo(object):\n    __metaclass__ = Meta
+    ATL236: class Foo(object):\n    foo=bar\n    __metaclass__ = Meta
+    ATL236: class Foo(object):\n    '''docstr.'''\n    __metaclass__ = Meta
+    ATL236: class Foo(object):\n    __metaclass__ = \\\n        Meta
+    Okay:   class Foo(object):\n    __metaclass__ = Meta  # noqa
+    """
+    if noqa:
+        return
+    split_line = logical_line.split()
+    if(len(split_line) > 2 and split_line[0] == '__metaclass__' and
+       split_line[1] == '='):
+        yield (logical_line.find('__metaclass__'),
+               "ATL236: Python 3.x incompatible __metaclass__, "
+               "use six.add_metaclass()")
+
+
+# NOTE(guochbo): This is removed module list:
+# http://python3porting.com/stdlib.html#removed-modules
+removed_modules = [
+    'audiodev', 'Bastion', 'bsddb185', 'bsddb3',
+    'Canvas', 'cfmfile', 'cl', 'commands', 'compiler'
+    'dircache', 'dl', 'exception', 'fpformat',
+    'htmllib', 'ihooks', 'imageop', 'imputil'
+    'linuxaudiodev', 'md5', 'mhlib', 'mimetools'
+    'MimeWriter', 'mimify', 'multifile', 'mutex',
+    'new', 'popen2', 'posixfile', 'pure', 'rexec'
+    'rfc822', 'sha', 'sgmllib', 'sre', 'stat'
+    'stringold', 'sunaudio' 'sv', 'test.testall',
+    'thread', 'timing', 'toaiff', 'user'
+]
+
+
+@utils.flake8_atlas
+def hacking_no_removed_module(logical_line, noqa):
+    r"""Check for removed modules in Python 3.
+
+    Examples:
+    Okay:   from os import path
+    Okay:   from os import path as p
+    Okay:   from os import (path as p)
+    Okay:   import os.path
+    ATL237: import thread
+    Okay:   import thread  # noqa
+    ATL237: import commands
+    ATL237: import md5 as std_md5
+    """
+    if noqa:
+        return
+    line = import_normalize(logical_line.strip())
+    if line and line.split()[0] == 'import':
+        module_name = line.split()[1].split('.')[0]
+        if module_name in removed_modules:
+            yield 0, ("ATL237: module %s is "
+                      "removed in Python 3" % module_name)
+
+
+RE_NEW_STYLE_CLASS = re.compile(r"^class[^(]+\(.+\):")
+
+@utils.flake8_atlas
+def no_old_style_class(logical_line, noqa):
+    r"""Check for old style classes.
+
+    Examples:
+    Okay:   class Foo(object):\n    pass
+    Okay:   class Foo(Bar, Baz):\n    pass
+    Okay:   class Foo(object, Baz):\n    pass
+    Okay:   class Foo(somefunc()):\n    pass
+    ATL238: class Bar:\n    pass
+    ATL238: class Bar():\n    pass
+    """
+    if noqa:
+        return
+    line = logical_line.replace(' ','')
+    if line.startswith("class") and not RE_NEW_STYLE_CLASS.match(line):
+       yield (0, "ATL238: old style class declaration, "
+              "use new style (inherit from `object`)")

External/flake8_atlas/flake8_atlas/test/test_OutputLevel.py

+# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+from flake8_atlas.checks import OutputLevel
+from flake8_atlas.test.testutils import Flake8Test
+
+class Test(Flake8Test):
+   """
+   Test OutputLevel checker
+   """
+   def test_assign(self):
+      """Setting of OutputLevel is not allowed"""
+      self.assertFail('c.OutputLevel = 0', OutputLevel)
+
+   def test_setattr(self):
+      """Use of setattr is not allowed"""
+      self.assertFail('setattr(c, "OutputLevel", 0)', OutputLevel)
+
+   def test_read(self):
+      """Reading of OutputLevel is OK"""
+      self.assertPass('if c.OutputLevel > 0: pass', OutputLevel)

External/flake8_atlas/flake8_atlas/test/test_logging.py

+# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+from flake8_atlas.test.testutils import Flake8Test
+
+class Test(Flake8Test):
+   """
+   Test proper use of loggging facilities
+   """
+
+   def test1(self):
+      """Check lazy logging"""
+      from flake8_atlas.checks import delayed_string_interpolation as checker
+
+      # These tests should be checking code like
+      #    log.info('logging %s' % 'bad')
+      # When run within flake8 it takes care of replacing the actual strings
+      # with placeholders to avoid accidental matches (here %). For the tests here
+      # we simply omit the string formatter.
+      #
+      # http://flake8.pycqa.org/en/latest/internal/checker.html#flake8.processor.mutate_string
+
+      self.assertFail("log.info('logging' % 'bad')", checker)
+      self.assertPass("log.info('logging', 'good')", checker)
+
+      self.assertFail("myfunnyloggername.warning('Hello' % 'world')", checker)
+
+      self.assertFail("log.info('logging' % ('bad',42))", checker)
+      self.assertPass("log.info('logging', 'good', 42)", checker)
+
+   def test2(self):
+      """Check use of print statements for logging"""
+      from flake8_atlas.checks import print_for_logging as checker
+      self.assertFail("print", checker)
+      self.assertFail("print('Hello')", checker)
+      self.assertFail("print 'Hello'", checker)
+
+      # Those should not trigger warnings:
+      self.assertPass("def myprint(): pass", checker)
+      self.assertPass("def printSummary(): pass", checker)
+      self.assertPass("def my_print_function(): pass", checker)

External/flake8_atlas/flake8_atlas/test/test_print_statement.py

+# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+from flake8_atlas.test.testutils import Flake8Test
+
+class Test(Flake8Test):
+   """
+   Test print statement checks
+   """
+
+   def test1(self):
+      """Check print statements"""
+      from flake8_atlas.python23 import print_statement as checker
+      self.assertFail('print "Hello"', checker)
+      self.assertPass('print("Hello")', checker)
+
+   def test2(self):
+      """Check incompatible print statements"""
+      from flake8_atlas.python23 import incompatible_print_statement as checker
+      self.assertFail('print("a","b")', checker)
+      self.assertPass('print(1)', checker)
+      self.assertPass('print("%s %s" % ("a","b"))', checker)
+      self.assertFail('print', checker)

External/flake8_atlas/flake8_atlas/test/testutils.py

+# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+
+import unittest
+import ast
+
+class Flake8Test(unittest.TestCase):
+   """Unit test case for flake8 checker"""
+
+   def _run(self, source, checker):
+      """Run AST or line-based checker and return list of errors"""
+      if hasattr(checker, 'run'):
+         chk = checker(ast.parse(source)).run()
+      else:
+         chk = checker(source)
+      return list(chk)
+
+   def assertFail(self, source, checker):
+      """Test that checker returns an error on source"""
+      codes = self._run(source, checker)
+      self.assertEqual(len(codes), 1)
+
+   def assertPass(self, source, checker):
+      """Test that checker passes on source"""
+      codes = self._run(source, checker)
+      self.assertListEqual(codes, [])

External/flake8_atlas/flake8_atlas/utils.py

+# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+"""Utilities for flake8 plugins"""
+
+from flake8_atlas import __version__
+
+def flake8_atlas(f):
+   """Default decorator for flake8 plugins"""
+   f.name = 'flake8_atlas'
+   f.version = __version__
+   if not hasattr(f, 'off_by_default'):
+      f.off_by_default = False
+   return f
+
+def off_by_default(f):
+   """Decorator to disable plugin by default (use --enable-extensions)"""
+   f.off_by_default = True
+   return f

External/flake8_atlas/setup.py

+# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+#
+# Register flake8 plugins:
+#   http://flake8.pycqa.org/en/latest/plugin-development/registering-plugins.html
+#
+import setuptools
+from flake8_atlas import __version__
+
+requires = [
+   "flake8 > 3.0.0",
+]
+
+setuptools.setup(
+   name="flake8_atlas",
+   version=__version__,
+   description="ATLAS plugins for flake8",
+   author='Frank Winklmeier',
+   url='https://gitlab.cern.ch/atlas/atlasexternals/tree/master/External/flake8_atlas',
+   test_loader='unittest:TestLoader',
+   packages=['flake8_atlas'],
+   zip_safe=True,
+   entry_points={
+      'flake8.extension': [
+         'ATL100 = flake8_atlas.checks:delayed_string_interpolation',
+         'ATL231 = flake8_atlas.python23:hacking_python3x_octal_literals',
+         'ATL232 = flake8_atlas.python23:incompatible_print_statement',
+         'ATL233 = flake8_atlas.python23:print_statement',
+         'ATL234 = flake8_atlas.python23:hacking_no_assert_equals',
+         'ATL235 = flake8_atlas.python23:hacking_no_assert_underscore',
+         'ATL236 = flake8_atlas.python23:hacking_python3x_metaclass',
+         'ATL237 = flake8_atlas.python23:hacking_no_removed_module',
+         'ATL238 = flake8_atlas.python23:no_old_style_class',
+         'ATL900 = flake8_atlas.checks:OutputLevel',
+         'ATL901 = flake8_atlas.checks:print_for_logging',
+      ],
+   }
+)

Projects/AthenaExternals/package_filters.txt

@@ -9,6 +9,7 @@
 + External/FFTW
 + External/FastJet
 + External/FastJetContrib
++ External/flake8_atlas
 + External/GPerfTools
 + External/Gdb
 + External/Geant4