flake8_atlas: Update print-related checks

Update the print-related Python3 checks:
- ATL232: Change to an AST-based plugin that finds all `print`
  statements that are truly incompatible with Py3. The previous version
  would e.g. also warn about `print("Hello")` while the new version
  will only warn for tuples with more than one entry, e.g.
  `print("Hello","world")`.
- ATL233: Only warn about `print` statements that do not look like
  function calls, i.e. without brackets.

Tools/PyUtils/CMakeLists.txt

@@ -80,3 +80,7 @@ atlas_add_test( flake8_OutputLevel
 atlas_add_test( flake8_logging
    SCRIPT flake8 --enable-extensions=ATL901 --select=ATL --stdin-display-name=flake8_logging.py
    --exit-zero --isolated - < ${CMAKE_CURRENT_SOURCE_DIR}/python/flake8_atlas/test/flake8_logging.py )
+
+atlas_add_test( flake8_print_statement
+   SCRIPT flake8 --select=ATL --stdin-display-name=flake8_print_statement.py
+   --exit-zero --isolated - < ${CMAKE_CURRENT_SOURCE_DIR}/python/flake8_atlas/test/flake8_print_statement.py )

Tools/PyUtils/python/flake8_atlas/README.md

@@ -14,4 +14,16 @@ flake8 --enable-extension=ATL901
 ```
 In addition to ATLAS specific plugins most of the `python23.py` plugins from
 the [hacking](https://github.com/openstack-dev/hacking) plugin by OpenStack
-were imported (and some modified).
+were imported (and some modified).
\ No newline at end of file
+
+## Addition of new plugins
+New plugins should be added following the existing examples:
+*  Add an entry_point to `setup.py`.
+*  Add the plugin code to one of the existing python files.
+*  Build the package (only needed when adding/changing entry points).
+*  If the plugin is enabled by default, make sure to run all existing flake8 unit tests
+   (`git grep -l flake8 '*/CMakeLists.txt'`) and apply the relevant fixes.
+
+### External Documentation:
+*  http://flake8.pycqa.org/en/latest/plugin-development
+*  AST-based plugins: https://greentreesnakes.readthedocs.io/en/latest/

Tools/PyUtils/python/flake8_atlas/python23.py

@@ -24,7 +24,7 @@
 from PyUtils.flake8_atlas import utils
 import re
 import tokenize
-
+import ast
 
 def import_normalize(line):
     # convert "from x import y" to "import x.y"
@@ -58,7 +58,7 @@ def hacking_python3x_octal_literals(logical_line, tokens, noqa):
     Okay:   f(0)
     Okay:   f(000)
     Okay:   MiB = 1.0415
-    ATL232: f(0755)
+    Fail:   f(0755)
     Okay:   f(0755)  # noqa
     """
     if noqa:
@@ -68,35 +68,53 @@ def hacking_python3x_octal_literals(logical_line, tokens, noqa):
         if token_type == tokenize.NUMBER:
             match = RE_OCTAL.match(text)
             if match:
-                yield 0, ("ATL232: Python 3.x incompatible octal %s should be "
+                yield 0, ("ATL231: Python 3.x incompatible octal %s should be "
                           "written as 0o%s " %
                           (match.group(0)[1:], match.group(1)))
 
 
-RE_PRINT = re.compile(r"\bprint(?:$|\s+[^\(])")
+@utils.flake8_atlas
+class incompatible_print_statement(object):
+    r"""Check if a Py3 incompatible print statement is used.
+
+    Check for the use of print statements. But only flag those that are
+    indeed Py3 incompatible. If print_function has been imported, by definition
+    there are no print statements in the code, and this check will never fire.
+
+    Okay: print msg       # but caught by ATL233
+    Okay: print(msg)
+    Fail: print("a","b")  # unless 'from __future__ import print_function'
+    Fail: print
+    """
+    msg = ('ATL232: Python 3.x incompatible use of print statement', 'ATL232')
+
+    def __init__(self, tree):
+        self.tree = tree
+
+    def run(self):
+        for node in ast.walk(self.tree):
+            if isinstance(node, ast.Print):   # Py2 print statement
+                # no arguments
+                if len(node.values)==0:
+                    yield (node.lineno, node.col_offset) + self.msg
+                # tuple as argument
+                if len(node.values)>0 and isinstance(node.values[0], ast.Tuple) and len(node.values[0].elts)>1:
+                    yield (node.lineno, node.col_offset) + self.msg
+
 
+RE_PRINT = re.compile(r"\bprint\b\s*[^\(]")
 @utils.flake8_atlas
-def hacking_python3x_print_function(logical_line, noqa):
-    r"""Check that all print occurrences look like print functions.
+def print_statement(logical_line):
+    r"""Check if a Py3 incompatible print statement is used.
 
-    Check that all occurrences of print look like functions, not
-    print operator. As of Python 3.x, the print operator has
-    been removed.
+    Check if a print statement without brackets is used. This check
+    complements ATL232.
 
     Okay:   print(msg)
-    Okay:   print (msg)
-    Okay:   print msg  # noqa
-    Okay:   print()
-    ATL233: print msg
-    ATL233: print >>sys.stderr, "hello"
-    ATL233: print msg,
-    ATL233: print
+    Fail:   print msg
     """
-    if noqa:
-        return
     for match in RE_PRINT.finditer(logical_line):
-        yield match.start(0), (
-            "ATL233: Python 3.x incompatible use of print operator")
+        yield match.start(0), ("ATL233: Python 3.x incompatible use of non function-like print statement")
 
 
 @utils.flake8_atlas
@@ -134,9 +152,8 @@ def hacking_no_assert_underscore(logical_line, tokens, noqa):
     for token_type, text, start_index, _, _ in tokens:
 
         if token_type == tokenize.NAME and text == "assert_":
-            yield (
-                start_index[1],
-                "ATL235: assert_ is deprecated, use assertTrue")
+            yield (start_index[1],
+                   "ATL235: assert_ is deprecated, use assertTrue")
 
 
 @utils.flake8_atlas

Tools/PyUtils/python/flake8_atlas/setup.py

@@ -15,8 +15,9 @@ setuptools.setup(
    entry_points={
       'flake8.extension': [
          'ATL100 = PyUtils.flake8_atlas.checks:delayed_string_interpolation',
-         'ATL232 = PyUtils.flake8_atlas.python23:hacking_python3x_octal_literals',
-         'ATL233 = PyUtils.flake8_atlas.python23:hacking_python3x_print_function',
+         'ATL231 = PyUtils.flake8_atlas.python23:hacking_python3x_octal_literals',
+         'ATL232 = PyUtils.flake8_atlas.python23:incompatible_print_statement',
+         'ATL233 = PyUtils.flake8_atlas.python23:print_statement',
          'ATL234 = PyUtils.flake8_atlas.python23:hacking_no_assert_equals',
          'ATL235 = PyUtils.flake8_atlas.python23:hacking_no_assert_underscore',
          'ATL236 = PyUtils.flake8_atlas.python23:hacking_python3x_metaclass',

Tools/PyUtils/python/flake8_atlas/test/flake8_print_statement.py

+#
+# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
+#
+# Test for Py3 incompatible print statements
+
+print "Hello"  # Fail
+print("Hello")
+print("a","b") # Fail
+print(1)
+print("%s %s" % ("a","b"))
+print  # Fail

Tools/PyUtils/share/flake8_logging.ref

@@ -2,5 +2,5 @@ flake8_logging.py:11:39: ATL100: use lazy string formatting in logging calls (',
 flake8_logging.py:13:37: ATL100: use lazy string formatting in logging calls (',' instead of '%')
 flake8_logging.py:15:43: ATL100: use lazy string formatting in logging calls (',' instead of '%')
 flake8_logging.py:19:1: ATL901: use 'AthenaCommon.Logging' instead of 'print'
-flake8_logging.py:21:1: ATL233: Python 3.x incompatible use of print operator
+flake8_logging.py:21:1: ATL233: Python 3.x incompatible use of non function-like print statement
 flake8_logging.py:21:1: ATL901: use 'AthenaCommon.Logging' instead of 'print'

Tools/PyUtils/share/flake8_print_statement.ref

+flake8_print_statement.py:6:1: ATL233: Python 3.x incompatible use of non function-like print statement
+flake8_print_statement.py:8:1: ATL232: Python 3.x incompatible use of print statement
+flake8_print_statement.py:11:1: ATL232: Python 3.x incompatible use of print statement