[79] linting: automation

.gitignore

@@ -16,7 +16,6 @@ secrets/
 
 *.key
 *.crt
-*.csr
 
 # Dump files
 
@@ -31,3 +30,6 @@ env/
 
 *.log
 *.pid
+
+# Gitlab CI Runner
+builds/

.gitlab-ci.yml

@@ -23,6 +23,7 @@ stages:
   - tag_image
   - import_image # This stage is only used when the built image is stored in the GitLab Registry
   - deploy
+  - test
 
 ### 'Build Base' stage
 ### Builds the base docker image and tags it according to the commit message.
@@ -157,3 +158,21 @@ deploy_production:
     OPENSHIFT_SERVER: ${OPENSHIFT_SERVER_PROD}
     NAMESPACE: ${NAMESPACE_PROD}
     GIT_STRATEGY: none
+
+########## MERGE REQUESTS ##########
+lint_test:
+  services:
+    - docker:dind
+  stage: test
+  image: tmaier/docker-compose:latest
+  script: make build-env lint test
+  before_script:
+    - docker info
+    - docker-compose --version
+    - apk add make
+  only:
+  - merge_requests
+  - tags
+  allow_failure: true
+  tags:
+    - docker-privileged-xl

.isort.cfg

+[settings]
+line_length=120

.pre-commit-config.yaml

+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.7.8
+    hooks:
+    -   id: flake8
+        additional_dependencies: [flake8-docstrings]
+
+-   repo: https://github.com/timothycrosley/isort
+    rev: 4.3.21-2
+    hooks:
+    -   id: isort

Makefile

@@ -11,7 +11,8 @@
 # make load-fixtures          # loads fixtures
 # make populate-instance      # create database, tables and indeces
 # make generate-certificates  # generate nginx certificates
-# make test                   # run test
+# make test                   # runs tests
+# make lint                   # runs linting tools
 
 SERVICE_NAME :=  cern-search-api
 DOCKER_FILE := docker-compose.full.yml
@@ -58,9 +59,18 @@ generate-certificates:
 .PHONY: generate-certificates
 
 test:
-	make pytest
+	@echo todo
+#	python pytest
 .PHONY: test
 
+lint:
+	docker-compose -f $(DOCKER_FILE) exec -T $(SERVICE_NAME) /bin/bash -c \
+		"echo running isort...; \
+		isort -rc -c -df; \
+		echo running flake8...; \
+		flake8 --max-complexity 10 --ignore E501,D401"
+.PHONY: lint
+
 ###################  Local development helpful directives  ####################
 ###################           (pipenv + docker)            ####################
 #
@@ -75,6 +85,8 @@ test:
 # make destroy-local-env        # stop and remove containers, networks, images, and volume and pipenv
 # make reload-local-env         # restart containers, networks, images, and volume and pipenv
 # make load-fixtures-local      # loads fixtures
+# make local-test               # runs tests
+# make local-lint               # runs linting tools
 
 PIPENV_DOTENV := .pipenv.env
 PYTHON_VERSION_FILE := .python-version
@@ -117,3 +129,15 @@ destroy-local-env:
 
 reload-local-env: destroy-local-env local-env
 .PHONY: reload-local-env
+
+local-test:
+	@echo todo
+#	python pytest
+.PHONY: test
+
+local-lint:
+	@echo running isort...;
+	pipenv run isort -rc -c -df;
+	@echo running flake8...;
+	pipenv run flake8 --max-complexity 10 --ignore E501,D401
+.PHONY: lint

Pipfile

@@ -5,8 +5,8 @@ verify_ssl = true
 
 [dev-packages]
 pytest = "*"
-flake8 = "*"
-flake8-docstrings = "*"
+flake8 = ">=3.7.8"
+flake8-docstrings = ">=1.5.0"
 isort = "==4.3.21"
 
 [packages]

Pipfile.lock

 {
     "_meta": {
         "hash": {
-            "sha256": "a17c1a5523fd862e252dd3f3ff653f1767960813634f20e8f1c183ad00dc3475"
+            "sha256": "c80aaa51a85f385ce614a1ccd3450cfb9559ccf37546221a664ab050d28996a9"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -1294,11 +1294,11 @@
         },
         "pytest": {
             "hashes": [
-                "sha256:7e4800063ccfc306a53c461442526c5571e1462f61583506ce97e4da6a1d88c8",
-                "sha256:ca563435f4941d0cb34767301c27bc65c510cb82e90b9ecf9cb52dc2c63caaa0"
+                "sha256:27abc3fef618a01bebb1f0d6d303d2816a99aa87a5968ebc32fe971be91eb1e6",
+                "sha256:58cee9e09242937e136dbb3dab466116ba20d6b7828c7620f23947f37eb4dae4"
             ],
             "index": "pypi",
-            "version": "==5.2.1"
+            "version": "==5.2.2"
         },
         "six": {
             "hashes": [

cern_search_rest_api/__init__.py

@@ -11,4 +11,4 @@ from __future__ import absolute_import, print_function
 
 from .version import __version__
 
-__all__ = ('__version__', )
\ No newline at end of file
+__all__ = ('__version__', )

cern_search_rest_api/config.py

@@ -12,16 +12,15 @@ from __future__ import absolute_import, print_function
 import ast
 import copy
 import os
+
 from flask import request
 from invenio_oauthclient.contrib import cern
 from invenio_records_rest import config as irr_config
 from invenio_records_rest.facets import terms_filter
 
-from .modules.cernsearch.permissions import (record_read_permission_factory,
-                                             record_create_permission_factory,
-                                             record_update_permission_factory,
-                                             record_delete_permission_factory,
-                                             record_list_permission_factory)
+from .modules.cernsearch.permissions import (record_create_permission_factory, record_delete_permission_factory,
+                                             record_list_permission_factory, record_read_permission_factory,
+                                             record_update_permission_factory)
 
 
 def _(x):

cern_search_rest_api/modules/__init__.py

@@ -5,4 +5,4 @@
 # Copyright (C) 2018-2019 CERN.
 #
 # CERN Search is free software; you can redistribute it and/or modify it
-# under the terms of the MIT License; see LICENSE file for more details.
\ No newline at end of file
+# under the terms of the MIT License; see LICENSE file for more details.

cern_search_rest_api/modules/cernsearch/__init__.py

@@ -5,4 +5,4 @@
 # Copyright (C) 2018-2019 CERN.
 #
 # CERN Search is free software; you can redistribute it and/or modify it
-# under the terms of the MIT License; see LICENSE file for more details.
\ No newline at end of file
+# under the terms of the MIT License; see LICENSE file for more details.

cern_search_rest_api/modules/cernsearch/api.py

@@ -8,8 +8,8 @@
 # under the terms of the MIT License; see LICENSE file for more details.
 
 """ Record API"""
-from invenio_records.api import Record
 from invenio_pidstore.models import PersistentIdentifier
+from invenio_records.api import Record
 
 from .fetchers import recid_fetcher
 
@@ -23,4 +23,4 @@ class CernSearchRecord(Record):
     def pid(self):
         """Return an instance of record PID."""
         pid = self.record_fetcher(self.id, self)
-        return PersistentIdentifier.get(pid.pid_type, pid.pid_value)
\ No newline at end of file
+        return PersistentIdentifier.get(pid.pid_type, pid.pid_value)

cern_search_rest_api/modules/cernsearch/errors.py

@@ -14,4 +14,4 @@ class InvalidRecordFormatError(RESTValidationError):
     """Invalid query syntax."""
 
     code = 400
-    description = 'Invalid query syntax.'
\ No newline at end of file
+    description = 'Invalid query syntax.'

cern_search_rest_api/modules/cernsearch/ext.py

@@ -7,10 +7,9 @@
 # CERN Search is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
 
-from invenio_indexer.signals import before_record_index
-
 from cern_search_rest_api.modules.cernsearch.indexer import csas_indexer_receiver
 from cern_search_rest_api.modules.cernsearch.views import build_blueprint
+from invenio_indexer.signals import before_record_index
 
 
 class CERNSearch(object):
@@ -34,4 +33,4 @@ class CERNSearch(object):
         # Set up API endpoints for records.
         for k in dir(app.config):
             if k.startswith('CERN_SEARCH'):
-                app.config.setdefault(k, getattr(app.config, k))
\ No newline at end of file
+                app.config.setdefault(k, getattr(app.config, k))

cern_search_rest_api/modules/cernsearch/fetchers.py

@@ -10,6 +10,7 @@
 """Persistent identifier fetcher."""
 
 from invenio_pidstore.fetchers import FetchedPID
+
 from .providers import CERNSearchRecordIdProvider
 
 
@@ -19,4 +20,4 @@ def recid_fetcher(record_uuid, data):
         # provider=CERNSearchRecordIdProvider,
         pid_type='recid',
         pid_value=str(data['recid'])
-    )
\ No newline at end of file
+    )

cern_search_rest_api/modules/cernsearch/handlers.py

@@ -11,22 +11,18 @@
 
 from __future__ import absolute_import, print_function
 
-from flask import current_app, redirect, session, url_for, g, after_this_request
+from cern_search_rest_api.modules.cernsearch.utils import get_user_provides
+from flask import after_this_request, current_app, g, redirect, session, url_for
 from flask_login import current_user, user_logged_in
 from flask_security import logout_user
 from flask_security.utils import get_post_logout_redirect
 from invenio_db import db
-
+from invenio_oauthclient.handlers import (get_session_next_url, oauth_error_handler, response_token_setter,
+                                          token_getter, token_session_key)
 from invenio_oauthclient.proxies import current_oauthclient
-from invenio_oauthclient.signals import account_info_received, account_setup_committed, \
-    account_setup_received
-from invenio_oauthclient.utils import oauth_authenticate, oauth_get_user, create_csrf_disabled_registrationform, \
-    fill_form, oauth_register
-from invenio_oauthclient.handlers import oauth_error_handler, token_session_key, response_token_setter, \
-    token_getter, get_session_next_url
-
-
-from cern_search_rest_api.modules.cernsearch.utils import get_user_provides
+from invenio_oauthclient.signals import account_info_received, account_setup_committed, account_setup_received
+from invenio_oauthclient.utils import (create_csrf_disabled_registrationform, fill_form, oauth_authenticate,
+                                       oauth_get_user, oauth_register)
 
 
 @oauth_error_handler

cern_search_rest_api/modules/cernsearch/jsonschemas/__init__.py

@@ -5,4 +5,4 @@
 # Copyright (C) 2018-2019 CERN.
 #
 # CERN Search is free software; you can redistribute it and/or modify it
-# under the terms of the MIT License; see LICENSE file for more details.
\ No newline at end of file
+# under the terms of the MIT License; see LICENSE file for more details.

cern_search_rest_api/modules/cernsearch/jsonschemas/edms/__init__.py

@@ -5,4 +5,4 @@
 # Copyright (C) 2018-2019 CERN.
 #
 # CERN Search is free software; you can redistribute it and/or modify it
-# under the terms of the MIT License; see LICENSE file for more details.
\ No newline at end of file
+# under the terms of the MIT License; see LICENSE file for more details.

cern_search_rest_api/modules/cernsearch/jsonschemas/indico/__init__.py

@@ -5,4 +5,4 @@
 # Copyright (C) 2018-2019 CERN.
 #
 # CERN Search is free software; you can redistribute it and/or modify it
-# under the terms of the MIT License; see LICENSE file for more details.
\ No newline at end of file
+# under the terms of the MIT License; see LICENSE file for more details.

cern_search_rest_api/modules/cernsearch/jsonschemas/test/__init__.py

@@ -5,4 +5,4 @@
 # Copyright (C) 2018-2019 CERN.
 #
 # CERN Search is free software; you can redistribute it and/or modify it
-# under the terms of the MIT License; see LICENSE file for more details.
\ No newline at end of file
+# under the terms of the MIT License; see LICENSE file for more details.