diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8b226d970b75713ee6688927ae43aa2134aa3423..72161e042b9e80397106314a3146439db8ed4660 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -61,7 +61,7 @@ lint: - if: $CI_MERGE_REQUEST_ID unit_tests: - image: docker:19.03.12 + image: tmaier/docker-compose:latest services: - docker:dind variables: @@ -74,7 +74,7 @@ unit_tests: # defaults to tcp://docker:2376 upon seeing the TLS certificate directory. #DOCKER_HOST: tcp://docker:2376/ stage: Test - script: make docker-build ci-test + script: make ci-test before_script: - docker info - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY diff --git a/Makefile b/Makefile index 5b78b5c29879bb46096300b57660075abd9de5ad..bb11a9ecbb186d05fbdda4a446f8c7e741ff6d83 100644 --- a/Makefile +++ b/Makefile @@ -3,11 +3,15 @@ ## ## make setup-env # sets up the environment ## make lint # runs linting tools outside docker +## make pytest # runs tests ## make docker-build # builds docker image +## make docker-build-test # builds docker image for tests ## make ci-lint # runs linting tools inside docker +## make ci-test # runs tests inside docker ## make docker-build-env # run docker-compose: creates images, containers, volumes and start the consumer ## make docker-rebuild-env # force create of docker environment ## make docker-shell-env # bash the main container +## make stop-test # stops containers, networks, images, and volume ## make docker-send-email # send a email notification to activeMQ ## make docker-run # run a consumer ## @@ -22,21 +26,27 @@ lint: .PHONY: lint pytest: - pytest tests -vv + docker-compose -f docker-compose.test.yml exec -T notifications-consumer /bin/bash -c "pytest tests -vv;" .PHONY: pytest docker-build: docker build -t notifications-consumer . --no-cache --build-arg build_env=development .PHONY: docker-build +docker-build-test: + docker-compose -f docker-compose.test.yml up -d --remove-orphans +.PHONY: docker-build-test + ci-lint: docker run notifications-consumer make lint .PHONY: ci-lint -ci-test: - docker run notifications-consumer make pytest +ci-test: docker-build-test pytest .PHONY: ci-test +test: stop-test ci-test +.PHONY: test + docker-build-env: docker-compose up --remove-orphans .PHONY: docker-build-env @@ -63,6 +73,10 @@ docker-stop: docker-compose rm -f .PHONY: docker-stop +stop-test: + docker-compose -f docker-compose.test.yml down --volumes +.PHONY: stop-test + docker-send-email: python scripts/docker-send-email.py .PHONY: docker-send-email diff --git a/README.md b/README.md index 619436239377c8a225ccbaf2c80e18cb865824d5..8ef9c761c2496ae1d8f6cf84931b6d60d88699b6 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,13 @@ Consume and process messages already pre-processed with notifications-routing. **Consumers:** - Email -- WebPush -- SafariPush +- Email Feed +- Web Push +- Safari Push +- Email Gateway +- Email Gateway Failure +- DLQ + ## Contribute @@ -129,6 +134,30 @@ See more on choosing [dependency constrains](https://python-poetry.org/docs/vers poetry update requests ``` +### Creating and Running Unit and Integration Tests + +#### Writing tests + +The folder ```tests``` contains all test cases and future tests should be placed into this folder. +Each test file needs to start with the prefix ```test_``` (e.g. ```test_postgres_data_source.py```) + + +Tests functions always start with the prefix ```test_``` e.g.: +```python + +def test_FUNCTION_OPERATION(self): + ... + +``` + +#### Execute Pytest Tests + +To manually execute the tests use the following command: + +```bash +make ci-test +``` + ### Running against cernmx Requirements: diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000000000000000000000000000000000000..b9d7661f3b13d11fa92a6c7f5eaa5847adc1ee56 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,83 @@ +version: '3.7' + +services: + notifications-consumer: + image: notifications-consumer + container_name: notifications-consumer + build: + context: . + dockerfile: Dockerfile + args: + build_env: development + networks: + - default + volumes: + - '.:/opt:delegated' + - './docker/activemq/email_publisher.conf:/etc/activemq-publisher-email_publisher.conf' + - './docker/activemq/email_consumer.conf:/etc/activemq-consumer-email_consumer.conf' + - './docker/activemq/email_gateway_publisher.conf:/etc/activemq-publisher-email_gateway_publisher.conf' + - './docker/activemq/email_gateway_consumer.conf:/etc/activemq-consumer-email_gateway_consumer.conf' + - './docker/activemq/email_gateway_failure_publisher.conf:/etc/activemq-publisher-email_gateway_failure_publisher.conf' + - './docker/activemq/email_gateway_failure_consumer.conf:/etc/activemq-consumer-email_gateway_failure_consumer.conf' + - './docker/activemq/webpush_publisher.conf:/etc/activemq-publisher-webpush_publisher.conf' + - './docker/activemq/webpush_consumer.conf:/etc/activemq-consumer-webpush_consumer.conf' + - './docker/activemq/safaripush_publisher.conf:/etc/activemq-publisher-safaripush_publisher.conf' + - './docker/activemq/safaripush_consumer.conf:/etc/activemq-consumer-safaripush_consumer.conf' + - './docker/activemq/email_daily_publisher.conf:/etc/activemq-publisher-email_daily_publisher.conf' + - './docker/activemq/email_daily_consumer.conf:/etc/activemq-consumer-email_daily_consumer.conf' + - './docker/activemq/email_dlq_consumer.conf:/etc/activemq-consumer-email_dlq_consumer.conf' + - './docker/activemq/email_daily_dlq_consumer.conf:/etc/activemq-consumer-email_daily_dlq_consumer.conf' + - './docker/activemq/email_gateway_failure_dlq_consumer.conf:/etc/activemq-consumer-email_gateway_failure_dlq_consumer.conf' + - './docker/activemq/email_gateway_critical_failure_dlq_consumer.conf:/etc/activemq-consumer-email_gateway_critical_failure_dlq_consumer.conf' + - './docker/activemq/webpush_dlq_consumer.conf:/etc/activemq-consumer-webpush_dlq_consumer.conf' + - './docker/activemq/safaripush_dlq_consumer.conf:/etc/activemq-consumer-safaripush_dlq_consumer.conf' + + ports: + - 8080:8080 + env_file: + - .env + depends_on: + pg_db: + condition: service_healthy + activemq: + condition: service_healthy + + pg_db: + image: postgres + volumes: + - pgsql-data:/var/lib/pgsql/data:rw + networks: + - default + ports: + - 5432:5432 + environment: + - POSTGRES_USER + - POSTGRES_PASSWORD + - POSTGRES_DB + healthcheck: + test: ["CMD-SHELL", "pg_isready -U test -d push_dev"] + interval: 10s + timeout: 5s + retries: 5 + + activemq: + build: ./docker/activemq/5.16.0-alpine + volumes: + - './docker/activemq/conf/activemq.xml:/opt/apache-activemq-5.16.0/conf/activemq.xml' + ports: + - 61613:61613 + - 8161:8161 + networks: + - default + healthcheck: + test: ["CMD", "nc", "-vz", "localhost", "8161"] + interval: 5s + timeout: 5s + retries: 5 + +volumes: + pgsql-data: + name: pgsql-data + +networks: + default: diff --git a/notifications_consumer/data_source/postgres/postgres_data_source.py b/notifications_consumer/data_source/postgres/postgres_data_source.py index 9f5d21544878efde289dbec7e078c07f1cbba1fb..f45591ad54d5f0e02ecb43d274f445477fd2a427 100644 --- a/notifications_consumer/data_source/postgres/postgres_data_source.py +++ b/notifications_consumer/data_source/postgres/postgres_data_source.py @@ -205,9 +205,17 @@ class Channel(PostgresDataSource.Base): ownerId = Column(UUID(as_uuid=True)) adminGroupId = Column(UUID(as_uuid=True)) adminGroup = relationship( - "Group", primaryjoin="Channel.adminGroupId == Group.id", foreign_keys="Group.id", uselist=False + "Group", + primaryjoin="Channel.adminGroupId == Group.id", + foreign_keys="Group.id", + uselist=False, + ) + owner = relationship( + "User", + primaryjoin="Channel.ownerId == User.id", + foreign_keys="User.id", + uselist=False, ) - owner = relationship("User", primaryjoin="Channel.ownerId == User.id", foreign_keys="User.id", uselist=False) class Group(PostgresDataSource.Base): diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..183a373a9129738ebe52e475acfba68698aee622 --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1,2 @@ +"""Initialize Test Package.""" +# FIX for Poetry: https://github.com/python-poetry/poetry/issues/87 diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..ed8f2ffb08a7fb14bb3915062c851bb1656392b8 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,178 @@ +"""Package's fixtures.""" + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.schema import CreateSchema + +from notifications_consumer.app import configure_logging +from notifications_consumer.config import Config, load_config +from notifications_consumer.data_source.postgres.postgres_data_source import ( + Channel, + Group, + Notification, + PostgresDataSource, + SubmissionByEmail, + User, +) + + +@pytest.fixture(scope="module") +def config(): + """Set up config.""" + config = load_config() + yield config + + +@pytest.fixture(scope="module") +def appctx(config): + """Set up app context.""" + configure_logging(config) + + +@pytest.fixture(scope="module") +def data_source(): + """Set up data source.""" + engine = create_engine(Config.SQLALCHEMY_DATABASE_URI) + if not engine.dialect.has_schema(engine, Config.DB_SCHEMA): + engine.execute(CreateSchema(Config.DB_SCHEMA)) + + # PostgresDataSource.Base.prepare(engine) + PostgresDataSource.Base.metadata.create_all(engine) + + db = PostgresDataSource() + yield db + + PostgresDataSource.Base.metadata.drop_all(bind=engine) + + +@pytest.fixture(scope="function") +def session(data_source): + """Return database session for test.""" + with data_source.session() as session: + yield session + + +@pytest.fixture(scope="function") +def user(session): + """Insert user to db.""" + user = User() + user.id = "16fd2706-8baf-433b-82eb-8c7fada847da" + user.email = "testuser@cern.ch" + + session.add(user) + session.commit() + + yield user + + session.delete(user) + session.commit() + + +@pytest.fixture(scope="function") +def group(session): + """Insert group to db.""" + group = Group() + group.id = "186d8dfc-2774-43a8-91b5-a887fcb6ba4a" + group.groupIdentifier = "test-group" + + session.add(group) + session.commit() + + yield group + + session.delete(group) + session.commit() + + +@pytest.fixture(scope="function") +def channel(session, group, user): + """Insert channel to db.""" + channel = Channel() + channel.id = "c3ccc15b-298f-4dc7-877f-2c8970331caf" + channel.slug = "test-channel" + channel.name = "Test Channel" + channel.incomingEmail = "testuser@cern.ch" + channel.groups = [group] + channel.members = [user] + channel.deleteDate = None + channel.submissionByEmail = [SubmissionByEmail.EMAIL] + channel.ownerId = user.id + channel.adminGroupId = group.id + + session.add(channel) + session.commit() + + yield channel + + # break relationships before deleting to avoid issues with default cascade save-update + channel.groups.remove(group) + channel.members.remove(user) + channel.ownerId = None + channel.adminGroupId = None + session.commit() + + session.delete(channel) + session.commit() + + +@pytest.fixture(scope="function") +def channel_submission_by_egroup(session, channel): + """Update channel with submission by egroup.""" + channel.incomingEgroup = "test-group" + channel.submissionByEmail = [SubmissionByEmail.EGROUP] + + session.commit() + + yield channel + + +@pytest.fixture(scope="function") +def channel_submission_by_administrators(session, channel): + """Update channel with submission by administrators.""" + channel.incomingEgroup = "test-group" + channel.submissionByEmail = [SubmissionByEmail.ADMINISTRATORS] + + session.commit() + + yield channel + + +@pytest.fixture(scope="function") +def channel_submission_by_members(session, channel): + """Update channel with submission by members.""" + channel.submissionByEmail = [SubmissionByEmail.MEMBERS] + + session.commit() + + yield channel + + +@pytest.fixture(scope="function") +def channel_no_submission_by_email(session, channel): + """Update channel with no submission by email.""" + channel.submissionByEmail = None + + session.commit() + + yield channel + + +@pytest.fixture(scope="function") +def notification(session, channel): + """Insert notification to db.""" + notification = Notification() + notification.id = "91be5d26-013c-4646-8809-61274645ea1d" + notification.body = "Test" + notification.summary = "Test Notification" + notification.sender = "dimitra.chatzichrysou@cern.ch" + notification.priority = "LOW" + notification.targetId = "c3ccc15b-298f-4dc7-877f-2c8970331caf" + notification.channel = channel + + session.add(notification) + session.commit() + + yield notification + + session.delete(notification) + session.commit() diff --git a/tests/integration/test_postgres_data_source.py b/tests/integration/test_postgres_data_source.py new file mode 100644 index 0000000000000000000000000000000000000000..010508737f830bd92f85874d1448073533187d00 --- /dev/null +++ b/tests/integration/test_postgres_data_source.py @@ -0,0 +1,66 @@ +"""Integration Tests for PostgresDataSource.""" + +from unittest import mock + + +def test_get_channel(appctx, data_source, channel): + """Test get channel.""" + assert data_source.get_channel(channel.slug) == { + "id": "c3ccc15b-298f-4dc7-877f-2c8970331caf", + "incoming_email": "testuser@cern.ch", + "owner_email": "testuser@cern.ch", + } + + +def test_can_send_to_channel_no_submission_by_email(appctx, data_source, channel_no_submission_by_email, user): + """Test can send to channel when submission by email is not set.""" + assert data_source.can_send_to_channel(channel_no_submission_by_email.id, user.email) is False + + +def test_can_send_to_channel_by_email(appctx, data_source, channel, user): + """Test can send to channel when submission by email is set and match.""" + assert data_source.can_send_to_channel(channel.id, user.email) is True + + +def test_can_send_to_channel_by_egroup(appctx, data_source, channel_submission_by_egroup, user, group): + """Test can send to channel when submission by egroup is set and match.""" + assert data_source.can_send_to_channel(channel_submission_by_egroup.id, user.email, group.groupIdentifier) is True + + +@mock.patch("notifications_consumer.data_source.postgres.postgres_data_source.get_group_users_api") +def test_can_send_to_channel_by_administrators( + mock_get_group_users_api, appctx, data_source, channel_submission_by_administrators, user, group +): + """Test can send to channel when submission by administrators is set and match.""" + mock_get_group_users_api.return_value = {"data": [{"upn": "testuser", "primaryAccountEmail": "testuser@cern.ch"}]} + assert ( + data_source.can_send_to_channel(channel_submission_by_administrators.id, user.email, group.groupIdentifier) + is True + ) + + +@mock.patch("notifications_consumer.data_source.postgres.postgres_data_source.get_group_users_api") +def test_can_send_to_channel_by_members( + mock_get_group_users_api, appctx, data_source, channel_submission_by_members, user, group +): + """Test can send to channel when submission by members is set and match.""" + mock_get_group_users_api.return_value = {"data": [{"upn": "testuser", "primaryAccountEmail": "testuser@cern.ch"}]} + assert data_source.can_send_to_channel(channel_submission_by_members.id, user.email, group.groupIdentifier) is True + + +def test_get_channel_notification(appctx, data_source, notification): + """Test get channel notifications.""" + notifications = data_source.get_channel_notifications([str(notification.id)]) + assert len(notifications) == 1 + assert str(notifications[0].id) == str(notification.id) + assert notifications[0].body == notification.body + assert notifications[0].summary == notification.summary + assert notifications[0].sender == notification.sender + assert notifications[0].priority == notification.priority + assert str(notifications[0].targetId) == notification.targetId + assert str(notifications[0].channel.id) == str(notification.channel.id) + + +def test_get_user_email(appctx, data_source, user): + """Test get user email.""" + assert data_source.get_user_email(user.id) == "testuser@cern.ch" diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..183a373a9129738ebe52e475acfba68698aee622 --- /dev/null +++ b/tests/unit/__init__.py @@ -0,0 +1,2 @@ +"""Initialize Test Package.""" +# FIX for Poetry: https://github.com/python-poetry/poetry/issues/87 diff --git a/tests/conftest.py b/tests/unit/conftest.py similarity index 100% rename from tests/conftest.py rename to tests/unit/conftest.py diff --git a/tests/test_postgres_data_source.py b/tests/unit/test_postgres_data_source.py similarity index 98% rename from tests/test_postgres_data_source.py rename to tests/unit/test_postgres_data_source.py index c4c35ec904a69fa87aa8919c155d254d8fc5d46d..fc06ac81a3c874f42583e83ad0a770e957fa7541 100644 --- a/tests/test_postgres_data_source.py +++ b/tests/unit/test_postgres_data_source.py @@ -301,7 +301,7 @@ def test_can_send_to_channel_submission_email_none( """Tests if submission by email is not set.""" mock_get_scalar.return_value = channel_no_incoming_email_no_submission_by_email - assert db_mock.can_send_to_channel("c3ccc15b-298f-4dc7-877f-2c8970331caf", "testuser@cern.ch") == False + assert db_mock.can_send_to_channel("c3ccc15b-298f-4dc7-877f-2c8970331caf", "testuser@cern.ch") is False mock_get_scalar.assert_called_once_with(ANY, Channel, id="c3ccc15b-298f-4dc7-877f-2c8970331caf", deleteDate=None) @@ -310,7 +310,7 @@ def test_can_send_to_channel_email(mock_get_scalar, db_mock, channel, app): """Tests if submission by email is set and match.""" mock_get_scalar.return_value = channel - assert db_mock.can_send_to_channel("c3ccc15b-298f-4dc7-877f-2c8970331caf", "testuser@cern.ch") == True + assert db_mock.can_send_to_channel("c3ccc15b-298f-4dc7-877f-2c8970331caf", "testuser@cern.ch") is True mock_get_scalar.assert_called_once_with(ANY, Channel, id="c3ccc15b-298f-4dc7-877f-2c8970331caf", deleteDate=None) @@ -323,7 +323,7 @@ def test_can_send_to_channel_administrators( mock_get_scalar.return_value = channel_with_administrators mock_get_group_users_api.return_value = {"data": [{"upn": "testuser", "primaryAccountEmail": "testuser@cern.ch"}]} - assert db_mock.can_send_to_channel("c3ccc15b-298f-4dc7-877f-2c8970331caf", "testuser@cern.ch") == True + assert db_mock.can_send_to_channel("c3ccc15b-298f-4dc7-877f-2c8970331caf", "testuser@cern.ch") is True mock_get_scalar.assert_called_once_with(ANY, Channel, id="c3ccc15b-298f-4dc7-877f-2c8970331caf", deleteDate=None) @@ -334,7 +334,7 @@ def test_can_send_to_channel_members(mock_get_scalar, mock_get_group_users_api, mock_get_scalar.return_value = channel_with_members mock_get_group_users_api.return_value = {"data": [{"upn": "testuser", "primaryAccountEmail": "testuser@cern.ch"}]} - assert db_mock.can_send_to_channel("c3ccc15b-298f-4dc7-877f-2c8970331caf", "testuser@cern.ch") == True + assert db_mock.can_send_to_channel("c3ccc15b-298f-4dc7-877f-2c8970331caf", "testuser@cern.ch") is True mock_get_scalar.assert_called_once_with(ANY, Channel, id="c3ccc15b-298f-4dc7-877f-2c8970331caf", deleteDate=None) diff --git a/tests/test_processors_email_gateway_utils.py b/tests/unit/test_processors_email_gateway_utils.py similarity index 100% rename from tests/test_processors_email_gateway_utils.py rename to tests/unit/test_processors_email_gateway_utils.py diff --git a/tests/test_processors_safaripush_utils.py b/tests/unit/test_processors_safaripush_utils.py similarity index 100% rename from tests/test_processors_safaripush_utils.py rename to tests/unit/test_processors_safaripush_utils.py diff --git a/tests/test_processors_webpush_utils.py b/tests/unit/test_processors_webpush_utils.py similarity index 100% rename from tests/test_processors_webpush_utils.py rename to tests/unit/test_processors_webpush_utils.py