From ce808840c8a5a5a9f49f051aff796e274ab3e764 Mon Sep 17 00:00:00 2001
From: Dimitra Chatzichrysou <dimitra.chatzichrysou@cern.ch>
Date: Thu, 17 Jun 2021 14:54:30 +0000
Subject: [PATCH] [#36] Implement monthly and weekly consumers
---
.env-email-daily | 4 +-
.env-email-monthly | 29 +++++++++++
.env-email-weekly | 30 ++++++++++++
.isort.cfg | 2 +-
docker-compose.full.yml | 48 +++++++++++++++++++
docker-compose.yml | 5 +-
docker/activemq/conf/activemq.xml | 39 +++++++++++++++
docker/activemq/email_monthly_consumer.conf | 14 ++++++
docker/activemq/email_monthly_publisher.conf | 12 +++++
docker/activemq/email_weekly_consumer.conf | 14 ++++++
docker/activemq/email_weekly_publisher.conf | 12 +++++
notifications_consumer/config.py | 5 +-
notifications_consumer/processors/__init__.py | 2 +-
.../{email_daily => email_feed}/processor.py | 10 ++--
.../templates/feed_notification.html} | 0
.../templates/feed_notification.txt} | 0
scripts/docker-send-weekly-email.py | 9 ++++
17 files changed, 225 insertions(+), 10 deletions(-)
create mode 100644 .env-email-monthly
create mode 100644 .env-email-weekly
create mode 100644 docker/activemq/email_monthly_consumer.conf
create mode 100644 docker/activemq/email_monthly_publisher.conf
create mode 100644 docker/activemq/email_weekly_consumer.conf
create mode 100644 docker/activemq/email_weekly_publisher.conf
rename notifications_consumer/processors/{email_daily => email_feed}/processor.py (87%)
rename notifications_consumer/processors/{email_daily/templates/daily_notification.html => email_feed/templates/feed_notification.html} (100%)
rename notifications_consumer/processors/{email_daily/templates/daily_notification.txt => email_feed/templates/feed_notification.txt} (100%)
create mode 100644 scripts/docker-send-weekly-email.py
diff --git a/.env-email-daily b/.env-email-daily
index 56b8f7f..da378cb 100644
--- a/.env-email-daily
+++ b/.env-email-daily
@@ -1,5 +1,5 @@
ENV=development
-PROCESSOR=email_daily
+PROCESSOR=email_feed
CONSUMER_NAME=email_daily_consumer
PUBLISHER_NAME=email_daily_publisher
@@ -25,3 +25,5 @@ EMAIL_USE_TLS=None
EMAIL_USE_SSL=None
EMAIL_TIMEOUT=10
EMAIL_WHITELIST=["user@cern.ch"]
+
+FEED_TITLE=Daily
diff --git a/.env-email-monthly b/.env-email-monthly
new file mode 100644
index 0000000..9648d9a
--- /dev/null
+++ b/.env-email-monthly
@@ -0,0 +1,29 @@
+ENV=development
+PROCESSOR=email_feed
+CONSUMER_NAME=email_monthly_consumer
+PUBLISHER_NAME=email_monthly_publisher
+
+POSTGRES_USER=admin
+POSTGRES_PASSWORD=password
+POSTGRES_DB=push_dev
+
+DB_USER=admin
+DB_PASSWORD=password
+DB_HOST=pg_db
+DB_PORT=5432
+DB_NAME=push_dev
+DB_SCHEMA=push
+
+# When using the SMTP backend, specify how to connect to the SMTP server.
+# Documentation for these settings can be found in the Django Email docs:
+# https://docs.djangoproject.com/en/2.2/topics/email/#smtp-backend
+EMAIL_HOST=localhost
+EMAIL_PORT=8025
+EMAIL_HOST_USER=user@cern.ch
+EMAIL_HOST_PASSWORD=password
+EMAIL_USE_TLS=None
+EMAIL_USE_SSL=None
+EMAIL_TIMEOUT=10
+EMAIL_WHITELIST=["user@cern.ch"]
+
+FEED_TITLE=Monthly
diff --git a/.env-email-weekly b/.env-email-weekly
new file mode 100644
index 0000000..d26d3b4
--- /dev/null
+++ b/.env-email-weekly
@@ -0,0 +1,30 @@
+ENV=development
+PROCESSOR=email_feed
+CONSUMER_NAME=email_weekly_consumer
+PUBLISHER_NAME=email_weekly_publisher
+FEED_SUMMARY=Weekly
+
+POSTGRES_USER=admin
+POSTGRES_PASSWORD=password
+POSTGRES_DB=push_dev
+
+DB_USER=admin
+DB_PASSWORD=password
+DB_HOST=pg_db
+DB_PORT=5432
+DB_NAME=push_dev
+DB_SCHEMA=push
+
+# When using the SMTP backend, specify how to connect to the SMTP server.
+# Documentation for these settings can be found in the Django Email docs:
+# https://docs.djangoproject.com/en/2.2/topics/email/#smtp-backend
+EMAIL_HOST=localhost
+EMAIL_PORT=8025
+EMAIL_HOST_USER=user@cern.ch
+EMAIL_HOST_PASSWORD=password
+EMAIL_USE_TLS=None
+EMAIL_USE_SSL=None
+EMAIL_TIMEOUT=10
+EMAIL_WHITELIST=["user@cern.ch"]
+
+FEED_TITLE=Weekly
diff --git a/.isort.cfg b/.isort.cfg
index f399f14..0ba4220 100644
--- a/.isort.cfg
+++ b/.isort.cfg
@@ -4,4 +4,4 @@ multi_line_output=3
include_trailing_comma=True
lines_after_imports=2
not_skip=__init__.py
-known_third_party = Crypto,apns2,dateutil,jinja2,megabus,pytest,pywebpush,requests,sqlalchemy,stomp,yaml
+known_third_party = Crypto,apns2,jinja2,megabus,pytest,pywebpush,requests,sqlalchemy,stomp,yaml
diff --git a/docker-compose.full.yml b/docker-compose.full.yml
index 0a5d17f..5c8b231 100644
--- a/docker-compose.full.yml
+++ b/docker-compose.full.yml
@@ -145,6 +145,54 @@ services:
activemq:
condition: service_healthy
+ notifications-consumer-email-weekly:
+ image: notifications-consumer
+ container_name: notifications-processor-email-weekly
+ build:
+ context: .
+ dockerfile: Dockerfile
+ args:
+ build_env: development
+ networks:
+ - default
+ volumes:
+ - '.:/opt:delegated'
+ - './docker/activemq/email_weekly_publisher.conf:/etc/activemq-publisher-email_weekly_publisher.conf'
+ - './docker/activemq/email_weekly_consumer.conf:/etc/activemq-consumer-email_weekly_consumer.conf'
+ ports:
+ - 8086:8080
+ env_file:
+ - .env-email-weekly
+ depends_on:
+ pg_db:
+ condition: service_healthy
+ activemq:
+ condition: service_healthy
+
+ notifications-consumer-email-monthly:
+ image: notifications-consumer
+ container_name: notifications-processor-email-monthly
+ build:
+ context: .
+ dockerfile: Dockerfile
+ args:
+ build_env: development
+ networks:
+ - default
+ volumes:
+ - '.:/opt:delegated'
+ - './docker/activemq/email_monthly_publisher.conf:/etc/activemq-publisher-email_monthly_publisher.conf'
+ - './docker/activemq/email_monthly_consumer.conf:/etc/activemq-consumer-email_monthly_consumer.conf'
+ ports:
+ - 8087:8080
+ env_file:
+ - .env-email-monthly
+ depends_on:
+ pg_db:
+ condition: service_healthy
+ activemq:
+ condition: service_healthy
+
pg_db:
image: postgres
volumes:
diff --git a/docker-compose.yml b/docker-compose.yml
index 9fa0bc2..1b64bc6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -31,7 +31,10 @@ services:
- './docker/activemq/email_gateway_critical_failure_dlq_consumer.conf:/etc/activemq-consumer-email_gateway_critical_failure_dlq_consumer.conf'
- './docker/activemq/webpush_dlq_consumer.conf:/etc/activemq-consumer-webpush_dlq_consumer.conf'
- './docker/activemq/safaripush_dlq_consumer.conf:/etc/activemq-consumer-safaripush_dlq_consumer.conf'
-
+ - './docker/activemq/email_weekly_publisher.conf:/etc/activemq-publisher-email_weekly_publisher.conf'
+ - './docker/activemq/email_weekly_consumer.conf:/etc/activemq-consumer-email_weekly_consumer.conf'
+ - './docker/activemq/email_monthly_publisher.conf:/etc/activemq-publisher-email_monthly_publisher.conf'
+ - './docker/activemq/email_monthly_consumer.conf:/etc/activemq-consumer-email_monthly_consumer.conf'
ports:
- 8080:8080
env_file:
diff --git a/docker/activemq/conf/activemq.xml b/docker/activemq/conf/activemq.xml
index 7123d50..3c85a27 100644
--- a/docker/activemq/conf/activemq.xml
+++ b/docker/activemq/conf/activemq.xml
@@ -99,6 +99,30 @@
maximumRedeliveryDelay="-1"
useExponentialBackOff="true"
/>
+ <redeliveryPolicy
+ queue="np.email-daily"
+ maximumRedeliveries="6"
+ initialRedeliveryDelay="10000"
+ backOffMultiplier="3"
+ maximumRedeliveryDelay="-1"
+ useExponentialBackOff="true"
+ />
+ <redeliveryPolicy
+ queue="np.email-weekly"
+ maximumRedeliveries="6"
+ initialRedeliveryDelay="10000"
+ backOffMultiplier="3"
+ maximumRedeliveryDelay="-1"
+ useExponentialBackOff="true"
+ />
+ <redeliveryPolicy
+ queue="np.email-monthly"
+ maximumRedeliveries="6"
+ initialRedeliveryDelay="10000"
+ backOffMultiplier="3"
+ maximumRedeliveryDelay="-1"
+ useExponentialBackOff="true"
+ />
</redeliveryPolicyEntries>
</redeliveryPolicyMap>
</redeliveryPolicyMap>
@@ -165,6 +189,21 @@
<individualDeadLetterStrategy queuePrefix="" queueSuffix=".dlq" useQueueForQueueMessages="true"/>
</deadLetterStrategy>
</policyEntry>
+ <policyEntry queue="np.email-daily">
+ <deadLetterStrategy>
+ <individualDeadLetterStrategy queuePrefix="" queueSuffix=".dlq" useQueueForQueueMessages="true"/>
+ </deadLetterStrategy>
+ </policyEntry>
+ <policyEntry queue="np.email-weekly">
+ <deadLetterStrategy>
+ <individualDeadLetterStrategy queuePrefix="" queueSuffix=".dlq" useQueueForQueueMessages="true"/>
+ </deadLetterStrategy>
+ </policyEntry>
+ <policyEntry queue="np.email-monthly">
+ <deadLetterStrategy>
+ <individualDeadLetterStrategy queuePrefix="" queueSuffix=".dlq" useQueueForQueueMessages="true"/>
+ </deadLetterStrategy>
+ </policyEntry>
</policyEntries>
</policyMap>
</destinationPolicy>
diff --git a/docker/activemq/email_monthly_consumer.conf b/docker/activemq/email_monthly_consumer.conf
new file mode 100644
index 0000000..ff071e4
--- /dev/null
+++ b/docker/activemq/email_monthly_consumer.conf
@@ -0,0 +1,14 @@
+[client]
+server:activemq
+auth_method:password
+port:61613
+destination:np
+destination_type:queue
+hostgroup_selector:
+host_selector:
+top_hostgroup:
+user:admin
+pass:admin
+use_multiple_brokers:False
+extension:email-weekly
+ack:client-individual
diff --git a/docker/activemq/email_monthly_publisher.conf b/docker/activemq/email_monthly_publisher.conf
new file mode 100644
index 0000000..7fab7ab
--- /dev/null
+++ b/docker/activemq/email_monthly_publisher.conf
@@ -0,0 +1,12 @@
+[client]
+server:activemq
+auth_method:password
+port:61613
+destination:np
+destination_type:queue
+hostgroup_selector:
+host_selector:
+top_hostgroup:
+user:admin
+pass:admin
+use_multiple_brokers:False
diff --git a/docker/activemq/email_weekly_consumer.conf b/docker/activemq/email_weekly_consumer.conf
new file mode 100644
index 0000000..ff071e4
--- /dev/null
+++ b/docker/activemq/email_weekly_consumer.conf
@@ -0,0 +1,14 @@
+[client]
+server:activemq
+auth_method:password
+port:61613
+destination:np
+destination_type:queue
+hostgroup_selector:
+host_selector:
+top_hostgroup:
+user:admin
+pass:admin
+use_multiple_brokers:False
+extension:email-weekly
+ack:client-individual
diff --git a/docker/activemq/email_weekly_publisher.conf b/docker/activemq/email_weekly_publisher.conf
new file mode 100644
index 0000000..7fab7ab
--- /dev/null
+++ b/docker/activemq/email_weekly_publisher.conf
@@ -0,0 +1,12 @@
+[client]
+server:activemq
+auth_method:password
+port:61613
+destination:np
+destination_type:queue
+hostgroup_selector:
+host_selector:
+top_hostgroup:
+user:admin
+pass:admin
+use_multiple_brokers:False
diff --git a/notifications_consumer/config.py b/notifications_consumer/config.py
index 68aba6a..f61f4bb 100644
--- a/notifications_consumer/config.py
+++ b/notifications_consumer/config.py
@@ -85,7 +85,7 @@ class Config:
loader=PrefixLoader(
{
"email": FileSystemLoader("notifications_consumer/processors/email/templates"),
- "email_daily": FileSystemLoader("notifications_consumer/processors/email_daily/templates"),
+ "email_feed": FileSystemLoader("notifications_consumer/processors/email_feed/templates"),
"dlq": FileSystemLoader("notifications_consumer/processors/dlq/templates"),
"email_gateway_failure": FileSystemLoader(
"notifications_consumer/processors/email_gateway_failure/templates"
@@ -117,6 +117,9 @@ class Config:
# Priority
CRITICAL_PRIORITY = os.getenv("CRITICAL_PRIORITY", "critical")
+ # Feed email title
+ FEED_TITLE = os.getenv("FEED_TITLE", "Daily")
+
class DevelopmentConfig(Config):
"""Development configuration overrides."""
diff --git a/notifications_consumer/processors/__init__.py b/notifications_consumer/processors/__init__.py
index 316fd2a..e09cd23 100644
--- a/notifications_consumer/processors/__init__.py
+++ b/notifications_consumer/processors/__init__.py
@@ -5,7 +5,7 @@ Required to auto register the processors: force interpreter to load them.
from notifications_consumer.processors.dlq.processor import DLQProcessor
from notifications_consumer.processors.email.processor import EmailProcessor
-from notifications_consumer.processors.email_daily.processor import EmailDailyProcessor
+from notifications_consumer.processors.email_feed.processor import EmailFeedProcessor
from notifications_consumer.processors.email_gateway.processor import MailGatewayProcessor
from notifications_consumer.processors.email_gateway_failure.processor import EmailGatewayFailureProcessor
from notifications_consumer.processors.safaripush.processor import SafariPushProcessor
diff --git a/notifications_consumer/processors/email_daily/processor.py b/notifications_consumer/processors/email_feed/processor.py
similarity index 87%
rename from notifications_consumer/processors/email_daily/processor.py
rename to notifications_consumer/processors/email_feed/processor.py
index ef59507..9f3eadd 100644
--- a/notifications_consumer/processors/email_daily/processor.py
+++ b/notifications_consumer/processors/email_feed/processor.py
@@ -12,10 +12,10 @@ from notifications_consumer.utils import NotificationPriority
@ProcessorRegistry.register
-class EmailDailyProcessor(Processor):
+class EmailFeedProcessor(Processor):
"""Send daily email notifications."""
- __id = "email_daily"
+ __id = "email_feed"
def __init__(self, **kwargs):
"""Initialize the Processor."""
@@ -41,7 +41,7 @@ class EmailDailyProcessor(Processor):
logging.info("Email recipient <%s> isn't on the EMAIL_WHITELIST. Email sent is skipped.", recipient_email)
return
- title = f'Daily Summary - {datetime.now().strftime("%d %B %Y")}'
+ title = f'{Config.FEED_TITLE} Summary - {datetime.now().strftime("%d %B %Y")}'
subject = f"[{Config.SERVICE_NAME}] {title}"
notification_ids = kwargs["notifications"]
@@ -56,8 +56,8 @@ class EmailDailyProcessor(Processor):
"sender": Config.SERVICE_NAME,
}
- text_template = Config.TEMPLATES.get_template("email_daily/daily_notification.txt")
- html_template = Config.TEMPLATES.get_template("email_daily/daily_notification.html")
+ text_template = Config.TEMPLATES.get_template("email_feed/feed_notification.txt")
+ html_template = Config.TEMPLATES.get_template("email_feed/feed_notification.html")
email = create_email(
[recipient_email],
diff --git a/notifications_consumer/processors/email_daily/templates/daily_notification.html b/notifications_consumer/processors/email_feed/templates/feed_notification.html
similarity index 100%
rename from notifications_consumer/processors/email_daily/templates/daily_notification.html
rename to notifications_consumer/processors/email_feed/templates/feed_notification.html
diff --git a/notifications_consumer/processors/email_daily/templates/daily_notification.txt b/notifications_consumer/processors/email_feed/templates/feed_notification.txt
similarity index 100%
rename from notifications_consumer/processors/email_daily/templates/daily_notification.txt
rename to notifications_consumer/processors/email_feed/templates/feed_notification.txt
diff --git a/scripts/docker-send-weekly-email.py b/scripts/docker-send-weekly-email.py
new file mode 100644
index 0000000..044d831
--- /dev/null
+++ b/scripts/docker-send-weekly-email.py
@@ -0,0 +1,9 @@
+"""Utility to send one message inside docker environment."""
+import stomp
+
+
+conn = stomp.Connection([("activemq", 61613)])
+conn.connect("admin", "admin", wait=True)
+message_body = open("scripts/activemq_messages/email_daily.json").read()
+conn.send(body=message_body, destination="/queue/np.email-daily", headers={"persistent": "true"})
+conn.disconnect()
--
GitLab