Update celery to 4.0 and remove reference to old boto package from sq…

…s class
ministryofjustice · Aug 10, 2023 · 8aa9c19 · 8aa9c19
1 parent ed10e6b
commit 8aa9c19
Show file tree

Hide file tree

Showing 7 changed files with 43 additions and 80 deletions.
diff --git a/cla_backend/settings/base.py b/cla_backend/settings/base.py
@@ -3,13 +3,11 @@
 import os
 
 import sentry_sdk
-from boto.s3.connection import NoHostProvided
 from cla_common.call_centre_availability import OpeningHours
 from cla_common.services import CacheAdapter
 from collections import defaultdict
 from kombu import transport
 from sentry_sdk.integrations.django import DjangoIntegration
-from django.conf.global_settings import TEMPLATE_CONTEXT_PROCESSORS
 from cla_backend.sqs import CLASQSChannel
 
 
@@ -116,10 +114,6 @@ def env_var_truthy_intention(name):
 AWS_DEFAULT_ACL = None
 AWS_QUERYSTRING_AUTH = False
 
-# Annoyingly the host parameter boto.s3.connection.S3Connection needs to be host string if it's not the default
-# value of boto.s3.connection.NoHostProvided class reference and not None
-AWS_S3_HOST = os.environ.get("AWS_S3_HOST", NoHostProvided)
-
 # This bucket needs to a private bucket as it will contain sensitive reports
 AWS_REPORTS_STORAGE_BUCKET_NAME = os.environ.get("AWS_REPORTS_STORAGE_BUCKET_NAME")
 # This bucket needs to a public bucket as it will serve public assets such as css,images and js
@@ -400,24 +394,11 @@ def traces_sampler(sampling_context):
 OBIEE_EMAIL_TO = os.environ.get("OBIEE_EMAIL_TO", DEFAULT_EMAIL_TO)
 OBIEE_ZIP_PASSWORD = os.environ.get("OBIEE_ZIP_PASSWORD")
 
-# celery
-if all([env_var_truthy_intention("SQS_ACCESS_KEY"), env_var_truthy_intention("SQS_SECRET_KEY")]):
-    import urllib
-
-    BROKER_URL = "sqs://{access_key}:{secret_key}@".format(
-        access_key=urllib.quote(os.environ.get("SQS_ACCESS_KEY"), safe=""),
-        secret_key=urllib.quote(os.environ.get("SQS_SECRET_KEY"), safe=""),
-    )
-else:
-    # if no BROKER_URL specified then don't try to use celery
-    # because it'll just cause errors
-    CELERY_ALWAYS_EAGER = True
-
 CLA_ENV = os.environ.get("CLA_ENV", "local")
 
 BROKER_TRANSPORT_OPTIONS = {
     "polling_interval": 10,
-    "region": os.environ.get("SQS_REGION", "eu-west-1"),
+    "region": os.environ.get("SQS_REGION", "eu-west-2"),
     "wait_time_seconds": 20,
 }
 
@@ -426,11 +407,15 @@ def traces_sampler(sampling_context):
     # This is to stop actions such as ListQueues being triggered
     # which we do not have on the cloud platform environments
     transport.SQS.Transport.Channel = CLASQSChannel
+    BROKER_URL = "sqs://"
 
     predefined_queue_url = os.environ.get("CELERY_PREDEFINED_QUEUE_URL")
     CELERY_DEFAULT_QUEUE = predefined_queue_url.split("/")[-1]
     BROKER_TRANSPORT_OPTIONS["predefined_queue_url"] = predefined_queue_url
 else:
+    # if no BROKER_URL specified then don't try to use celery
+    # because it'll just cause errors
+    CELERY_ALWAYS_EAGER = True
     BROKER_TRANSPORT_OPTIONS["queue_name_prefix"] = "env-%(env)s-" % {"env": CLA_ENV}
 
 CELERY_ACCEPT_CONTENT = ["yaml"]  # because json serializer doesn't support dates

diff --git a/cla_backend/sqs.py b/cla_backend/sqs.py
@@ -1,39 +1,11 @@
-import collections
-
-from boto.sqs.queue import Queue
-from boto.sqs.message import Message
-
 from kombu.transport.SQS import Channel
-from kombu.transport import virtual
-from kombu.utils import cached_property
 
 
 class CLASQSChannel(Channel):
-    @cached_property
-    def predefined_queues(self):
-        # We are using a strict sqs setup which we are given only list of predefined queues and
+    def _update_queue_cache(self, queue_name_prefix):
         url = self.transport_options.get("predefined_queue_url", None)
-        q = Queue(connection=self.sqs, url=url, message_class=Message)
-        return [q]
-
-    def __init__(self, *args, **kwargs):
-        # CLA Change - On cloud platforms we don't have permissions to perform actions such as ListQueues
-        # So instead lets use a list of predefined queue names
-        # Remove call to direct parent as that will perform the ListQueues action
-        # super(CLASQSChannel, self).__init__(*args, **kwargs)
-        virtual.Channel.__init__(self, *args, **kwargs)
-
-        queues = self.predefined_queues  # self.sqs.get_all_queues(prefix=self.queue_name_prefix)
-
-        for queue in queues:
-            self._queue_cache[queue.name] = queue
-        self._fanout_queues = set()
-
-        # The drain_events() method stores extra messages in a local
-        # Deque object. This allows multiple messages to be requested from
-        # SQS at once for performance, but maintains the same external API
-        # to the caller of the drain_events() method.
-        self._queue_message_cache = collections.deque()
+        queue_name = url.split("/")[-1]
+        self._queue_cache[queue_name] = url
 
     def _new_queue(self, queue, **kwargs):
         # Translate to SQS name for consistency with initial

diff --git a/requirements/generated/requirements-dev.txt b/requirements/generated/requirements-dev.txt
@@ -4,13 +4,11 @@
 #
 #    pip-compile --output-file=requirements/generated/requirements-dev.txt requirements/source/requirements-dev.in
 #
-amqp==1.4.9
-    # via kombu
-anyjson==0.3.3
+amqp==2.6.1
     # via kombu
 aspy.yaml==1.3.0
     # via pre-commit
-billiard==3.3.0.23
+billiard==3.5.0.5
     # via celery
 bleach==3.3.0
     # via -r requirements/source/requirements-base.in
@@ -20,7 +18,7 @@ botocore==1.20.112
     # via
     #   boto3
     #   s3transfer
-celery==3.1.18
+celery==4.0.2
     # via -r requirements/source/requirements-base.in
 certifi==2021.10.8
     # via
@@ -154,6 +152,7 @@ idna==2.10
     #   urllib3
 importlib-metadata==2.1.3
     # via
+    #   kombu
     #   pre-commit
     #   sqlalchemy
     #   virtualenv
@@ -177,7 +176,7 @@ jsonpatch==1.9
     # via -r requirements/source/requirements-base.in
 jsonpointer==2.3
     # via jsonpatch
-kombu==3.0.37
+kombu==4.6.11
     # via celery
 logstash-formatter==0.5.9
     # via -r requirements/source/requirements-base.in
@@ -308,6 +307,8 @@ urllib3[secure]==1.26.14
     #   sentry-sdk
 uwsgi==2.0.18
     # via -r requirements/source/requirements-base.in
+vine==1.3.0
+    # via amqp
 virtualenv==20.15.1
     # via pre-commit
 webdriver-manager==1.6.2

diff --git a/requirements/generated/requirements-docs.txt b/requirements/generated/requirements-docs.txt
@@ -6,13 +6,11 @@
 #
 alabaster==0.7.12
     # via sphinx
-amqp==1.4.9
-    # via kombu
-anyjson==0.3.3
+amqp==2.6.1
     # via kombu
 babel==2.9.1
     # via sphinx
-billiard==3.3.0.23
+billiard==3.5.0.5
     # via celery
 bleach==3.3.0
     # via -r requirements/source/requirements-base.in
@@ -22,7 +20,7 @@ botocore==1.20.112
     # via
     #   boto3
     #   s3transfer
-celery==3.1.18
+celery==4.0.2
     # via -r requirements/source/requirements-base.in
 certifi==2021.10.8
     # via
@@ -116,7 +114,9 @@ idna==2.10
 imagesize==1.4.1
     # via sphinx
 importlib-metadata==2.1.3
-    # via sqlalchemy
+    # via
+    #   kombu
+    #   sqlalchemy
 jdcal==1.4.1
     # via openpyxl
 jinja2==2.11.3
@@ -131,7 +131,7 @@ jsonpatch==1.9
     # via -r requirements/source/requirements-base.in
 jsonpointer==2.3
     # via jsonpatch
-kombu==3.0.37
+kombu==4.6.11
     # via celery
 logstash-formatter==0.5.9
     # via -r requirements/source/requirements-base.in
@@ -230,6 +230,8 @@ urllib3==1.26.14
     #   sentry-sdk
 uwsgi==2.0.18
     # via -r requirements/source/requirements-base.in
+vine==1.3.0
+    # via amqp
 webencodings==0.5.1
     # via bleach
 xlrd==2.0.1

diff --git a/requirements/generated/requirements-production.txt b/requirements/generated/requirements-production.txt
@@ -4,11 +4,9 @@
 #
 #    pip-compile --output-file=requirements/generated/requirements-production.txt requirements/source/requirements-production.in
 #
-amqp==1.4.9
+amqp==2.6.1
     # via kombu
-anyjson==0.3.3
-    # via kombu
-billiard==3.3.0.23
+billiard==3.5.0.5
     # via celery
 bleach==3.3.0
     # via -r requirements/source/requirements-base.in
@@ -18,7 +16,7 @@ botocore==1.20.112
     # via
     #   boto3
     #   s3transfer
-celery==3.1.18
+celery==4.0.2
     # via -r requirements/source/requirements-base.in
 certifi==2021.10.8
     # via
@@ -104,7 +102,9 @@ futures==3.4.0
 idna==2.10
     # via requests
 importlib-metadata==2.1.3
-    # via sqlalchemy
+    # via
+    #   kombu
+    #   sqlalchemy
 jdcal==1.4.1
     # via openpyxl
 jmespath==0.10.0
@@ -117,7 +117,7 @@ jsonpatch==1.9
     # via -r requirements/source/requirements-base.in
 jsonpointer==2.3
     # via jsonpatch
-kombu==3.0.37
+kombu==4.6.11
     # via celery
 logstash-formatter==0.5.9
     # via -r requirements/source/requirements-base.in
@@ -197,6 +197,8 @@ urllib3==1.26.14
     #   sentry-sdk
 uwsgi==2.0.18
     # via -r requirements/source/requirements-base.in
+vine==1.3.0
+    # via amqp
 webencodings==0.5.1
     # via bleach
 xlrd==2.0.1

diff --git a/requirements/generated/requirements-testing.txt b/requirements/generated/requirements-testing.txt
@@ -4,11 +4,9 @@
 #
 #    pip-compile --output-file=requirements/generated/requirements-testing.txt requirements/source/requirements-testing.in
 #
-amqp==1.4.9
+amqp==2.6.1
     # via kombu
-anyjson==0.3.3
-    # via kombu
-billiard==3.3.0.23
+billiard==3.5.0.5
     # via celery
 bleach==3.3.0
     # via -r requirements/source/requirements-base.in
@@ -18,7 +16,7 @@ botocore==1.20.112
     # via
     #   boto3
     #   s3transfer
-celery==3.1.18
+celery==4.0.2
     # via -r requirements/source/requirements-base.in
 certifi==2021.10.8
     # via
@@ -133,7 +131,9 @@ idna==2.10
     #   requests
     #   urllib3
 importlib-metadata==2.1.3
-    # via sqlalchemy
+    # via
+    #   kombu
+    #   sqlalchemy
 ipaddress==1.0.23
     # via
     #   cryptography
@@ -150,7 +150,7 @@ jsonpatch==1.9
     # via -r requirements/source/requirements-base.in
 jsonpointer==2.3
     # via jsonpatch
-kombu==3.0.37
+kombu==4.6.11
     # via celery
 logstash-formatter==0.5.9
     # via -r requirements/source/requirements-base.in
@@ -254,6 +254,8 @@ urllib3[secure]==1.26.14
     #   sentry-sdk
 uwsgi==2.0.18
     # via -r requirements/source/requirements-base.in
+vine==1.3.0
+    # via amqp
 webdriver-manager==1.6.2
     # via -r requirements/source/requirements-testing.in
 webencodings==0.5.1

diff --git a/requirements/source/requirements-base.in b/requirements/source/requirements-base.in
@@ -40,7 +40,7 @@ polib==1.0.6
 
 # Fork PgFulltext - PR added
 https://github.com/ministryofjustice/djorm-ext-pgfulltext/archive/refs/tags/0.1.0.tar.gz
-celery==3.1.18
+celery==4.0.2
 PyYAML==5.4
 pyminizip==0.2.3
 
@@ -59,4 +59,3 @@ django-session-security==2.2.3
 
 # Govuk Notify Service
 notifications-python-client==6.0.0
-