Ensures that signals are actually consumed forever (#3874)

Netflix · Oct 16, 2023 · 2a7e051 · 2a7e051
1 parent 2ed3927
commit 2a7e051
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 41 deletions.
diff --git a/src/dispatch/cli.py b/src/dispatch/cli.py
@@ -836,13 +836,13 @@ def consume_signals():
 
                 for plugin in plugins:
                     log.debug(f"Consuming signals for plugin: {plugin.plugin.slug}")
-                    p = multiprocessing.Process(
-                        target=_run_consume,
-                        args=(plugin.plugin.slug, organization.slug, project.id, running),
-                    )
-                    p.start()
-                    workers.append(p)
-                    print(workers)
+                    for _ in range(5):  # TODO add plugin.instance.concurrency:
+                        p = multiprocessing.Process(
+                            target=_run_consume,
+                            args=(plugin.plugin.slug, organization.slug, project.id, running),
+                        )
+                        p.start()
+                        workers.append(p)
 
         def terminate_processes(signum, frame):
             print("Terminating main process...")

diff --git a/src/dispatch/plugins/dispatch_aws/plugin.py b/src/dispatch/plugins/dispatch_aws/plugin.py
@@ -39,39 +39,40 @@ def consume(self, db_session, project):
             QueueOwnerAWSAccountId=self.configuration.queue_owner,
         )["QueueUrl"]
 
-        response = client.receive_message(
-            QueueUrl=queue_url,
-            MaxNumberOfMessages=self.configuration.batch_size,
-            VisibilityTimeout=40,
-            WaitTimeSeconds=20,
-        )
-        if response.get("Messages") and len(response.get("Messages")) > 0:
-            entries = []
-            for message in response["Messages"]:
-                try:
-                    body = json.loads(message["Body"])
-                    signal_data = json.loads(body["Message"])
+        while True:
+            response = client.receive_message(
+                QueueUrl=queue_url,
+                MaxNumberOfMessages=self.configuration.batch_size,
+                VisibilityTimeout=40,
+                WaitTimeSeconds=20,
+            )
+            if response.get("Messages") and len(response.get("Messages")) > 0:
+                entries = []
+                for message in response["Messages"]:
+                    try:
+                        body = json.loads(message["Body"])
+                        signal_data = json.loads(body["Message"])
 
-                    signal_instance = signal_service.create_signal_instance(
-                        db_session=db_session,
-                        signal_instance_in=SignalInstanceCreate(
-                            project=project, raw=signal_data, **signal_data
-                        ),
-                    )
-                    metrics_provider.counter(
-                        "aws-sqs-signal-consumer.signal.received",
-                        tags={
-                            "signalName": signal_instance.signal.name,
-                            "externalId": signal_instance.signal.external_id,
-                        },
-                    )
-                    log.debug(
-                        f"Received signal: SignalName: {signal_instance.signal.name} ExernalId: {signal_instance.signal.external_id}"
-                    )
-                    entries.append(
-                        {"Id": message["MessageId"], "ReceiptHandle": message["ReceiptHandle"]}
-                    )
-                except Exception as e:
-                    log.exception(e)
+                        signal_instance = signal_service.create_signal_instance(
+                            db_session=db_session,
+                            signal_instance_in=SignalInstanceCreate(
+                                project=project, raw=signal_data, **signal_data
+                            ),
+                        )
+                        metrics_provider.counter(
+                            "aws-sqs-signal-consumer.signal.received",
+                            tags={
+                                "signalName": signal_instance.signal.name,
+                                "externalId": signal_instance.signal.external_id,
+                            },
+                        )
+                        log.debug(
+                            f"Received signal: SignalName: {signal_instance.signal.name} ExernalId: {signal_instance.signal.external_id}"
+                        )
+                        entries.append(
+                            {"Id": message["MessageId"], "ReceiptHandle": message["ReceiptHandle"]}
+                        )
+                    except Exception as e:
+                        log.exception(e)
 
-            client.delete_message_batch(QueueUrl=queue_url, Entries=entries)
+                client.delete_message_batch(QueueUrl=queue_url, Entries=entries)