From e9ae3a07e3d833f9517afe1c7dd7401d77574239 Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Wed, 23 Aug 2023 08:47:10 -0700 Subject: [PATCH 1/6] Bump helm chart version/app version --- helm-chart/banzai/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/banzai/Chart.yaml b/helm-chart/banzai/Chart.yaml index 881c29c8..9788557c 100644 --- a/helm-chart/banzai/Chart.yaml +++ b/helm-chart/banzai/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: "1.10.0" +appVersion: "1.11.0" description: A Helm chart to deploy the BANZAI pipeline name: banzai -version: 1.10.0 +version: 1.11.0 From 53f4ecbfe0c3f6159fbd0350f754b69e84d41031 Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Tue, 23 Jan 2024 09:04:58 -0800 Subject: [PATCH 2/6] Quick hack to get processing working again. --- banzai/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/banzai/main.py b/banzai/main.py index fbbcbb5b..e0fc4e66 100755 --- a/banzai/main.py +++ b/banzai/main.py @@ -48,7 +48,9 @@ def get_consumers(self, Consumer, channel): def on_message(self, body, message): instrument = LCOFrameFactory.get_instrument_from_header(body, self.runtime_context.db_address) - if instrument is not None and instrument.nx * instrument.ny > self.runtime_context.LARGE_WORKER_THRESHOLD: + if instrument is None or instrument.nx is None: + queue_name = self.runtime_context.CELERY_TASK_QUEUE_NAME + elif instrument.nx * instrument.ny > self.runtime_context.LARGE_WORKER_THRESHOLD: queue_name = self.runtime_context.LARGE_WORKER_QUEUE else: queue_name = self.runtime_context.CELERY_TASK_QUEUE_NAME From 66bbe89918c96f93aa4bb8bfad4ba6758e839bbf Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Mon, 29 Jan 2024 15:05:18 -0800 Subject: [PATCH 3/6] Fix for null values in instrument and instrument size Guard against this so the listener can gracefully continue after encountering any weird data --- banzai/main.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/banzai/main.py b/banzai/main.py index e0fc4e66..115123da 100755 --- a/banzai/main.py +++ b/banzai/main.py @@ -48,12 +48,13 @@ def get_consumers(self, Consumer, channel): def on_message(self, body, message): instrument = LCOFrameFactory.get_instrument_from_header(body, self.runtime_context.db_address) - if instrument is None or instrument.nx is None: - queue_name = self.runtime_context.CELERY_TASK_QUEUE_NAME - elif instrument.nx * instrument.ny > self.runtime_context.LARGE_WORKER_THRESHOLD: - queue_name = self.runtime_context.LARGE_WORKER_QUEUE - else: - queue_name = self.runtime_context.CELERY_TASK_QUEUE_NAME + queue_name = self.runtime_context.CELERY_TASK_QUEUE_NAME + try: + if instrument.nx * instrument.ny > self.runtime_context.LARGE_WORKER_THRESHOLD: + queue_name = self.runtime_context.LARGE_WORKER_QUEUE + except Exception as e: + logger.warning(f"Instrument not found in DB, or instrument size not defined: {e}", extra_tags={'instrument': body.get('INSTRUME')}) + process_image.apply_async(args=(body, vars(self.runtime_context)), queue=queue_name) message.ack() # acknowledge to the sender we got this message (it can be popped) From 3d608b6100532b48733a58cf4635ecb4515a983f Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Mon, 29 Jan 2024 15:15:44 -0800 Subject: [PATCH 4/6] Revert "Fix for null values in instrument and instrument size" This reverts commit 66bbe89918c96f93aa4bb8bfad4ba6758e839bbf. --- banzai/main.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/banzai/main.py b/banzai/main.py index 115123da..e0fc4e66 100755 --- a/banzai/main.py +++ b/banzai/main.py @@ -48,13 +48,12 @@ def get_consumers(self, Consumer, channel): def on_message(self, body, message): instrument = LCOFrameFactory.get_instrument_from_header(body, self.runtime_context.db_address) - queue_name = self.runtime_context.CELERY_TASK_QUEUE_NAME - try: - if instrument.nx * instrument.ny > self.runtime_context.LARGE_WORKER_THRESHOLD: - queue_name = self.runtime_context.LARGE_WORKER_QUEUE - except Exception as e: - logger.warning(f"Instrument not found in DB, or instrument size not defined: {e}", extra_tags={'instrument': body.get('INSTRUME')}) - + if instrument is None or instrument.nx is None: + queue_name = self.runtime_context.CELERY_TASK_QUEUE_NAME + elif instrument.nx * instrument.ny > self.runtime_context.LARGE_WORKER_THRESHOLD: + queue_name = self.runtime_context.LARGE_WORKER_QUEUE + else: + queue_name = self.runtime_context.CELERY_TASK_QUEUE_NAME process_image.apply_async(args=(body, vars(self.runtime_context)), queue=queue_name) message.ack() # acknowledge to the sender we got this message (it can be popped) From fa80dbd17146d41574b62974556121b935a009f1 Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Mon, 29 Jan 2024 15:18:25 -0800 Subject: [PATCH 5/6] Update changelog, up large worker volume to 40g The large workers need more disk space! --- CHANGES.md | 4 ++++ helm-chart/banzai/Chart.yaml | 4 ++-- helm-chart/banzai/templates/workers-large.yaml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1fc61e74..c84f005d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +1.14.1 (2024-01-29) +------------------- +- Fix for null instrument/instrument size to allow processing to continue + 1.14.0 (2024-01-22) ------------------- - Split worker queues for small and large images diff --git a/helm-chart/banzai/Chart.yaml b/helm-chart/banzai/Chart.yaml index 9788557c..8b4fd7bf 100644 --- a/helm-chart/banzai/Chart.yaml +++ b/helm-chart/banzai/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: "1.11.0" +appVersion: "1.14.1" description: A Helm chart to deploy the BANZAI pipeline name: banzai -version: 1.11.0 +version: 1.14.1 \ No newline at end of file diff --git a/helm-chart/banzai/templates/workers-large.yaml b/helm-chart/banzai/templates/workers-large.yaml index bd8cd15c..f9784fa5 100644 --- a/helm-chart/banzai/templates/workers-large.yaml +++ b/helm-chart/banzai/templates/workers-large.yaml @@ -57,7 +57,7 @@ spec: volumes: - name: tmp emptyDir: - sizeLimit: 20Gi + sizeLimit: 40Gi {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} From b3611208e565f7d5d8aff9e84bd024be36d5c38f Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Mon, 29 Jan 2024 15:19:38 -0800 Subject: [PATCH 6/6] Add newline at EOF --- helm-chart/banzai/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-chart/banzai/Chart.yaml b/helm-chart/banzai/Chart.yaml index 8b4fd7bf..77555cac 100644 --- a/helm-chart/banzai/Chart.yaml +++ b/helm-chart/banzai/Chart.yaml @@ -2,4 +2,4 @@ apiVersion: v1 appVersion: "1.14.1" description: A Helm chart to deploy the BANZAI pipeline name: banzai -version: 1.14.1 \ No newline at end of file +version: 1.14.1