diff --git a/charts/lightcurve-step/templates/configmap.yaml b/charts/lightcurve-step/templates/configmap.yaml index 0547d4fc8..3f3efae8c 100644 --- a/charts/lightcurve-step/templates/configmap.yaml +++ b/charts/lightcurve-step/templates/configmap.yaml @@ -20,3 +20,4 @@ data: sql-secret-name: {{ .Values.configmap.psqlSecretName }} mongo-secret: {{ .Values.configmap.mongoSecret }} logging-debug: "{{ .Values.configmap.loggingDebug }}" + skip-mjd-filter: "{{ .Values.configmap.skipMjdFilter }}" diff --git a/charts/lightcurve-step/templates/deployment.yaml b/charts/lightcurve-step/templates/deployment.yaml index 0dc8a9d63..60b9d61ad 100644 --- a/charts/lightcurve-step/templates/deployment.yaml +++ b/charts/lightcurve-step/templates/deployment.yaml @@ -101,6 +101,11 @@ spec: configMapKeyRef: name: {{ include "lightcurve-step.fullname" . }} key: logging-debug + - name: SKIP_MJD_FILTER + valueFrom: + configMapKeyRef: + name: {{ include "lightcurve-step.fullname" . }} + key: skip-mjd-filter {{- if .Values.secrets.kafkaAuth.consumer.enabled }} - name: CONSUMER_KAFKA_USERNAME valueFrom: diff --git a/charts/lightcurve-step/values.yaml b/charts/lightcurve-step/values.yaml index 587657f50..1f0f954bd 100644 --- a/charts/lightcurve-step/values.yaml +++ b/charts/lightcurve-step/values.yaml @@ -58,6 +58,7 @@ configmap: useProfiling: "" mongoSecret: "" pyroscopeServer: "http://pyroscope.pyroscope:4040" + skipMjdFilter: "" secrets: kafkaAuth: diff --git a/correction_step/correction/_step/step.py b/correction_step/correction/_step/step.py index 56cf91055..6b7eb5a68 100644 --- a/correction_step/correction/_step/step.py +++ b/correction_step/correction/_step/step.py @@ -11,6 +11,7 @@ from pprint import pprint + class CorrectionStep(GenericStep): """Step that applies magnitude correction to new alert and previous candidates. @@ -90,13 +91,10 @@ def pre_execute(cls, messages: list[dict]) -> dict: @classmethod def execute(cls, message: dict) -> dict: - pprint(message) corrector = Corrector(message["detections"]) detections = corrector.corrected_as_records() non_detections = pd.DataFrame(message["non_detections"]).drop_duplicates(["oid", "fid", "mjd"]) coords = corrector.coordinates_as_records() - # delet dis - pprint(coords) return { "detections": detections, "non_detections": non_detections.to_dict("records"), diff --git a/lightcurve-step/lightcurve_step/step.py b/lightcurve-step/lightcurve_step/step.py index 9a60b7669..97737cfe6 100644 --- a/lightcurve-step/lightcurve_step/step.py +++ b/lightcurve-step/lightcurve_step/step.py @@ -1,5 +1,6 @@ import logging import numpy as np +import os import pandas as pd import pickle from typing import List @@ -53,6 +54,7 @@ def pre_execute(cls, messages: List[dict]) -> dict: last_mjds[aid] = max(last_mjds.get(aid, 0), msg["detections"][0]["mjd"]) detections.extend([det | {"new": True} for det in msg["detections"]]) non_detections.extend(msg["non_detections"]) + logger = logging.getLogger("alerce.LightcurveStep") logger.debug(f"Received {len(detections)} detections from messages") return { @@ -231,8 +233,9 @@ def serialize_dia_object(ef: dict): except KeyError: nd = [] dets["extra_fields"] = dets["extra_fields"].apply(serialize_dia_object) - mjds = result["last_mjds"] - dets = dets[dets["mjd"] <= mjds[aid]] + if not os.getenv("SKIP_MJD_FILTER", "false") == "true": + mjds = result["last_mjds"] + dets = dets[dets["mjd"] <= mjds[aid]] output.append( { "aid": aid,