DM-43722 EFD Transform Implementation

.github/workflows/build.yaml

@@ -31,3 +31,10 @@ jobs:
         image: ${{ github.repository }}-pq
         github_token: ${{ secrets.GITHUB_TOKEN }}
         dockerfile: Dockerfile.pqserver
+
+    - name: Build efdtransform
+      uses: lsst-sqre/build-and-push-to-ghcr@v1
+      with:
+        image: ${{ github.repository }}-efdtransform
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        dockerfile: Dockerfile.efdtransform

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
         # supported by your project here, or alternatively use
         # pre-commit's default_language_version, see
         # https://pre-commit.com/#top_level-default_language_version
-        language_version: python3.12
+        language_version: python3.11
   - repo: https://github.com/pycqa/isort
     rev: 5.13.2
     hooks:

.vscode/settings.json

@@ -0,0 +1,15 @@
+{
+    "editor.formatOnSave": true,
+    "[python]": {
+        "editor.tabSize": 4,
+        "editor.rulers": [
+            79,
+            110
+        ],
+    },
+    "python.analysis.extraPaths": [
+        "./python",
+        "./python",
+        "./python/lsst"
+    ]
+}

Dockerfile.efdtransform

@@ -0,0 +1,51 @@
+ARG OBS_LSST_VERSION=w_2024_47
+FROM lsstsqre/centos:7-stack-lsst_distrib-${OBS_LSST_VERSION}
+USER lsst
+RUN source loadLSST.bash && mamba install aiokafka httpx
+RUN source loadLSST.bash && pip install \
+    kafkit==0.2.1 \
+    lsst_efd_client==0.12.0
+
+# Python code
+COPY --chown=lsst:lsst python/lsst/consdb/efd_transform ./consdb/efd_transform
+
+RUN mkdir data
+
+# TODO: SQLITE TEST DATABASE
+COPY --chown=lsst:lsst tmp/efd_transform/LATISS.db ./data/
+COPY --chown=lsst:lsst tmp/efd_transform/LSSTComCam.db ./data/
+COPY --chown=lsst:lsst tmp/efd_transform/LSSTComCamSim.db ./data/
+
+# Environment variables that must be set:
+# ------------------------------
+ENV CONFIG_FILE="/opt/lsst/software/stack/consdb/efd_transform/config_LATISS.yml"
+ENV INSTRUMENT="LATISS"
+
+# Buttler Access Variables
+ENV BUTLER_REPO="s3://rubin-summit-users/butler.yaml"
+ENV S3_ENDPOINT_URL="https://s3dfrgw.slac.stanford.edu/"
+ENV LSST_RESOURCES_S3_PROFILE_embargo="https://sdfembs3.sdf.slac.stanford.edu"
+
+# ENV AWS_ACCESS_KEY_ID="placeholder"
+# ENV AWS_SECRET_ACCESS_KEY="placeholder"
+ENV PGUSER="rubin"
+# ENV PGPASSWORD="placeholder"
+
+# USDF EFD API access Variables
+ENV EFD="usdf_efd"
+ENV EFD_USERNAME="efdreader"
+# ENV EFD_PASSWORD="placeholder"
+
+# Consdb Transform DATABASE Variables
+ENV CONSDB_URL="sqlite:////opt/lsst/software/stack/data/test.db"
+
+# Processing time interval in minutes
+ENV TIMEDELTA="5"
+
+ENV LOG_FILE="/opt/lsst/software/stack/data/transform.log"
+
+CMD ["bash", "-c", "source loadLSST.bash; setup lsst_distrib; python ./consdb/efd_transform/transform_efd.py -c \"$CONFIG_FILE\" -i \"$INSTRUMENT\" -r \"$BUTLER_REPO\" -d \"$CONSDB_URL\"  -E \"$EFD\" -t \"$TIMEDELTA\" -l \"$LOG_FILE\""]
+
+
+# Exemple of command used to execute transform_efd with docker
+# docker run --rm -it --volume $PWD/data:/opt/lsst/software/stack/data  -e CONFIG_FILE=config_LATISS.yaml -e ...  consdb/efd_transform:latest

pyproject.toml

@@ -44,4 +44,4 @@ asyncio_mode = "auto"
 test = ["pytest"]
 dev = [
     "documenteer[guide] < 2",
-]
+]

python/lsst/consdb/efd_transform/cdb_transformed_efd_LATISS.yaml

@@ -0,0 +1,233 @@
+---
+name: cdb_latiss
+"@id": "#cdb_latiss"
+description: Transformed EFD Consolidated Database for LATISS
+tables:
+- name: exposure_efd
+  "@id": "#exposure_efd"
+  description: Transformed EFD topics by exposure.
+  primaryKey:
+  - "#exposure_efd.exposure_id"
+  - "#exposure_efd.instrument"
+  constraints:
+  - name: un_exposure_id_instrument
+    "@id": "#exposure_efd.un_exposure_id_instrument"
+    "@type": Unique
+    description: Ensure exposure_id is unique.
+    columns:
+    - "#exposure_efd.exposure_id"
+    - "#exposure_efd.instrument"
+  columns:
+  - name: exposure_id
+    "@id": "#exposure_efd.exposure_id"
+    datatype: long
+    description: Exposure unique ID.
+  - name: created_at
+    "@id": "#exposure_efd.created_at"
+    datatype: timestamp
+    value: 'CURRENT_TIMESTAMP'
+    description: Timestamp when the record was created, default is the current timestamp
+  - name: instrument
+    "@id": "#exposure_efd.instrument"
+    datatype: char
+    length: 20
+    description: Instrument name.
+
+- name: exposure_efd_unpivoted
+  "@id": "#exposure_efd_unpivoted"
+  description: Unpivoted EFD exposure data.
+  primaryKey:
+  - "#exposure_efd_unpivoted.exposure_id"
+  - "#exposure_efd_unpivoted.property"
+  - "#exposure_efd_unpivoted.field"
+  constraints:
+  - name: un_exposure_property_field
+    "@id": "#exposure_efd_unpivoted.un_exposure_property_field"
+    "@type": Unique
+    description: Ensure the combination of exposure_id, property, and field is unique.
+    columns:
+    - "#exposure_efd_unpivoted.exposure_id"
+    - "#exposure_efd_unpivoted.property"
+    - "#exposure_efd_unpivoted.field"
+  columns:
+  - name: exposure_id
+    "@id": "#exposure_efd_unpivoted.exposure_id"
+    datatype: long
+    nullable: False
+    description: Unique identifier for the exposure
+  - name: property
+    "@id": "#exposure_efd_unpivoted.property"
+    datatype: string
+    length: 64
+    nullable: False
+    value: default_property
+    description: Property name for the unpivoted data
+  - name: field
+    "@id": "#exposure_efd_unpivoted.field"
+    datatype: string
+    length: 32
+    nullable: False
+    value: default_field
+    description: Field name for the unpivoted data
+  - name: value
+    "@id": "#exposure_efd_unpivoted.value"
+    datatype: float
+    nullable: True
+    description: Value corresponding to the parameter
+  - name: created_at
+    "@id": "#exposure_efd_unpivoted.created_at"
+    datatype: timestamp
+    value: 'CURRENT_TIMESTAMP'
+    description: Timestamp when the record was created, default is the current timestamp
+
+- name: visit1_efd
+  "@id": "#visit1_efd"
+  description: Transformed EFD topics by visit.
+  primaryKey:
+  - "#visit1_efd.visit_id"
+  - "#visit1_efd.instrument"
+  constraints:
+  - name: un_visit_id_instrument
+    "@id": "#visit1_efd.un_visit_id_instrument"
+    "@type": Unique
+    description: Ensure visit_id is unique.
+    columns:
+    - "#visit1_efd.visit_id"
+    - "#visit1_efd.instrument"
+  columns:
+  - name: visit_id
+    "@id": "#visit1_efd.visit_id"
+    datatype: long
+    description: Visit unique ID.
+  - name: created_at
+    "@id": "#visit1_efd.created_at"
+    datatype: timestamp
+    value: 'CURRENT_TIMESTAMP'
+    description: Timestamp when the record was created, default is the current timestamp
+  - name: instrument
+    "@id": "#visit1_efd.instrument"
+    datatype: char
+    length: 20
+    description: Instrument name.
+
+- name: visit1_efd_unpivoted
+  "@id": "#visit1_efd_unpivoted"
+  description: Unpivoted EFD visit data.
+  primaryKey:
+  - "#visit1_efd_unpivoted.visit_id"
+  - "#visit1_efd_unpivoted.property"
+  - "#visit1_efd_unpivoted.field"
+  constraints:
+  - name: un_visit_property_field
+    "@id": "#visit1_efd_unpivoted.un_visit_property_field"
+    "@type": Unique
+    description: Ensure the combination of visit_id, property, and field is unique.
+    columns:
+    - "#visit1_efd_unpivoted.visit_id"
+    - "#visit1_efd_unpivoted.property"
+    - "#visit1_efd_unpivoted.field"
+  columns:
+  - name: visit_id
+    "@id": "#visit1_efd_unpivoted.visit_id"
+    datatype: long
+    nullable: False
+    description: Unique identifier for the visit
+  - name: property
+    "@id": "#visit1_efd_unpivoted.property"
+    datatype: string
+    length: 64
+    nullable: False
+    value: default_property
+    description: Property name for the unpivoted data
+  - name: field
+    "@id": "#visit1_efd_unpivoted.field"
+    datatype: string
+    length: 32
+    nullable: False
+    value: default_field
+    description: Field name for the unpivoted data
+  - name: value
+    "@id": "#visit1_efd_unpivoted.value"
+    datatype: float
+    nullable: True
+    description: Value corresponding to the parameter
+  - name: created_at
+    "@id": "#visit1_efd_unpivoted.created_at"
+    datatype: timestamp
+    value: 'CURRENT_TIMESTAMP'
+    description: Timestamp when the record was created, default is the current timestamp
+
+- name: transformed_efd_scheduler
+  "@id": "#transformed_efd_scheduler"
+  description: Transformed EFD scheduler.
+  primaryKey:
+  - "#transformed_efd_scheduler.id"
+  constraints:
+  - name: un_id
+    "@id": "#transformed_efd_scheduler.un_id"
+    "@type": Unique
+    description: Ensure id is unique.
+    columns:
+    - "#transformed_efd_scheduler.id"
+  columns:
+  - name: id
+    "@id": "#transformed_efd_scheduler.id"
+    datatype: int
+    nullable: False
+    autoincrement: True
+    description: Unique ID, auto-incremented
+  - name: start_time
+    "@id": "#transformed_efd_scheduler.start_time"
+    datatype: timestamp
+    description: Start time of the transformation interval, must be provided
+  - name: end_time
+    "@id": "#transformed_efd_scheduler.end_time"
+    datatype: timestamp
+    description: End time of the transformation interval, must be provided
+  - name: timewindow
+    "@id": "#transformed_efd_scheduler.timewindow"
+    datatype: int
+    description: Time window used to expand start and end times by, in minutes
+  - name: status
+    "@id": "#transformed_efd_scheduler.status"
+    datatype: char
+    length: 20
+    value: "pending"
+    description: Status of the process, default is 'pending'
+  - name: process_start_time
+    "@id": "#transformed_efd_scheduler.process_start_time"
+    datatype: timestamp
+    description: Timestamp when the process started
+  - name: process_end_time
+    "@id": "#transformed_efd_scheduler.process_end_time"
+    datatype: timestamp
+    description: Timestamp when the process ended
+  - name: process_exec_time
+    "@id": "#transformed_efd_scheduler.process_exec_time"
+    datatype: int
+    value: 0
+    description: Execution time of the process in seconds, default is 0
+  - name: exposures
+    "@id": "#transformed_efd_scheduler.exposures"
+    datatype: int
+    value: 0
+    description: Number of exposures processed, default is 0
+  - name: visits1
+    "@id": "#transformed_efd_scheduler.visits1"
+    datatype: int
+    value: 0
+    description: Number of visits recorded, default is 0
+  - name: retries
+    "@id": "#transformed_efd_scheduler.retries"
+    datatype: int
+    value: 0
+    description: Number of retries attempted, default is 0
+  - name: error
+    "@id": "#transformed_efd_scheduler.error"
+    datatype: text
+    description: "Error message, if any"
+  - name: created_at
+    "@id": "#transformed_efd_scheduler.created_at"
+    datatype: timestamp
+    value: 'CURRENT_TIMESTAMP'
+    description: Timestamp when the record was created, default is the current timestamp