Incorporate Kafka and handle all instruments.

lsst-dm · Feb 28, 2024 · b925ea1 · b925ea1
1 parent 4e8419d
commit b925ea1
Show file tree

Hide file tree

Showing 4 changed files with 317 additions and 272 deletions.
diff --git a/Dockerfile.hinfo b/Dockerfile.hinfo
@@ -3,10 +3,17 @@ FROM lsstsqre/newinstall:${RUBINENV_VERSION}
 ARG OBS_LSST_VERSION
 ENV OBS_LSST_VERSION=${OBS_LSST_VERSION:-w_2024_06}
 USER lsst
-RUN source loadLSST.bash && pip install confluent-kafka
+RUN source loadLSST.bash && mamba install aiokafka httpx
+RUN source loadLSST.bash && pip install kafkit
 RUN source loadLSST.bash && eups distrib install -t "${OBS_LSST_VERSION}" obs_lsst
-COPY python/lsst/consdb/hinfo*.py ./hinfo/
+COPY python/lsst/consdb/hinfo.py ./hinfo/
+
 # Environment variables that must be set:
-# POSTGRES_URL INSTRUMENT SITE
+#   INSTRUMENT: LATISS, LSSTComCam, LSSTComCamSim, LSSTCam
+#   POSTGRES_URL: SQLAlchemy connection URL
+#   KAFKA_BOOTSTRAP: AIOKafkaConsumer bootstrap server specification
+#   SCHEMA_URL: Kafkit registry schema URL
+# Optional environment variable:
+#   BUCKET_PREFIX: "rubin:" at USDF
 
-ENTRYPOINT [ "bash", "-c", "source loadLSST.bash; setup obs_lsst; python .hinfo/hinfo-latiss.py" ]
+ENTRYPOINT [ "bash", "-c", "source loadLSST.bash; setup obs_lsst; python ./hinfo/hinfo.py" ]
diff --git a/python/lsst/consdb/header_proc.py b/python/lsst/consdb/header_proc.py
diff --git a/python/lsst/consdb/hinfo-latiss.py b/python/lsst/consdb/hinfo-latiss.py