DM-44099: Major revamp of publish/query client/server.

.github/workflows/build.yaml

@@ -24,3 +24,10 @@ jobs:
         image: ${{ github.repository }}-hinfo
         github_token: ${{ secrets.GITHUB_TOKEN }}
         dockerfile: Dockerfile.hinfo
+
+    - name: Build pqserver
+      uses: lsst-sqre/build-and-push-to-ghcr@v1
+      with:
+        image: ${{ github.repository }}-pq
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        dockerfile: Dockerfile.pqserver

Dockerfile.pqserver

@@ -0,0 +1,12 @@
+FROM python:3.11
+RUN pip install flask gunicorn sqlalchemy psycopg2
+WORKDIR /consdb-pq
+COPY python/lsst/consdb/pqserver.py python/lsst/consdb/utils.py /consdb-pq/
+# Environment variables that must be set:
+# DB_HOST DB_PASS DB_USER DB_NAME or POSTGRES_URL
+
+# Expose the port.
+EXPOSE 8080
+
+ENTRYPOINT [ "gunicorn", "-b", "0.0.0.0:8080", "-w", "2", "pqserver:app" ]
+

pyproject.toml

@@ -4,9 +4,9 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "consdb"
-description = "consdb is a Commandable SAL Component for the `Vera C. Rubin Observatory <https://lsst.org>`_."
+description = "consdb provides support for the Consolidated Database for the `Vera C. Rubin Observatory <https://lsst.org>`_."
 license = { text = "GPL" }
-dependencies = ["pymodbus"]
+dependencies = ["flask", "requests"]
 readme = "README.rst"
 urls = { documentation = "https://consdb.lsst.io", source_code = "https://github.com/lsst-dm/consdb"}
 dynamic = ["version"]
@@ -17,9 +17,6 @@ version = { attr = "setuptools_scm.get_version" }
 [tool.setuptools.packages.find]
 where = [ "python" ]
 
-[project.scripts]
-run_consdb = "lsst.consdb.cli:execute_csc"
-
 [tool.setuptools_scm]
 write_to = "python/lsst/consdb/version.py"
 write_to_template = """

python/lsst/consdb/__init__.py

@@ -19,4 +19,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-# from .version import *  # Generated by sconsUtils
+# from .version import *  # Generated by sconsUtils

python/lsst/consdb/hinfo.py

@@ -16,8 +16,9 @@
 import yaml
 from astro_metadata_translator import ObservationInfo
 from lsst.resources import ResourcePath
-from sqlalchemy import MetaData, Table, create_engine
+from sqlalchemy import MetaData, Table
 from sqlalchemy.dialects.postgresql import insert
+from utils import setup_postgres
 
 ###############################
 # Header Processing Functions #
@@ -56,36 +57,36 @@ def logical_or(*bools: int | str | None) -> bool:
     "controller": "CONTRLLR",
     "seq_num": "SEQNUM",
     "band": "FILTBAND",
-    "ra": "RA",
-    "decl": "DEC",
-    "skyrotation": "ROTPA",
+    "s_ra": "RA",
+    "s_decl": "DEC",
+    "sky_rotation": "ROTPA",
     "azimuth_start": "AZSTART",
     "azimuth_end": "AZEND",
     "azimuth": (mean, "AZSTART", "AZEND"),
     "altitude_start": (ninety_minus, "ELSTART"),
     "altitude_end": (ninety_minus, "ELEND"),
     "altitude": (mean, (ninety_minus, "ELSTART"), (ninety_minus, "ELEND")),
-    "zenithdistance_start": "ELSTART",
-    "zenithdistance_end": "ELEND",
-    "zenithdistance": (mean, "ELSTART", "ELEND"),
-    "expmidpt": (tai_mean, "DATE-BEG", "DATE-END"),
-    "expmidptmjd": (mean, "MJD-BEG", "MJD-END"),
-    "obsstart": (tai_convert, "DATE-BEG"),
-    "obsstartmjd": "MJD-BEG",
-    "obsend": (tai_convert, "DATE-END"),
-    "obsendmjd": "MJD-END",
-    "exptime": "EXPTIME",
-    "shuttime": "SHUTTIME",
-    "darktime": "DARKTIME",
+    "zenith_distance_start": "ELSTART",
+    "zenith_distance_end": "ELEND",
+    "zenith_distance": (mean, "ELSTART", "ELEND"),
+    "exp_midpt": (tai_mean, "DATE-BEG", "DATE-END"),
+    "exp_midpt_mjd": (mean, "MJD-BEG", "MJD-END"),
+    "obs_start": (tai_convert, "DATE-BEG"),
+    "obs_start_mjd": "MJD-BEG",
+    "obs_end": (tai_convert, "DATE-END"),
+    "obs_endmjd": "MJD-END",
+    "exp_time": "EXPTIME",
+    "shut_time": "SHUTTIME",
+    "dark_time": "DARKTIME",
     "group_id": "GROUPID",
-    "curindex": "CURINDEX",
-    "maxindex": "MAXINDEX",
-    "imgtype": "IMGTYPE",
+    "cur_index": "CURINDEX",
+    "max_index": "MAXINDEX",
+    "img_type": "IMGTYPE",
     "emulated": (logical_or, "EMUIMAGE"),
     "science_program": "PROGRAM",
     "observation_reason": "REASON",
     "target_name": "OBJECT",
-    "airtemp": "AIRTEMP",
+    "air_temp": "AIRTEMP",
     "pressure": "PRESSURE",
     "humidity": "HUMIDITY",
     "wind_speed": "WINDSPD",
@@ -161,10 +162,10 @@ def process_column(column_def: str | Sequence, info: dict) -> Any:
         The value to use for the column.
         None if any input value is missing.
     """
-    if type(column_def) is str:
+    if isinstance(column_def, str):
         if column_def in info:
             return info[column_def]
-    elif type(column_def) is tuple:
+    elif isinstance(column_def, tuple):
         fn = column_def[0]
         arg_values = [process_column(a, info) for a in column_def[1:]]
         if all(arg_values):
@@ -287,20 +288,7 @@ def get_kafka_config() -> KafkaConfig:
         instrument_mapping = LSSTCAM_MAPPING
 logging.info(f"Instrument = {instrument}")
 
-host = os.environ.get("DB_HOST")
-passwd = os.environ.get("DB_PASS")
-user = os.environ.get("DB_USER")
-dbname = os.environ.get("DB_NAME")
-pg_url = ""
-if host and passwd and user and dbname:
-    logging.info(f"Connecting to {host} as {user} to {dbname}")
-    pg_url = f"postgresql://{user}:{passwd}@{host}/{dbname}"
-else:
-    pg_url = os.environ.get(
-        "POSTGRES_URL", "postgresql://usdf-butler.slac.stanford.edu:5432/lsstdb1"
-    )
-    logging.info(f"Using POSTGRES_URL {user} {host} {dbname}")
-engine = create_engine(pg_url)
+engine = setup_postgres()
 metadata_obj = MetaData(schema=f"cdb_{instrument.lower()}")
 exposure_table = Table("exposure", metadata_obj, autoload_with=engine)