Fix column name bug unpivoted data

lsst-dm · Dec 8, 2024 · 59d2e2a · 59d2e2a
1 parent 28a900c
commit 59d2e2a
Show file tree

Hide file tree

Showing 17 changed files with 56 additions and 28 deletions.
diff --git a/python/lsst/consdb/efd_transform/cdb_transformed_efd_LATISS.yaml b/python/lsst/consdb/efd_transform/cdb_transformed_efd_LATISS.yaml
@@ -60,12 +60,14 @@ tables:
     datatype: char
     length: 255
     nullable: False
+    value: default_property
     description: Property name for the unpivoted data
   - name: field
     "@id": "#exposure_efd_unpivoted.field"
     datatype: char
     length: 255
     nullable: False
+    value: default_field
     description: Field name for the unpivoted data
   - name: value
     "@id": "#exposure_efd_unpivoted.value"
@@ -135,12 +137,14 @@ tables:
     datatype: char
     length: 255
     nullable: False
+    value: default_property
     description: Property name for the unpivoted data
   - name: field
     "@id": "#visit1_efd_unpivoted.field"
     datatype: char
     length: 255
     nullable: False
+    value: default_field
     description: Field name for the unpivoted data
   - name: value
     "@id": "#visit1_efd_unpivoted.value"

diff --git a/python/lsst/consdb/efd_transform/cdb_transformed_efd_LSSTComCam.yaml b/python/lsst/consdb/efd_transform/cdb_transformed_efd_LSSTComCam.yaml
@@ -2135,12 +2135,14 @@ tables:
     datatype: char
     length: 255
     nullable: False
+    value: default_property
     description: Property name for the unpivoted data
   - name: field
     "@id": "#exposure_efd_unpivoted.field"
     datatype: char
     length: 255
     nullable: False
+    value: default_field
     description: Field name for the unpivoted data
   - name: value
     "@id": "#exposure_efd_unpivoted.value"
@@ -4245,12 +4247,14 @@ tables:
     datatype: char
     length: 255
     nullable: False
+    value: default_property
     description: Property name for the unpivoted data
   - name: field
     "@id": "#visit1_efd_unpivoted.field"
     datatype: char
     length: 255
     nullable: False
+    value: default_field
     description: Field name for the unpivoted data
   - name: value
     "@id": "#visit1_efd_unpivoted.value"

diff --git a/python/lsst/consdb/efd_transform/cdb_transformed_efd_LSSTComCamSim.yaml b/python/lsst/consdb/efd_transform/cdb_transformed_efd_LSSTComCamSim.yaml
@@ -2135,12 +2135,14 @@ tables:
     datatype: char
     length: 255
     nullable: False
+    value: default_property
     description: Property name for the unpivoted data
   - name: field
     "@id": "#exposure_efd_unpivoted.field"
     datatype: char
     length: 255
     nullable: False
+    value: default_field
     description: Field name for the unpivoted data
   - name: value
     "@id": "#exposure_efd_unpivoted.value"
@@ -4245,12 +4247,14 @@ tables:
     datatype: char
     length: 255
     nullable: False
+    value: default_property
     description: Property name for the unpivoted data
   - name: field
     "@id": "#visit1_efd_unpivoted.field"
     datatype: char
     length: 255
     nullable: False
+    value: default_field
     description: Field name for the unpivoted data
   - name: value
     "@id": "#visit1_efd_unpivoted.value"

diff --git a/python/lsst/consdb/efd_transform/generate_schema.py b/python/lsst/consdb/efd_transform/generate_schema.py
@@ -124,12 +124,14 @@ def build_argparser() -> argparse.ArgumentParser:
         f.write("    datatype: char\n")
         f.write("    length: 255\n")
         f.write("    nullable: False\n")
+        f.write("    value: default_property\n")
         f.write("    description: Property name for the unpivoted data\n")
         f.write("  - name: field\n")
         f.write('    "@id": "#exposure_efd_unpivoted.field"\n')
         f.write("    datatype: char\n")
         f.write("    length: 255\n")
         f.write("    nullable: False\n")
+        f.write("    value: default_field\n")
         f.write("    description: Field name for the unpivoted data\n")
         f.write("  - name: value\n")
         f.write('    "@id": "#exposure_efd_unpivoted.value"\n')
@@ -214,12 +216,14 @@ def build_argparser() -> argparse.ArgumentParser:
         f.write("    datatype: char\n")
         f.write("    length: 255\n")
         f.write("    nullable: False\n")
+        f.write("    value: default_property\n")
         f.write("    description: Property name for the unpivoted data\n")
         f.write("  - name: field\n")
         f.write('    "@id": "#visit1_efd_unpivoted.field"\n')
         f.write("    datatype: char\n")
         f.write("    length: 255\n")
         f.write("    nullable: False\n")
+        f.write("    value: default_field\n")
         f.write("    description: Field name for the unpivoted data\n")
         f.write("  - name: value\n")
         f.write('    "@id": "#visit1_efd_unpivoted.value"\n')

diff --git a/python/lsst/consdb/efd_transform/transform.py b/python/lsst/consdb/efd_transform/transform.py
@@ -254,8 +254,8 @@ def process_interval(
                                     result_exp_unpivoted.append(
                                         {
                                             "exposure_id": exposure["id"],
-                                            "topic": column["name"],
-                                            "column": col,
+                                            "property": column["name"],
+                                            "field": col,
                                             "value": column_value,
                                         }
                                     )
@@ -295,8 +295,8 @@ def process_interval(
                                     result_vis_unpivoted.append(
                                         {
                                             "visit_id": visit["id"],
-                                            "topic": topic,
-                                            "column": col,
+                                            "property": topic,
+                                            "field": col,
                                             "value": column_value,
                                         }
                                     )

diff --git a/tests/efd_transform/LATISS-pg.sql b/tests/efd_transform/LATISS-pg.sql
@@ -16,8 +16,8 @@ COMMENT ON CONSTRAINT un_exposure_id_instrument ON cdb_latiss.exposure_efd IS 'E
 
 CREATE TABLE cdb_latiss.exposure_efd_unpivoted (
 	exposure_id INTEGER NOT NULL,
-	property CHAR(255) NOT NULL,
-	field CHAR(255) NOT NULL,
+	property CHAR(255) DEFAULT 'default_property' NOT NULL,
+	field CHAR(255) DEFAULT 'default_field' NOT NULL,
 	value DOUBLE PRECISION,
 	created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP,
 	PRIMARY KEY (exposure_id, property, field),
@@ -50,8 +50,8 @@ COMMENT ON CONSTRAINT un_visit_id_instrument ON cdb_latiss.visit1_efd IS 'Ensure
 
 CREATE TABLE cdb_latiss.visit1_efd_unpivoted (
 	visit_id INTEGER NOT NULL,
-	property CHAR(255) NOT NULL,
-	field CHAR(255) NOT NULL,
+	property CHAR(255) DEFAULT 'default_property' NOT NULL,
+	field CHAR(255) DEFAULT 'default_field' NOT NULL,
 	value DOUBLE PRECISION,
 	created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP,
 	PRIMARY KEY (visit_id, property, field),

diff --git a/tests/efd_transform/LATISS-sqlite.sql b/tests/efd_transform/LATISS-sqlite.sql
@@ -11,8 +11,8 @@ CREATE TABLE main.exposure_efd (
 
 CREATE TABLE main.exposure_efd_unpivoted (
 	exposure_id INTEGER NOT NULL,
-	property CHAR(255) NOT NULL,
-	field CHAR(255) NOT NULL,
+	property CHAR(255) DEFAULT 'default_property' NOT NULL,
+	field CHAR(255) DEFAULT 'default_field' NOT NULL,
 	value DOUBLE,
 	created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
 	PRIMARY KEY (exposure_id, property, field),
@@ -33,8 +33,8 @@ CREATE TABLE main.visit1_efd (
 
 CREATE TABLE main.visit1_efd_unpivoted (
 	visit_id INTEGER NOT NULL,
-	property CHAR(255) NOT NULL,
-	field CHAR(255) NOT NULL,
+	property CHAR(255) DEFAULT 'default_property' NOT NULL,
+	field CHAR(255) DEFAULT 'default_field' NOT NULL,
 	value DOUBLE,
 	created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
 	PRIMARY KEY (visit_id, property, field),

diff --git a/tests/efd_transform/LATISS.db b/tests/efd_transform/LATISS.db
diff --git a/tests/efd_transform/LSSTComCam-pg.sql b/tests/efd_transform/LSSTComCam-pg.sql
@@ -846,8 +846,8 @@ COMMENT ON CONSTRAINT un_exposure_id_instrument ON cdb_lsstcomcam.exposure_efd I
 
 CREATE TABLE cdb_lsstcomcam.exposure_efd_unpivoted (
 	exposure_id INTEGER NOT NULL,
-	property CHAR(255) NOT NULL,
-	field CHAR(255) NOT NULL,
+	property CHAR(255) DEFAULT 'default_property' NOT NULL,
+	field CHAR(255) DEFAULT 'default_field' NOT NULL,
 	value DOUBLE PRECISION,
 	created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP,
 	PRIMARY KEY (exposure_id, property, field),
@@ -1694,8 +1694,8 @@ COMMENT ON CONSTRAINT un_visit_id_instrument ON cdb_lsstcomcam.visit1_efd IS 'En
 
 CREATE TABLE cdb_lsstcomcam.visit1_efd_unpivoted (
 	visit_id INTEGER NOT NULL,
-	property CHAR(255) NOT NULL,
-	field CHAR(255) NOT NULL,
+	property CHAR(255) DEFAULT 'default_property' NOT NULL,
+	field CHAR(255) DEFAULT 'default_field' NOT NULL,
 	value DOUBLE PRECISION,
 	created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP,
 	PRIMARY KEY (visit_id, property, field),

diff --git a/tests/efd_transform/LSSTComCam-sqlite.sql b/tests/efd_transform/LSSTComCam-sqlite.sql
@@ -426,8 +426,8 @@ CREATE TABLE main.exposure_efd (
 
 CREATE TABLE main.exposure_efd_unpivoted (
 	exposure_id INTEGER NOT NULL,
-	property CHAR(255) NOT NULL,
-	field CHAR(255) NOT NULL,
+	property CHAR(255) DEFAULT 'default_property' NOT NULL,
+	field CHAR(255) DEFAULT 'default_field' NOT NULL,
 	value DOUBLE,
 	created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
 	PRIMARY KEY (exposure_id, property, field),
@@ -855,8 +855,8 @@ CREATE TABLE main.visit1_efd (
 
 CREATE TABLE main.visit1_efd_unpivoted (
 	visit_id INTEGER NOT NULL,
-	property CHAR(255) NOT NULL,
-	field CHAR(255) NOT NULL,
+	property CHAR(255) DEFAULT 'default_property' NOT NULL,
+	field CHAR(255) DEFAULT 'default_field' NOT NULL,
 	value DOUBLE,
 	created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
 	PRIMARY KEY (visit_id, property, field),

diff --git a/tests/efd_transform/LSSTComCam.db b/tests/efd_transform/LSSTComCam.db
diff --git a/tests/efd_transform/LSSTComCamSim-pg.sql b/tests/efd_transform/LSSTComCamSim-pg.sql
@@ -846,8 +846,8 @@ COMMENT ON CONSTRAINT un_exposure_id_instrument ON cdb_lsstcomcamsim.exposure_ef
 
 CREATE TABLE cdb_lsstcomcamsim.exposure_efd_unpivoted (
 	exposure_id INTEGER NOT NULL,
-	property CHAR(255) NOT NULL,
-	field CHAR(255) NOT NULL,
+	property CHAR(255) DEFAULT 'default_property' NOT NULL,
+	field CHAR(255) DEFAULT 'default_field' NOT NULL,
 	value DOUBLE PRECISION,
 	created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP,
 	PRIMARY KEY (exposure_id, property, field),
@@ -1694,8 +1694,8 @@ COMMENT ON CONSTRAINT un_visit_id_instrument ON cdb_lsstcomcamsim.visit1_efd IS
 
 CREATE TABLE cdb_lsstcomcamsim.visit1_efd_unpivoted (
 	visit_id INTEGER NOT NULL,
-	property CHAR(255) NOT NULL,
-	field CHAR(255) NOT NULL,
+	property CHAR(255) DEFAULT 'default_property' NOT NULL,
+	field CHAR(255) DEFAULT 'default_field' NOT NULL,
 	value DOUBLE PRECISION,
 	created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP,
 	PRIMARY KEY (visit_id, property, field),

diff --git a/tests/efd_transform/LSSTComCamSim-sqlite.sql b/tests/efd_transform/LSSTComCamSim-sqlite.sql
@@ -426,8 +426,8 @@ CREATE TABLE main.exposure_efd (
 
 CREATE TABLE main.exposure_efd_unpivoted (
 	exposure_id INTEGER NOT NULL,
-	property CHAR(255) NOT NULL,
-	field CHAR(255) NOT NULL,
+	property CHAR(255) DEFAULT 'default_property' NOT NULL,
+	field CHAR(255) DEFAULT 'default_field' NOT NULL,
 	value DOUBLE,
 	created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
 	PRIMARY KEY (exposure_id, property, field),
@@ -855,8 +855,8 @@ CREATE TABLE main.visit1_efd (
 
 CREATE TABLE main.visit1_efd_unpivoted (
 	visit_id INTEGER NOT NULL,
-	property CHAR(255) NOT NULL,
-	field CHAR(255) NOT NULL,
+	property CHAR(255) DEFAULT 'default_property' NOT NULL,
+	field CHAR(255) DEFAULT 'default_field' NOT NULL,
 	value DOUBLE,
 	created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
 	PRIMARY KEY (visit_id, property, field),

diff --git a/tests/efd_transform/LSSTComCamSim.db b/tests/efd_transform/LSSTComCamSim.db
diff --git a/tests/efd_transform/cdb_transformed_efd_LATISS.yaml b/tests/efd_transform/cdb_transformed_efd_LATISS.yaml
@@ -60,12 +60,14 @@ tables:
     datatype: char
     length: 255
     nullable: False
+    value: default_property
     description: Property name for the unpivoted data
   - name: field
     "@id": "#exposure_efd_unpivoted.field"
     datatype: char
     length: 255
     nullable: False
+    value: default_field
     description: Field name for the unpivoted data
   - name: value
     "@id": "#exposure_efd_unpivoted.value"
@@ -135,12 +137,14 @@ tables:
     datatype: char
     length: 255
     nullable: False
+    value: default_property
     description: Property name for the unpivoted data
   - name: field
     "@id": "#visit1_efd_unpivoted.field"
     datatype: char
     length: 255
     nullable: False
+    value: default_field
     description: Field name for the unpivoted data
   - name: value
     "@id": "#visit1_efd_unpivoted.value"

diff --git a/tests/efd_transform/cdb_transformed_efd_LSSTComCam.yaml b/tests/efd_transform/cdb_transformed_efd_LSSTComCam.yaml
@@ -2135,12 +2135,14 @@ tables:
     datatype: char
     length: 255
     nullable: False
+    value: default_property
     description: Property name for the unpivoted data
   - name: field
     "@id": "#exposure_efd_unpivoted.field"
     datatype: char
     length: 255
     nullable: False
+    value: default_field
     description: Field name for the unpivoted data
   - name: value
     "@id": "#exposure_efd_unpivoted.value"
@@ -4245,12 +4247,14 @@ tables:
     datatype: char
     length: 255
     nullable: False
+    value: default_property
     description: Property name for the unpivoted data
   - name: field
     "@id": "#visit1_efd_unpivoted.field"
     datatype: char
     length: 255
     nullable: False
+    value: default_field
     description: Field name for the unpivoted data
   - name: value
     "@id": "#visit1_efd_unpivoted.value"

diff --git a/tests/efd_transform/cdb_transformed_efd_LSSTComCamSim.yaml b/tests/efd_transform/cdb_transformed_efd_LSSTComCamSim.yaml
@@ -2135,12 +2135,14 @@ tables:
     datatype: char
     length: 255
     nullable: False
+    value: default_property
     description: Property name for the unpivoted data
   - name: field
     "@id": "#exposure_efd_unpivoted.field"
     datatype: char
     length: 255
     nullable: False
+    value: default_field
     description: Field name for the unpivoted data
   - name: value
     "@id": "#exposure_efd_unpivoted.value"
@@ -4245,12 +4247,14 @@ tables:
     datatype: char
     length: 255
     nullable: False
+    value: default_property
     description: Property name for the unpivoted data
   - name: field
     "@id": "#visit1_efd_unpivoted.field"
     datatype: char
     length: 255
     nullable: False
+    value: default_field
     description: Field name for the unpivoted data
   - name: value
     "@id": "#visit1_efd_unpivoted.value"