Test files updated

python/lsst/consdb/efd_transform/generate_schema.py

@@ -168,7 +168,7 @@ def build_argparser() -> argparse.ArgumentParser:
         f.write("    description: End time of the transformation interval, must be provided\n")
         f.write("  - name: timewindow\n")
         f.write('    "@id": "#transformed_efd_scheduler.timewindow"\n')
-        f.write("    datatype: float\n")
+        f.write("    datatype: int\n")
         f.write("    description: Time window used to expand start and end times by, in minutes\n")
         f.write("  - name: status\n")
         f.write('    "@id": "#transformed_efd_scheduler.status"\n')

tests/efd_transform/LATISS.sql → tests/efd_transform/LATISS-sqlite.sql

@@ -528,6 +528,7 @@ CREATE TABLE main.transformed_efd_scheduler (
 	id INTEGER NOT NULL,
 	start_time TIMESTAMP,
 	end_time TIMESTAMP,
+	timewindow INTEGER,
 	status CHAR(20) DEFAULT 'pending',
 	process_start_time TIMESTAMP,
 	process_end_time TIMESTAMP,

tests/efd_transform/LSSTComCam.sql → tests/efd_transform/LSSTComCam-sqlite.sql

@@ -737,6 +737,7 @@ CREATE TABLE main.transformed_efd_scheduler (
 	id INTEGER NOT NULL,
 	start_time TIMESTAMP,
 	end_time TIMESTAMP,
+	timewindow INTEGER,
 	status CHAR(20) DEFAULT 'pending',
 	process_start_time TIMESTAMP,
 	process_end_time TIMESTAMP,

tests/efd_transform/LSSTComCamSim.sql → tests/efd_transform/LSSTComCamSim-sqlite.sql

@@ -737,6 +737,7 @@ CREATE TABLE main.transformed_efd_scheduler (
 	id INTEGER NOT NULL,
 	start_time TIMESTAMP,
 	end_time TIMESTAMP,
+	timewindow INTEGER,
 	status CHAR(20) DEFAULT 'pending',
 	process_start_time TIMESTAMP,
 	process_end_time TIMESTAMP,

tests/efd_transform/cdb_transformed_efd_LATISS.yml → tests/efd_transform/cdb_transformed_efd_LATISS.yaml

@@ -2615,6 +2615,10 @@ tables:
     "@id": "#transformed_efd_scheduler.end_time"
     datatype: timestamp
     description: End time of the transformation interval, must be provided
+  - name: timewindow
+    "@id": "#transformed_efd_scheduler.timewindow"
+    datatype: int
+    description: Time window used to expand start and end times by, in minutes
   - name: status
     "@id": "#transformed_efd_scheduler.status"
     datatype: char

tests/efd_transform/cdb_transformed_efd_LSSTComCam.yml → tests/efd_transform/cdb_transformed_efd_LSSTComCam.yaml

@@ -1,6 +1,6 @@
 ---
-name: cdb_latiss
-"@id": "#cdb_latiss"
+name: cdb_lsstcomcam
+"@id": "#cdb_lsstcomcam"
 description: Transformed EFD Consolidated Database for LSSTComCam
 tables:
 - name: exposure_efd
@@ -3660,6 +3660,10 @@ tables:
     "@id": "#transformed_efd_scheduler.end_time"
     datatype: timestamp
     description: End time of the transformation interval, must be provided
+  - name: timewindow
+    "@id": "#transformed_efd_scheduler.timewindow"
+    datatype: int
+    description: Time window used to expand start and end times by, in minutes
   - name: status
     "@id": "#transformed_efd_scheduler.status"
     datatype: char

tests/efd_transform/cdb_transformed_efd_LSSTComCamSim.yml → tests/efd_transform/cdb_transformed_efd_LSSTComCamSim.yaml

@@ -1,6 +1,6 @@
 ---
-name: cdb_latiss
-"@id": "#cdb_latiss"
+name: cdb_lsstcomcamsim
+"@id": "#cdb_lsstcomcamsim"
 description: Transformed EFD Consolidated Database for LSSTComCamSim
 tables:
 - name: exposure_efd
@@ -3660,6 +3660,10 @@ tables:
     "@id": "#transformed_efd_scheduler.end_time"
     datatype: timestamp
     description: End time of the transformation interval, must be provided
+  - name: timewindow
+    "@id": "#transformed_efd_scheduler.timewindow"
+    datatype: int
+    description: Time window used to expand start and end times by, in minutes
   - name: status
     "@id": "#transformed_efd_scheduler.status"
     datatype: char

tests/efd_transform/create_sqlite_db.sh

@@ -22,7 +22,7 @@ case "$INSTRUMENT" in
 esac
 
 # Set the configuration file based on the instrument
-CONFIG="config_${INSTRUMENT}.yml"
+CONFIG="config_${INSTRUMENT}.yaml"
 echo "Using configuration file: $CONFIG"
 
 # Navigate to the directory containing the schema generation script
@@ -34,19 +34,19 @@ echo "Generating schema for $INSTRUMENT..."
 python generate_schema.py --config "$CONFIG" --instrument "$INSTRUMENT"
 
 # Check if the schema generation was successful
-if [ ! -f "cdb_transformed_efd_${INSTRUMENT}.yml" ]; then
-  echo "Error: Failed to generate cdb_transformed_efd_${INSTRUMENT}.yml"
+if [ ! -f "cdb_transformed_efd_${INSTRUMENT}.yaml" ]; then
+  echo "Error: Failed to generate cdb_transformed_efd_${INSTRUMENT}.yaml"
   exit 1
 fi
-echo "Schema generated: cdb_transformed_efd_${INSTRUMENT}.yml"
+echo "Schema generated: cdb_transformed_efd_${INSTRUMENT}.yaml"
 
 # Remove the old transformed file in the test directory
 echo "Removing old transformed schema file (if any)..."
-rm -f ../../../../tests/efd_transform/cdb_transformed_efd_${INSTRUMENT}.yml
+rm -f ../../../../tests/efd_transform/cdb_transformed_efd_${INSTRUMENT}.yaml
 
 # Copy the new transformed file to the test directory
 echo "Copying the new transformed schema file to the test directory..."
-cp "cdb_transformed_efd_${INSTRUMENT}.yml" ../../../../tests/efd_transform/
+cp "cdb_transformed_efd_${INSTRUMENT}.yaml" ../../../../tests/efd_transform/
 
 # Navigate to the test directory
 echo "Navigating to the test directory..."
@@ -58,23 +58,9 @@ rm -f "${INSTRUMENT}.db"
 
 # Create the SQL file with Felis
 echo "Creating the SQL file with Felis..."
-felis create --dry-run --engine-url sqlite:/// "cdb_transformed_efd_${INSTRUMENT}.yml" > "${INSTRUMENT}.sql"
+felis create --dry-run --engine-url sqlite://// "cdb_transformed_efd_${INSTRUMENT}.yaml" > "${INSTRUMENT}-sqlite.sql"
+felis create --engine-url sqlite:///${INSTRUMENT}.db "cdb_transformed_efd_${INSTRUMENT}.yaml"
 
-# Check if the SQL file creation was successful
-if [ ! -f "${INSTRUMENT}.sql" ]; then
-  echo "Error: Failed to create ${INSTRUMENT}.sql"
-  exit 1
-fi
-echo "SQL file created: ${INSTRUMENT}.sql"
-
-# Load the SQL file into a new SQLite database
-echo "Loading SQL into the new SQLite database..."
-sqlite3 "${INSTRUMENT}.db" < "${INSTRUMENT}.sql"
-
-# Confirm successful completion
-if [ $? -eq 0 ]; then
-  echo "Database ${INSTRUMENT}.db created successfully."
-else
-  echo "Error: Failed to create the SQLite database."
-  exit 1
-fi
+# Create postgresql code for the database
+echo "Creating the SQL file with Felis for postgres..."
+felis create --dry-run --engine-url postgresql+psycopg2://username:password@localhost/database "cdb_transformed_efd_${INSTRUMENT}.yaml" > "${INSTRUMENT}-pg.sql"

tests/efd_transform/run_transform_efd.sh

@@ -7,7 +7,7 @@
 #     -r s3://rubin-summit-users/butler.yaml \
 #     -d sqlite:///$PWD/tests/efd_transform/LATISS.db \
 #     -E usdf_efd \
-#     -c python/lsst/consdb/efd_transform/config_LATISS.yml \
+#     -c python/lsst/consdb/efd_transform/config_LATISS.yaml \
 #     -t 5 \
 #     -w 1 \
 #     -l $PWD/tmp/transform.log
@@ -18,7 +18,7 @@
 #     -r /repo/embargo_new \
 #     -d sqlite:///$PWD/tests/efd_transform/LSSTComCam.db \
 #     -E usdf_efd \
-#     -c python/lsst/consdb/efd_transform/config_LSSTComCam.yml \
+#     -c python/lsst/consdb/efd_transform/config_LSSTComCam.yaml \
 #     -t 5 \
 #     -w 1 \
 #     -l $PWD/tmp/transform.log
@@ -29,7 +29,7 @@
 #     -r /repo/embargo_new \
 #     -d sqlite:///$PWD/tests/efd_transform/LSSTComCamSim.db \
 #     -E usdf_efd \
-#     -c python/lsst/consdb/efd_transform/config_LSSTComCamSim.yml \
+#     -c python/lsst/consdb/efd_transform/config_LSSTComCamSim.yaml \
 #     -t 5 \
 #     -w 1 \
 #     -l $PWD/tmp/transform.log
@@ -46,17 +46,17 @@
 #     -l $PWD/tmp/transform.log
 
 
-# python python/lsst/consdb/efd_transform/transform_efd.py \
-#     -i LATISS \
-#     -s 2024-10-17T12:00:00 \
-#     -e 2024-10-18T12:00:00 \
-#     -r s3://rubin-summit-users/butler.yaml \
-#     -d sqlite:///$PWD/tests/efd_transform/LATISS.db \
-#     -E usdf_efd \
-#     -c python/lsst/consdb/efd_transform/config_LATISS.yml \
-#     -t 60 \
-#     -w 1 \
-#     -l $PWD/tmp/transform.log
+python python/lsst/consdb/efd_transform/transform_efd.py \
+    -i LATISS \
+    -s 2024-10-17T12:00:00 \
+    -e 2024-10-18T12:00:00 \
+    -r s3://rubin-summit-users/butler.yaml \
+    -d sqlite:///$PWD/tests/efd_transform/LATISS.db \
+    -E usdf_efd \
+    -c python/lsst/consdb/efd_transform/config_LATISS.yaml \
+    -t 60 \
+    -w 1 \
+    -l $PWD/tmp/transform.log
 
 
 # python python/lsst/consdb/efd_transform/transform_efd.py \
@@ -66,20 +66,20 @@
 #     -r /repo/embargo_new \
 #     -d sqlite:///$PWD/tests/efd_transform/LSSTComCam.db \
 #     -E usdf_efd \
-#     -c python/lsst/consdb/efd_transform/config_LSSTComCam.yml \
+#     -c python/lsst/consdb/efd_transform/config_LSSTComCam.yaml \
 #     -t 5 \
 #     -w 1 \
 #     -l $PWD/tmp/transform.log
 
 
-python python/lsst/consdb/efd_transform/transform_efd.py \
-    -i LSSTComCamSim \
-    -r /repo/embargo_new \
-    -d sqlite:///$PWD/tests/efd_transform/LSSTComCamSim.db \
-    -E usdf_efd \
-    -c python/lsst/consdb/efd_transform/config_LSSTComCamSim.yml \
-    -t 5 \
-    -w 1 \
-    -l $PWD/tmp/transform.log
-    # -s 2024-10-17T12:00:00 \
-    # -e 2024-10-18T12:00:00 \
+# python python/lsst/consdb/efd_transform/transform_efd.py \
+#     -i LSSTComCamSim \
+#     -r /repo/embargo_new \
+#     -d sqlite:///$PWD/tests/efd_transform/LSSTComCamSim.db \
+#     -E usdf_efd \
+#     -c python/lsst/consdb/efd_transform/config_LSSTComCamSim.yaml \
+#     -t 5 \
+#     -w 1 \
+#     -l $PWD/tmp/transform.log
+#     # -s 2024-10-17T12:00:00 \
+#     # -e 2024-10-18T12:00:00 \