Skip to content

Commit

Permalink
Updated schema structures
Browse files Browse the repository at this point in the history
  • Loading branch information
@rcboufleur
rcboufleur committed Oct 25, 2024
1 parent e4ce479 commit 8928f5c
Show file tree
Hide file tree
Showing 18 changed files with 12,834 additions and 1,081 deletions.
1,259 changes: 1,258 additions & 1 deletion python/lsst/consdb/efd_transform/cdb_transformed_efd_LATISS.yml
View file

Large diffs are not rendered by default.

1,769 changes: 1,768 additions & 1 deletion python/lsst/consdb/efd_transform/cdb_transformed_efd_LSSTComCam.yml
View file

Large diffs are not rendered by default.

1,769 changes: 1,768 additions & 1 deletion python/lsst/consdb/efd_transform/cdb_transformed_efd_LSSTComCamSim.yml
View file

Large diffs are not rendered by default.

503 changes: 251 additions & 252 deletions python/lsst/consdb/efd_transform/config_LATISS.yml
View file

Large diffs are not rendered by default.

707 changes: 353 additions & 354 deletions python/lsst/consdb/efd_transform/config_LSSTComCam.yml
View file

Large diffs are not rendered by default.

707 changes: 353 additions & 354 deletions python/lsst/consdb/efd_transform/config_LSSTComCamSim.yml
View file

Large diffs are not rendered by default.

232 changes: 118 additions & 114 deletions python/lsst/consdb/efd_transform/generate_schema.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import yaml
import argparse

from transform_efd import read_config


def build_argparser() -> argparse.ArgumentParser:
"""
Build the argument parser for the script.
Expand All @@ -25,8 +26,9 @@ def build_argparser() -> argparse.ArgumentParser:

return parser


if __name__ == "__main__":

# run example: python generate_schema.py --config config.yml --instrument latiss

Check failure on line 32 in python/lsst/consdb/efd_transform/generate_schema.py

View workflow job for this annotation

GitHub Actions / call-workflow / lint

W505 

doc line too long (84 > 79 characters)

parse = build_argparser()
Expand All @@ -37,166 +39,168 @@ def build_argparser() -> argparse.ArgumentParser:

with open(output, "w") as f:
# Generate the schema for the EFD transform.
f.write('---\n')
f.write('name: cdb_latiss\n')
f.write("---\n")
f.write("name: cdb_latiss\n")
f.write('"@id": "#cdb_latiss"\n')
f.write(f'description: Transformed EFD Consolidated Database for {args.instrument}\n')
f.write('tables:\n')
f.write(f"description: Transformed EFD Consolidated Database for {args.instrument}\n")
f.write("tables:\n")

# Generate exposure_efd table.
f.write('- name: exposure_efd\n')
f.write("- name: exposure_efd\n")
f.write(' "@id": "#exposure_efd"\n')
f.write(' description: Transformed EFD topics by exposure.\n')
f.write(' primaryKey:\n')
f.write(" description: Transformed EFD topics by exposure.\n")
f.write(" primaryKey:\n")
f.write(' - "#exposure_efd.exposure_id"\n')
f.write(' - "#exposure_efd.instrument"\n')
f.write(' constraints:\n')
f.write(' - name: un_exposure_id_instrument\n')
f.write(" constraints:\n")
f.write(" - name: un_exposure_id_instrument\n")
f.write(' "@id": "#exposure_efd.un_exposure_id_instrument"\n')
f.write(' "@type": Unique\n')
f.write(' description: Ensure exposure_id is unique.\n')
f.write(' columns:\n')
f.write(" description: Ensure exposure_id is unique.\n")
f.write(" columns:\n")
f.write(' - "#exposure_efd.exposure_id"\n')
f.write(' - "#exposure_efd.instrument"\n')
f.write(' columns:\n')
f.write(' - name: exposure_id\n')
f.write(" columns:\n")
f.write(" - name: exposure_id\n")
f.write(' "@id": "#exposure_efd.exposure_id"\n')
f.write(' datatype: long\n')
f.write(' description: Exposure unique ID.\n')
f.write(' - name: created_at\n')
f.write(" datatype: long\n")
f.write(" description: Exposure unique ID.\n")
f.write(" - name: created_at\n")
f.write(' "@id": "#exposure_efd.created_at"\n')
f.write(' datatype: timestamp\n')
f.write(' value: \'CURRENT_TIMESTAMP\'\n')
f.write(' description: Timestamp when the record was created, default is the current timestamp\n')
f.write(' - name: instrument\n')
f.write(" datatype: timestamp\n")
f.write(" value: 'CURRENT_TIMESTAMP'\n")
f.write(" description: Timestamp when the record was created, default is the current timestamp\n")
f.write(" - name: instrument\n")
f.write(' "@id": "#exposure_efd.instrument"\n')
f.write(' datatype: char\n')
f.write(' length: 20\n')
f.write(' description: Instrument name.\n')
f.write(" datatype: char\n")
f.write(" length: 20\n")
f.write(" description: Instrument name.\n")
# Iterate over columns in the config file
for column in config['columns']:
if "exposure_efd" in column['tables']:
column_name = column['name']
for column in config["columns"]:
if "exposure_efd" in column["tables"]:
column_name = column["name"]
f.write(f' - name: "{column_name}"\n')
f.write(f' "@id": "#exposure_efd.{column_name}"\n')
datatype = column['datatype']
f.write(f' datatype: {datatype}\n')
f.write(f' nullable: True\n')
column_description = column['description']
f.write(f' description: {column_description}\n')
f.write('\n')
datatype = column["datatype"]
f.write(f" datatype: {datatype}\n")
f.write(" nullable: True\n")
column_description = column["description"]
f.write(f" description: {column_description}\n")
f.write("\n")

# Generate visit1_efd table.
f.write('- name: visit1_efd\n')
f.write("- name: visit1_efd\n")
f.write(' "@id": "#visit1_efd"\n')
f.write(' description: Transformed EFD topics by visit.\n')
f.write(' primaryKey:\n')
f.write(" description: Transformed EFD topics by visit.\n")
f.write(" primaryKey:\n")
f.write(' - "#visit1_efd.visit_id"\n')
f.write(' - "#visit1_efd.instrument"\n')
f.write(' constraints:\n')
f.write(' - name: un_visit_id_instrument\n')
f.write(" constraints:\n")
f.write(" - name: un_visit_id_instrument\n")
f.write(' "@id": "#visit1_efd.un_visit_id_instrument"\n')
f.write(' "@type": Unique\n')
f.write(' description: Ensure visit_id is unique.\n')
f.write(' columns:\n')
f.write(" description: Ensure visit_id is unique.\n")
f.write(" columns:\n")
f.write(' - "#visit1_efd.visit_id"\n')
f.write(' - "#visit1_efd.instrument"\n')
f.write(' columns:\n')
f.write(' - name: visit_id\n')
f.write(" columns:\n")
f.write(" - name: visit_id\n")
f.write(' "@id": "#visit1_efd.visit_id"\n')
f.write(' datatype: long\n')
f.write(' description: Visit unique ID.\n')
f.write(' - name: created_at\n')
f.write(" datatype: long\n")
f.write(" description: Visit unique ID.\n")
f.write(" - name: created_at\n")
f.write(' "@id": "#visit1_efd.created_at"\n')
f.write(' datatype: timestamp\n')
f.write(' value: \'CURRENT_TIMESTAMP\'\n')
f.write(' description: Timestamp when the record was created, default is the current timestamp\n')
f.write(' - name: instrument\n')
f.write(" datatype: timestamp\n")
f.write(" value: 'CURRENT_TIMESTAMP'\n")
f.write(" description: Timestamp when the record was created, default is the current timestamp\n")
f.write(" - name: instrument\n")
f.write(' "@id": "#visit1_efd.instrument"\n')
f.write(' datatype: char\n')
f.write(' length: 20\n')
f.write(' description: Instrument name.\n')
f.write(" datatype: char\n")
f.write(" length: 20\n")
f.write(" description: Instrument name.\n")
# Iterate over columns in the config file
for column in config['columns']:
if "visit1_efd" in column['tables']:
column_name = column['name']
for column in config["columns"]:
if "visit1_efd" in column["tables"]:
column_name = column["name"]
f.write(f' - name: "{column_name}"\n')
f.write(f' "@id": "#visit1_efd.{column_name}"\n')
datatype = column['datatype']
f.write(f' datatype: {datatype}\n')
f.write(f' nullable: True\n')
column_description = column['description']
f.write(f' description: {column_description}\n')
f.write('\n')
datatype = column["datatype"]
f.write(f" datatype: {datatype}\n")
f.write(" nullable: True\n")
column_description = column["description"]
f.write(f" description: {column_description}\n")
f.write("\n")

# Generate transformed_efd scheduler table.
f.write('- name: transformed_efd_scheduler\n')
f.write("- name: transformed_efd_scheduler\n")
f.write(' "@id": "#transformed_efd_scheduler"\n')
f.write(' description: Transformed EFD scheduler.\n')
f.write(' primaryKey:\n')
f.write(" description: Transformed EFD scheduler.\n")
f.write(" primaryKey:\n")
f.write(' - "#transformed_efd_scheduler.id"\n')
f.write(' constraints:\n')
f.write(' - name: un_id\n')
f.write(" constraints:\n")
f.write(" - name: un_id\n")
f.write(' "@id": "#transformed_efd_scheduler.un_id"\n')
f.write(' "@type": Unique\n')
f.write(' description: Ensure id is unique.\n')
f.write(' columns:\n')
f.write(" description: Ensure id is unique.\n")
f.write(" columns:\n")
f.write(' - "#transformed_efd_scheduler.id"\n')
f.write(' columns:\n')
f.write(' - name: id\n')
f.write(" columns:\n")
f.write(" - name: id\n")
f.write(' "@id": "#transformed_efd_scheduler.id"\n')
f.write(' datatype: long\n')
f.write(' description: Unique ID, auto-incremented\n')
f.write(' - name: start_time\n')
f.write(" datatype: int\n")
f.write(" nullable: False\n")
f.write(" autoincrement: True\n")
f.write(" description: Unique ID, auto-incremented\n")
f.write(" - name: start_time\n")
f.write(' "@id": "#transformed_efd_scheduler.start_time"\n')
f.write(' datatype: timestamp\n')
f.write(' description: Start time of the transformation interval, must be provided\n')
f.write(' - name: end_time\n')
f.write(" datatype: timestamp\n")
f.write(" description: Start time of the transformation interval, must be provided\n")
f.write(" - name: end_time\n")
f.write(' "@id": "#transformed_efd_scheduler.end_time"\n')
f.write(' datatype: timestamp\n')
f.write(' description: End time of the transformation interval, must be provided\n')
f.write(' - name: status\n')
f.write(" datatype: timestamp\n")
f.write(" description: End time of the transformation interval, must be provided\n")
f.write(" - name: status\n")
f.write(' "@id": "#transformed_efd_scheduler.status"\n')
f.write(' datatype: char\n')
f.write(' length: 20\n')
f.write(" datatype: char\n")
f.write(" length: 20\n")
f.write(' value: "pending"\n')
f.write(' description: Status of the process, default is \'pending\'\n')
f.write(' - name: process_start_time\n')
f.write(" description: Status of the process, default is 'pending'\n")
f.write(" - name: process_start_time\n")
f.write(' "@id": "#transformed_efd_scheduler.process_start_time"\n')
f.write(' datatype: timestamp\n')
f.write(' description: Timestamp when the process started\n')
f.write(' - name: process_end_time\n')
f.write(" datatype: timestamp\n")
f.write(" description: Timestamp when the process started\n")
f.write(" - name: process_end_time\n")
f.write(' "@id": "#transformed_efd_scheduler.process_end_time"\n')
f.write(' datatype: timestamp\n')
f.write(' description: Timestamp when the process ended\n')
f.write(' - name: process_exec_time\n')
f.write(" datatype: timestamp\n")
f.write(" description: Timestamp when the process ended\n")
f.write(" - name: process_exec_time\n")
f.write(' "@id": "#transformed_efd_scheduler.process_exec_time"\n')
f.write(' datatype: int\n')
f.write(' value: 0\n')
f.write(' description: Execution time of the process in seconds, default is 0\n')
f.write(' - name: exposures\n')
f.write(" datatype: int\n")
f.write(" value: 0\n")
f.write(" description: Execution time of the process in seconds, default is 0\n")
f.write(" - name: exposures\n")
f.write(' "@id": "#transformed_efd_scheduler.exposures"\n')
f.write(' datatype: int\n')
f.write(' value: 0\n')
f.write(' description: Number of exposures processed, default is 0\n')
f.write(' - name: visits1\n')
f.write(" datatype: int\n")
f.write(" value: 0\n")
f.write(" description: Number of exposures processed, default is 0\n")
f.write(" - name: visits1\n")
f.write(' "@id": "#transformed_efd_scheduler.visits1"\n')
f.write(' datatype: int\n')
f.write(' value: 0\n')
f.write(' description: Number of visits recorded, default is 0\n')
f.write(' - name: retries\n')
f.write(" datatype: int\n")
f.write(" value: 0\n")
f.write(" description: Number of visits recorded, default is 0\n")
f.write(" - name: retries\n")
f.write(' "@id": "#transformed_efd_scheduler.retries"\n')
f.write(' datatype: int\n')
f.write(' value: 0\n')
f.write(' description: Number of retries attempted, default is 0\n')
f.write(' - name: error\n')
f.write(" datatype: int\n")
f.write(" value: 0\n")
f.write(" description: Number of retries attempted, default is 0\n")
f.write(" - name: error\n")
f.write(' "@id": "#transformed_efd_scheduler.error"\n')
f.write(' datatype: text\n')
f.write(" datatype: text\n")
f.write(' description: "Error message, if any"\n')
f.write(' - name: created_at\n')
f.write(" - name: created_at\n")
f.write(' "@id": "#transformed_efd_scheduler.created_at"\n')
f.write(' datatype: timestamp\n')
f.write(' value: \'CURRENT_TIMESTAMP\'\n')
f.write(' description: Timestamp when the record was created, default is the current timestamp\n')
f.close()
f.write(" datatype: timestamp\n")
f.write(" value: 'CURRENT_TIMESTAMP'\n")
f.write(" description: Timestamp when the record was created, default is the current timestamp\n")

f.close()
Binary file modified tests/efd_transform/LATISS.db
View file
Binary file not shown.
Loading

0 comments on commit 8928f5c

Please sign in to comment.