From 4c0ca57ad690e7f260395e2595f882449bcfbffd Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Wed, 28 Aug 2024 14:56:57 -0700 Subject: [PATCH 01/10] Additional unit tests --- python/lsst/consdb/hinfo.py | 22 +- python/lsst/consdb/pqserver.py | 47 ++- tests/lsstcomcamsim/ccdexposure.ecsv | 19 ++ tests/lsstcomcamsim/ccdvisit1_quicklook.ecsv | 46 +++ tests/lsstcomcamsim/cdb_lsstcomcamsim.sql | 323 +++++++++++++++++++ tests/lsstcomcamsim/exposure.ecsv | 55 ++++ tests/lsstcomcamsim/exposure2.ecsv | 81 +++++ tests/lsstcomcamsim/visit1_quicklook.ecsv | 96 ++++++ tests/test_pqserver.py | 256 +++++++++++++++ 9 files changed, 938 insertions(+), 7 deletions(-) create mode 100644 tests/lsstcomcamsim/ccdexposure.ecsv create mode 100644 tests/lsstcomcamsim/ccdvisit1_quicklook.ecsv create mode 100644 tests/lsstcomcamsim/cdb_lsstcomcamsim.sql create mode 100644 tests/lsstcomcamsim/exposure.ecsv create mode 100644 tests/lsstcomcamsim/exposure2.ecsv create mode 100644 tests/lsstcomcamsim/visit1_quicklook.ecsv diff --git a/python/lsst/consdb/hinfo.py b/python/lsst/consdb/hinfo.py index 271facd7..0ac4dc45 100644 --- a/python/lsst/consdb/hinfo.py +++ b/python/lsst/consdb/hinfo.py @@ -398,6 +398,19 @@ def process_resource(resource: ResourcePath, instrument_dict: dict, update: bool conn.commit() +def process_local_path(path: str) -> None: + # If a directory is provided on the command line, process all yaml + # files recursively. + if os.path.isdir(path): + for _, _, files in os.walk(path): + for file in files: + if file.endswith(".yaml"): + process_resource(ResourcePath(file), get_instrument_dict(instrument)) + # If a yaml file is provided on the command line, process it. + elif os.path.isfile(path) and path.endswith(".yaml"): + process_resource(ResourcePath(path), get_instrument_dict(instrument)) + + def process_date(day_obs: str, instrument_dict: dict, update: bool = False) -> None: """Process all headers from a given observation day (as YYYY-MM-DD). @@ -527,7 +540,7 @@ def get_instrument_dict(instrument: str) -> dict: ), } else: - raise ValueError("Unrecognized instrument: {instrument}") + raise ValueError(f"Unrecognized instrument: {instrument}") return instrument_dict @@ -578,5 +591,10 @@ async def main() -> None: if __name__ == "__main__": + import sys + engine = setup_postgres() - asyncio.run(main()) + if len(sys.argv) > 1: + process_local_path(sys.argv[1]) + else: + asyncio.run(main()) diff --git a/python/lsst/consdb/pqserver.py b/python/lsst/consdb/pqserver.py index 847ca001..92b2d05b 100644 --- a/python/lsst/consdb/pqserver.py +++ b/python/lsst/consdb/pqserver.py @@ -123,6 +123,7 @@ def __init__(self): self.schemas = dict() self.flexible_metadata_schemas = dict() self.obs_id_column = dict() + self.timestamp_columns = dict() for instrument in self.instrument_list: md = sqlalchemy.MetaData(schema=f"cdb_{instrument}") md.reflect(engine) @@ -131,11 +132,20 @@ def __init__(self): self.obs_id_column[instrument] = dict() self.flexible_metadata_schemas[instrument] = dict() for table in md.tables: + # Find all timestamp columns in the table + self.timestamp_columns[table] = set( + [ + column.name for column in md.tables[table].columns + if isinstance(column.type, sqlalchemy.DateTime) + ] + ) + for col_name in ObsIdColname: col_name = col_name.value if col_name in md.tables[table].columns: self.obs_id_column[instrument][table] = col_name break + for obs_type in ObsTypeEnum: obs_type = obs_type.value table_name = f"cdb_{instrument}.{obs_type}_flexdata" @@ -150,6 +160,26 @@ def __init__(self): schema[row[0]] = row[1:] self.flexible_metadata_schemas[instrument][obs_type] = schema + def get_timestamp_columns(self, table: str) -> set[str]: + """Returns a set containing all timestamp columns. + + Given the instrument and table name, returns a set + of strings listing all the columns in the table that + are a timestamp. + + Parameters + ---------- + table : `str` + The name of the table, e.g., "cdb_latiss.exposure". + + Returns + ------- + `set[str]` + The names of all timestamp columns in the table. + """ + columns = self.timestamp_columns[table] + return columns + def refresh_flexible_metadata_schema(self, instrument: str, obs_type: str): schema = dict() schema_table = self.get_flexible_metadata_schema(instrument, obs_type) @@ -467,7 +497,7 @@ class AddKeyRequestModel(BaseModel): ) @field_validator("unit") - def validate_unit(v): + def validate_unit(v: str): try: _ = astropy.units.Unit(v) except ValueError: @@ -475,7 +505,7 @@ def validate_unit(v): return v @field_validator("ucd") - def validate_ucd(v): + def validate_ucd(v: str): if not astropy.io.votable.ucd.check_ucd(v): raise ValueError(f"'{v}' is not a valid IVOA UCD.") return v @@ -723,7 +753,6 @@ class InsertMultipleRequestModel(BaseModel): class InsertMultipleResponseModel(BaseModel): message: str = Field(title="Human-readable response message") instrument: str = Field(title="Instrument name (e.g., ``LATISS``)") - obs_type: ObsTypeEnum = Field(title="The observation type (e.g., ``exposure``)") obs_id: ObservationIdType | list[ObservationIdType] = Field(title="Observation ID") table: str = Field(title="Table name") @@ -757,13 +786,21 @@ def insert_multiple( table_name = schema + table_name table_obj = instrument_tables.schemas[instrument_l].tables[table_name] table_name = f"cdb_{instrument_l}." + table.lower() - table = instrument_tables.schemas[instrument_l].tables[table_name] obs_id_colname = instrument_tables.obs_id_column[instrument_l][table_name] + timestamp_columns = instrument_tables.get_timestamp_columns(table_name) + with engine.connect() as conn: for obs_id, valdict in data.obs_dict.items(): valdict[obs_id_colname] = obs_id + # Convert timestamps in the input from string to datetime + for column in timestamp_columns: + if column in valdict and valdict[column] is not None: + timestamp = valdict[column] + timestamp = astropy.time.Time(timestamp, format="isot", scale="tai") + valdict[column] = timestamp.to_datetime() + stmt: sqlalchemy.sql.dml.Insert stmt = sqlalchemy.dialects.postgresql.insert(table_obj).values(valdict) if u != 0: @@ -775,7 +812,7 @@ def insert_multiple( return InsertMultipleResponseModel( message="Data inserted", - table=table_name, + table=table, instrument=instrument, obs_id=data.obs_dict.keys(), ) diff --git a/tests/lsstcomcamsim/ccdexposure.ecsv b/tests/lsstcomcamsim/ccdexposure.ecsv new file mode 100644 index 00000000..d9944d9e --- /dev/null +++ b/tests/lsstcomcamsim/ccdexposure.ecsv @@ -0,0 +1,19 @@ +# %ECSV 1.0 +# --- +# datatype: +# - {name: ccdexposure_id, datatype: int64} +# - {name: exposure_id, datatype: int64} +# - {name: detector, datatype: int64} +# - {name: s_region, datatype: string, subtype: json} +# schema: astropy-2.0 +ccdexposure_id exposure_id detector s_region +730865976064 7024040300451 0 null +730865976065 7024040300451 1 null +730865976066 7024040300451 2 null +730865976067 7024040300451 3 null +730865976068 7024040300451 4 null +730865976069 7024040300451 5 null +730865976070 7024040300451 6 null +730865976071 7024040300451 7 null +730865976072 7024040300451 8 null + diff --git a/tests/lsstcomcamsim/ccdvisit1_quicklook.ecsv b/tests/lsstcomcamsim/ccdvisit1_quicklook.ecsv new file mode 100644 index 00000000..d55eb0f0 --- /dev/null +++ b/tests/lsstcomcamsim/ccdvisit1_quicklook.ecsv @@ -0,0 +1,46 @@ +# %ECSV 1.0 +# --- +# datatype: +# - {name: ccdvisit_id, datatype: int64} +# - {name: s_ra, datatype: string, subtype: json} +# - {name: s_dec, datatype: string, subtype: json} +# - {name: zenith_distance, datatype: float64} +# - {name: photometric_calib, datatype: string, subtype: json} +# - {name: psf_sigma, datatype: float64} +# - {name: sky_bg, datatype: float64} +# - {name: sky_noise, datatype: float64} +# - {name: astrom_offset_mean, datatype: float64} +# - {name: astrom_offset_std, datatype: float64} +# - {name: n_psf_star, datatype: int64} +# - {name: psf_star_delta_e1_median, datatype: float64} +# - {name: psf_star_delta_e2_median, datatype: float64} +# - {name: psf_star_delta_e1_scatter, datatype: float64} +# - {name: psf_star_delta_e2_scatter, datatype: float64} +# - {name: psf_star_delta_size_median, datatype: float64} +# - {name: psf_star_delta_size_scatter, datatype: float64} +# - {name: psf_star_scaled_delta_size_scatter, datatype: float64} +# - {name: psf_trace_radius_delta, datatype: float64} +# - {name: max_dist_to_nearest_psf, datatype: float64} +# - {name: zero_point, datatype: float64} +# - {name: seeing_zenith_500nm, datatype: string, subtype: json} +# - {name: mean_var, datatype: float64} +# - {name: eff_time, datatype: string, subtype: json} +# - {name: eff_time_psf_sigma_scale, datatype: string, subtype: json} +# - {name: eff_time_sky_bg_scale, datatype: string, subtype: json} +# - {name: eff_time_zero_point_scale, datatype: string, subtype: json} +# - {name: psf_area, datatype: string, subtype: json} +# - {name: psf_ixx, datatype: string, subtype: json} +# - {name: psf_ixy, datatype: string, subtype: json} +# - {name: psf_iyy, datatype: string, subtype: json} +# schema: astropy-2.0 +ccdvisit_id s_ra s_dec zenith_distance photometric_calib psf_sigma sky_bg sky_noise astrom_offset_mean astrom_offset_std n_psf_star psf_star_delta_e1_median psf_star_delta_e2_median psf_star_delta_e1_scatter psf_star_delta_e2_scatter psf_star_delta_size_median psf_star_delta_size_scatter psf_star_scaled_delta_size_scatter psf_trace_radius_delta max_dist_to_nearest_psf zero_point seeing_zenith_500nm mean_var eff_time eff_time_psf_sigma_scale eff_time_sky_bg_scale eff_time_zero_point_scale psf_area psf_ixx psf_ixy psf_iyy +730865976064 null null 19.908431369156446 null 1.4783571153492203 325.756928877905 18.8443699448574 0.008262017001223145 0.004718156295271102 217 0.00016299915786729263 0.00011438961203464899 0.011962956264587885 0.010951664970417795 -0.0016228945823550944 0.014533274780719559 0.009831288919328727 0.012445912191268338 514.4628010947106 31.38873096483804 null 353.2532882820847 null null null null null null null null +730865976065 null null 20.101420893447923 null 1.4775241728998028 326.24286852031946 16.168533669485356 0.008345244097560052 0.004644486211030871 205 0.0007096202011189533 -6.279953687492439e-05 0.010025114280863591 0.010869131022067498 -0.00013780353456638306 0.011419562132594574 0.00773608179738651 0.01826433405243688 495.26432725169224 31.391624077205588 null 261.2827813341214 null null null null null null null null +730865976066 null null 20.295261471307356 null 1.481360383220493 327.9174807704985 17.117237088100463 0.008037184779008407 0.004509928499067784 240 0.00020827345612088252 -0.0009301515235772186 0.012141335021302824 0.01204704147640473 -0.0001849443492182301 0.013509237523051394 0.009123403884200907 0.010163010271619921 575.3210439920313 31.397368599773657 null 291.87838481228005 null null null null null null null null +730865976067 null null 20.04385574654343 null 1.4819077958347195 327.42115081846714 18.195719319686763 0.007306209156368198 0.003911773454604958 203 -0.0005030365146762635 -0.0002584813719447203 0.010691458148750244 0.009808981209256286 -0.0017265064325687751 0.011395387470475023 0.007700107967144595 0.01040220001654757 462.0034884059603 31.395384340242202 null 329.06359432923387 null null null null null null null null +730865976068 null null 20.235612462876873 null 1.4796918868486453 330.20515786483884 16.721598318052813 0.006869899714417714 0.004140340915062614 228 0.00043941160380314096 2.30499724320982e-05 0.010656812118126873 0.00926258473422664 -0.001182418450328404 0.012192983396004468 0.00824411396790928 0.009473662935317151 453.98200950766716 31.403766055859773 null 279.144652089946 null null null null null null null null +730865976069 null null 20.428235446662057 null 1.4808431755577847 327.98930598422885 18.01940772728306 0.007426047832899482 0.00407317264847736 219 0.0007112101408741738 0.00025940136765663716 0.008677658245206814 0.011400476261642561 -0.0007948797192933732 0.012432583745027833 0.008408350955801825 0.01922813661572187 562.6383678700041 31.398043978205017 null 323.94810586398717 null null null null null null null null +730865976070 null null 20.181037254777436 null 1.4786767957703044 324.92875876277685 18.04389147144221 0.007065422075437092 0.0040108591151109966 204 -0.0005667716476274514 -0.000498213881041874 0.01082692511868196 0.011470010573759007 0.0006602196068982158 0.01102648496164427 0.0074530341952774885 0.011156733720228518 501.3851614050777 31.387959747786027 null 324.84230133614454 null null null null null null null null +730865976071 null null 20.37155386161841 null 1.4825542676900207 332.9820567816496 17.435919459383122 0.007608150512381772 0.0040053085159124395 209 0.0008352543379669189 -0.0007314137363722759 0.009076289733708769 0.008944283164177712 0.0011724778113302303 0.012661660598416875 0.008543480730366567 0.02383131519393955 531.2477601301455 31.415341164852908 null 303.0631709313133 null null null null null null null null +730865976072 null null 20.562948229915577 null 1.481181714112671 316.3169114049524 18.965357862481312 0.006607995409089556 0.0038569409113254414 177 0.0003617365886151129 -9.827991873108399e-05 0.00876221677148559 0.010159400087041156 -0.000808027201299022 0.011083649183102768 0.007473748468841092 0.018365489162744053 730.3504366105043 31.359705721805007 null 357.5558361986811 null null null null null null null null + diff --git a/tests/lsstcomcamsim/cdb_lsstcomcamsim.sql b/tests/lsstcomcamsim/cdb_lsstcomcamsim.sql new file mode 100644 index 00000000..54da548e --- /dev/null +++ b/tests/lsstcomcamsim/cdb_lsstcomcamsim.sql @@ -0,0 +1,323 @@ + +CREATE TABLE main.exposure ( + exposure_id BIGINT NOT NULL, + exposure_name VARCHAR(20) NOT NULL, + controller VARCHAR(1) NOT NULL, + day_obs INTEGER NOT NULL, + seq_num INTEGER NOT NULL, + physical_filter VARCHAR(32), + band VARCHAR(32), + s_ra DOUBLE, + s_dec DOUBLE, + sky_rotation DOUBLE, + azimuth_start FLOAT, + azimuth_end FLOAT, + azimuth FLOAT, + altitude_start FLOAT, + altitude_end FLOAT, + altitude FLOAT, + zenith_distance_start FLOAT, + zenith_distance_end FLOAT, + zenith_distance FLOAT, + airmass FLOAT, + exp_midpt TIMESTAMP, + exp_midpt_mjd DOUBLE, + obs_start TIMESTAMP, + obs_start_mjd DOUBLE, + obs_end TIMESTAMP, + obs_end_mjd DOUBLE, + exp_time FLOAT, + shut_time FLOAT, + dark_time FLOAT, + group_id VARCHAR(64), + cur_index INTEGER, + max_index INTEGER, + img_type VARCHAR(64), + emulated BOOLEAN, + science_program VARCHAR(64), + observation_reason VARCHAR(68), + target_name VARCHAR(64), + air_temp FLOAT, + pressure FLOAT, + humidity FLOAT, + wind_speed FLOAT, + wind_dir FLOAT, + dimm_seeing FLOAT, + focus_z FLOAT, + simulated BOOLEAN, + vignette VARCHAR(10), + vignette_min VARCHAR(10), + s_region VARCHAR(1024), + PRIMARY KEY (exposure_id), + CONSTRAINT un_day_obs_seq_num UNIQUE (day_obs, seq_num) +) + +; + +CREATE TABLE main.exposure_flexdata_schema ( + "key" VARCHAR(128) NOT NULL, + dtype VARCHAR(64) NOT NULL, + doc TEXT NOT NULL, + unit VARCHAR(128), + ucd VARCHAR(128), + PRIMARY KEY ("key") +) + +; + +CREATE TABLE main.ccdexposure_flexdata_schema ( + "key" VARCHAR(128) NOT NULL, + dtype VARCHAR(64) NOT NULL, + doc TEXT NOT NULL, + unit VARCHAR(128), + ucd VARCHAR(128), + PRIMARY KEY ("key") +) + +; + +CREATE TABLE main.visit1 ( + visit_id BIGINT NOT NULL, + exposure_name VARCHAR(20) NOT NULL, + controller VARCHAR(1) NOT NULL, + day_obs INTEGER NOT NULL, + seq_num INTEGER NOT NULL, + physical_filter VARCHAR(32), + band VARCHAR(32), + s_ra DOUBLE, + s_dec DOUBLE, + sky_rotation DOUBLE, + azimuth_start FLOAT, + azimuth_end FLOAT, + azimuth FLOAT, + altitude_start FLOAT, + altitude_end FLOAT, + altitude FLOAT, + zenith_distance_start FLOAT, + zenith_distance_end FLOAT, + zenith_distance FLOAT, + airmass FLOAT, + exp_midpt TIMESTAMP, + exp_midpt_mjd DOUBLE, + obs_start TIMESTAMP, + obs_start_mjd DOUBLE, + obs_end TIMESTAMP, + obs_end_mjd DOUBLE, + exp_time FLOAT, + shut_time FLOAT, + dark_time FLOAT, + group_id VARCHAR(64), + cur_index INTEGER, + max_index INTEGER, + img_type VARCHAR(64), + emulated BOOLEAN, + science_program VARCHAR(64), + observation_reason VARCHAR(68), + target_name VARCHAR(64), + air_temp FLOAT, + pressure FLOAT, + humidity FLOAT, + wind_speed FLOAT, + wind_dir FLOAT, + dimm_seeing FLOAT, + focus_z FLOAT, + simulated BOOLEAN, + vignette VARCHAR(10), + vignette_min VARCHAR(10), + s_region VARCHAR(1024), + PRIMARY KEY (visit_id) +) + +; + +CREATE TABLE main.ccdvisit1 ( + ccdvisit_id BIGINT NOT NULL, + visit_id BIGINT NOT NULL, + detector INTEGER NOT NULL, + s_region VARCHAR(1024), + PRIMARY KEY (ccdvisit_id) +) + +; + +CREATE TABLE main.exposure_flexdata ( + obs_id BIGINT NOT NULL, + "key" VARCHAR(128) NOT NULL, + value TEXT, + PRIMARY KEY (obs_id, "key"), + CONSTRAINT fk_exposure_flexdata_obs_id FOREIGN KEY(obs_id) REFERENCES exposure (exposure_id), + CONSTRAINT fk_exposure_flexdata_key FOREIGN KEY("key") REFERENCES exposure_flexdata_schema ("key") +) + +; + +CREATE TABLE main.ccdexposure ( + ccdexposure_id BIGINT NOT NULL, + exposure_id BIGINT NOT NULL, + detector INTEGER NOT NULL, + s_region VARCHAR(1024), + PRIMARY KEY (ccdexposure_id), + CONSTRAINT un_exposure_id_detector UNIQUE (exposure_id, detector), + CONSTRAINT fk_exposure_id FOREIGN KEY(exposure_id) REFERENCES exposure (exposure_id) +) + +; + +CREATE TABLE main.visit1_quicklook ( + visit_id BIGINT NOT NULL, + n_inputs INTEGER, + astrom_offset_mean_min FLOAT, + astrom_offset_mean_max FLOAT, + astrom_offset_mean_median FLOAT, + astrom_offset_std_min FLOAT, + astrom_offset_std_max FLOAT, + astrom_offset_std_median FLOAT, + eff_time_min FLOAT, + eff_time_max FLOAT, + eff_time_median FLOAT, + eff_time_psf_sigma_scale_min FLOAT, + eff_time_psf_sigma_scale_max FLOAT, + eff_time_psf_sigma_scale_median FLOAT, + eff_time_sky_bg_scale_min FLOAT, + eff_time_sky_bg_scale_max FLOAT, + eff_time_sky_bg_scale_median FLOAT, + eff_time_zero_point_scale_min FLOAT, + eff_time_zero_point_scale_max FLOAT, + eff_time_zero_point_scale_median FLOAT, + max_dist_to_nearest_psf_min FLOAT, + max_dist_to_nearest_psf_max FLOAT, + max_dist_to_nearest_psf_median FLOAT, + mean_var_min FLOAT, + mean_var_max FLOAT, + mean_var_median FLOAT, + n_psf_star_min INTEGER, + n_psf_star_max INTEGER, + n_psf_star_median INTEGER, + n_psf_star_total INTEGER, + psf_area_min FLOAT, + psf_area_max FLOAT, + psf_area_median FLOAT, + psf_ixx_min FLOAT, + psf_ixx_max FLOAT, + psf_ixx_median FLOAT, + psf_ixy_min FLOAT, + psf_ixy_max FLOAT, + psf_ixy_median FLOAT, + psf_iyy_min FLOAT, + psf_iyy_max FLOAT, + psf_iyy_median FLOAT, + psf_sigma_min FLOAT, + psf_sigma_max FLOAT, + psf_sigma_median FLOAT, + psf_star_delta_e1_median_min FLOAT, + psf_star_delta_e1_median_max FLOAT, + psf_star_delta_e1_median_median FLOAT, + psf_star_delta_e1_scatter_min FLOAT, + psf_star_delta_e1_scatter_max FLOAT, + psf_star_delta_e1_scatter_median FLOAT, + psf_star_delta_e2_median_min FLOAT, + psf_star_delta_e2_median_max FLOAT, + psf_star_delta_e2_median_median FLOAT, + psf_star_delta_e2_scatter_min FLOAT, + psf_star_delta_e2_scatter_max FLOAT, + psf_star_delta_e2_scatter_median FLOAT, + psf_star_delta_size_median_min FLOAT, + psf_star_delta_size_median_max FLOAT, + psf_star_delta_size_median_median FLOAT, + psf_star_delta_size_scatter_min FLOAT, + psf_star_delta_size_scatter_max FLOAT, + psf_star_delta_size_scatter_median FLOAT, + psf_star_scaled_delta_size_scatter_min FLOAT, + psf_star_scaled_delta_size_scatter_max FLOAT, + psf_star_scaled_delta_size_scatter_median FLOAT, + psf_trace_radius_delta_min FLOAT, + psf_trace_radius_delta_max FLOAT, + psf_trace_radius_delta_median FLOAT, + sky_bg_min FLOAT, + sky_bg_max FLOAT, + sky_bg_median FLOAT, + sky_noise_min FLOAT, + sky_noise_max FLOAT, + sky_noise_median FLOAT, + seeing_zenith_500nm_min DOUBLE, + seeing_zenith_500nm_max DOUBLE, + seeing_zenith_500nm_median DOUBLE, + zero_point_min FLOAT, + zero_point_max FLOAT, + zero_point_median FLOAT, + low_snr_source_count_min INTEGER, + low_snr_source_count_max INTEGER, + low_snr_source_count_median INTEGER, + low_snr_source_count_total INTEGER, + high_snr_source_count_min INTEGER, + high_snr_source_count_max INTEGER, + high_snr_source_count_median INTEGER, + high_snr_source_count_total INTEGER, + postisr_pixel_median_median FLOAT, + postisr_pixel_median_mean FLOAT, + postisr_pixel_median_max FLOAT, + PRIMARY KEY (visit_id), + CONSTRAINT fk_visit1_quicklook_obs_id FOREIGN KEY(visit_id) REFERENCES exposure (exposure_id) +) + +; + +CREATE TABLE main.ccdexposure_camera ( + ccdexposure_id BIGINT NOT NULL, + temp_set FLOAT, + ccd_temp FLOAT, + PRIMARY KEY (ccdexposure_id), + CONSTRAINT fk_ccdexposure_camera_ccdexposure_id FOREIGN KEY(ccdexposure_id) REFERENCES ccdexposure (ccdexposure_id) +) + +; + +CREATE TABLE main.ccdexposure_flexdata ( + obs_id BIGINT NOT NULL, + "key" VARCHAR(128) NOT NULL, + value TEXT, + PRIMARY KEY (obs_id, "key"), + CONSTRAINT fk_ccdexposure_flexdata_obs_id FOREIGN KEY(obs_id) REFERENCES ccdexposure (ccdexposure_id), + CONSTRAINT fk_key FOREIGN KEY("key") REFERENCES ccdexposure_flexdata_schema ("key") +) + +; + +CREATE TABLE main.ccdvisit1_quicklook ( + ccdvisit_id BIGINT NOT NULL, + s_ra DOUBLE, + s_dec DOUBLE, + zenith_distance FLOAT, + photometric_calib FLOAT, + psf_sigma FLOAT, + sky_bg FLOAT, + sky_noise FLOAT, + zero_point FLOAT, + seeing_zenith_500nm DOUBLE, + astrom_offset_mean FLOAT, + astrom_offset_std FLOAT, + eff_time FLOAT, + eff_time_psf_sigma_scale FLOAT, + eff_time_sky_bg_scale FLOAT, + eff_time_zero_point_scale FLOAT, + mean_var FLOAT, + n_psf_star INTEGER, + psf_area FLOAT, + psf_ixx FLOAT, + psf_ixy FLOAT, + psf_iyy FLOAT, + psf_star_delta_e1_median FLOAT, + psf_star_delta_e2_median FLOAT, + psf_star_delta_e1_scatter FLOAT, + psf_star_delta_e2_scatter FLOAT, + psf_star_delta_size_median FLOAT, + psf_star_delta_size_scatter FLOAT, + psf_star_scaled_delta_size_scatter FLOAT, + psf_trace_radius_delta FLOAT, + max_dist_to_nearest_psf FLOAT, + postisr_pixel_median FLOAT, + PRIMARY KEY (ccdvisit_id), + CONSTRAINT fk_ccdvisit1_quicklook_obs_id FOREIGN KEY(ccdvisit_id) REFERENCES ccdexposure (ccdexposure_id) +) + +; diff --git a/tests/lsstcomcamsim/exposure.ecsv b/tests/lsstcomcamsim/exposure.ecsv new file mode 100644 index 00000000..843b4d5f --- /dev/null +++ b/tests/lsstcomcamsim/exposure.ecsv @@ -0,0 +1,55 @@ +# %ECSV 1.0 +# --- +# datatype: +# - {name: exposure_id, datatype: int64} +# - {name: exposure_name, datatype: string} +# - {name: controller, datatype: string} +# - {name: day_obs, datatype: int64} +# - {name: seq_num, datatype: int64} +# - {name: physical_filter, datatype: string, subtype: json} +# - {name: band, datatype: string, subtype: json} +# - {name: s_ra, datatype: string, subtype: json} +# - {name: s_dec, datatype: string, subtype: json} +# - {name: sky_rotation, datatype: string, subtype: json} +# - {name: azimuth_start, datatype: string, subtype: json} +# - {name: azimuth_end, datatype: string, subtype: json} +# - {name: azimuth, datatype: string, subtype: json} +# - {name: altitude_start, datatype: string, subtype: json} +# - {name: altitude_end, datatype: string, subtype: json} +# - {name: altitude, datatype: string, subtype: json} +# - {name: zenith_distance_start, datatype: string, subtype: json} +# - {name: zenith_distance_end, datatype: string, subtype: json} +# - {name: zenith_distance, datatype: string, subtype: json} +# - {name: airmass, datatype: string, subtype: json} +# - {name: exp_midpt, datatype: string, nullable: true} +# - {name: exp_midpt_mjd, datatype: string, subtype: json} +# - {name: obs_start, datatype: string, nullable: true} +# - {name: obs_start_mjd, datatype: string, subtype: json} +# - {name: obs_end, datatype: string, nullable: true} +# - {name: obs_end_mjd, datatype: string, subtype: json} +# - {name: exp_time, datatype: string, subtype: json} +# - {name: shut_time, datatype: string, subtype: json} +# - {name: dark_time, datatype: string, subtype: json} +# - {name: group_id, datatype: string, subtype: json} +# - {name: cur_index, datatype: string, subtype: json} +# - {name: max_index, datatype: string, subtype: json} +# - {name: img_type, datatype: string, subtype: json} +# - {name: emulated, datatype: string, subtype: json} +# - {name: science_program, datatype: string, subtype: json} +# - {name: observation_reason, datatype: string, subtype: json} +# - {name: target_name, datatype: string, subtype: json} +# - {name: air_temp, datatype: string, subtype: json} +# - {name: pressure, datatype: string, subtype: json} +# - {name: humidity, datatype: string, subtype: json} +# - {name: wind_speed, datatype: string, subtype: json} +# - {name: wind_dir, datatype: string, subtype: json} +# - {name: dimm_seeing, datatype: string, subtype: json} +# - {name: focus_z, datatype: string, subtype: json} +# - {name: simulated, datatype: string, subtype: json} +# - {name: s_region, datatype: string, subtype: json} +# - {name: vignette, datatype: string, subtype: json} +# - {name: vignette_min, datatype: string, subtype: json} +# schema: astropy-2.0 +exposure_id exposure_name controller day_obs seq_num physical_filter band s_ra s_dec sky_rotation azimuth_start azimuth_end azimuth altitude_start altitude_end altitude zenith_distance_start zenith_distance_end zenith_distance airmass exp_midpt exp_midpt_mjd obs_start obs_start_mjd obs_end obs_end_mjd exp_time shut_time dark_time group_id cur_index max_index img_type emulated science_program observation_reason target_name air_temp pressure humidity wind_speed wind_dir dimm_seeing focus_z simulated s_region vignette vignette_min +7024040300451 CC_S_20240403_000451 S 20240403 451 null null null null null null null null null null null null null null null null null null null null null null null null null null null null null null null null null null null null null null null null null null null + diff --git a/tests/lsstcomcamsim/exposure2.ecsv b/tests/lsstcomcamsim/exposure2.ecsv new file mode 100644 index 00000000..ff965586 --- /dev/null +++ b/tests/lsstcomcamsim/exposure2.ecsv @@ -0,0 +1,81 @@ +# %ECSV 1.0 +# --- +# datatype: +# - {name: exposure_id, datatype: int64} +# - {name: exposure_name, datatype: string} +# - {name: controller, datatype: string} +# - {name: day_obs, datatype: int64} +# - {name: seq_num, datatype: int64} +# - {name: physical_filter, datatype: string} +# - {name: band, datatype: string, subtype: json} +# - {name: s_ra, datatype: string, subtype: json} +# - {name: s_dec, datatype: string, subtype: json} +# - {name: sky_rotation, datatype: string, subtype: json} +# - {name: azimuth_start, datatype: string, subtype: json} +# - {name: azimuth_end, datatype: string, subtype: json} +# - {name: azimuth, datatype: string, subtype: json} +# - {name: altitude_start, datatype: string, subtype: json} +# - {name: altitude_end, datatype: string, subtype: json} +# - {name: altitude, datatype: string, subtype: json} +# - {name: zenith_distance_start, datatype: string, subtype: json} +# - {name: zenith_distance_end, datatype: string, subtype: json} +# - {name: zenith_distance, datatype: string, subtype: json} +# - {name: airmass, datatype: string, subtype: json} +# - {name: exp_midpt, datatype: string} +# - {name: exp_midpt_mjd, datatype: float64} +# - {name: obs_start, datatype: string} +# - {name: obs_start_mjd, datatype: float64} +# - {name: obs_end, datatype: string} +# - {name: obs_end_mjd, datatype: float64} +# - {name: exp_time, datatype: string, subtype: json} +# - {name: shut_time, datatype: string, subtype: json} +# - {name: dark_time, datatype: string, subtype: json} +# - {name: group_id, datatype: string, subtype: json} +# - {name: cur_index, datatype: string, subtype: json} +# - {name: max_index, datatype: string, subtype: json} +# - {name: img_type, datatype: string, subtype: json} +# - {name: emulated, datatype: bool} +# - {name: science_program, datatype: string, subtype: json} +# - {name: observation_reason, datatype: string, subtype: json} +# - {name: target_name, datatype: string, subtype: json} +# - {name: air_temp, datatype: string, subtype: json} +# - {name: pressure, datatype: string, subtype: json} +# - {name: humidity, datatype: string, subtype: json} +# - {name: wind_speed, datatype: string, subtype: json} +# - {name: wind_dir, datatype: string, subtype: json} +# - {name: dimm_seeing, datatype: string, subtype: json} +# - {name: focus_z, datatype: string, subtype: json} +# - {name: simulated, datatype: string, subtype: json} +# - {name: s_region, datatype: string, subtype: json} +# - {name: vignette, datatype: string, subtype: json} +# - {name: vignette_min, datatype: string, subtype: json} +# meta: !!omap +# - __serialized_columns__: +# exp_midpt: +# __class__: astropy.time.core.Time +# format: isot +# in_subfmt: '*' +# out_subfmt: '*' +# precision: 6 +# scale: tai +# value: !astropy.table.SerializedColumn {name: exp_midpt} +# obs_start: +# __class__: astropy.time.core.Time +# format: isot +# in_subfmt: '*' +# out_subfmt: '*' +# precision: 6 +# scale: tai +# value: !astropy.table.SerializedColumn {name: obs_start} +# obs_end: +# __class__: astropy.time.core.Time +# format: isot +# in_subfmt: '*' +# out_subfmt: '*' +# precision: 6 +# scale: tai +# value: !astropy.table.SerializedColumn {name: obs_end} +# schema: astropy-2.0 +exposure_id exposure_name controller day_obs seq_num physical_filter band s_ra s_dec sky_rotation azimuth_start azimuth_end azimuth altitude_start altitude_end altitude zenith_distance_start zenith_distance_end zenith_distance airmass exp_midpt exp_midpt_mjd obs_start obs_start_mjd obs_end obs_end_mjd exp_time shut_time dark_time group_id cur_index max_index img_type emulated science_program observation_reason target_name air_temp pressure humidity wind_speed wind_dir dimm_seeing focus_z simulated s_region vignette vignette_min +7024052800003 CC_S_20240528_000003 S 20240528 3 i_06 null null null null null null null null null null null null null null 2024-05-28T22:18:26.874000 60458.92947770489 2024-05-28T22:18:26.326000 60458.929471360774 2024-05-28T22:18:27.422000 60458.92948404902 null null null null null null null False null null null null null null null null null null null null null null + diff --git a/tests/lsstcomcamsim/visit1_quicklook.ecsv b/tests/lsstcomcamsim/visit1_quicklook.ecsv new file mode 100644 index 00000000..e61168e3 --- /dev/null +++ b/tests/lsstcomcamsim/visit1_quicklook.ecsv @@ -0,0 +1,96 @@ +# %ECSV 1.0 +# --- +# datatype: +# - {name: visit_id, datatype: int64} +# - {name: astrom_offset_mean_min, datatype: float64} +# - {name: astrom_offset_mean_max, datatype: float64} +# - {name: astrom_offset_mean_median, datatype: float64} +# - {name: astrom_offset_std_min, datatype: float64} +# - {name: astrom_offset_std_max, datatype: float64} +# - {name: astrom_offset_std_median, datatype: float64} +# - {name: eff_time_min, datatype: float64} +# - {name: eff_time_max, datatype: float64} +# - {name: eff_time_median, datatype: float64} +# - {name: eff_time_psf_sigma_scale_min, datatype: float64} +# - {name: eff_time_psf_sigma_scale_max, datatype: float64} +# - {name: eff_time_psf_sigma_scale_median, datatype: float64} +# - {name: eff_time_sky_bg_scale_min, datatype: float64} +# - {name: eff_time_sky_bg_scale_max, datatype: float64} +# - {name: eff_time_sky_bg_scale_median, datatype: float64} +# - {name: eff_time_zero_point_scale_min, datatype: float64} +# - {name: eff_time_zero_point_scale_max, datatype: float64} +# - {name: eff_time_zero_point_scale_median, datatype: float64} +# - {name: max_dist_to_nearest_psf_min, datatype: float64} +# - {name: max_dist_to_nearest_psf_max, datatype: float64} +# - {name: max_dist_to_nearest_psf_median, datatype: float64} +# - {name: mean_var_min, datatype: float64} +# - {name: mean_var_max, datatype: float64} +# - {name: mean_var_median, datatype: float64} +# - {name: n_psf_star_min, datatype: int64} +# - {name: n_psf_star_max, datatype: int64} +# - {name: n_psf_star_median, datatype: int64} +# - {name: n_psf_star_total, datatype: int64} +# - {name: psf_area_min, datatype: float64} +# - {name: psf_area_max, datatype: float64} +# - {name: psf_area_median, datatype: float64} +# - {name: psf_ixx_min, datatype: float64} +# - {name: psf_ixx_max, datatype: float64} +# - {name: psf_ixx_median, datatype: float64} +# - {name: psf_ixy_min, datatype: float64} +# - {name: psf_ixy_max, datatype: float64} +# - {name: psf_ixy_median, datatype: float64} +# - {name: psf_iyy_min, datatype: float64} +# - {name: psf_iyy_max, datatype: float64} +# - {name: psf_iyy_median, datatype: float64} +# - {name: psf_sigma_min, datatype: float64} +# - {name: psf_sigma_max, datatype: float64} +# - {name: psf_sigma_median, datatype: float64} +# - {name: psf_star_delta_e1_median_min, datatype: float64} +# - {name: psf_star_delta_e1_median_max, datatype: float64} +# - {name: psf_star_delta_e1_median_median, datatype: float64} +# - {name: psf_star_delta_e1_scatter_min, datatype: float64} +# - {name: psf_star_delta_e1_scatter_max, datatype: float64} +# - {name: psf_star_delta_e1_scatter_median, datatype: float64} +# - {name: psf_star_delta_e2_median_min, datatype: float64} +# - {name: psf_star_delta_e2_median_max, datatype: float64} +# - {name: psf_star_delta_e2_median_median, datatype: float64} +# - {name: psf_star_delta_e2_scatter_min, datatype: float64} +# - {name: psf_star_delta_e2_scatter_max, datatype: float64} +# - {name: psf_star_delta_e2_scatter_median, datatype: float64} +# - {name: psf_star_delta_size_median_min, datatype: float64} +# - {name: psf_star_delta_size_median_max, datatype: float64} +# - {name: psf_star_delta_size_median_median, datatype: float64} +# - {name: psf_star_delta_size_scatter_min, datatype: float64} +# - {name: psf_star_delta_size_scatter_max, datatype: float64} +# - {name: psf_star_delta_size_scatter_median, datatype: float64} +# - {name: psf_star_scaled_delta_size_scatter_min, datatype: float64} +# - {name: psf_star_scaled_delta_size_scatter_max, datatype: float64} +# - {name: psf_star_scaled_delta_size_scatter_median, datatype: float64} +# - {name: psf_trace_radius_delta_min, datatype: float64} +# - {name: psf_trace_radius_delta_max, datatype: float64} +# - {name: psf_trace_radius_delta_median, datatype: float64} +# - {name: sky_bg_min, datatype: float64} +# - {name: sky_bg_max, datatype: float64} +# - {name: sky_bg_median, datatype: float64} +# - {name: sky_noise_min, datatype: float64} +# - {name: sky_noise_max, datatype: float64} +# - {name: sky_noise_median, datatype: float64} +# - {name: zero_point_min, datatype: float64} +# - {name: zero_point_max, datatype: float64} +# - {name: zero_point_median, datatype: float64} +# - {name: low_snr_source_count_min, datatype: string, subtype: json} +# - {name: low_snr_source_count_max, datatype: string, subtype: json} +# - {name: low_snr_source_count_median, datatype: string, subtype: json} +# - {name: low_snr_source_count_total, datatype: string, subtype: json} +# - {name: high_snr_source_count_min, datatype: string, subtype: json} +# - {name: high_snr_source_count_max, datatype: string, subtype: json} +# - {name: high_snr_source_count_median, datatype: string, subtype: json} +# - {name: high_snr_source_count_total, datatype: string, subtype: json} +# - {name: seeing_zenith_500nm_min, datatype: string, subtype: json} +# - {name: seeing_zenith_500nm_max, datatype: string, subtype: json} +# - {name: seeing_zenith_500nm_median, datatype: string, subtype: json} +# - {name: n_inputs, datatype: int64} +# schema: astropy-2.0 +visit_id astrom_offset_mean_min astrom_offset_mean_max astrom_offset_mean_median astrom_offset_std_min astrom_offset_std_max astrom_offset_std_median eff_time_min eff_time_max eff_time_median eff_time_psf_sigma_scale_min eff_time_psf_sigma_scale_max eff_time_psf_sigma_scale_median eff_time_sky_bg_scale_min eff_time_sky_bg_scale_max eff_time_sky_bg_scale_median eff_time_zero_point_scale_min eff_time_zero_point_scale_max eff_time_zero_point_scale_median max_dist_to_nearest_psf_min max_dist_to_nearest_psf_max max_dist_to_nearest_psf_median mean_var_min mean_var_max mean_var_median n_psf_star_min n_psf_star_max n_psf_star_median n_psf_star_total psf_area_min psf_area_max psf_area_median psf_ixx_min psf_ixx_max psf_ixx_median psf_ixy_min psf_ixy_max psf_ixy_median psf_iyy_min psf_iyy_max psf_iyy_median psf_sigma_min psf_sigma_max psf_sigma_median psf_star_delta_e1_median_min psf_star_delta_e1_median_max psf_star_delta_e1_median_median psf_star_delta_e1_scatter_min psf_star_delta_e1_scatter_max psf_star_delta_e1_scatter_median psf_star_delta_e2_median_min psf_star_delta_e2_median_max psf_star_delta_e2_median_median psf_star_delta_e2_scatter_min psf_star_delta_e2_scatter_max psf_star_delta_e2_scatter_median psf_star_delta_size_median_min psf_star_delta_size_median_max psf_star_delta_size_median_median psf_star_delta_size_scatter_min psf_star_delta_size_scatter_max psf_star_delta_size_scatter_median psf_star_scaled_delta_size_scatter_min psf_star_scaled_delta_size_scatter_max psf_star_scaled_delta_size_scatter_median psf_trace_radius_delta_min psf_trace_radius_delta_max psf_trace_radius_delta_median sky_bg_min sky_bg_max sky_bg_median sky_noise_min sky_noise_max sky_noise_median zero_point_min zero_point_max zero_point_median low_snr_source_count_min low_snr_source_count_max low_snr_source_count_median low_snr_source_count_total high_snr_source_count_min high_snr_source_count_max high_snr_source_count_median high_snr_source_count_total seeing_zenith_500nm_min seeing_zenith_500nm_max seeing_zenith_500nm_median n_inputs +7024040300451 0.007426047697663307 0.007426047697663307 0.007426047697663307 0.0040731728076934814 0.0040731728076934814 0.0040731728076934814 36.148712158203125 36.148712158203125 36.148712158203125 1.3490833044052124 1.3490833044052124 1.3490833044052124 0.8557785749435425 0.8557785749435425 0.8557785749435425 1.042473316192627 1.042473316192627 1.042473316192627 514.4628295898438 514.4628295898438 514.4628295898438 323.9481201171875 323.9481201171875 323.9481201171875 209 209 209 1902 33.13351821899414 33.13351821899414 33.13351821899414 2.22502064704895 2.22502064704895 2.22502064704895 -0.03166781738400459 -0.03166781738400459 -0.03166781738400459 2.159698009490967 2.159698009490967 2.159698009490967 1.480843186378479 1.480843186378479 1.480843186378479 0.00036173660191707313 0.00036173660191707313 0.00036173660191707313 0.010656812228262424 0.010656812228262424 0.010656812228262424 -9.827991743804887e-05 -9.827991743804887e-05 -9.827991743804887e-05 0.010869131423532963 0.010869131423532963 0.010869131423532963 -0.000794879742898047 -0.000794879742898047 -0.000794879742898047 0.0121929831802845 0.0121929831802845 0.0121929831802845 0.008244113996624947 0.008244113996624947 0.008244113996624947 0.012445911765098572 0.012445911765098572 0.012445911765098572 327.421142578125 327.421142578125 327.421142578125 18.019407272338867 18.019407272338867 18.019407272338867 31.395383834838867 31.395383834838867 31.395383834838867 null null null null null null null null null null null 9 + diff --git a/tests/test_pqserver.py b/tests/test_pqserver.py index 7c163b30..7484f7e5 100644 --- a/tests/test_pqserver.py +++ b/tests/test_pqserver.py @@ -1,9 +1,15 @@ +import datetime import os import shutil import sqlite3 import tempfile from pathlib import Path +from astropy.table import Table +from astropy.time import Time +import numpy as np +import sqlalchemy as sa + import pytest from fastapi.testclient import TestClient from requests import Response @@ -52,6 +58,77 @@ def db(tmpdir, scope="module"): return db_path +@pytest.fixture +def lsstcomcamsim(tmpdir, request, scope="module"): + schema = "cdb_lsstcomcamsim" + db_path = tmpdir / "test.db" + schema_path = tmpdir / f"{schema}.db" + + data_path = Path(__file__).parent / "lsstcomcamsim" + sql_path = data_path / f"{schema}.sql" + + # Build the main db file to specify where to look for + # specific schemas. + with sqlite3.connect(db_path) as conn: + conn.execute("CREATE TABLE schemas (name text, path text)") + conn.execute(f"INSERT INTO schemas VALUES ('{schema}', '{schema_path}')") + + # Set up the lsstcomcamsim schema with the schema from the SQL file + # generated with felis, and with sample data interrogated from consdb, + # and with fake made up flex data + with sqlite3.connect(schema_path) as conn: + conn.executescript(sql_path.read_text()) + import time + time.sleep(10) + + engine = sa.create_engine(f"sqlite:///{schema_path}") + + table_dict = { + "exposure2.ecsv": "exposure", + "ccdexposure.ecsv": "ccdexposure", + "ccdvisit1_quicklook.ecsv": "ccdvisit1_quicklook", + "exposure.ecsv": "exposure", + "visit1_quicklook.ecsv": "visit1_quicklook", + } + + request.astropy_tables = dict() + for file_name, table_name in table_dict.items(): + astropy_table = Table.read(data_path / file_name) + request.astropy_tables[file_name] = astropy_table + metadata = sa.MetaData() + + sql_table = sa.Table(table_name, metadata, autoload_with=engine) + + # Convert the Astropy table to a list of dictionaries, one per row + rows = [ + { + k: ( + None if v is None or v == "null" else + v.to_datetime() if isinstance(v, Time) else + bool(v) if isinstance(v, np.bool_) else + str(v) + ) + for k, v in dict(row).items() + } + for row in astropy_table + ] + + # Insert rows into the SQL table + with engine.begin() as connection: + connection.execute(sql_table.insert(), rows) + + engine.dispose() + + print(f"{db_path=}") + os.environ["POSTGRES_URL"] = f"sqlite:///{db_path}" + from lsst.consdb import pqserver, utils + #pqserver.engine = utils.setup_postgres() + try: + yield TestClient(pqserver.app) + finally: + pqserver.engine.dispose() + + @pytest.fixture def app(db, scope="module"): os.environ["POSTGRES_URL"] = f"sqlite:///{db}" @@ -88,6 +165,185 @@ def test_root2(client): assert "dtypes" in result assert set(result["dtypes"]) == {"bool", "int", "float", "str"} +def test_insert_multiple(lsstcomcamsim, request): + data = { + 7024052800012: { + "exposure_name": "CC_S_20240528_000012", + "controller": "S", + "day_obs": 20240528, + "seq_num": 12, + "physical_filter": "i_06", + "exp_midpt": "2024-05-28T22:19:04.847500", + "exp_midpt_mjd": 60458.92991721521, + "obs_start": "2024-05-28T22:19:04.300000", + "obs_start_mjd": 60458.92991088214, + "obs_end": "2024-05-28T22:19:05.395000", + "obs_end_mjd": 60458.929923548276, + "emulated": False, + }, + 7024052800011: + { + "exposure_name": "CC_S_20240528_000011", + "controller": "S", + "day_obs": 20240528, + "seq_num": 11, + "physical_filter": "i_06", + "exp_midpt": "2024-05-28T22:19:00.632500", + "exp_midpt_mjd": 60458.929868432606, + "obs_start": "2024-05-28T22:19:00.086000", + "obs_start_mjd": 60458.929862109704, + "obs_end": "2024-05-28T22:19:01.179000", + "obs_end_mjd": 60458.92987475551, + "emulated": False, + }, + } + response = lsstcomcamsim.post( + "/consdb/insert/lsstcomcamsim/exposure", + json={"obs_dict": data}, + ) + _assert_http_status(response, 200) + result = response.json() + assert "Data inserted" in result["message"] + assert result["table"] == "exposure" + assert result["instrument"] == "lsstcomcamsim" + assert result["obs_id"] == list(data.keys()) + + +def test_insert_multiple_update(lsstcomcamsim, request): + data = { + 7024052800012: { + "exposure_name": "CC_S_20240528_000012", + "controller": "S", + "day_obs": 20240528, + "seq_num": 12, + "physical_filter": "i_06", + "exp_midpt": "2024-05-28T22:19:04.847500", + "exp_midpt_mjd": 60458.92991721521, + "obs_start": "2024-05-28T22:19:04.300000", + "obs_start_mjd": 60458.92991088214, + "obs_end": "2024-05-28T22:19:05.395000", + "obs_end_mjd": 60458.929923548276, + "emulated": False, + }, + 7024052800011: + { + "exposure_name": "CC_S_20240528_000011", + "controller": "S", + "day_obs": 20240528, + "seq_num": 11, + "physical_filter": "i_06", + "exp_midpt": "2024-05-28T22:19:00.632500", + "exp_midpt_mjd": 60458.929868432606, + "obs_start": "2024-05-28T22:19:00.086000", + "obs_start_mjd": 60458.929862109704, + "obs_end": "2024-05-28T22:19:01.179000", + "obs_end_mjd": 60458.92987475551, + "emulated": False, + }, + } + response = lsstcomcamsim.post( + "/consdb/insert/lsstcomcamsim/exposure", + json={"obs_dict": data}, + ) + _assert_http_status(response, 200) + + data[7024052800012]["exposure_name"] = "fred" + data[7024052800011]["exposure_name"] = "sally" + response = lsstcomcamsim.post( + "/consdb/insert/lsstcomcamsim/exposure", + json={"obs_dict": data}, + ) + _assert_http_status(response, 500) + + + response = lsstcomcamsim.post( + "/consdb/insert/lsstcomcamsim/exposure?u=1", + json={"obs_dict": data}, + ) + _assert_http_status(response, 200) + assert response.json()["obs_id"] == list(data.keys()) + + +def test_schema(lsstcomcamsim, request): + response = lsstcomcamsim.get( + "/consdb/schema/lsstcomcamsim" + ) + _assert_http_status(response, 200) + + +def test_schema_non_instrument(lsstcomcamsim, request): + response = lsstcomcamsim.get( + "/consdb/schema/asdf" + ) + _assert_http_status(response, 404) + result = response.json() + assert "Unknown instrument" in result["message"] + + +def test_schema_instrument(lsstcomcamsim, request): + response = lsstcomcamsim.get( + "/consdb/schema/lsstcomcamsim" + ) + _assert_http_status(response, 200) + result = response.json() + assert isinstance(result, list) + tables = [ + "exposure", + "ccdexposure", + "ccdexposure_camera", + "ccdvisit1_quicklook", + "visit1_quicklook", + "exposure_flexdata", + "exposure_flexdata_schema", + "ccdexposure_flexdata", + "ccdexposure_flexdata_schema", + ] + tables = [f"cdb_lsstcomcamsim.{t}" for t in tables] + for t in tables: + assert t in result + + +def test_schema_non_table(lsstcomcamsim, request): + response = lsstcomcamsim.get( + "/consdb/schema/lsstcomcamsim/asdf" + ) + _assert_http_status(response, 404) + assert "Unknown table" in response.json()["message"] + + +def test_schema_table(lsstcomcamsim, request): + response = lsstcomcamsim.get( + "/consdb/schema/lsstcomcamsim/exposure" + ) + astropy_table = request.astropy_tables["exposure.ecsv"] + _assert_http_status(response, 200) + result = response.json() + for column in astropy_table.columns: + assert column in result.keys() + + +def test_validate_unit(): + os.environ["POSTGRES_URL"] = "sqlite://" + from lsst.consdb import pqserver + assert pqserver.AddKeyRequestModel.validate_unit("s") == "s" + assert pqserver.AddKeyRequestModel.validate_unit("km/s") == "km/s" + assert pqserver.AddKeyRequestModel.validate_unit("km s-1") == "km s-1" + + with pytest.raises(ValueError): + pqserver.AddKeyRequestModel.validate_unit("tacos / s") + +def test_validate_ucd(): + os.environ["POSTGRES_URL"] = "sqlite://" + from lsst.consdb import pqserver + assert pqserver.AddKeyRequestModel.validate_ucd("this.is.a.valid.ucd") + assert pqserver.AddKeyRequestModel.validate_ucd("THIS-ONE.IS.TOO") + + with pytest.raises(ValueError): + pqserver.AddKeyRequestModel.validate_ucd("but this is not") + + with pytest.raises(ValueError): + pqserver.AddKeyRequestModel.validate_ucd("neither#is@this[one]!") + def test_flexible_metadata(client): response = client.post( From da385f46929074683941b769c44d80e67994d647 Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Wed, 28 Aug 2024 15:53:49 -0700 Subject: [PATCH 02/10] Fix failing tests --- python/lsst/consdb/pqserver.py | 12 +++-- tests/test_pqserver.py | 83 +++++++++++++++++----------------- 2 files changed, 51 insertions(+), 44 deletions(-) diff --git a/python/lsst/consdb/pqserver.py b/python/lsst/consdb/pqserver.py index 92b2d05b..bb62cbec 100644 --- a/python/lsst/consdb/pqserver.py +++ b/python/lsst/consdb/pqserver.py @@ -105,7 +105,7 @@ def _missing_(cls, value): docs_url=f"{path_prefix}/docs", redoc_url=f"{path_prefix}/redoc", ) -engine = setup_postgres() +engine = None logger = setup_logging(__name__) ######################## @@ -135,7 +135,8 @@ def __init__(self): # Find all timestamp columns in the table self.timestamp_columns[table] = set( [ - column.name for column in md.tables[table].columns + column.name + for column in md.tables[table].columns if isinstance(column.type, sqlalchemy.DateTime) ] ) @@ -317,7 +318,7 @@ def compute_wide_view_name(self, instrument: str, obs_type: str) -> str: return view_name -instrument_tables = InstrumentTables() +instrument_tables = None ################## @@ -973,3 +974,8 @@ def schema( if table not in schema.tables: raise BadValueException("table", table, list(schema.tables.keys())) return {c.name: [str(c.type), c.doc] for c in schema.tables[table].columns} + + +if __name__ == "__main__": + engine = setup_postgres() + instrument_tables = InstrumentTables() diff --git a/tests/test_pqserver.py b/tests/test_pqserver.py index 7484f7e5..a7d8dc2f 100644 --- a/tests/test_pqserver.py +++ b/tests/test_pqserver.py @@ -1,16 +1,14 @@ -import datetime import os import shutil import sqlite3 import tempfile from pathlib import Path -from astropy.table import Table -from astropy.time import Time import numpy as np -import sqlalchemy as sa - import pytest +import sqlalchemy as sa +from astropy.table import Table +from astropy.time import Time from fastapi.testclient import TestClient from requests import Response @@ -59,7 +57,13 @@ def db(tmpdir, scope="module"): @pytest.fixture -def lsstcomcamsim(tmpdir, request, scope="module"): +def astropy_tables(scope="module"): + t = dict() + return t + + +@pytest.fixture +def lsstcomcamsim(tmpdir, astropy_tables, scope="module"): schema = "cdb_lsstcomcamsim" db_path = tmpdir / "test.db" schema_path = tmpdir / f"{schema}.db" @@ -79,6 +83,7 @@ def lsstcomcamsim(tmpdir, request, scope="module"): with sqlite3.connect(schema_path) as conn: conn.executescript(sql_path.read_text()) import time + time.sleep(10) engine = sa.create_engine(f"sqlite:///{schema_path}") @@ -91,10 +96,9 @@ def lsstcomcamsim(tmpdir, request, scope="module"): "visit1_quicklook.ecsv": "visit1_quicklook", } - request.astropy_tables = dict() for file_name, table_name in table_dict.items(): astropy_table = Table.read(data_path / file_name) - request.astropy_tables[file_name] = astropy_table + astropy_tables[file_name] = astropy_table metadata = sa.MetaData() sql_table = sa.Table(table_name, metadata, autoload_with=engine) @@ -103,10 +107,13 @@ def lsstcomcamsim(tmpdir, request, scope="module"): rows = [ { k: ( - None if v is None or v == "null" else - v.to_datetime() if isinstance(v, Time) else - bool(v) if isinstance(v, np.bool_) else - str(v) + None + if v is None or v == "null" + else ( + v.to_datetime() + if isinstance(v, Time) + else bool(v) if isinstance(v, np.bool_) else str(v) + ) ) for k, v in dict(row).items() } @@ -122,7 +129,9 @@ def lsstcomcamsim(tmpdir, request, scope="module"): print(f"{db_path=}") os.environ["POSTGRES_URL"] = f"sqlite:///{db_path}" from lsst.consdb import pqserver, utils - #pqserver.engine = utils.setup_postgres() + + pqserver.engine = utils.setup_postgres() + pqserver.instrument_tables = pqserver.InstrumentTables() try: yield TestClient(pqserver.app) finally: @@ -135,6 +144,7 @@ def app(db, scope="module"): from lsst.consdb import pqserver, utils pqserver.engine = utils.setup_postgres() + pqserver.instrument_tables = pqserver.InstrumentTables() try: yield pqserver.app finally: @@ -165,7 +175,8 @@ def test_root2(client): assert "dtypes" in result assert set(result["dtypes"]) == {"bool", "int", "float", "str"} -def test_insert_multiple(lsstcomcamsim, request): + +def test_insert_multiple(lsstcomcamsim): data = { 7024052800012: { "exposure_name": "CC_S_20240528_000012", @@ -181,8 +192,7 @@ def test_insert_multiple(lsstcomcamsim, request): "obs_end_mjd": 60458.929923548276, "emulated": False, }, - 7024052800011: - { + 7024052800011: { "exposure_name": "CC_S_20240528_000011", "controller": "S", "day_obs": 20240528, @@ -209,7 +219,7 @@ def test_insert_multiple(lsstcomcamsim, request): assert result["obs_id"] == list(data.keys()) -def test_insert_multiple_update(lsstcomcamsim, request): +def test_insert_multiple_update(lsstcomcamsim): data = { 7024052800012: { "exposure_name": "CC_S_20240528_000012", @@ -225,8 +235,7 @@ def test_insert_multiple_update(lsstcomcamsim, request): "obs_end_mjd": 60458.929923548276, "emulated": False, }, - 7024052800011: - { + 7024052800011: { "exposure_name": "CC_S_20240528_000011", "controller": "S", "day_obs": 20240528, @@ -255,7 +264,6 @@ def test_insert_multiple_update(lsstcomcamsim, request): ) _assert_http_status(response, 500) - response = lsstcomcamsim.post( "/consdb/insert/lsstcomcamsim/exposure?u=1", json={"obs_dict": data}, @@ -264,26 +272,20 @@ def test_insert_multiple_update(lsstcomcamsim, request): assert response.json()["obs_id"] == list(data.keys()) -def test_schema(lsstcomcamsim, request): - response = lsstcomcamsim.get( - "/consdb/schema/lsstcomcamsim" - ) +def test_schema(lsstcomcamsim): + response = lsstcomcamsim.get("/consdb/schema/lsstcomcamsim") _assert_http_status(response, 200) -def test_schema_non_instrument(lsstcomcamsim, request): - response = lsstcomcamsim.get( - "/consdb/schema/asdf" - ) +def test_schema_non_instrument(lsstcomcamsim): + response = lsstcomcamsim.get("/consdb/schema/asdf") _assert_http_status(response, 404) result = response.json() assert "Unknown instrument" in result["message"] -def test_schema_instrument(lsstcomcamsim, request): - response = lsstcomcamsim.get( - "/consdb/schema/lsstcomcamsim" - ) +def test_schema_instrument(lsstcomcamsim): + response = lsstcomcamsim.get("/consdb/schema/lsstcomcamsim") _assert_http_status(response, 200) result = response.json() assert isinstance(result, list) @@ -303,19 +305,15 @@ def test_schema_instrument(lsstcomcamsim, request): assert t in result -def test_schema_non_table(lsstcomcamsim, request): - response = lsstcomcamsim.get( - "/consdb/schema/lsstcomcamsim/asdf" - ) +def test_schema_non_table(lsstcomcamsim): + response = lsstcomcamsim.get("/consdb/schema/lsstcomcamsim/asdf") _assert_http_status(response, 404) assert "Unknown table" in response.json()["message"] -def test_schema_table(lsstcomcamsim, request): - response = lsstcomcamsim.get( - "/consdb/schema/lsstcomcamsim/exposure" - ) - astropy_table = request.astropy_tables["exposure.ecsv"] +def test_schema_table(lsstcomcamsim, astropy_tables): + astropy_table = astropy_tables["exposure.ecsv"] + response = lsstcomcamsim.get("/consdb/schema/lsstcomcamsim/exposure") _assert_http_status(response, 200) result = response.json() for column in astropy_table.columns: @@ -325,6 +323,7 @@ def test_schema_table(lsstcomcamsim, request): def test_validate_unit(): os.environ["POSTGRES_URL"] = "sqlite://" from lsst.consdb import pqserver + assert pqserver.AddKeyRequestModel.validate_unit("s") == "s" assert pqserver.AddKeyRequestModel.validate_unit("km/s") == "km/s" assert pqserver.AddKeyRequestModel.validate_unit("km s-1") == "km s-1" @@ -332,9 +331,11 @@ def test_validate_unit(): with pytest.raises(ValueError): pqserver.AddKeyRequestModel.validate_unit("tacos / s") + def test_validate_ucd(): os.environ["POSTGRES_URL"] = "sqlite://" from lsst.consdb import pqserver + assert pqserver.AddKeyRequestModel.validate_ucd("this.is.a.valid.ucd") assert pqserver.AddKeyRequestModel.validate_ucd("THIS-ONE.IS.TOO") From 380ad95d778e28ae8c4c849bbd1f007e0ba5a44b Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Thu, 29 Aug 2024 09:03:14 -0700 Subject: [PATCH 03/10] Fix typo --- python/lsst/consdb/pqserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lsst/consdb/pqserver.py b/python/lsst/consdb/pqserver.py index bb62cbec..b5b1d001 100644 --- a/python/lsst/consdb/pqserver.py +++ b/python/lsst/consdb/pqserver.py @@ -302,7 +302,7 @@ def compute_wide_view_name(self, instrument: str, obs_type: str) -> str: Returns ------- - view_nae: `str` + view_name: `str` Name of the appropriate wide view. """ instrument = instrument.lower() From 914f123550bef098fd957de9853490edd0d5655f Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Fri, 6 Sep 2024 09:45:08 -0700 Subject: [PATCH 04/10] Remove sleep during test --- tests/test_pqserver.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_pqserver.py b/tests/test_pqserver.py index a7d8dc2f..4316e524 100644 --- a/tests/test_pqserver.py +++ b/tests/test_pqserver.py @@ -82,9 +82,6 @@ def lsstcomcamsim(tmpdir, astropy_tables, scope="module"): # and with fake made up flex data with sqlite3.connect(schema_path) as conn: conn.executescript(sql_path.read_text()) - import time - - time.sleep(10) engine = sa.create_engine(f"sqlite:///{schema_path}") From c9e0e6335c2b6809fda6e404a8eb356daa5d4e55 Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Fri, 6 Sep 2024 09:45:49 -0700 Subject: [PATCH 05/10] Convert inline comment for function to a docstring Co-authored-by: Valerie --- python/lsst/consdb/hinfo.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/lsst/consdb/hinfo.py b/python/lsst/consdb/hinfo.py index 0ac4dc45..dbf7c4c5 100644 --- a/python/lsst/consdb/hinfo.py +++ b/python/lsst/consdb/hinfo.py @@ -399,7 +399,12 @@ def process_resource(resource: ResourcePath, instrument_dict: dict, update: bool def process_local_path(path: str) -> None: - # If a directory is provided on the command line, process all yaml + """ If a directory is provided on the command line, process all yaml files recursively + + Parameters + ----------- + path : `str` + Path to directory that contains yaml files # files recursively. if os.path.isdir(path): for _, _, files in os.walk(path): From 84185ac58364a0395009f3fe3ef78c0c579e52e2 Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Fri, 6 Sep 2024 09:47:10 -0700 Subject: [PATCH 06/10] ... and even include a close quote --- python/lsst/consdb/hinfo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/lsst/consdb/hinfo.py b/python/lsst/consdb/hinfo.py index dbf7c4c5..52fb140c 100644 --- a/python/lsst/consdb/hinfo.py +++ b/python/lsst/consdb/hinfo.py @@ -399,13 +399,13 @@ def process_resource(resource: ResourcePath, instrument_dict: dict, update: bool def process_local_path(path: str) -> None: - """ If a directory is provided on the command line, process all yaml files recursively + """If a directory is provided on the command line, process all yaml files recursively. Parameters ----------- path : `str` - Path to directory that contains yaml files - # files recursively. + Path to directory that contains yaml files. + """ if os.path.isdir(path): for _, _, files in os.walk(path): for file in files: From 6106d1426009d8fbc6d65aa9cfedc9cbf4be13f5 Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Fri, 6 Sep 2024 09:52:13 -0700 Subject: [PATCH 07/10] Tweak the docstring --- python/lsst/consdb/hinfo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lsst/consdb/hinfo.py b/python/lsst/consdb/hinfo.py index 52fb140c..81763c50 100644 --- a/python/lsst/consdb/hinfo.py +++ b/python/lsst/consdb/hinfo.py @@ -399,8 +399,8 @@ def process_resource(resource: ResourcePath, instrument_dict: dict, update: bool def process_local_path(path: str) -> None: - """If a directory is provided on the command line, process all yaml files recursively. - + """Processes all yaml files in the specified path recursively. + Parameters ----------- path : `str` From 34bae8da18a31cda4b7ff9ddb14ba2b5d6c93e46 Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Fri, 6 Sep 2024 16:48:52 -0700 Subject: [PATCH 08/10] Convert to felis/postgresql for testing h/t Jeremy McCormick --- Dockerfile.pytest | 2 +- tests/test_pqserver.py | 119 ++++++++++++++++++++--------------------- 2 files changed, 58 insertions(+), 63 deletions(-) diff --git a/Dockerfile.pytest b/Dockerfile.pytest index edab7cd2..c6ad6895 100644 --- a/Dockerfile.pytest +++ b/Dockerfile.pytest @@ -2,7 +2,7 @@ ARG OBS_LSST_VERSION=w_2024_21 FROM lsstsqre/centos:7-stack-lsst_distrib-${OBS_LSST_VERSION} USER lsst RUN source loadLSST.bash && mamba install aiokafka httpx -RUN source loadLSST.bash && pip install kafkit aiokafka httpx pytest-asyncio +RUN source loadLSST.bash && pip install kafkit aiokafka httpx pytest-asyncio testing.postgresql WORKDIR /home/lsst/ COPY --chown=lsst . ./consdb/ diff --git a/tests/test_pqserver.py b/tests/test_pqserver.py index 4316e524..9fdb7f85 100644 --- a/tests/test_pqserver.py +++ b/tests/test_pqserver.py @@ -11,6 +11,14 @@ from astropy.time import Time from fastapi.testclient import TestClient from requests import Response +import yaml + +from felis.datamodel import Schema +from felis.db.utils import DatabaseContext +from felis.metadata import MetaDataBuilder +from felis.tests.postgresql import setup_postgres_test_db +from lsst.consdb import pqserver +import lsst.utils def _assert_http_status(response: Response, status: int): @@ -64,75 +72,62 @@ def astropy_tables(scope="module"): @pytest.fixture def lsstcomcamsim(tmpdir, astropy_tables, scope="module"): - schema = "cdb_lsstcomcamsim" - db_path = tmpdir / "test.db" - schema_path = tmpdir / f"{schema}.db" - + schema_name = "cdb_lsstcomcamsim" data_path = Path(__file__).parent / "lsstcomcamsim" - sql_path = data_path / f"{schema}.sql" - - # Build the main db file to specify where to look for - # specific schemas. - with sqlite3.connect(db_path) as conn: - conn.execute("CREATE TABLE schemas (name text, path text)") - conn.execute(f"INSERT INTO schemas VALUES ('{schema}', '{schema_path}')") - - # Set up the lsstcomcamsim schema with the schema from the SQL file - # generated with felis, and with sample data interrogated from consdb, - # and with fake made up flex data - with sqlite3.connect(schema_path) as conn: - conn.executescript(sql_path.read_text()) - - engine = sa.create_engine(f"sqlite:///{schema_path}") - - table_dict = { - "exposure2.ecsv": "exposure", - "ccdexposure.ecsv": "ccdexposure", - "ccdvisit1_quicklook.ecsv": "ccdvisit1_quicklook", - "exposure.ecsv": "exposure", - "visit1_quicklook.ecsv": "visit1_quicklook", - } - for file_name, table_name in table_dict.items(): - astropy_table = Table.read(data_path / file_name) - astropy_tables[file_name] = astropy_table - metadata = sa.MetaData() - - sql_table = sa.Table(table_name, metadata, autoload_with=engine) - - # Convert the Astropy table to a list of dictionaries, one per row - rows = [ - { - k: ( - None - if v is None or v == "null" - else ( - v.to_datetime() - if isinstance(v, Time) - else bool(v) if isinstance(v, np.bool_) else str(v) + schema_file = os.path.join(lsst.utils.getPackageDir("sdm_schemas"), "yml", schema_name + ".yaml") + + with open(schema_file) as f: + yaml_data = yaml.safe_load(f) + schema = Schema.model_validate(yaml_data) + md = MetaDataBuilder(schema).build() + + with setup_postgres_test_db() as instance: + context = DatabaseContext(md, instance.engine) + print(f"{type(instance.engine)=}") + context.initialize() + context.create_all() + + table_dict = { + "exposure.ecsv": "exposure", + "exposure2.ecsv": "exposure", + "ccdexposure.ecsv": "ccdexposure", + "ccdvisit1_quicklook.ecsv": "ccdvisit1_quicklook", + "visit1_quicklook.ecsv": "visit1_quicklook", + } + + for file_name, table_name in table_dict.items(): + astropy_table = Table.read(data_path / file_name) + astropy_tables[file_name] = astropy_table + metadata = sa.MetaData() + + sql_table = sa.Table(table_name, metadata, schema=schema_name, autoload_with=instance.engine) + + # Convert the Astropy table to a list of dictionaries, one per row + rows = [ + { + k: ( + None + if v is None or v == "null" + else ( + v.to_datetime() + if isinstance(v, Time) + else bool(v) if isinstance(v, np.bool_) else str(v) + ) ) - ) - for k, v in dict(row).items() - } - for row in astropy_table - ] + for k, v in dict(row).items() + } + for row in astropy_table + ] - # Insert rows into the SQL table - with engine.begin() as connection: - connection.execute(sql_table.insert(), rows) + # Insert rows into the SQL table + with instance.engine.begin() as connection: + connection.execute(sql_table.insert(), rows) - engine.dispose() + pqserver.engine = instance.engine + pqserver.instrument_tables = pqserver.InstrumentTables() - print(f"{db_path=}") - os.environ["POSTGRES_URL"] = f"sqlite:///{db_path}" - from lsst.consdb import pqserver, utils - - pqserver.engine = utils.setup_postgres() - pqserver.instrument_tables = pqserver.InstrumentTables() - try: yield TestClient(pqserver.app) - finally: - pqserver.engine.dispose() @pytest.fixture From bc3326d000e62c2e7f6f66aaa53378aeec34e090 Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Fri, 6 Sep 2024 16:51:15 -0700 Subject: [PATCH 09/10] Convert to felis/postgresql for testing h/t Jeremy McCormick --- tests/test_pqserver.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_pqserver.py b/tests/test_pqserver.py index 9fdb7f85..bea25925 100644 --- a/tests/test_pqserver.py +++ b/tests/test_pqserver.py @@ -4,21 +4,20 @@ import tempfile from pathlib import Path +import lsst.utils import numpy as np import pytest import sqlalchemy as sa +import yaml from astropy.table import Table from astropy.time import Time from fastapi.testclient import TestClient -from requests import Response -import yaml - from felis.datamodel import Schema from felis.db.utils import DatabaseContext from felis.metadata import MetaDataBuilder from felis.tests.postgresql import setup_postgres_test_db from lsst.consdb import pqserver -import lsst.utils +from requests import Response def _assert_http_status(response: Response, status: int): From 7d4794739a7d439b39ff94b0b176d6737ea6a6aa Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Mon, 9 Sep 2024 16:16:17 -0700 Subject: [PATCH 10/10] Additions to the docker file to test with felis --- Dockerfile.pytest | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Dockerfile.pytest b/Dockerfile.pytest index c6ad6895..12f6ada1 100644 --- a/Dockerfile.pytest +++ b/Dockerfile.pytest @@ -1,12 +1,18 @@ -ARG OBS_LSST_VERSION=w_2024_21 +ARG OBS_LSST_VERSION=w_2024_32 FROM lsstsqre/centos:7-stack-lsst_distrib-${OBS_LSST_VERSION} +USER root + +# testing.postgresql looks for pg in /usr/local... +RUN yum install -y postgresql-server postgresql && rmdir /usr/local/bin && ln -s /usr/bin /usr/local + USER lsst -RUN source loadLSST.bash && mamba install aiokafka httpx -RUN source loadLSST.bash && pip install kafkit aiokafka httpx pytest-asyncio testing.postgresql +RUN source loadLSST.bash && mamba install -y aiokafka httpx +RUN source loadLSST.bash && pip install kafkit aiokafka httpx pytest-asyncio testing.postgresql lsst-felis WORKDIR /home/lsst/ + COPY --chown=lsst . ./consdb/ WORKDIR /home/lsst/consdb/ RUN source /opt/lsst/software/stack/loadLSST.bash && pip install -e . -ENTRYPOINT [ "/bin/bash", "-c", "source /opt/lsst/software/stack/loadLSST.bash; setup obs_lsst; pytest ." ] +ENTRYPOINT [ "/bin/bash", "-c", "source /opt/lsst/software/stack/loadLSST.bash; setup obs_lsst; setup felis; pytest ." ]