From cbbe269f5c3ed6a72255d0fce42713e73302b36e Mon Sep 17 00:00:00 2001 From: Meg Schwamb Date: Wed, 24 Apr 2024 10:53:06 +0100 Subject: [PATCH 01/19] Smoke tests (#108) * Update smoke-test.yml * Update testing-and-coverage.yml --- .github/workflows/smoke-test.yml | 2 +- .github/workflows/testing-and-coverage.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml index 9bd50e2..128ed0f 100644 --- a/.github/workflows/smoke-test.yml +++ b/.github/workflows/smoke-test.yml @@ -20,7 +20,7 @@ jobs: strategy: matrix: os: ['macos-latest','ubuntu-latest'] - python-version: ['3.9', '3.10', '3.11'] + python-version: ['3.10', '3.11', '3.12'] runs-on: ${{ matrix.os }} steps: diff --git a/.github/workflows/testing-and-coverage.yml b/.github/workflows/testing-and-coverage.yml index c39f18d..de5d7d4 100644 --- a/.github/workflows/testing-and-coverage.yml +++ b/.github/workflows/testing-and-coverage.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: os: ['macos-latest','ubuntu-latest'] - python-version: ['3.9', '3.10', '3.11'] + python-version: ['3.10', '3.11', '3.12'] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 From ccc2731b50b124d428efd2e2229c27800bf309d0 Mon Sep 17 00:00:00 2001 From: Stephanie Merritt Date: Wed, 24 Apr 2024 13:58:56 +0000 Subject: [PATCH 02/19] Suppressing warning, code tidyup --- src/adler/adler.py | 7 ++- src/adler/dataclasses/MPCORB.py | 51 ++++++++----------- src/adler/dataclasses/Observations.py | 45 +++++++---------- src/adler/dataclasses/SSObject.py | 53 +++++++------------- src/adler/dataclasses/dataclass_utilities.py | 49 +++++++++++------- 5 files changed, 94 insertions(+), 111 deletions(-) diff --git a/src/adler/adler.py b/src/adler/adler.py index bc24b29..2163304 100644 --- a/src/adler/adler.py +++ b/src/adler/adler.py @@ -10,10 +10,13 @@ def runAdler(args): # now let's do some phase curves! + # get the r filter SSObject metadata + sso_r = planetoid.SSObject_in_filter("r") + # get the RSP r filter model pc = PhaseCurve( - abs_mag=planetoid.SSObject.H[2] * u.mag, - phase_param=planetoid.SSObject.G12[2], + abs_mag=sso_r.H * u.mag, + phase_param=sso_r.G12, model_name="HG12_Pen16", ) print(pc) diff --git a/src/adler/dataclasses/MPCORB.py b/src/adler/dataclasses/MPCORB.py index 483acd7..9854f7b 100644 --- a/src/adler/dataclasses/MPCORB.py +++ b/src/adler/dataclasses/MPCORB.py @@ -2,9 +2,22 @@ from adler.dataclasses.dataclass_utilities import get_from_table +MPCORB_KEYS = {"mpcDesignation": str, + "mpcNumber": int, + "mpcH": float, + "mpcG": float, + "epoch": float, + "peri": float, + "node": float, + "incl": float, + "e": float, + "n": float, + "q": float, + "uncertaintyParameter": str, + "flags": str} @dataclass -class MPCORB: +class MPCORB(): """Object information from MPCORB. All attributes carry the same names as the column names from the MPCORB table. Attributes: @@ -87,33 +100,9 @@ def construct_from_data_table(cls, ssObjectId, data_table): """ - mpcDesignation = get_from_table(data_table, "mpcDesignation", "str") - mpcNumber = get_from_table(data_table, "mpcNumber", "int") - mpcH = get_from_table(data_table, "mpcH", "float") - mpcG = get_from_table(data_table, "mpcG", "float") - epoch = get_from_table(data_table, "epoch", "float") - peri = get_from_table(data_table, "peri", "float") - node = get_from_table(data_table, "node", "float") - incl = get_from_table(data_table, "incl", "float") - e = get_from_table(data_table, "e", "float") - n = get_from_table(data_table, "n", "float") - q = get_from_table(data_table, "q", "float") - uncertaintyParameter = get_from_table(data_table, "uncertaintyParameter", "str") - flags = get_from_table(data_table, "flags", "str") - - return cls( - ssObjectId, - mpcDesignation, - mpcNumber, - mpcH, - mpcG, - epoch, - peri, - node, - incl, - e, - n, - q, - uncertaintyParameter, - flags, - ) + mpcorb_dict = {"ssObjectId": ssObjectId} + + for mpcorb_key, mpcorb_type in MPCORB_KEYS.items(): + mpcorb_dict[mpcorb_key] = get_from_table(data_table, mpcorb_key, mpcorb_type, "MPCORB") + + return cls(**mpcorb_dict) diff --git a/src/adler/dataclasses/Observations.py b/src/adler/dataclasses/Observations.py index 77768f6..ae420b9 100644 --- a/src/adler/dataclasses/Observations.py +++ b/src/adler/dataclasses/Observations.py @@ -3,6 +3,14 @@ from adler.dataclasses.dataclass_utilities import get_from_table +OBSERVATIONS_KEYS = {"mag": np.ndarray, + "magErr": np.ndarray, + "midPointMjdTai": np.ndarray, + "ra": np.ndarray, + "dec": np.ndarray, + "phaseAngle": np.ndarray, + "topocentricDist": np.ndarray, + "heliocentricDist": np.ndarray} @dataclass class Observations: @@ -24,7 +32,7 @@ class Observations: magErr: array_like of floats Magnitude error. This is a placeholder and will be replaced by flux error. - midpointMjdTai: array_like of floats + midPointMjdTai: array_like of floats Effective mid-visit time for this diaSource, expressed as Modified Julian Date, International Atomic Time. ra: array_like of floats @@ -54,7 +62,7 @@ class Observations: filter_name: str = "" mag: np.ndarray = field(default_factory=lambda: np.zeros(0)) magErr: np.ndarray = field(default_factory=lambda: np.zeros(0)) - midpointMjdTai: np.ndarray = field(default_factory=lambda: np.zeros(0)) + midPointMjdTai: np.ndarray = field(default_factory=lambda: np.zeros(0)) ra: np.ndarray = field(default_factory=lambda: np.zeros(0)) dec: np.ndarray = field(default_factory=lambda: np.zeros(0)) phaseAngle: np.ndarray = field(default_factory=lambda: np.zeros(0)) @@ -85,31 +93,14 @@ def construct_from_data_table(cls, ssObjectId, filter_name, data_table): """ - mag = get_from_table(data_table, "mag", "array") - magErr = get_from_table(data_table, "magErr", "array") - midpointMjdTai = get_from_table(data_table, "midPointMjdTai", "array") - ra = get_from_table(data_table, "ra", "array") - dec = get_from_table(data_table, "dec", "array") - phaseAngle = get_from_table(data_table, "phaseAngle", "array") - topocentricDist = get_from_table(data_table, "topocentricDist", "array") - heliocentricDist = get_from_table(data_table, "heliocentricDist", "array") - - reduced_mag = cls.calculate_reduced_mag(cls, mag, topocentricDist, heliocentricDist) - - return cls( - ssObjectId, - filter_name, - mag, - magErr, - midpointMjdTai, - ra, - dec, - phaseAngle, - topocentricDist, - heliocentricDist, - reduced_mag, - len(data_table), - ) + obs_dict = {"ssObjectId": ssObjectId, "filter_name": filter_name, "num_obs": len(data_table)} + + for obs_key, obs_type in OBSERVATIONS_KEYS.items(): + obs_dict[obs_key] = get_from_table(data_table, obs_key, obs_type, "SSSource/DIASource") + + obs_dict["reduced_mag"] = cls.calculate_reduced_mag(cls, obs_dict["mag"], obs_dict["topocentricDist"], obs_dict["heliocentricDist"]) + + return cls(**obs_dict) def calculate_reduced_mag(self, mag, topocentric_dist, heliocentric_dist): """ diff --git a/src/adler/dataclasses/SSObject.py b/src/adler/dataclasses/SSObject.py index 2207c51..5478108 100644 --- a/src/adler/dataclasses/SSObject.py +++ b/src/adler/dataclasses/SSObject.py @@ -3,6 +3,13 @@ from adler.dataclasses.dataclass_utilities import get_from_table +SSO_KEYS = {"discoverySubmissionDate": float, + "firstObservationDate": float, + "arc": float, + "numObs": int, + "maxExtendedness": float, + "minExtendedness": float, + "medianExtendedness": float} @dataclass class SSObject: @@ -57,47 +64,25 @@ class SSObject: @classmethod def construct_from_data_table(cls, ssObjectId, filter_list, data_table): - discoverySubmissionDate = get_from_table(data_table, "discoverySubmissionDate", "float") - firstObservationDate = get_from_table(data_table, "firstObservationDate", "float") - arc = get_from_table(data_table, "arc", "float") - numObs = get_from_table(data_table, "numObs", "int") - H = np.zeros(len(filter_list)) - G12 = np.zeros(len(filter_list)) - Herr = np.zeros(len(filter_list)) - G12err = np.zeros(len(filter_list)) - nData = np.zeros(len(filter_list)) - - filter_dependent_values = [] + sso_dict = {"ssObjectId": ssObjectId, "filter_list": filter_list, "filter_dependent_values": []} + + for sso_key, sso_type in SSO_KEYS.items(): + sso_dict[sso_key] = get_from_table(data_table, sso_key, sso_type, "SSObject") for i, filter_name in enumerate(filter_list): filter_dept_object = FilterDependentSSO( filter_name=filter_name, - H=get_from_table(data_table, filter_name + "_H", "float"), - G12=get_from_table(data_table, filter_name + "_G12", "float"), - Herr=get_from_table(data_table, filter_name + "_HErr", "float"), - G12err=get_from_table(data_table, filter_name + "_G12Err", "float"), - nData=get_from_table(data_table, filter_name + "_Ndata", "int"), + H=get_from_table(data_table, filter_name + "_H", float, "SSObject"), + G12=get_from_table(data_table, filter_name + "_G12", float, "SSObject"), + Herr=get_from_table(data_table, filter_name + "_HErr", float, "SSObject"), + G12err=get_from_table(data_table, filter_name + "_G12Err", float, "SSObject"), + nData=get_from_table(data_table, filter_name + "_Ndata", float, "SSObject"), ) - filter_dependent_values.append(filter_dept_object) - - maxExtendedness = get_from_table(data_table, "maxExtendedness", "float") - minExtendedness = get_from_table(data_table, "minExtendedness", "float") - medianExtendedness = get_from_table(data_table, "medianExtendedness", "float") - - return cls( - ssObjectId, - filter_list, - discoverySubmissionDate, - firstObservationDate, - arc, - numObs, - filter_dependent_values, - maxExtendedness, - minExtendedness, - medianExtendedness, - ) + sso_dict["filter_dependent_values"].append(filter_dept_object) + + return cls(**sso_dict) @dataclass diff --git a/src/adler/dataclasses/dataclass_utilities.py b/src/adler/dataclasses/dataclass_utilities.py index ebba3d8..c0cd046 100644 --- a/src/adler/dataclasses/dataclass_utilities.py +++ b/src/adler/dataclasses/dataclass_utilities.py @@ -48,8 +48,8 @@ def get_data_table(sql_query, service=None, sql_filename=None): return data_table -def get_from_table(data_table, column_name, data_type): - """Retrieves information from the data_table class variable and forces it to be a specified type. +def get_from_table(data_table, column_name, data_type, table_name): + """Retrieves information from the data_table and forces it to be a specified type. Parameters ----------- @@ -64,18 +64,33 @@ def get_from_table(data_table, column_name, data_type): The data requested from the table cast to the type required. """ - try: - if data_type == "str": - return str(data_table[column_name][0]) - elif data_type == "float": - return float(data_table[column_name][0]) - elif data_type == "int": - return int(data_table[column_name][0]) - elif data_type == "array": - return np.array(data_table[column_name]) - else: - raise TypeError( - "Type for argument data_type not recognised: must be one of 'str', 'float', 'int', 'array'." - ) - except ValueError: - raise ValueError("Could not cast column name to type.") + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning) # RSP tables mask unpopulated elements, which get converted to NaN here and trigger a warning we don't care about. + try: + if data_type == str: + data_val = str(data_table[column_name][0]) + elif data_type == float: + data_val = float(data_table[column_name][0]) + elif data_type == int: + data_val = int(data_table[column_name][0]) + elif data_type == np.ndarray: + data_val = np.array(data_table[column_name]) + else: + raise TypeError( + "Type for argument data_type not recognised for column {} in table {}: must be str, float, int or np.ndarray.".format(column_name, table_name) + ) + except ValueError: + raise ValueError("Could not cast column name to type.") + + # here we alert the user if one of the values is NaN + check_value_for_nan(column_name, data_val, data_type, table_name) + + return data_val + + +def check_value_for_nan(column_name, data_val, data_type, table_name): + + if data_type == np.ndarray and np.isnan(data_val).any(): + print("WARNING: {} unpopulated in {} table for this object. Storing NaN instead.".format(column_name, table_name)) + elif data_type in [float, int] and np.isnan(data_val): + print("WARNING: {} unpopulated in {} table for this object. Storing NaN instead.".format(column_name, table_name)) From b561591567e719308480c38239735f0a97cc1bb0 Mon Sep 17 00:00:00 2001 From: Steph Merritt Date: Wed, 24 Apr 2024 15:29:42 +0100 Subject: [PATCH 03/19] Linting and fixing tests. --- src/adler/dataclasses/Observations.py | 26 +++++++++------ src/adler/dataclasses/dataclass_utilities.py | 33 ++++++++++++++----- .../adler/dataclasses/test_AdlerPlanetoid.py | 2 +- tests/adler/dataclasses/test_Observations.py | 2 +- .../dataclasses/test_dataclass_utilities.py | 13 ++++---- 5 files changed, 49 insertions(+), 27 deletions(-) diff --git a/src/adler/dataclasses/Observations.py b/src/adler/dataclasses/Observations.py index ae420b9..8689a84 100644 --- a/src/adler/dataclasses/Observations.py +++ b/src/adler/dataclasses/Observations.py @@ -3,14 +3,17 @@ from adler.dataclasses.dataclass_utilities import get_from_table -OBSERVATIONS_KEYS = {"mag": np.ndarray, - "magErr": np.ndarray, - "midPointMjdTai": np.ndarray, - "ra": np.ndarray, - "dec": np.ndarray, - "phaseAngle": np.ndarray, - "topocentricDist": np.ndarray, - "heliocentricDist": np.ndarray} +OBSERVATIONS_KEYS = { + "mag": np.ndarray, + "magErr": np.ndarray, + "midPointMjdTai": np.ndarray, + "ra": np.ndarray, + "dec": np.ndarray, + "phaseAngle": np.ndarray, + "topocentricDist": np.ndarray, + "heliocentricDist": np.ndarray, +} + @dataclass class Observations: @@ -96,9 +99,12 @@ def construct_from_data_table(cls, ssObjectId, filter_name, data_table): obs_dict = {"ssObjectId": ssObjectId, "filter_name": filter_name, "num_obs": len(data_table)} for obs_key, obs_type in OBSERVATIONS_KEYS.items(): + print(obs_key, obs_type) obs_dict[obs_key] = get_from_table(data_table, obs_key, obs_type, "SSSource/DIASource") - - obs_dict["reduced_mag"] = cls.calculate_reduced_mag(cls, obs_dict["mag"], obs_dict["topocentricDist"], obs_dict["heliocentricDist"]) + + obs_dict["reduced_mag"] = cls.calculate_reduced_mag( + cls, obs_dict["mag"], obs_dict["topocentricDist"], obs_dict["heliocentricDist"] + ) return cls(**obs_dict) diff --git a/src/adler/dataclasses/dataclass_utilities.py b/src/adler/dataclasses/dataclass_utilities.py index c0cd046..20fb1d6 100644 --- a/src/adler/dataclasses/dataclass_utilities.py +++ b/src/adler/dataclasses/dataclass_utilities.py @@ -48,7 +48,7 @@ def get_data_table(sql_query, service=None, sql_filename=None): return data_table -def get_from_table(data_table, column_name, data_type, table_name): +def get_from_table(data_table, column_name, data_type, table_name="default"): """Retrieves information from the data_table and forces it to be a specified type. Parameters @@ -65,7 +65,9 @@ def get_from_table(data_table, column_name, data_type, table_name): """ with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=UserWarning) # RSP tables mask unpopulated elements, which get converted to NaN here and trigger a warning we don't care about. + warnings.filterwarnings( + "ignore", category=UserWarning + ) # RSP tables mask unpopulated elements, which get converted to NaN here and trigger a warning we don't care about. try: if data_type == str: data_val = str(data_table[column_name][0]) @@ -77,20 +79,33 @@ def get_from_table(data_table, column_name, data_type, table_name): data_val = np.array(data_table[column_name]) else: raise TypeError( - "Type for argument data_type not recognised for column {} in table {}: must be str, float, int or np.ndarray.".format(column_name, table_name) + "Type for argument data_type not recognised for column {} in table {}: must be str, float, int or np.ndarray.".format( + column_name, table_name + ) ) except ValueError: raise ValueError("Could not cast column name to type.") - # here we alert the user if one of the values is NaN - check_value_for_nan(column_name, data_val, data_type, table_name) + # here we alert the user if one of the values is unpopulated and change it to a NaN + data_val = check_value_for_nan(column_name, data_val, data_type, table_name) return data_val def check_value_for_nan(column_name, data_val, data_type, table_name): - - if data_type == np.ndarray and np.isnan(data_val).any(): - print("WARNING: {} unpopulated in {} table for this object. Storing NaN instead.".format(column_name, table_name)) + if data_type == np.ndarray and len(data_val) == 0: + print( + "WARNING: {} unpopulated in {} table for this object. Storing NaN instead.".format( + column_name, table_name + ) + ) + data_val = np.nan elif data_type in [float, int] and np.isnan(data_val): - print("WARNING: {} unpopulated in {} table for this object. Storing NaN instead.".format(column_name, table_name)) + print( + "WARNING: {} unpopulated in {} table for this object. Storing NaN instead.".format( + column_name, table_name + ) + ) + data_val = np.nan + + return data_val diff --git a/tests/adler/dataclasses/test_AdlerPlanetoid.py b/tests/adler/dataclasses/test_AdlerPlanetoid.py index 288c5cc..a011540 100644 --- a/tests/adler/dataclasses/test_AdlerPlanetoid.py +++ b/tests/adler/dataclasses/test_AdlerPlanetoid.py @@ -86,7 +86,7 @@ def test_construct_with_date_range(): ] ) - assert_almost_equal(test_planetoid.observations_by_filter[0].midpointMjdTai, expected_dates) + assert_almost_equal(test_planetoid.observations_by_filter[0].midPointMjdTai, expected_dates) with pytest.raises(ValueError) as error_info_1: test_planetoid = AdlerPlanetoid.construct_from_SQL( diff --git a/tests/adler/dataclasses/test_Observations.py b/tests/adler/dataclasses/test_Observations.py index 24c2b25..44a3717 100644 --- a/tests/adler/dataclasses/test_Observations.py +++ b/tests/adler/dataclasses/test_Observations.py @@ -161,7 +161,7 @@ def test_construct_observations_from_data_table(): assert_almost_equal(test_observations.mag, expected_mag) assert_almost_equal(test_observations.magErr, expected_magerr) - assert_almost_equal(test_observations.midpointMjdTai, expected_mjd) + assert_almost_equal(test_observations.midPointMjdTai, expected_mjd) assert_almost_equal(test_observations.ra, expected_ra) assert_almost_equal(test_observations.dec, expected_dec) assert_almost_equal(test_observations.phaseAngle, expected_phaseangle) diff --git a/tests/adler/dataclasses/test_dataclass_utilities.py b/tests/adler/dataclasses/test_dataclass_utilities.py index dfae4b7..6fca494 100644 --- a/tests/adler/dataclasses/test_dataclass_utilities.py +++ b/tests/adler/dataclasses/test_dataclass_utilities.py @@ -1,5 +1,6 @@ import pytest import pandas as pd +import numpy as np from pandas.testing import assert_frame_equal from numpy.testing import assert_equal @@ -37,13 +38,13 @@ def test_get_from_table(): {"string_col": "a test string", "int_col": 4, "float_col": 4.5, "array_col": [5, 6]} ) - assert get_from_table(test_table, "string_col", "str") == "a test string" - assert get_from_table(test_table, "int_col", "int") == 4 - assert get_from_table(test_table, "float_col", "float") == 4.5 - assert_equal(get_from_table(test_table, "array_col", "array"), [5, 6]) + assert get_from_table(test_table, "string_col", str) == "a test string" + assert get_from_table(test_table, "int_col", int) == 4 + assert get_from_table(test_table, "float_col", float) == 4.5 + assert_equal(get_from_table(test_table, "array_col", np.ndarray), [5, 6]) with pytest.raises(ValueError) as error_info_1: - get_from_table(test_table, "string_col", "int") + get_from_table(test_table, "string_col", int) assert error_info_1.value.args[0] == "Could not cast column name to type." @@ -52,5 +53,5 @@ def test_get_from_table(): assert ( error_info_2.value.args[0] - == "Type for argument data_type not recognised: must be one of 'str', 'float', 'int', 'array'." + == "Type for argument data_type not recognised for column string_col in table default: must be str, float, int or np.ndarray." ) From 9dd5bfd41372cb8be98f3dcb5ed9a16d7d7f2042 Mon Sep 17 00:00:00 2001 From: Steph Merritt Date: Wed, 24 Apr 2024 16:15:02 +0100 Subject: [PATCH 04/19] Adding class for CLI arguments. --- src/adler/adler.py | 9 ++++--- src/adler/utilities/AdlerCLIArguments.py | 30 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 src/adler/utilities/AdlerCLIArguments.py diff --git a/src/adler/adler.py b/src/adler/adler.py index 2163304..6ebef27 100644 --- a/src/adler/adler.py +++ b/src/adler/adler.py @@ -3,10 +3,13 @@ from adler.dataclasses.AdlerPlanetoid import AdlerPlanetoid from adler.science.PhaseCurve import PhaseCurve +from adler.utilities.AdlerCLIArguments import AdlerCLIArguments def runAdler(args): - planetoid = AdlerPlanetoid.construct_from_RSP(args.ssoid, args.filter_list, args.date_range) + planetoid = AdlerPlanetoid.construct_from_RSP( + cli_args.ssObjectId, cli_args.filter_list, cli_args.date_range + ) # now let's do some phase curves! @@ -51,9 +54,9 @@ def main(): args = parser.parse_args() - args.filter_list = args.filters.split(",") + cli_args = AdlerCLIArguments(args) - runAdler(args) + runAdler(cli_args) if __name__ == "__main__": diff --git a/src/adler/utilities/AdlerCLIArguments.py b/src/adler/utilities/AdlerCLIArguments.py new file mode 100644 index 0000000..6d96068 --- /dev/null +++ b/src/adler/utilities/AdlerCLIArguments.py @@ -0,0 +1,30 @@ +class AdlerCLIArguments: + def __init__(self, args): + self.ssObjectId = args.ssoid + self.filter_list = args.filters.split(",") + self.date_range = args.date_range + + self.validate_arguments() + + def validate_arguments(self): + self._validate_filter_list() + self._validate_ssObjectId() + self._validate_date_range() + + def _validate_filter_list(self): + expected_filters = ["u", "g", "r", "i", "z", "y"] + + if not set(self.filter_list).issubset(expected_filters): + raise ValueError( + "Unexpected filters found in filter_list command-line argument. filter_list must be a comma-separated list of LSST filters." + ) + + def _validate_ssObjectId(self): + try: + int(self.ssObjectId) + except ValueError: + raise ValueError("ssoid command-line argument does not appear to be a valid ssObjectId.") + + def _validate_date_range(self): + if len(self.date_range) != 2: + raise ValueError("date_range command-line argument must be of length 2.") From 0ad2d39a7fe31da93b3a02767b00f6cd98a59d08 Mon Sep 17 00:00:00 2001 From: Steph Merritt Date: Wed, 24 Apr 2024 16:23:10 +0100 Subject: [PATCH 05/19] Removing print statement. --- src/adler/dataclasses/Observations.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/adler/dataclasses/Observations.py b/src/adler/dataclasses/Observations.py index 8689a84..64b3900 100644 --- a/src/adler/dataclasses/Observations.py +++ b/src/adler/dataclasses/Observations.py @@ -99,7 +99,6 @@ def construct_from_data_table(cls, ssObjectId, filter_name, data_table): obs_dict = {"ssObjectId": ssObjectId, "filter_name": filter_name, "num_obs": len(data_table)} for obs_key, obs_type in OBSERVATIONS_KEYS.items(): - print(obs_key, obs_type) obs_dict[obs_key] = get_from_table(data_table, obs_key, obs_type, "SSSource/DIASource") obs_dict["reduced_mag"] = cls.calculate_reduced_mag( From d34c5edfbfbafa8d01b8d5da109201860b61c3df Mon Sep 17 00:00:00 2001 From: Stephanie Merritt Date: Wed, 24 Apr 2024 15:33:45 +0000 Subject: [PATCH 06/19] Checking it works on the RSP. --- src/adler/adler.py | 4 ++-- src/adler/utilities/AdlerCLIArguments.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/adler/adler.py b/src/adler/adler.py index 6ebef27..f2b797c 100644 --- a/src/adler/adler.py +++ b/src/adler/adler.py @@ -6,7 +6,7 @@ from adler.utilities.AdlerCLIArguments import AdlerCLIArguments -def runAdler(args): +def runAdler(cli_args): planetoid = AdlerPlanetoid.construct_from_RSP( cli_args.ssObjectId, cli_args.filter_list, cli_args.date_range ) @@ -41,7 +41,7 @@ def main(): parser.add_argument("-s", "--ssoid", help="SSObject ID of planetoid.", type=str, required=True) parser.add_argument( - "-f", "--filters", help="Comma-separated list of filters required.", type=str, default="u,g,r,i,z,y" + "-f", "--filter_list", help="Filters required.", nargs="*", type=str, default=["u", "g", "r", "i", "z", "y"] ) parser.add_argument( "-d", diff --git a/src/adler/utilities/AdlerCLIArguments.py b/src/adler/utilities/AdlerCLIArguments.py index 6d96068..0f65a8b 100644 --- a/src/adler/utilities/AdlerCLIArguments.py +++ b/src/adler/utilities/AdlerCLIArguments.py @@ -1,7 +1,7 @@ class AdlerCLIArguments: def __init__(self, args): self.ssObjectId = args.ssoid - self.filter_list = args.filters.split(",") + self.filter_list = args.filter_list self.date_range = args.date_range self.validate_arguments() From 14f65619c9297792a7963cf9a273a7ddd2cd8f11 Mon Sep 17 00:00:00 2001 From: Steph Merritt Date: Wed, 24 Apr 2024 17:34:10 +0100 Subject: [PATCH 07/19] Unit tests and docstrings. --- src/adler/adler.py | 9 ++- src/adler/dataclasses/dataclass_utilities.py | 54 ++++++++++---- src/adler/utilities/AdlerCLIArguments.py | 28 ++++++- tests/adler/dataclasses/test_MPCORB.py | 2 +- .../dataclasses/test_dataclass_utilities.py | 14 ++++ .../adler/utilities/test_AdlerCLIArguments.py | 73 +++++++++++++++++++ 6 files changed, 160 insertions(+), 20 deletions(-) create mode 100644 tests/adler/utilities/test_AdlerCLIArguments.py diff --git a/src/adler/adler.py b/src/adler/adler.py index f2b797c..7d29f86 100644 --- a/src/adler/adler.py +++ b/src/adler/adler.py @@ -39,9 +39,14 @@ def runAdler(cli_args): def main(): parser = argparse.ArgumentParser(description="Runs Adler for a select planetoid and given user input.") - parser.add_argument("-s", "--ssoid", help="SSObject ID of planetoid.", type=str, required=True) + parser.add_argument("-s", "--ssObjectId", help="SSObject ID of planetoid.", type=str, required=True) parser.add_argument( - "-f", "--filter_list", help="Filters required.", nargs="*", type=str, default=["u", "g", "r", "i", "z", "y"] + "-f", + "--filter_list", + help="Filters required.", + nargs="*", + type=str, + default=["u", "g", "r", "i", "z", "y"], ) parser.add_argument( "-d", diff --git a/src/adler/dataclasses/dataclass_utilities.py b/src/adler/dataclasses/dataclass_utilities.py index 20fb1d6..172a264 100644 --- a/src/adler/dataclasses/dataclass_utilities.py +++ b/src/adler/dataclasses/dataclass_utilities.py @@ -53,14 +53,21 @@ def get_from_table(data_table, column_name, data_type, table_name="default"): Parameters ----------- + data_table : DALResultsTable or Pandas dataframe + Data table containing columns of interest. + column_name : str Column name under which the data of interest is stored. - type : str - String delineating data type. Should be "str", "float", "int" or "array". + + data_type : type + Data type. Should be int, float, str or np.ndarray. + + table_name : str + Name of the table. This is mostly for more informative error messages. Default="default". Returns ----------- - data : any type + data_val : str, float, int or nd.array The data requested from the table cast to the type required. """ @@ -87,20 +94,41 @@ def get_from_table(data_table, column_name, data_type, table_name="default"): raise ValueError("Could not cast column name to type.") # here we alert the user if one of the values is unpopulated and change it to a NaN - data_val = check_value_for_nan(column_name, data_val, data_type, table_name) + data_val = check_value_populated(data_val, data_type, column_name, table_name) return data_val -def check_value_for_nan(column_name, data_val, data_type, table_name): - if data_type == np.ndarray and len(data_val) == 0: - print( - "WARNING: {} unpopulated in {} table for this object. Storing NaN instead.".format( - column_name, table_name - ) - ) - data_val = np.nan - elif data_type in [float, int] and np.isnan(data_val): +def check_value_populated(data_val, data_type, column_name, table_name): + """Checks to see if data_val populated properly and prints a helpful warning if it didn't. + Usually this will trigger because the RSP hasn't populated that field for this particular object. + + Parameters + ----------- + data_val : str, float, int or nd.array + The value to check. + + data_type: type + Data type. Should be int, float, str or np.ndarray. + + column_name: str + Column name under which the data of interest is stored. + + table_name : str + Name of the table. This is mostly for more informative error messages. Default="default". + + Returns + ----------- + data_val : str, float, int, nd.array or np.nan + Either returns the original data_val or an np.nan if it detected that the value was not populated. + + """ + + array_length_zero = data_type == np.ndarray and len(data_val) == 0 + number_is_nan = data_type in [float, int] and np.isnan(data_val) + str_is_empty = data_type == str and len(data_val) == 0 + + if array_length_zero or number_is_nan or str_is_empty: print( "WARNING: {} unpopulated in {} table for this object. Storing NaN instead.".format( column_name, table_name diff --git a/src/adler/utilities/AdlerCLIArguments.py b/src/adler/utilities/AdlerCLIArguments.py index 0f65a8b..7a30066 100644 --- a/src/adler/utilities/AdlerCLIArguments.py +++ b/src/adler/utilities/AdlerCLIArguments.py @@ -1,6 +1,16 @@ class AdlerCLIArguments: + """ + Class for storing abd validating Adler command-line arguments. + + Attributes: + ----------- + args : argparse.Namespace object + argparse.Namespace object created by calling parse_args(). + + """ + def __init__(self, args): - self.ssObjectId = args.ssoid + self.ssObjectId = args.ssObjectId self.filter_list = args.filter_list self.date_range = args.date_range @@ -16,7 +26,7 @@ def _validate_filter_list(self): if not set(self.filter_list).issubset(expected_filters): raise ValueError( - "Unexpected filters found in filter_list command-line argument. filter_list must be a comma-separated list of LSST filters." + "Unexpected filters found in filter_list command-line argument. filter_list must be a list of LSST filters." ) def _validate_ssObjectId(self): @@ -26,5 +36,15 @@ def _validate_ssObjectId(self): raise ValueError("ssoid command-line argument does not appear to be a valid ssObjectId.") def _validate_date_range(self): - if len(self.date_range) != 2: - raise ValueError("date_range command-line argument must be of length 2.") + for d in self.date_range: + try: + float(d) + except ValueError: + raise ValueError( + "One or both of the values for the date_range command-line argument do not seem to be valid numbers." + ) + + if any(d > 250000 for d in self.date_range): + raise ValueError( + "Dates for date_range command-line argument seem rather large. Did you input JD instead of MJD?" + ) diff --git a/tests/adler/dataclasses/test_MPCORB.py b/tests/adler/dataclasses/test_MPCORB.py index f94db75..d139d00 100644 --- a/tests/adler/dataclasses/test_MPCORB.py +++ b/tests/adler/dataclasses/test_MPCORB.py @@ -36,5 +36,5 @@ def test_construct_MPCORB_from_data_table(): assert_almost_equal(test_MPCORB.e, 0.7168805704972735, decimal=6) assert np.isnan(test_MPCORB.n) assert_almost_equal(test_MPCORB.q, 0.5898291078470536, decimal=6) - assert test_MPCORB.uncertaintyParameter == "" + assert np.isnan(test_MPCORB.uncertaintyParameter) assert test_MPCORB.flags == "0" diff --git a/tests/adler/dataclasses/test_dataclass_utilities.py b/tests/adler/dataclasses/test_dataclass_utilities.py index 6fca494..b087e0e 100644 --- a/tests/adler/dataclasses/test_dataclass_utilities.py +++ b/tests/adler/dataclasses/test_dataclass_utilities.py @@ -6,6 +6,7 @@ from adler.dataclasses.dataclass_utilities import get_data_table from adler.dataclasses.dataclass_utilities import get_from_table +from adler.dataclasses.dataclass_utilities import check_value_populated from adler.utilities.tests_utilities import get_test_data_filepath @@ -55,3 +56,16 @@ def test_get_from_table(): error_info_2.value.args[0] == "Type for argument data_type not recognised for column string_col in table default: must be str, float, int or np.ndarray." ) + + +def test_check_value_populated(): + populated_value = check_value_populated(3, int, "column", "table") + assert populated_value == 3 + + array_length_zero = check_value_populated(np.array([]), np.ndarray, "column", "table") + number_is_nan = check_value_populated(np.nan, float, "column", "table") + str_is_empty = check_value_populated("", str, "column", "table") + + assert np.isnan(array_length_zero) + assert np.isnan(number_is_nan) + assert np.isnan(str_is_empty) diff --git a/tests/adler/utilities/test_AdlerCLIArguments.py b/tests/adler/utilities/test_AdlerCLIArguments.py new file mode 100644 index 0000000..90c4e5f --- /dev/null +++ b/tests/adler/utilities/test_AdlerCLIArguments.py @@ -0,0 +1,73 @@ +import pytest +from adler.utilities.AdlerCLIArguments import AdlerCLIArguments + + +# AdlerCLIArguments object takes an object as input, so we define a quick one here +class args: + def __init__(self, ssObjectId, filter_list, date_range): + self.ssObjectId = ssObjectId + self.filter_list = filter_list + self.date_range = date_range + + +def test_AdlerCLIArguments(): + # test correct population + good_input_dict = {"ssObjectId": "666", "filter_list": ["g", "r", "i"], "date_range": [60000.0, 67300.0]} + good_arguments = args(**good_input_dict) + good_arguments_object = AdlerCLIArguments(good_arguments) + + assert good_arguments_object.__dict__ == good_input_dict + + # test that a bad ssObjectId triggers the right error + bad_ssoid_arguments = args("hello!", ["g", "r", "i"], [60000.0, 67300.0]) + + with pytest.raises(ValueError) as bad_ssoid_error: + bad_ssoid_object = AdlerCLIArguments(bad_ssoid_arguments) + + assert ( + bad_ssoid_error.value.args[0] + == "ssoid command-line argument does not appear to be a valid ssObjectId." + ) + + # test that non-LSST or unexpected filters trigger the right error + bad_filter_arguments = args("666", ["g", "r", "i", "m"], [60000.0, 67300.0]) + + with pytest.raises(ValueError) as bad_filter_error: + bad_filter_object = AdlerCLIArguments(bad_filter_arguments) + + assert ( + bad_filter_error.value.args[0] + == "Unexpected filters found in filter_list command-line argument. filter_list must be a list of LSST filters." + ) + + bad_filter_arguments_2 = args("666", ["pony"], [60000.0, 67300.0]) + + with pytest.raises(ValueError) as bad_filter_error_2: + bad_filter_object = AdlerCLIArguments(bad_filter_arguments_2) + + assert ( + bad_filter_error_2.value.args[0] + == "Unexpected filters found in filter_list command-line argument. filter_list must be a list of LSST filters." + ) + + # test that overly-large dates trigger the right error + big_date_arguments = args("666", ["g", "r", "i"], [260000.0, 267300.0]) + + with pytest.raises(ValueError) as big_date_error: + big_date_object = AdlerCLIArguments(big_date_arguments) + + assert ( + big_date_error.value.args[0] + == "Dates for date_range command-line argument seem rather large. Did you input JD instead of MJD?" + ) + + # test that unexpected date values trigger the right error + bad_date_arguments = args("666", ["g", "r", "i"], [260000.0, "cheese"]) + + with pytest.raises(ValueError) as bad_date_error: + bad_date_object = AdlerCLIArguments(bad_date_arguments) + + assert ( + bad_date_error.value.args[0] + == "One or both of the values for the date_range command-line argument do not seem to be valid numbers." + ) From d9f2106864c1f5efd9346d9d7951ddf44bcce9c9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Wed, 24 Apr 2024 16:48:37 +0000 Subject: [PATCH 08/19] [pre-commit.ci lite] apply automatic fixes --- src/adler/dataclasses/MPCORB.py | 33 +++++++++++++++++-------------- src/adler/dataclasses/SSObject.py | 10 ++++++---- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/adler/dataclasses/MPCORB.py b/src/adler/dataclasses/MPCORB.py index 9854f7b..a17bd18 100644 --- a/src/adler/dataclasses/MPCORB.py +++ b/src/adler/dataclasses/MPCORB.py @@ -2,22 +2,25 @@ from adler.dataclasses.dataclass_utilities import get_from_table -MPCORB_KEYS = {"mpcDesignation": str, - "mpcNumber": int, - "mpcH": float, - "mpcG": float, - "epoch": float, - "peri": float, - "node": float, - "incl": float, - "e": float, - "n": float, - "q": float, - "uncertaintyParameter": str, - "flags": str} +MPCORB_KEYS = { + "mpcDesignation": str, + "mpcNumber": int, + "mpcH": float, + "mpcG": float, + "epoch": float, + "peri": float, + "node": float, + "incl": float, + "e": float, + "n": float, + "q": float, + "uncertaintyParameter": str, + "flags": str, +} + @dataclass -class MPCORB(): +class MPCORB: """Object information from MPCORB. All attributes carry the same names as the column names from the MPCORB table. Attributes: @@ -101,7 +104,7 @@ def construct_from_data_table(cls, ssObjectId, data_table): """ mpcorb_dict = {"ssObjectId": ssObjectId} - + for mpcorb_key, mpcorb_type in MPCORB_KEYS.items(): mpcorb_dict[mpcorb_key] = get_from_table(data_table, mpcorb_key, mpcorb_type, "MPCORB") diff --git a/src/adler/dataclasses/SSObject.py b/src/adler/dataclasses/SSObject.py index 5478108..9ec0443 100644 --- a/src/adler/dataclasses/SSObject.py +++ b/src/adler/dataclasses/SSObject.py @@ -3,13 +3,16 @@ from adler.dataclasses.dataclass_utilities import get_from_table -SSO_KEYS = {"discoverySubmissionDate": float, +SSO_KEYS = { + "discoverySubmissionDate": float, "firstObservationDate": float, "arc": float, "numObs": int, "maxExtendedness": float, "minExtendedness": float, - "medianExtendedness": float} + "medianExtendedness": float, +} + @dataclass class SSObject: @@ -64,9 +67,8 @@ class SSObject: @classmethod def construct_from_data_table(cls, ssObjectId, filter_list, data_table): - sso_dict = {"ssObjectId": ssObjectId, "filter_list": filter_list, "filter_dependent_values": []} - + for sso_key, sso_type in SSO_KEYS.items(): sso_dict[sso_key] = get_from_table(data_table, sso_key, sso_type, "SSObject") From 5def5ff47fe9fdc18b2220298c8df2b2ac214c49 Mon Sep 17 00:00:00 2001 From: Steph Merritt Date: Wed, 24 Apr 2024 17:50:15 +0100 Subject: [PATCH 09/19] More linting. --- src/adler/dataclasses/MPCORB.py | 33 +++++++++++++++++-------------- src/adler/dataclasses/SSObject.py | 10 ++++++---- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/adler/dataclasses/MPCORB.py b/src/adler/dataclasses/MPCORB.py index 9854f7b..a17bd18 100644 --- a/src/adler/dataclasses/MPCORB.py +++ b/src/adler/dataclasses/MPCORB.py @@ -2,22 +2,25 @@ from adler.dataclasses.dataclass_utilities import get_from_table -MPCORB_KEYS = {"mpcDesignation": str, - "mpcNumber": int, - "mpcH": float, - "mpcG": float, - "epoch": float, - "peri": float, - "node": float, - "incl": float, - "e": float, - "n": float, - "q": float, - "uncertaintyParameter": str, - "flags": str} +MPCORB_KEYS = { + "mpcDesignation": str, + "mpcNumber": int, + "mpcH": float, + "mpcG": float, + "epoch": float, + "peri": float, + "node": float, + "incl": float, + "e": float, + "n": float, + "q": float, + "uncertaintyParameter": str, + "flags": str, +} + @dataclass -class MPCORB(): +class MPCORB: """Object information from MPCORB. All attributes carry the same names as the column names from the MPCORB table. Attributes: @@ -101,7 +104,7 @@ def construct_from_data_table(cls, ssObjectId, data_table): """ mpcorb_dict = {"ssObjectId": ssObjectId} - + for mpcorb_key, mpcorb_type in MPCORB_KEYS.items(): mpcorb_dict[mpcorb_key] = get_from_table(data_table, mpcorb_key, mpcorb_type, "MPCORB") diff --git a/src/adler/dataclasses/SSObject.py b/src/adler/dataclasses/SSObject.py index 5478108..9ec0443 100644 --- a/src/adler/dataclasses/SSObject.py +++ b/src/adler/dataclasses/SSObject.py @@ -3,13 +3,16 @@ from adler.dataclasses.dataclass_utilities import get_from_table -SSO_KEYS = {"discoverySubmissionDate": float, +SSO_KEYS = { + "discoverySubmissionDate": float, "firstObservationDate": float, "arc": float, "numObs": int, "maxExtendedness": float, "minExtendedness": float, - "medianExtendedness": float} + "medianExtendedness": float, +} + @dataclass class SSObject: @@ -64,9 +67,8 @@ class SSObject: @classmethod def construct_from_data_table(cls, ssObjectId, filter_list, data_table): - sso_dict = {"ssObjectId": ssObjectId, "filter_list": filter_list, "filter_dependent_values": []} - + for sso_key, sso_type in SSO_KEYS.items(): sso_dict[sso_key] = get_from_table(data_table, sso_key, sso_type, "SSObject") From c796ccda59aff8cc3e8c5e6d4c0c1c5a3355564b Mon Sep 17 00:00:00 2001 From: Steph Merritt Date: Thu, 25 Apr 2024 11:14:33 +0100 Subject: [PATCH 10/19] Fixing old reference. --- src/adler/utilities/AdlerCLIArguments.py | 2 +- tests/adler/utilities/test_AdlerCLIArguments.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/adler/utilities/AdlerCLIArguments.py b/src/adler/utilities/AdlerCLIArguments.py index 7a30066..03cf41c 100644 --- a/src/adler/utilities/AdlerCLIArguments.py +++ b/src/adler/utilities/AdlerCLIArguments.py @@ -33,7 +33,7 @@ def _validate_ssObjectId(self): try: int(self.ssObjectId) except ValueError: - raise ValueError("ssoid command-line argument does not appear to be a valid ssObjectId.") + raise ValueError("ssObjectId command-line argument does not appear to be a valid ssObjectId.") def _validate_date_range(self): for d in self.date_range: diff --git a/tests/adler/utilities/test_AdlerCLIArguments.py b/tests/adler/utilities/test_AdlerCLIArguments.py index 90c4e5f..e99d3b5 100644 --- a/tests/adler/utilities/test_AdlerCLIArguments.py +++ b/tests/adler/utilities/test_AdlerCLIArguments.py @@ -26,7 +26,7 @@ def test_AdlerCLIArguments(): assert ( bad_ssoid_error.value.args[0] - == "ssoid command-line argument does not appear to be a valid ssObjectId." + == "ssObjectId command-line argument does not appear to be a valid ssObjectId." ) # test that non-LSST or unexpected filters trigger the right error From 9fc14f6e3e50fae3aa81be91d707b26ff7df0816 Mon Sep 17 00:00:00 2001 From: Stephanie Merritt Date: Thu, 9 May 2024 16:49:06 +0000 Subject: [PATCH 11/19] Adding error handling, missing filters. --- src/adler/dataclasses/AdlerPlanetoid.py | 28 ++++++++-- .../adler/dataclasses/test_AdlerPlanetoid.py | 52 ++++++++++++------- 2 files changed, 59 insertions(+), 21 deletions(-) diff --git a/src/adler/dataclasses/AdlerPlanetoid.py b/src/adler/dataclasses/AdlerPlanetoid.py index 8b03781..2b1d6a3 100644 --- a/src/adler/dataclasses/AdlerPlanetoid.py +++ b/src/adler/dataclasses/AdlerPlanetoid.py @@ -86,6 +86,12 @@ def construct_from_SQL( cls, ssObjectId, filter_list, date_range, sql_filename=sql_filename, schema=schema ) + if len(observations_by_filter) == 0: + raise Exception("No observations found for this object in the given filter(s). Check SSOID and try again.") + + # redo the filter list based on the available filters in observations_by_filter + filter_list = [obs_object.filter_name for obs_object in observations_by_filter] + mpcorb = cls.populate_MPCORB(cls, ssObjectId, sql_filename=sql_filename, schema=schema) ssobject = cls.populate_SSObject( cls, ssObjectId, filter_list, sql_filename=sql_filename, schema=schema @@ -122,6 +128,13 @@ def construct_from_RSP( observations_by_filter = cls.populate_observations( cls, ssObjectId, filter_list, date_range, service=service ) + + if len(observations_by_filter) == 0: + raise Exception("No observations found for this object in the given filter(s). Check SSOID and try again.") + + # redo the filter list based on the available filters in observations_by_filter + filter_list = [obs_object.filter_name for obs_object in observations_by_filter] + mpcorb = cls.populate_MPCORB(cls, ssObjectId, service=service) ssobject = cls.populate_SSObject(cls, ssObjectId, filter_list, service=service) @@ -185,9 +198,12 @@ def populate_observations( data_table = get_data_table(observations_sql_query, service=service, sql_filename=sql_filename) - observations_by_filter.append( - Observations.construct_from_data_table(ssObjectId, filter_name, data_table) - ) + if len(data_table) == 0: + print("WARNING: No observations found in {} filter for this object. Skipping this filter.".format(filter_name)) + else: + observations_by_filter.append( + Observations.construct_from_data_table(ssObjectId, filter_name, data_table) + ) return observations_by_filter @@ -228,6 +244,9 @@ def populate_MPCORB(self, ssObjectId, service=None, sql_filename=None, schema="d data_table = get_data_table(MPCORB_sql_query, service=service, sql_filename=sql_filename) + if len(data_table) == 0: + raise Exception("No MPCORB data for this object could be found for this SSObjectId.") + return MPCORB.construct_from_data_table(ssObjectId, data_table) def populate_SSObject( @@ -282,6 +301,9 @@ def populate_SSObject( data_table = get_data_table(SSObject_sql_query, service=service, sql_filename=sql_filename) + if len(data_table) == 0: + raise Exception("No SSObject data for this object could be found for this SSObjectId.") + return SSObject.construct_from_data_table(ssObjectId, filter_list, data_table) def observations_in_filter(self, filter_name): diff --git a/tests/adler/dataclasses/test_AdlerPlanetoid.py b/tests/adler/dataclasses/test_AdlerPlanetoid.py index a011540..6c978c6 100644 --- a/tests/adler/dataclasses/test_AdlerPlanetoid.py +++ b/tests/adler/dataclasses/test_AdlerPlanetoid.py @@ -11,13 +11,13 @@ def test_construct_from_SQL(): - test_planetoid = AdlerPlanetoid.construct_from_SQL(ssoid, test_db_path) + test_planetoid = AdlerPlanetoid.construct_from_SQL(ssoid, test_db_path, filter_list=["u", "g", "r", "i", "z", "y"]) # testing just a few values here to ensure correct setup: these objects have their own unit tests assert test_planetoid.MPCORB.mpcH == 19.8799991607666 assert test_planetoid.SSObject.discoverySubmissionDate == 60218.0 assert_almost_equal( - test_planetoid.observations_by_filter[1].mag, + test_planetoid.observations_by_filter[0].mag, [ 21.33099937, 22.67099953, @@ -31,10 +31,10 @@ def test_construct_from_SQL(): ], ) - # did we pick up all the filters? - assert len(test_planetoid.observations_by_filter) == 6 - assert len(test_planetoid.SSObject.filter_dependent_values) == 6 - assert test_planetoid.filter_list == ["u", "g", "r", "i", "z", "y"] + # did we pick up all the filters? note we ask for ugrizy but u and y are unpopulated in DP0.3, so the code should eliminate them + assert len(test_planetoid.observations_by_filter) == 4 + assert len(test_planetoid.SSObject.filter_dependent_values) == 4 + assert test_planetoid.filter_list == ["g", "r", "i", "z"] # checking the date range to ensure it's the default assert test_planetoid.date_range == [60000.0, 67300.0] @@ -100,12 +100,10 @@ def test_observations_in_filter(): test_planetoid = AdlerPlanetoid.construct_from_SQL(ssoid, test_db_path) # Python dataclasses create an __eq__ for you so object-to-object comparison just works, isn't that nice? - assert test_planetoid.observations_in_filter("u") == test_planetoid.observations_by_filter[0] - assert test_planetoid.observations_in_filter("g") == test_planetoid.observations_by_filter[1] - assert test_planetoid.observations_in_filter("r") == test_planetoid.observations_by_filter[2] - assert test_planetoid.observations_in_filter("i") == test_planetoid.observations_by_filter[3] - assert test_planetoid.observations_in_filter("z") == test_planetoid.observations_by_filter[4] - assert test_planetoid.observations_in_filter("y") == test_planetoid.observations_by_filter[5] + assert test_planetoid.observations_in_filter("g") == test_planetoid.observations_by_filter[0] + assert test_planetoid.observations_in_filter("r") == test_planetoid.observations_by_filter[1] + assert test_planetoid.observations_in_filter("i") == test_planetoid.observations_by_filter[2] + assert test_planetoid.observations_in_filter("z") == test_planetoid.observations_by_filter[3] with pytest.raises(ValueError) as error_info_1: test_planetoid.observations_in_filter("f") @@ -116,14 +114,32 @@ def test_observations_in_filter(): def test_SSObject_in_filter(): test_planetoid = AdlerPlanetoid.construct_from_SQL(ssoid, test_db_path) - assert test_planetoid.SSObject_in_filter("u") == test_planetoid.SSObject.filter_dependent_values[0] - assert test_planetoid.SSObject_in_filter("g") == test_planetoid.SSObject.filter_dependent_values[1] - assert test_planetoid.SSObject_in_filter("r") == test_planetoid.SSObject.filter_dependent_values[2] - assert test_planetoid.SSObject_in_filter("i") == test_planetoid.SSObject.filter_dependent_values[3] - assert test_planetoid.SSObject_in_filter("z") == test_planetoid.SSObject.filter_dependent_values[4] - assert test_planetoid.SSObject_in_filter("y") == test_planetoid.SSObject.filter_dependent_values[5] + assert test_planetoid.SSObject_in_filter("g") == test_planetoid.SSObject.filter_dependent_values[0] + assert test_planetoid.SSObject_in_filter("r") == test_planetoid.SSObject.filter_dependent_values[1] + assert test_planetoid.SSObject_in_filter("i") == test_planetoid.SSObject.filter_dependent_values[2] + assert test_planetoid.SSObject_in_filter("z") == test_planetoid.SSObject.filter_dependent_values[3] with pytest.raises(ValueError) as error_info_1: test_planetoid.SSObject_in_filter("f") assert error_info_1.value.args[0] == "Filter f is not in AdlerPlanetoid.filter_list." + + +def test_no_observations(): + with pytest.raises(Exception) as error_info: + test_planetoid = AdlerPlanetoid.construct_from_SQL(826857066833589477, test_db_path) + + assert error_info.value.args[0] == "No observations found for this object in the given filter(s). Check SSOID and try again." + + +def test_for_warnings(capsys): + + test_planetoid = AdlerPlanetoid.construct_from_SQL(ssoid, test_db_path, filter_list=["u", "g"]) + captured = capsys.readouterr() + + expected = ("WARNING: No observations found in u filter for this object. Skipping this filter.\n" + + "WARNING: n unpopulated in MPCORB table for this object. Storing NaN instead.\n" + + "WARNING: uncertaintyParameter unpopulated in MPCORB table for this object. Storing NaN instead.\n") + + assert captured.out == expected + \ No newline at end of file From ad9a4f775ea7be97c06120850c817dc4745d726b Mon Sep 17 00:00:00 2001 From: Steph Merritt Date: Thu, 9 May 2024 17:50:16 +0100 Subject: [PATCH 12/19] Linting. --- src/adler/dataclasses/AdlerPlanetoid.py | 16 ++++++++++++---- .../adler/dataclasses/test_AdlerPlanetoid.py | 19 ++++++++++++------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/adler/dataclasses/AdlerPlanetoid.py b/src/adler/dataclasses/AdlerPlanetoid.py index 2b1d6a3..7d0e381 100644 --- a/src/adler/dataclasses/AdlerPlanetoid.py +++ b/src/adler/dataclasses/AdlerPlanetoid.py @@ -87,7 +87,9 @@ def construct_from_SQL( ) if len(observations_by_filter) == 0: - raise Exception("No observations found for this object in the given filter(s). Check SSOID and try again.") + raise Exception( + "No observations found for this object in the given filter(s). Check SSOID and try again." + ) # redo the filter list based on the available filters in observations_by_filter filter_list = [obs_object.filter_name for obs_object in observations_by_filter] @@ -130,11 +132,13 @@ def construct_from_RSP( ) if len(observations_by_filter) == 0: - raise Exception("No observations found for this object in the given filter(s). Check SSOID and try again.") + raise Exception( + "No observations found for this object in the given filter(s). Check SSOID and try again." + ) # redo the filter list based on the available filters in observations_by_filter filter_list = [obs_object.filter_name for obs_object in observations_by_filter] - + mpcorb = cls.populate_MPCORB(cls, ssObjectId, service=service) ssobject = cls.populate_SSObject(cls, ssObjectId, filter_list, service=service) @@ -199,7 +203,11 @@ def populate_observations( data_table = get_data_table(observations_sql_query, service=service, sql_filename=sql_filename) if len(data_table) == 0: - print("WARNING: No observations found in {} filter for this object. Skipping this filter.".format(filter_name)) + print( + "WARNING: No observations found in {} filter for this object. Skipping this filter.".format( + filter_name + ) + ) else: observations_by_filter.append( Observations.construct_from_data_table(ssObjectId, filter_name, data_table) diff --git a/tests/adler/dataclasses/test_AdlerPlanetoid.py b/tests/adler/dataclasses/test_AdlerPlanetoid.py index 6c978c6..b72ea15 100644 --- a/tests/adler/dataclasses/test_AdlerPlanetoid.py +++ b/tests/adler/dataclasses/test_AdlerPlanetoid.py @@ -11,7 +11,9 @@ def test_construct_from_SQL(): - test_planetoid = AdlerPlanetoid.construct_from_SQL(ssoid, test_db_path, filter_list=["u", "g", "r", "i", "z", "y"]) + test_planetoid = AdlerPlanetoid.construct_from_SQL( + ssoid, test_db_path, filter_list=["u", "g", "r", "i", "z", "y"] + ) # testing just a few values here to ensure correct setup: these objects have their own unit tests assert test_planetoid.MPCORB.mpcH == 19.8799991607666 @@ -129,17 +131,20 @@ def test_no_observations(): with pytest.raises(Exception) as error_info: test_planetoid = AdlerPlanetoid.construct_from_SQL(826857066833589477, test_db_path) - assert error_info.value.args[0] == "No observations found for this object in the given filter(s). Check SSOID and try again." + assert ( + error_info.value.args[0] + == "No observations found for this object in the given filter(s). Check SSOID and try again." + ) def test_for_warnings(capsys): - test_planetoid = AdlerPlanetoid.construct_from_SQL(ssoid, test_db_path, filter_list=["u", "g"]) captured = capsys.readouterr() - expected = ("WARNING: No observations found in u filter for this object. Skipping this filter.\n" - + "WARNING: n unpopulated in MPCORB table for this object. Storing NaN instead.\n" - + "WARNING: uncertaintyParameter unpopulated in MPCORB table for this object. Storing NaN instead.\n") + expected = ( + "WARNING: No observations found in u filter for this object. Skipping this filter.\n" + + "WARNING: n unpopulated in MPCORB table for this object. Storing NaN instead.\n" + + "WARNING: uncertaintyParameter unpopulated in MPCORB table for this object. Storing NaN instead.\n" + ) assert captured.out == expected - \ No newline at end of file From 0a2fb2c46c0c05e9453239ae3b5f159182176631 Mon Sep 17 00:00:00 2001 From: Steph Merritt Date: Thu, 9 May 2024 17:59:51 +0100 Subject: [PATCH 13/19] Extra unit test. --- .../adler/dataclasses/test_AdlerPlanetoid.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/adler/dataclasses/test_AdlerPlanetoid.py b/tests/adler/dataclasses/test_AdlerPlanetoid.py index b72ea15..3ac451d 100644 --- a/tests/adler/dataclasses/test_AdlerPlanetoid.py +++ b/tests/adler/dataclasses/test_AdlerPlanetoid.py @@ -148,3 +148,23 @@ def test_for_warnings(capsys): ) assert captured.out == expected + + +def test_failed_SQL_queries(): + test_planetoid = AdlerPlanetoid.construct_from_SQL( + ssoid, test_db_path, filter_list=["u", "g", "r", "i", "z", "y"] + ) + + with pytest.raises(Exception) as error_info_1: + test_planetoid.populate_MPCORB("826857066833589477", sql_filename=test_db_path, schema="") + + assert error_info_1.value.args[0] == "No MPCORB data for this object could be found for this SSObjectId." + + with pytest.raises(Exception) as error_info_2: + test_planetoid.populate_SSObject( + "826857066833589477", filter_list=["u"], sql_filename=test_db_path, schema="" + ) + + assert ( + error_info_2.value.args[0] == "No SSObject data for this object could be found for this SSObjectId." + ) From cc8f07662fc53ac6ba8066cb705cc2050d2dea05 Mon Sep 17 00:00:00 2001 From: Steph Merritt Date: Fri, 10 May 2024 13:04:02 +0100 Subject: [PATCH 14/19] Adding output CLIs, setting up logging. --- src/adler/adler.py | 21 ++++++- src/adler/dataclasses/AdlerData.py | 2 + src/adler/dataclasses/AdlerPlanetoid.py | 3 + src/adler/dataclasses/dataclass_utilities.py | 2 + src/adler/utilities/AdlerCLIArguments.py | 21 +++++-- src/adler/utilities/adler_logging.py | 39 +++++++++++++ .../adler/utilities/test_AdlerCLIArguments.py | 57 ++++++++++++++----- 7 files changed, 127 insertions(+), 18 deletions(-) create mode 100644 src/adler/utilities/adler_logging.py diff --git a/src/adler/adler.py b/src/adler/adler.py index 7d29f86..be7fbe4 100644 --- a/src/adler/adler.py +++ b/src/adler/adler.py @@ -4,6 +4,7 @@ from adler.dataclasses.AdlerPlanetoid import AdlerPlanetoid from adler.science.PhaseCurve import PhaseCurve from adler.utilities.AdlerCLIArguments import AdlerCLIArguments +from adler.utilities.adler_logging import setup_adler_logging def runAdler(cli_args): @@ -37,7 +38,7 @@ def runAdler(cli_args): def main(): - parser = argparse.ArgumentParser(description="Runs Adler for a select planetoid and given user input.") + parser = argparse.ArgumentParser(description="Runs Adler for select planetoid(s) and given user input.") parser.add_argument("-s", "--ssObjectId", help="SSObject ID of planetoid.", type=str, required=True) parser.add_argument( @@ -56,11 +57,29 @@ def main(): type=float, default=[60000.0, 67300.0], ) + parser.add_argument( + "-o", + "--outpath", + help="Output path location. Default is current working directory.", + type=str, + default="./", + ) + parser.add_argument( + "-n", + "--db_name", + help="Stem filename of output database. If this doesn't exist, it will be created. Default: adler_out.", + type=str, + default="adler_out", + ) args = parser.parse_args() cli_args = AdlerCLIArguments(args) + adler_logger = setup_adler_logging(cli_args.outpath) + + cli_args.logger = adler_logger + runAdler(cli_args) diff --git a/src/adler/dataclasses/AdlerData.py b/src/adler/dataclasses/AdlerData.py index 1453cda..0170e38 100644 --- a/src/adler/dataclasses/AdlerData.py +++ b/src/adler/dataclasses/AdlerData.py @@ -15,6 +15,8 @@ "phase_parameter_2_err", ] +logger = logging.getLogger(__name__) + @dataclass class AdlerData: diff --git a/src/adler/dataclasses/AdlerPlanetoid.py b/src/adler/dataclasses/AdlerPlanetoid.py index 7d0e381..9c5a24c 100644 --- a/src/adler/dataclasses/AdlerPlanetoid.py +++ b/src/adler/dataclasses/AdlerPlanetoid.py @@ -1,5 +1,6 @@ from lsst.rsp import get_tap_service import pandas as pd +import logging from adler.dataclasses.Observations import Observations from adler.dataclasses.MPCORB import MPCORB @@ -7,6 +8,8 @@ from adler.dataclasses.AdlerData import AdlerData from adler.dataclasses.dataclass_utilities import get_data_table +logger = logging.getLogger(__name__) + class AdlerPlanetoid: """AdlerPlanetoid class. Contains the Observations, MPCORB and SSObject dataclass objects.""" diff --git a/src/adler/dataclasses/dataclass_utilities.py b/src/adler/dataclasses/dataclass_utilities.py index 172a264..8abf9d0 100644 --- a/src/adler/dataclasses/dataclass_utilities.py +++ b/src/adler/dataclasses/dataclass_utilities.py @@ -3,6 +3,8 @@ import sqlite3 import warnings +logger = logging.getLogger(__name__) + def get_data_table(sql_query, service=None, sql_filename=None): """Gets a table of data based on a SQL query. Table is pulled from either the RSP or a local SQL database: diff --git a/src/adler/utilities/AdlerCLIArguments.py b/src/adler/utilities/AdlerCLIArguments.py index 03cf41c..dea7e98 100644 --- a/src/adler/utilities/AdlerCLIArguments.py +++ b/src/adler/utilities/AdlerCLIArguments.py @@ -1,3 +1,6 @@ +import os + + class AdlerCLIArguments: """ Class for storing abd validating Adler command-line arguments. @@ -13,6 +16,8 @@ def __init__(self, args): self.ssObjectId = args.ssObjectId self.filter_list = args.filter_list self.date_range = args.date_range + self.outpath = args.outpath + self.db_name = args.db_name self.validate_arguments() @@ -20,20 +25,21 @@ def validate_arguments(self): self._validate_filter_list() self._validate_ssObjectId() self._validate_date_range() + self._validate_outpath() def _validate_filter_list(self): expected_filters = ["u", "g", "r", "i", "z", "y"] if not set(self.filter_list).issubset(expected_filters): raise ValueError( - "Unexpected filters found in filter_list command-line argument. filter_list must be a list of LSST filters." + "Unexpected filters found in --filter_list command-line argument. --filter_list must be a list of LSST filters." ) def _validate_ssObjectId(self): try: int(self.ssObjectId) except ValueError: - raise ValueError("ssObjectId command-line argument does not appear to be a valid ssObjectId.") + raise ValueError("--ssObjectId command-line argument does not appear to be a valid ssObjectId.") def _validate_date_range(self): for d in self.date_range: @@ -41,10 +47,17 @@ def _validate_date_range(self): float(d) except ValueError: raise ValueError( - "One or both of the values for the date_range command-line argument do not seem to be valid numbers." + "One or both of the values for the --date_range command-line argument do not seem to be valid numbers." ) if any(d > 250000 for d in self.date_range): raise ValueError( - "Dates for date_range command-line argument seem rather large. Did you input JD instead of MJD?" + "Dates for --date_range command-line argument seem rather large. Did you input JD instead of MJD?" ) + + def _validate_outpath(self): + # make it an absolute path if it's relative! + self.outpath = os.path.abspath(self.outpath) + + if not os.path.isdir(self.outpath): + raise ValueError("The output path for the command-line argument --outpath cannot be found.") diff --git a/src/adler/utilities/adler_logging.py b/src/adler/utilities/adler_logging.py new file mode 100644 index 0000000..6b8ebfe --- /dev/null +++ b/src/adler/utilities/adler_logging.py @@ -0,0 +1,39 @@ +import logging +import os +from datetime import datetime + + +def setup_adler_logging( + log_location, + log_format="%(asctime)s %(name)-12s %(levelname)-8s %(message)s ", + log_name="", + log_file_info="adler.log", + log_file_error="adler.err", +): + log = logging.getLogger(log_name) + log_formatter = logging.Formatter(log_format) + + # comment this to suppress console output + # stream_handler = logging.StreamHandler() + # stream_handler.setFormatter(log_formatter) + # log.addHandler(stream_handler) + + dstr = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + cpid = os.getpid() + + log_file_info = os.path.join(log_location, dstr + "-p" + str(cpid) + "-" + log_file_info) + log_file_error = os.path.join(log_location, dstr + "-p" + str(cpid) + "-" + log_file_error) + + file_handler_info = logging.FileHandler(log_file_info, mode="w") + file_handler_info.setFormatter(log_formatter) + file_handler_info.setLevel(logging.INFO) + log.addHandler(file_handler_info) + + file_handler_error = logging.FileHandler(log_file_error, mode="w") + file_handler_error.setFormatter(log_formatter) + file_handler_error.setLevel(logging.ERROR) + log.addHandler(file_handler_error) + + log.setLevel(logging.INFO) + + return log diff --git a/tests/adler/utilities/test_AdlerCLIArguments.py b/tests/adler/utilities/test_AdlerCLIArguments.py index e99d3b5..5dff38c 100644 --- a/tests/adler/utilities/test_AdlerCLIArguments.py +++ b/tests/adler/utilities/test_AdlerCLIArguments.py @@ -1,73 +1,104 @@ +import os import pytest from adler.utilities.AdlerCLIArguments import AdlerCLIArguments # AdlerCLIArguments object takes an object as input, so we define a quick one here class args: - def __init__(self, ssObjectId, filter_list, date_range): + def __init__(self, ssObjectId, filter_list, date_range, outpath, db_name): self.ssObjectId = ssObjectId self.filter_list = filter_list self.date_range = date_range + self.outpath = outpath + self.db_name = db_name -def test_AdlerCLIArguments(): +def test_AdlerCLIArguments_population(): # test correct population - good_input_dict = {"ssObjectId": "666", "filter_list": ["g", "r", "i"], "date_range": [60000.0, 67300.0]} + good_input_dict = { + "ssObjectId": "666", + "filter_list": ["g", "r", "i"], + "date_range": [60000.0, 67300.0], + "outpath": "./", + "db_name": "output", + } good_arguments = args(**good_input_dict) good_arguments_object = AdlerCLIArguments(good_arguments) + good_input_dict["outpath"] = os.path.abspath("./") + assert good_arguments_object.__dict__ == good_input_dict + +def test_AdlerCLIArguments_badSSOID(): # test that a bad ssObjectId triggers the right error - bad_ssoid_arguments = args("hello!", ["g", "r", "i"], [60000.0, 67300.0]) + bad_ssoid_arguments = args("hello!", ["g", "r", "i"], [60000.0, 67300.0], "./", "output") with pytest.raises(ValueError) as bad_ssoid_error: bad_ssoid_object = AdlerCLIArguments(bad_ssoid_arguments) assert ( bad_ssoid_error.value.args[0] - == "ssObjectId command-line argument does not appear to be a valid ssObjectId." + == "--ssObjectId command-line argument does not appear to be a valid ssObjectId." ) + +def test_AdlerCLIArguments_badfilters(): # test that non-LSST or unexpected filters trigger the right error - bad_filter_arguments = args("666", ["g", "r", "i", "m"], [60000.0, 67300.0]) + bad_filter_arguments = args("666", ["g", "r", "i", "m"], [60000.0, 67300.0], "./", "output") with pytest.raises(ValueError) as bad_filter_error: bad_filter_object = AdlerCLIArguments(bad_filter_arguments) assert ( bad_filter_error.value.args[0] - == "Unexpected filters found in filter_list command-line argument. filter_list must be a list of LSST filters." + == "Unexpected filters found in --filter_list command-line argument. --filter_list must be a list of LSST filters." ) - bad_filter_arguments_2 = args("666", ["pony"], [60000.0, 67300.0]) + bad_filter_arguments_2 = args("666", ["pony"], [60000.0, 67300.0], "./", "output") with pytest.raises(ValueError) as bad_filter_error_2: bad_filter_object = AdlerCLIArguments(bad_filter_arguments_2) assert ( bad_filter_error_2.value.args[0] - == "Unexpected filters found in filter_list command-line argument. filter_list must be a list of LSST filters." + == "Unexpected filters found in --filter_list command-line argument. --filter_list must be a list of LSST filters." ) + +def test_AdlerCLIArguments_baddates(): # test that overly-large dates trigger the right error - big_date_arguments = args("666", ["g", "r", "i"], [260000.0, 267300.0]) + big_date_arguments = args("666", ["g", "r", "i"], [260000.0, 267300.0], "./", "output") with pytest.raises(ValueError) as big_date_error: big_date_object = AdlerCLIArguments(big_date_arguments) assert ( big_date_error.value.args[0] - == "Dates for date_range command-line argument seem rather large. Did you input JD instead of MJD?" + == "Dates for --date_range command-line argument seem rather large. Did you input JD instead of MJD?" ) # test that unexpected date values trigger the right error - bad_date_arguments = args("666", ["g", "r", "i"], [260000.0, "cheese"]) + bad_date_arguments = args("666", ["g", "r", "i"], [60000.0, "cheese"], "./", "output") with pytest.raises(ValueError) as bad_date_error: bad_date_object = AdlerCLIArguments(bad_date_arguments) assert ( bad_date_error.value.args[0] - == "One or both of the values for the date_range command-line argument do not seem to be valid numbers." + == "One or both of the values for the --date_range command-line argument do not seem to be valid numbers." + ) + + +def test_AdlerCLIArguments_badoutput(): + bad_output_arguments = args( + "666", ["g", "r", "i"], [60000.0, 67300.0], "./definitely_fake_folder/", "output" + ) + + with pytest.raises(ValueError) as bad_output_error: + bad_output_object = AdlerCLIArguments(bad_output_arguments) + + assert ( + bad_output_error.value.args[0] + == "The output path for the command-line argument --outpath cannot be found." ) From 5b630e11f86bea94fed4a5ff84c43ec4bc9f3ff6 Mon Sep 17 00:00:00 2001 From: Steph Merritt Date: Fri, 10 May 2024 13:09:22 +0100 Subject: [PATCH 15/19] Missing imports. --- src/adler/dataclasses/AdlerData.py | 1 + src/adler/dataclasses/dataclass_utilities.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/adler/dataclasses/AdlerData.py b/src/adler/dataclasses/AdlerData.py index 0170e38..a6709e2 100644 --- a/src/adler/dataclasses/AdlerData.py +++ b/src/adler/dataclasses/AdlerData.py @@ -1,5 +1,6 @@ import os import sqlite3 +import logging import numpy as np from dataclasses import dataclass, field from datetime import datetime, timezone diff --git a/src/adler/dataclasses/dataclass_utilities.py b/src/adler/dataclasses/dataclass_utilities.py index 8abf9d0..fe7a514 100644 --- a/src/adler/dataclasses/dataclass_utilities.py +++ b/src/adler/dataclasses/dataclass_utilities.py @@ -2,6 +2,7 @@ import pandas as pd import sqlite3 import warnings +import logging logger = logging.getLogger(__name__) From dc1ad78ab682d55674fdb28fc306d49e476cdb11 Mon Sep 17 00:00:00 2001 From: Stephanie Merritt Date: Fri, 10 May 2024 13:24:45 +0000 Subject: [PATCH 16/19] Adding logging messages. --- src/adler/adler.py | 9 ++++++++ src/adler/dataclasses/AdlerData.py | 8 +++++++ src/adler/dataclasses/AdlerPlanetoid.py | 23 ++++++++++++++++---- src/adler/dataclasses/dataclass_utilities.py | 3 +++ src/adler/utilities/adler_logging.py | 5 ++++- 5 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/adler/adler.py b/src/adler/adler.py index be7fbe4..2c9a703 100644 --- a/src/adler/adler.py +++ b/src/adler/adler.py @@ -1,3 +1,4 @@ +import logging import argparse import astropy.units as u @@ -6,12 +7,20 @@ from adler.utilities.AdlerCLIArguments import AdlerCLIArguments from adler.utilities.adler_logging import setup_adler_logging +logger = logging.getLogger(__name__) def runAdler(cli_args): + + logger.info("Beginning Adler.") + logger.info("Ingesting all data for object {} from RSP...".format(cli_args.ssObjectId)) + planetoid = AdlerPlanetoid.construct_from_RSP( cli_args.ssObjectId, cli_args.filter_list, cli_args.date_range ) + logger.info("Data successfully ingested.") + logger.info("Calculating phase curves...") + # now let's do some phase curves! # get the r filter SSObject metadata diff --git a/src/adler/dataclasses/AdlerData.py b/src/adler/dataclasses/AdlerData.py index a6709e2..e1e5e1c 100644 --- a/src/adler/dataclasses/AdlerData.py +++ b/src/adler/dataclasses/AdlerData.py @@ -73,10 +73,12 @@ def populate_phase_parameters(self, filter_name, **kwargs): try: filter_index = self.filter_list.index(filter_name) except ValueError: + logger.error("ValueError: Filter {} does not exist in AdlerData.filter_list.".format(filter_name)) raise ValueError("Filter {} does not exist in AdlerData.filter_list.".format(filter_name)) # if model-dependent parameters exist without a model name, return an error if not kwargs.get("model_name") and any(name in kwargs for name in MODEL_DEPENDENT_KEYS): + logger.error("NameError: No model name given. Cannot update model-specific phase parameters.") raise NameError("No model name given. Cannot update model-specific phase parameters.") # update the value if it's in **kwargs @@ -166,6 +168,7 @@ def get_phase_parameters_in_filter(self, filter_name, model_name=None): try: filter_index = self.filter_list.index(filter_name) except ValueError: + logger.error("ValueError: Filter {} does not exist in AdlerData.filter_list.".format(filter_name)) raise ValueError("Filter {} does not exist in AdlerData.filter_list.".format(filter_name)) output_obj = PhaseParameterOutput() @@ -176,11 +179,16 @@ def get_phase_parameters_in_filter(self, filter_name, model_name=None): output_obj.arc = self.filter_dependent_values[filter_index].arc if not model_name: + logger.warn("No model name was specified. Returning non-model-dependent phase parameters.") print("No model name specified. Returning non-model-dependent phase parameters.") else: try: model_index = self.filter_dependent_values[filter_index].model_list.index(model_name) except ValueError: + logger.error("ValueError: Model {} does not exist for filter {} in AdlerData.model_lists.".format( + model_name, filter_name + ) + ) raise ValueError( "Model {} does not exist for filter {} in AdlerData.model_lists.".format( model_name, filter_name diff --git a/src/adler/dataclasses/AdlerPlanetoid.py b/src/adler/dataclasses/AdlerPlanetoid.py index 9c5a24c..30b5875 100644 --- a/src/adler/dataclasses/AdlerPlanetoid.py +++ b/src/adler/dataclasses/AdlerPlanetoid.py @@ -83,6 +83,7 @@ def construct_from_SQL( """ if len(date_range) != 2: + logger.error("ValueError: date_range attribute must be of length 2.") raise ValueError("date_range attribute must be of length 2.") observations_by_filter = cls.populate_observations( @@ -90,12 +91,15 @@ def construct_from_SQL( ) if len(observations_by_filter) == 0: + logger.error("No observations found for this object in the given filter(s). Check SSOID and try again.") raise Exception( "No observations found for this object in the given filter(s). Check SSOID and try again." ) - # redo the filter list based on the available filters in observations_by_filter - filter_list = [obs_object.filter_name for obs_object in observations_by_filter] + if len(filter_list) > len(observations_by_filter): + logger.info("Not all specified filters have observations. Recalculating filter list based on past observations.") + filter_list = [obs_object.filter_name for obs_object in observations_by_filter] + logger.info("New filter list is: {}".format(filter_list)) mpcorb = cls.populate_MPCORB(cls, ssObjectId, sql_filename=sql_filename, schema=schema) ssobject = cls.populate_SSObject( @@ -130,19 +134,25 @@ def construct_from_RSP( raise Exception("date_range argument must be of length 2.") service = get_tap_service("ssotap") + logger.info("Getting past observations from DIASource/SSSource...") observations_by_filter = cls.populate_observations( cls, ssObjectId, filter_list, date_range, service=service ) if len(observations_by_filter) == 0: + logger.error("No observations found for this object in the given filter(s). Check SSOID and try again.") raise Exception( "No observations found for this object in the given filter(s). Check SSOID and try again." ) - # redo the filter list based on the available filters in observations_by_filter - filter_list = [obs_object.filter_name for obs_object in observations_by_filter] + if len(filter_list) > len(observations_by_filter): + logger.info("Not all specified filters have observations. Recalculating filter list based on past observations.") + filter_list = [obs_object.filter_name for obs_object in observations_by_filter] + logger.info("New filter list is: {}".format(filter_list)) + logger.info("Populating MPCORB metadata...") mpcorb = cls.populate_MPCORB(cls, ssObjectId, service=service) + logger.info("Populating SSObject metadata...") ssobject = cls.populate_SSObject(cls, ssObjectId, filter_list, service=service) adler_data = AdlerData(ssObjectId, filter_list) @@ -206,6 +216,7 @@ def populate_observations( data_table = get_data_table(observations_sql_query, service=service, sql_filename=sql_filename) if len(data_table) == 0: + logger.warning("No observations found in {} filter for this object. Skipping this filter.".format(filter_name)) print( "WARNING: No observations found in {} filter for this object. Skipping this filter.".format( filter_name @@ -256,6 +267,7 @@ def populate_MPCORB(self, ssObjectId, service=None, sql_filename=None, schema="d data_table = get_data_table(MPCORB_sql_query, service=service, sql_filename=sql_filename) if len(data_table) == 0: + logger.error("No MPCORB data for this object could be found for this SSObjectId.") raise Exception("No MPCORB data for this object could be found for this SSObjectId.") return MPCORB.construct_from_data_table(ssObjectId, data_table) @@ -313,6 +325,7 @@ def populate_SSObject( data_table = get_data_table(SSObject_sql_query, service=service, sql_filename=sql_filename) if len(data_table) == 0: + logger.error("No SSObject data for this object could be found for this SSObjectId.") raise Exception("No SSObject data for this object could be found for this SSObjectId.") return SSObject.construct_from_data_table(ssObjectId, filter_list, data_table) @@ -335,6 +348,7 @@ def observations_in_filter(self, filter_name): try: filter_index = self.filter_list.index(filter_name) except ValueError: + logger.error("ValueError: Filter {} is not in AdlerPlanetoid.filter_list.".format(filter_name)) raise ValueError("Filter {} is not in AdlerPlanetoid.filter_list.".format(filter_name)) return self.observations_by_filter[filter_index] @@ -357,6 +371,7 @@ def SSObject_in_filter(self, filter_name): try: filter_index = self.filter_list.index(filter_name) except ValueError: + logger.error("ValueError: Filter {} is not in AdlerPlanetoid.filter_list.".format(filter_name)) raise ValueError("Filter {} is not in AdlerPlanetoid.filter_list.".format(filter_name)) return self.SSObject.filter_dependent_values[filter_index] diff --git a/src/adler/dataclasses/dataclass_utilities.py b/src/adler/dataclasses/dataclass_utilities.py index fe7a514..1eb3fbb 100644 --- a/src/adler/dataclasses/dataclass_utilities.py +++ b/src/adler/dataclasses/dataclass_utilities.py @@ -88,12 +88,14 @@ def get_from_table(data_table, column_name, data_type, table_name="default"): elif data_type == np.ndarray: data_val = np.array(data_table[column_name]) else: + logger.error("TypeError: Type for argument data_type not recognised for column {} in table {}: must be str, float, int or np.ndarray.".format(column_name, table_name)) raise TypeError( "Type for argument data_type not recognised for column {} in table {}: must be str, float, int or np.ndarray.".format( column_name, table_name ) ) except ValueError: + logger.error("ValueError: Could not cast column name to type.") raise ValueError("Could not cast column name to type.") # here we alert the user if one of the values is unpopulated and change it to a NaN @@ -132,6 +134,7 @@ def check_value_populated(data_val, data_type, column_name, table_name): str_is_empty = data_type == str and len(data_val) == 0 if array_length_zero or number_is_nan or str_is_empty: + logger.warning("{} unpopulated in {} table for this object. Storing NaN instead.".format(column_name, table_name)) print( "WARNING: {} unpopulated in {} table for this object. Storing NaN instead.".format( column_name, table_name diff --git a/src/adler/utilities/adler_logging.py b/src/adler/utilities/adler_logging.py index 6b8ebfe..d885c79 100644 --- a/src/adler/utilities/adler_logging.py +++ b/src/adler/utilities/adler_logging.py @@ -24,16 +24,19 @@ def setup_adler_logging( log_file_info = os.path.join(log_location, dstr + "-p" + str(cpid) + "-" + log_file_info) log_file_error = os.path.join(log_location, dstr + "-p" + str(cpid) + "-" + log_file_error) + # this log will log pretty much everything: basic info, but also warnings and errors file_handler_info = logging.FileHandler(log_file_info, mode="w") file_handler_info.setFormatter(log_formatter) file_handler_info.setLevel(logging.INFO) log.addHandler(file_handler_info) + # this log only logs warnings and errors, so they can be looked at quickly without a lot of scrolling file_handler_error = logging.FileHandler(log_file_error, mode="w") file_handler_error.setFormatter(log_formatter) - file_handler_error.setLevel(logging.ERROR) + file_handler_error.setLevel(logging.WARN) log.addHandler(file_handler_error) + # I don't know why we need this line but info logging doesn't work without it, upsettingly log.setLevel(logging.INFO) return log From 5684e3571a55ae8a29e6857d05f6a8e6a197e1b3 Mon Sep 17 00:00:00 2001 From: Steph Merritt Date: Fri, 10 May 2024 14:26:09 +0100 Subject: [PATCH 17/19] Linting. --- src/adler/adler.py | 6 +++--- src/adler/dataclasses/AdlerData.py | 3 ++- src/adler/dataclasses/AdlerPlanetoid.py | 22 +++++++++++++++----- src/adler/dataclasses/dataclass_utilities.py | 10 +++++++-- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/adler/adler.py b/src/adler/adler.py index 2c9a703..cdf1d09 100644 --- a/src/adler/adler.py +++ b/src/adler/adler.py @@ -9,18 +9,18 @@ logger = logging.getLogger(__name__) -def runAdler(cli_args): +def runAdler(cli_args): logger.info("Beginning Adler.") logger.info("Ingesting all data for object {} from RSP...".format(cli_args.ssObjectId)) - + planetoid = AdlerPlanetoid.construct_from_RSP( cli_args.ssObjectId, cli_args.filter_list, cli_args.date_range ) logger.info("Data successfully ingested.") logger.info("Calculating phase curves...") - + # now let's do some phase curves! # get the r filter SSObject metadata diff --git a/src/adler/dataclasses/AdlerData.py b/src/adler/dataclasses/AdlerData.py index e1e5e1c..86c2e2a 100644 --- a/src/adler/dataclasses/AdlerData.py +++ b/src/adler/dataclasses/AdlerData.py @@ -185,7 +185,8 @@ def get_phase_parameters_in_filter(self, filter_name, model_name=None): try: model_index = self.filter_dependent_values[filter_index].model_list.index(model_name) except ValueError: - logger.error("ValueError: Model {} does not exist for filter {} in AdlerData.model_lists.".format( + logger.error( + "ValueError: Model {} does not exist for filter {} in AdlerData.model_lists.".format( model_name, filter_name ) ) diff --git a/src/adler/dataclasses/AdlerPlanetoid.py b/src/adler/dataclasses/AdlerPlanetoid.py index 30b5875..93177e0 100644 --- a/src/adler/dataclasses/AdlerPlanetoid.py +++ b/src/adler/dataclasses/AdlerPlanetoid.py @@ -91,13 +91,17 @@ def construct_from_SQL( ) if len(observations_by_filter) == 0: - logger.error("No observations found for this object in the given filter(s). Check SSOID and try again.") + logger.error( + "No observations found for this object in the given filter(s). Check SSOID and try again." + ) raise Exception( "No observations found for this object in the given filter(s). Check SSOID and try again." ) if len(filter_list) > len(observations_by_filter): - logger.info("Not all specified filters have observations. Recalculating filter list based on past observations.") + logger.info( + "Not all specified filters have observations. Recalculating filter list based on past observations." + ) filter_list = [obs_object.filter_name for obs_object in observations_by_filter] logger.info("New filter list is: {}".format(filter_list)) @@ -140,13 +144,17 @@ def construct_from_RSP( ) if len(observations_by_filter) == 0: - logger.error("No observations found for this object in the given filter(s). Check SSOID and try again.") + logger.error( + "No observations found for this object in the given filter(s). Check SSOID and try again." + ) raise Exception( "No observations found for this object in the given filter(s). Check SSOID and try again." ) if len(filter_list) > len(observations_by_filter): - logger.info("Not all specified filters have observations. Recalculating filter list based on past observations.") + logger.info( + "Not all specified filters have observations. Recalculating filter list based on past observations." + ) filter_list = [obs_object.filter_name for obs_object in observations_by_filter] logger.info("New filter list is: {}".format(filter_list)) @@ -216,7 +224,11 @@ def populate_observations( data_table = get_data_table(observations_sql_query, service=service, sql_filename=sql_filename) if len(data_table) == 0: - logger.warning("No observations found in {} filter for this object. Skipping this filter.".format(filter_name)) + logger.warning( + "No observations found in {} filter for this object. Skipping this filter.".format( + filter_name + ) + ) print( "WARNING: No observations found in {} filter for this object. Skipping this filter.".format( filter_name diff --git a/src/adler/dataclasses/dataclass_utilities.py b/src/adler/dataclasses/dataclass_utilities.py index 1eb3fbb..497e1a6 100644 --- a/src/adler/dataclasses/dataclass_utilities.py +++ b/src/adler/dataclasses/dataclass_utilities.py @@ -88,7 +88,11 @@ def get_from_table(data_table, column_name, data_type, table_name="default"): elif data_type == np.ndarray: data_val = np.array(data_table[column_name]) else: - logger.error("TypeError: Type for argument data_type not recognised for column {} in table {}: must be str, float, int or np.ndarray.".format(column_name, table_name)) + logger.error( + "TypeError: Type for argument data_type not recognised for column {} in table {}: must be str, float, int or np.ndarray.".format( + column_name, table_name + ) + ) raise TypeError( "Type for argument data_type not recognised for column {} in table {}: must be str, float, int or np.ndarray.".format( column_name, table_name @@ -134,7 +138,9 @@ def check_value_populated(data_val, data_type, column_name, table_name): str_is_empty = data_type == str and len(data_val) == 0 if array_length_zero or number_is_nan or str_is_empty: - logger.warning("{} unpopulated in {} table for this object. Storing NaN instead.".format(column_name, table_name)) + logger.warning( + "{} unpopulated in {} table for this object. Storing NaN instead.".format(column_name, table_name) + ) print( "WARNING: {} unpopulated in {} table for this object. Storing NaN instead.".format( column_name, table_name From 30694b5b50bedec60d5bf8f774fb40c3e4fa57b1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Fri, 10 May 2024 13:26:53 +0000 Subject: [PATCH 18/19] [pre-commit.ci lite] apply automatic fixes --- src/adler/adler.py | 6 +++--- src/adler/dataclasses/AdlerData.py | 3 ++- src/adler/dataclasses/AdlerPlanetoid.py | 22 +++++++++++++++----- src/adler/dataclasses/dataclass_utilities.py | 10 +++++++-- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/adler/adler.py b/src/adler/adler.py index 2c9a703..cdf1d09 100644 --- a/src/adler/adler.py +++ b/src/adler/adler.py @@ -9,18 +9,18 @@ logger = logging.getLogger(__name__) -def runAdler(cli_args): +def runAdler(cli_args): logger.info("Beginning Adler.") logger.info("Ingesting all data for object {} from RSP...".format(cli_args.ssObjectId)) - + planetoid = AdlerPlanetoid.construct_from_RSP( cli_args.ssObjectId, cli_args.filter_list, cli_args.date_range ) logger.info("Data successfully ingested.") logger.info("Calculating phase curves...") - + # now let's do some phase curves! # get the r filter SSObject metadata diff --git a/src/adler/dataclasses/AdlerData.py b/src/adler/dataclasses/AdlerData.py index e1e5e1c..86c2e2a 100644 --- a/src/adler/dataclasses/AdlerData.py +++ b/src/adler/dataclasses/AdlerData.py @@ -185,7 +185,8 @@ def get_phase_parameters_in_filter(self, filter_name, model_name=None): try: model_index = self.filter_dependent_values[filter_index].model_list.index(model_name) except ValueError: - logger.error("ValueError: Model {} does not exist for filter {} in AdlerData.model_lists.".format( + logger.error( + "ValueError: Model {} does not exist for filter {} in AdlerData.model_lists.".format( model_name, filter_name ) ) diff --git a/src/adler/dataclasses/AdlerPlanetoid.py b/src/adler/dataclasses/AdlerPlanetoid.py index 30b5875..93177e0 100644 --- a/src/adler/dataclasses/AdlerPlanetoid.py +++ b/src/adler/dataclasses/AdlerPlanetoid.py @@ -91,13 +91,17 @@ def construct_from_SQL( ) if len(observations_by_filter) == 0: - logger.error("No observations found for this object in the given filter(s). Check SSOID and try again.") + logger.error( + "No observations found for this object in the given filter(s). Check SSOID and try again." + ) raise Exception( "No observations found for this object in the given filter(s). Check SSOID and try again." ) if len(filter_list) > len(observations_by_filter): - logger.info("Not all specified filters have observations. Recalculating filter list based on past observations.") + logger.info( + "Not all specified filters have observations. Recalculating filter list based on past observations." + ) filter_list = [obs_object.filter_name for obs_object in observations_by_filter] logger.info("New filter list is: {}".format(filter_list)) @@ -140,13 +144,17 @@ def construct_from_RSP( ) if len(observations_by_filter) == 0: - logger.error("No observations found for this object in the given filter(s). Check SSOID and try again.") + logger.error( + "No observations found for this object in the given filter(s). Check SSOID and try again." + ) raise Exception( "No observations found for this object in the given filter(s). Check SSOID and try again." ) if len(filter_list) > len(observations_by_filter): - logger.info("Not all specified filters have observations. Recalculating filter list based on past observations.") + logger.info( + "Not all specified filters have observations. Recalculating filter list based on past observations." + ) filter_list = [obs_object.filter_name for obs_object in observations_by_filter] logger.info("New filter list is: {}".format(filter_list)) @@ -216,7 +224,11 @@ def populate_observations( data_table = get_data_table(observations_sql_query, service=service, sql_filename=sql_filename) if len(data_table) == 0: - logger.warning("No observations found in {} filter for this object. Skipping this filter.".format(filter_name)) + logger.warning( + "No observations found in {} filter for this object. Skipping this filter.".format( + filter_name + ) + ) print( "WARNING: No observations found in {} filter for this object. Skipping this filter.".format( filter_name diff --git a/src/adler/dataclasses/dataclass_utilities.py b/src/adler/dataclasses/dataclass_utilities.py index 1eb3fbb..497e1a6 100644 --- a/src/adler/dataclasses/dataclass_utilities.py +++ b/src/adler/dataclasses/dataclass_utilities.py @@ -88,7 +88,11 @@ def get_from_table(data_table, column_name, data_type, table_name="default"): elif data_type == np.ndarray: data_val = np.array(data_table[column_name]) else: - logger.error("TypeError: Type for argument data_type not recognised for column {} in table {}: must be str, float, int or np.ndarray.".format(column_name, table_name)) + logger.error( + "TypeError: Type for argument data_type not recognised for column {} in table {}: must be str, float, int or np.ndarray.".format( + column_name, table_name + ) + ) raise TypeError( "Type for argument data_type not recognised for column {} in table {}: must be str, float, int or np.ndarray.".format( column_name, table_name @@ -134,7 +138,9 @@ def check_value_populated(data_val, data_type, column_name, table_name): str_is_empty = data_type == str and len(data_val) == 0 if array_length_zero or number_is_nan or str_is_empty: - logger.warning("{} unpopulated in {} table for this object. Storing NaN instead.".format(column_name, table_name)) + logger.warning( + "{} unpopulated in {} table for this object. Storing NaN instead.".format(column_name, table_name) + ) print( "WARNING: {} unpopulated in {} table for this object. Storing NaN instead.".format( column_name, table_name From 112b4983d3a90dbf6f5b83563925b50d54ae8859 Mon Sep 17 00:00:00 2001 From: Steph Merritt Date: Fri, 10 May 2024 14:33:46 +0100 Subject: [PATCH 19/19] Unit test for log creation. --- tests/adler/utilities/test_adler_logging.py | 40 +++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 tests/adler/utilities/test_adler_logging.py diff --git a/tests/adler/utilities/test_adler_logging.py b/tests/adler/utilities/test_adler_logging.py new file mode 100644 index 0000000..eefd91a --- /dev/null +++ b/tests/adler/utilities/test_adler_logging.py @@ -0,0 +1,40 @@ +import glob +import os +import pytest +import tempfile + + +def test_setup_adler_logging(): + from adler.utilities.adler_logging import setup_adler_logging + + with tempfile.TemporaryDirectory() as dir_name: + logger = setup_adler_logging(dir_name) + + # Check that the files get created. + errlog = glob.glob(os.path.join(dir_name, "*-adler.err")) + datalog = glob.glob(os.path.join(dir_name, "*-adler.log")) + + assert os.path.exists(errlog[0]) + assert os.path.exists(datalog[0]) + + # Log some information. + logger.info("Test1") + logger.info("Test2") + logger.error("Error1") + logger.info("Test3") + + # Check that all five lines exist in the INFO file. + with open(datalog[0], "r") as f_info: + log_data = f_info.read() + assert "Test1" in log_data + assert "Test2" in log_data + assert "Error1" in log_data + assert "Test3" in log_data + + # Check that only error and critical lines exist in the ERROR file. + with open(errlog[0], "r") as f_err: + log_data = f_err.read() + assert "Test1" not in log_data + assert "Test2" not in log_data + assert "Error1" in log_data + assert "Test3" not in log_data