diff --git a/day_integration_test.yaml b/day_integration_test.yaml index 14a9520..66eb04a 100644 --- a/day_integration_test.yaml +++ b/day_integration_test.yaml @@ -16,13 +16,33 @@ modules: url: "localhost" port: 8080 - # This is overridden by the test module measurement_log: - url: "" - bucket: "" - org: "" - token: "" - measurement_name: "measurement" + # This is overridden by the test module + + logger: + type: "influx" + url: "" + bucket: "" + org: "" + token: "" + + measurements: + cloud_zenith: + - type: "zenith" + name: "cloudcover" + altitude: 80.0 + cloud_total: + - type: "total" + name: "cloudcover" + coverage: + - type: "change" + name: "coverage-change" + - type: "sun" + name: "sun-coverage" + radius: 20.0 + - type: "moon" + name: "moon-coverage" + radius: 20.0 pipelines: day: diff --git a/example.yaml b/example.yaml index 96412fa..3403dc5 100644 --- a/example.yaml +++ b/example.yaml @@ -12,11 +12,31 @@ server: # Influx DB measurement_log: - url: "" - bucket: "" - org: "" - token: "" - measurement_name: "measurement" + logger: + type: "influx" + url: "" + bucket: "" + org: "" + token: "" + + measurements: + cloud_zenith: + - type: "zenith" + name: "cloudcover" + altitude: 80.0 + cloud_total: + - type: "total" + name: "cloudcover" + coverage: + - type: "change" + name: "coverage-change" + - type: "sun" + name: "sun-coverage" + radius: 20.0 + - type: "moon" + name: "moon-coverage" + radius: 20.0 + # Different image analysis pipelines can be used for different solar altitude intervals # Currently only the "night" pipeline is available, which works best after the astronomical dawn diff --git a/night_integration_test.yaml b/night_integration_test.yaml index 2398db6..50ab430 100644 --- a/night_integration_test.yaml +++ b/night_integration_test.yaml @@ -16,13 +16,32 @@ modules: url: "localhost" port: 8080 - # This is overridden by the test module measurement_log: - url: "" - bucket: "" - org: "" - token: "" - measurement_name: "measurement" + # This is overridden by the test module + logger: + type: "influx" + url: "" + bucket: "" + org: "" + token: "" + + measurements: + cloud_zenith: + - type: "zenith" + name: "cloudcover" + altitude: 80.0 + cloud_total: + - type: "total" + name: "cloudcover" + coverage: + - type: "change" + name: "coverage-change" + - type: "sun" + name: "sun-coverage" + radius: 20.0 + - type: "moon" + name: "moon-coverage" + radius: 20.0 pipelines: night: diff --git a/pyobs_cloudcover/application.py b/pyobs_cloudcover/application.py index 1ffb218..919939d 100644 --- a/pyobs_cloudcover/application.py +++ b/pyobs_cloudcover/application.py @@ -5,7 +5,8 @@ from pyobs.events import NewImageEvent, Event from pyobs.modules import Module -from pyobs_cloudcover.measurement_log.influx import Influx +from pyobs_cloudcover.measurement_log.logger_strategies.influx import Influx +from pyobs_cloudcover.measurement_log.measurment_logger_factory import MeasurementLoggerFactory from pyobs_cloudcover.pipeline.pipeline_controller_factory import PipelineControllerFactory from pyobs_cloudcover.web_api.server_factory import ServerFactory @@ -26,7 +27,8 @@ def __init__(self, server_factory = ServerFactory(self.observer) self._server = server_factory(server) - self._measurement_log = Influx(**measurement_log) + measurement_logger_factory = MeasurementLoggerFactory(self.observer) + self._measurement_log = measurement_logger_factory(measurement_log) pipeline_controller_factory = PipelineControllerFactory(self.observer) self._pipeline_controller = pipeline_controller_factory(pipelines) @@ -53,8 +55,6 @@ async def process_new_image(self, event: Event, sender: str) -> None: measurement = self._pipeline_controller(image.data, obs_time) if measurement is not None: - log.debug(f"{measurement.total_cover}, {measurement.zenith_cover}, {measurement.change}") - self._server.set_measurement(measurement) self._measurement_log(measurement) diff --git a/pyobs_cloudcover/cloud_coverage_info.py b/pyobs_cloudcover/cloud_coverage_info.py index 8182122..f6b09a9 100644 --- a/pyobs_cloudcover/cloud_coverage_info.py +++ b/pyobs_cloudcover/cloud_coverage_info.py @@ -5,10 +5,7 @@ class CloudCoverageInfo(object): - def __init__(self, cloud_cover_query: SkyPixelQuery, - total_cover: float, zenith_cover: Optional[float], change: Optional[float], obs_time: datetime.datetime) -> None: + def __init__(self, cloud_cover_query: SkyPixelQuery, change: Optional[float], obs_time: datetime.datetime) -> None: self.cloud_cover_query = cloud_cover_query - self.total_cover = total_cover - self.zenith_cover = zenith_cover self.change = change self.obs_time = obs_time diff --git a/pyobs_cloudcover/cloud_info_calculator/__init__.py b/pyobs_cloudcover/cloud_info_calculator/__init__.py index f8519f0..fa5879a 100644 --- a/pyobs_cloudcover/cloud_info_calculator/__init__.py +++ b/pyobs_cloudcover/cloud_info_calculator/__init__.py @@ -1,6 +1,5 @@ from pyobs_cloudcover.cloud_info_calculator.coverage_info_calculator import CoverageInfoCalculator -from pyobs_cloudcover.cloud_info_calculator.zenith_cloud_coverage_calculator import ZenithCloudCoverageCalculator +from pyobs_cloudcover.measurement_log.field_evaluators.zenith_cloud_coverage_calculator import ZenithCloudCoverageCalculator from pyobs_cloudcover.cloud_info_calculator.cloud_info_calculator_factory import CloudInfoCalculatorFactory -from pyobs_cloudcover.cloud_info_calculator.cloud_info_calculator_options import CloudInfoCalculatorOptions -__all__ = ["CoverageInfoCalculator", "ZenithCloudCoverageCalculator", "CloudInfoCalculatorFactory", "CloudInfoCalculatorOptions"] \ No newline at end of file +__all__ = ["CoverageInfoCalculator", "ZenithCloudCoverageCalculator", "CloudInfoCalculatorFactory"] \ No newline at end of file diff --git a/pyobs_cloudcover/cloud_info_calculator/cloud_info_calculator_factory.py b/pyobs_cloudcover/cloud_info_calculator/cloud_info_calculator_factory.py index f671dac..000275c 100644 --- a/pyobs_cloudcover/cloud_info_calculator/cloud_info_calculator_factory.py +++ b/pyobs_cloudcover/cloud_info_calculator/cloud_info_calculator_factory.py @@ -1,18 +1,12 @@ -from pyobs_cloudcover.cloud_info_calculator.cloud_info_calculator_options import \ - CloudInfoCalculatorOptions from pyobs_cloudcover.cloud_info_calculator.coverage_change_calculator import \ CoverageChangeCalculator from pyobs_cloudcover.cloud_info_calculator.coverage_info_calculator import CoverageInfoCalculator -from pyobs_cloudcover.cloud_info_calculator.zenith_cloud_coverage_calculator import ZenithCloudCoverageCalculator class CloudInfoCalculatorFactory(object): - def __init__(self, options: CloudInfoCalculatorOptions): - self._options = options def __call__(self) -> CoverageInfoCalculator: coverage_change_calculator = CoverageChangeCalculator() - zenith_masker = ZenithCloudCoverageCalculator(self._options.zenith_altitude) - cloud_coverage_info_calculator = CoverageInfoCalculator(coverage_change_calculator, zenith_masker) + cloud_coverage_info_calculator = CoverageInfoCalculator(coverage_change_calculator) return cloud_coverage_info_calculator diff --git a/pyobs_cloudcover/cloud_info_calculator/cloud_info_calculator_options.py b/pyobs_cloudcover/cloud_info_calculator/cloud_info_calculator_options.py deleted file mode 100644 index 70073d2..0000000 --- a/pyobs_cloudcover/cloud_info_calculator/cloud_info_calculator_options.py +++ /dev/null @@ -1,12 +0,0 @@ -from __future__ import annotations - -from typing import Dict, Any - - -class CloudInfoCalculatorOptions(object): - def __init__(self, zenith_altitude: float) -> None: - self.zenith_altitude = zenith_altitude - - @classmethod - def from_dict(cls, options: Dict[str, Any]) -> CloudInfoCalculatorOptions: - return CloudInfoCalculatorOptions(**options) diff --git a/pyobs_cloudcover/cloud_info_calculator/coverage_info_calculator.py b/pyobs_cloudcover/cloud_info_calculator/coverage_info_calculator.py index d1da9e0..217fa1e 100644 --- a/pyobs_cloudcover/cloud_info_calculator/coverage_info_calculator.py +++ b/pyobs_cloudcover/cloud_info_calculator/coverage_info_calculator.py @@ -6,19 +6,15 @@ from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo from pyobs_cloudcover.cloud_info_calculator.coverage_change_calculator import \ CoverageChangeCalculator -from pyobs_cloudcover.cloud_info_calculator.zenith_cloud_coverage_calculator import \ +from pyobs_cloudcover.measurement_log.field_evaluators.zenith_cloud_coverage_calculator import \ ZenithCloudCoverageCalculator class CoverageInfoCalculator: - def __init__(self, coverage_change_calculator: CoverageChangeCalculator, zenith_coverage: ZenithCloudCoverageCalculator) -> None: + def __init__(self, coverage_change_calculator: CoverageChangeCalculator) -> None: self._coverage_change_calculator = coverage_change_calculator - self._zenith_coverage = zenith_coverage def __call__(self, sky_query: SkyPixelQuery, obs_time: datetime.datetime) -> CloudCoverageInfo: - coverage = sky_query.query_radius(AltAzCoord(np.pi/2, 0), np.pi/2) change = self._coverage_change_calculator(sky_query.get_pixels()) - zenith_coverage = self._zenith_coverage(sky_query) - - return CloudCoverageInfo(sky_query, coverage, zenith_coverage, change, obs_time) + return CloudCoverageInfo(sky_query, change, obs_time) diff --git a/pyobs_cloudcover/cloud_info_calculator/zenith_cloud_coverage_calculator.py b/pyobs_cloudcover/cloud_info_calculator/zenith_cloud_coverage_calculator.py deleted file mode 100644 index 267c54b..0000000 --- a/pyobs_cloudcover/cloud_info_calculator/zenith_cloud_coverage_calculator.py +++ /dev/null @@ -1,15 +0,0 @@ -from typing import Optional, cast - -import numpy as np -from cloudmap_rs import AltAzCoord, SkyPixelQuery - - -class ZenithCloudCoverageCalculator(object): - def __init__(self, altitude: float) -> None: - self._altitude = altitude - - def __call__(self, sky_query: SkyPixelQuery) -> Optional[float]: - radius = np.pi/2 - np.deg2rad(self._altitude) - return cast(Optional[float], sky_query.query_radius(AltAzCoord(np.pi/2, 0), radius)) - - diff --git a/pyobs_cloudcover/measurement_log/field_evaluators/__init__.py b/pyobs_cloudcover/measurement_log/field_evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyobs_cloudcover/measurement_log/field_evaluators/cloud_change.py b/pyobs_cloudcover/measurement_log/field_evaluators/cloud_change.py new file mode 100644 index 0000000..d7cd3d0 --- /dev/null +++ b/pyobs_cloudcover/measurement_log/field_evaluators/cloud_change.py @@ -0,0 +1,9 @@ +from typing import Optional + +from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo +from pyobs_cloudcover.measurement_log.field_evaluators.field_evaluator import FieldEvaluator + + +class CloudChangeFieldEvaluator(FieldEvaluator): + def __call__(self, cloud_info: CloudCoverageInfo) -> Optional[float]: + return cloud_info.change diff --git a/pyobs_cloudcover/measurement_log/field_evaluators/field_evaluator.py b/pyobs_cloudcover/measurement_log/field_evaluators/field_evaluator.py new file mode 100644 index 0000000..750c339 --- /dev/null +++ b/pyobs_cloudcover/measurement_log/field_evaluators/field_evaluator.py @@ -0,0 +1,11 @@ +import abc +from typing import Optional + +from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo + + +class FieldEvaluator(object, metaclass=abc.ABCMeta): + + @abc.abstractmethod + def __call__(self, cloud_info: CloudCoverageInfo) -> Optional[float]: + ... diff --git a/pyobs_cloudcover/measurement_log/field_evaluators/field_evaluators_factory.py b/pyobs_cloudcover/measurement_log/field_evaluators/field_evaluators_factory.py new file mode 100644 index 0000000..952f7d1 --- /dev/null +++ b/pyobs_cloudcover/measurement_log/field_evaluators/field_evaluators_factory.py @@ -0,0 +1,35 @@ +from typing import Dict + +from astroplan import Observer + +from pyobs_cloudcover.measurement_log.field_evaluators.cloud_change import CloudChangeFieldEvaluator +from pyobs_cloudcover.measurement_log.field_evaluators.field_evaluator import FieldEvaluator +from pyobs_cloudcover.measurement_log.field_evaluators.moon_cloud_coverage_calculator import MoonCloudCoverageCalculator +from pyobs_cloudcover.measurement_log.field_evaluators.sun_cloud_coverage_calculator import SunCloudCoverageCalculator +from pyobs_cloudcover.measurement_log.field_evaluators.total_cloud_coverage_calculator import \ + TotalCloudCoverageCalculator +from pyobs_cloudcover.measurement_log.field_evaluators.zenith_cloud_coverage_calculator import \ + ZenithCloudCoverageCalculator + + +class FieldEvaluatorFactory: + def __init__(self, observer: Observer): + self._observer = observer + + def __call__(self, config: Dict[str, str]) -> FieldEvaluator: + if config["type"] == "total": + return TotalCloudCoverageCalculator() + + if config["type"] == "zenith": + return ZenithCloudCoverageCalculator(float(config["altitude"])) + + if config["type"] == "change": + return CloudChangeFieldEvaluator() + + if config["type"] == "sun": + return SunCloudCoverageCalculator(self._observer, float(config["radius"])) + + if config["type"] == "moon": + return MoonCloudCoverageCalculator(self._observer, float(config["radius"])) + + raise ValueError("Invalid Type") diff --git a/pyobs_cloudcover/measurement_log/field_evaluators/moon_cloud_coverage_calculator.py b/pyobs_cloudcover/measurement_log/field_evaluators/moon_cloud_coverage_calculator.py new file mode 100644 index 0000000..96fef07 --- /dev/null +++ b/pyobs_cloudcover/measurement_log/field_evaluators/moon_cloud_coverage_calculator.py @@ -0,0 +1,23 @@ +from typing import Optional, cast + +import numpy as np +from astroplan import Observer +from cloudmap_rs import AltAzCoord + +from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo +from pyobs_cloudcover.measurement_log.field_evaluators.field_evaluator import FieldEvaluator + + +class MoonCloudCoverageCalculator(FieldEvaluator): + def __init__(self, observer: Observer, radius: float): + self._observer = observer + self._radius = radius + + def __call__(self, cloud_info: CloudCoverageInfo) -> Optional[float]: + sun_alt_az = self._observer.moon_altaz(cloud_info.obs_time) + coverage = cloud_info.cloud_cover_query.query_radius( + AltAzCoord(sun_alt_az.alt.rad, sun_alt_az.az.rad), + np.deg2rad(self._radius) + ) + + return cast(Optional[float], coverage) diff --git a/pyobs_cloudcover/measurement_log/field_evaluators/sun_cloud_coverage_calculator.py b/pyobs_cloudcover/measurement_log/field_evaluators/sun_cloud_coverage_calculator.py new file mode 100644 index 0000000..8f0c98a --- /dev/null +++ b/pyobs_cloudcover/measurement_log/field_evaluators/sun_cloud_coverage_calculator.py @@ -0,0 +1,23 @@ +from typing import Optional, cast + +import numpy as np +from astroplan import Observer +from cloudmap_rs import AltAzCoord + +from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo +from pyobs_cloudcover.measurement_log.field_evaluators.field_evaluator import FieldEvaluator + + +class SunCloudCoverageCalculator(FieldEvaluator): + def __init__(self, observer: Observer, radius: float): + self._observer = observer + self._radius = radius + + def __call__(self, cloud_info: CloudCoverageInfo) -> Optional[float]: + sun_alt_az = self._observer.sun_altaz(cloud_info.obs_time) + coverage = cloud_info.cloud_cover_query.query_radius( + AltAzCoord(sun_alt_az.alt.rad, sun_alt_az.az.rad), + np.deg2rad(self._radius) + ) + + return cast(Optional[float], coverage) diff --git a/pyobs_cloudcover/measurement_log/field_evaluators/total_cloud_coverage_calculator.py b/pyobs_cloudcover/measurement_log/field_evaluators/total_cloud_coverage_calculator.py new file mode 100644 index 0000000..1f11f4e --- /dev/null +++ b/pyobs_cloudcover/measurement_log/field_evaluators/total_cloud_coverage_calculator.py @@ -0,0 +1,13 @@ +from typing import Optional, cast + +import numpy as np +from cloudmap_rs import AltAzCoord + +from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo +from pyobs_cloudcover.measurement_log.field_evaluators.field_evaluator import FieldEvaluator + + +class TotalCloudCoverageCalculator(FieldEvaluator): + def __call__(self, cloud_info: CloudCoverageInfo) -> Optional[float]: + coverage = cloud_info.cloud_cover_query.query_radius(AltAzCoord(np.pi / 2, 0), np.pi / 2) + return cast(Optional[float], coverage) diff --git a/pyobs_cloudcover/measurement_log/field_evaluators/zenith_cloud_coverage_calculator.py b/pyobs_cloudcover/measurement_log/field_evaluators/zenith_cloud_coverage_calculator.py new file mode 100644 index 0000000..b0154d2 --- /dev/null +++ b/pyobs_cloudcover/measurement_log/field_evaluators/zenith_cloud_coverage_calculator.py @@ -0,0 +1,17 @@ +from typing import Optional, cast + +import numpy as np +from cloudmap_rs import AltAzCoord, SkyPixelQuery + +from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo +from pyobs_cloudcover.measurement_log.field_evaluators.field_evaluator import FieldEvaluator + + +class ZenithCloudCoverageCalculator(FieldEvaluator): + def __init__(self, altitude: float) -> None: + self._altitude = altitude + + def __call__(self, cloud_info: CloudCoverageInfo) -> Optional[float]: + radius = np.pi/2 - np.deg2rad(self._altitude) + zenith_cover = cloud_info.cloud_cover_query.query_radius(AltAzCoord(np.pi/2, 0), radius) + return cast(Optional[float], zenith_cover) diff --git a/pyobs_cloudcover/measurement_log/influx.py b/pyobs_cloudcover/measurement_log/influx.py deleted file mode 100644 index 5604d80..0000000 --- a/pyobs_cloudcover/measurement_log/influx.py +++ /dev/null @@ -1,32 +0,0 @@ -import datetime - -import influxdb_client -from influxdb_client.client.write_api import SYNCHRONOUS - - -from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo -from pyobs_cloudcover.measurement_log.measurement_log import MeasurementLog - - -class Influx(MeasurementLog): - def __init__(self, url: str, bucket: str, org: str, token: str, measurement_name: str) -> None: - self._client = influxdb_client.InfluxDBClient( - url=url, - token=token, - org=org - ) - self._bucket = bucket - self._org = org - self._measurement_name = measurement_name - - def __call__(self, measurement: CloudCoverageInfo) -> None: - data = influxdb_client.Point(self._measurement_name) - - data.time(measurement.obs_time) - - data.field("total-cover", measurement.total_cover) - data.field("zenith-cover", measurement.zenith_cover) - data.field("change", measurement.change) - - with self._client.write_api(write_options=SYNCHRONOUS) as write_api: - write_api.write(bucket=self._bucket, org=self._org, record=data) diff --git a/pyobs_cloudcover/measurement_log/logger_strategies/__init__.py b/pyobs_cloudcover/measurement_log/logger_strategies/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyobs_cloudcover/measurement_log/logger_strategies/influx.py b/pyobs_cloudcover/measurement_log/logger_strategies/influx.py new file mode 100644 index 0000000..166b46f --- /dev/null +++ b/pyobs_cloudcover/measurement_log/logger_strategies/influx.py @@ -0,0 +1,36 @@ +import datetime +from typing import Dict, Optional + +import influxdb_client +from influxdb_client.client.write_api import SYNCHRONOUS + +from pyobs_cloudcover.measurement_log.logger_strategies.measurement_log import LoggerStrategy + + +class Influx(LoggerStrategy): + def __init__(self, url: str, bucket: str, org: str, token: str) -> None: + self._client = influxdb_client.InfluxDBClient( + url=url, + token=token, + org=org + ) + self._bucket = bucket + self._org = org + + def __call__(self, measurements: Dict[str, Dict[str, Optional[float]]], obs_time: datetime.datetime) -> None: + points = [ + self._to_point(name, measurements, obs_time) for name, measurements in measurements.items() + ] + + with self._client.write_api(write_options=SYNCHRONOUS) as write_api: + write_api.write(bucket=self._bucket, org=self._org, record=points) + + @staticmethod + def _to_point(name: str, measurements: Dict[str, Optional[float]], obs_time: datetime.datetime) -> influxdb_client.Point: + data = influxdb_client.Point(name) + data.time(obs_time) + + for field_name, field_value in measurements.items(): + data.field(field_name, field_value) + + return data diff --git a/pyobs_cloudcover/measurement_log/logger_strategies/measurement_log.py b/pyobs_cloudcover/measurement_log/logger_strategies/measurement_log.py new file mode 100644 index 0000000..b90d2fe --- /dev/null +++ b/pyobs_cloudcover/measurement_log/logger_strategies/measurement_log.py @@ -0,0 +1,9 @@ +import abc +import datetime +from typing import Dict, Optional + + +class LoggerStrategy(object, metaclass=abc.ABCMeta): + @abc.abstractmethod + def __call__(self, measurements: Dict[str, Dict[str, Optional[float]]], obs_time: datetime.datetime) -> None: + ... diff --git a/pyobs_cloudcover/measurement_log/logger_strategies/measurment_log_factory.py b/pyobs_cloudcover/measurement_log/logger_strategies/measurment_log_factory.py new file mode 100644 index 0000000..a83fe11 --- /dev/null +++ b/pyobs_cloudcover/measurement_log/logger_strategies/measurment_log_factory.py @@ -0,0 +1,18 @@ +from typing import Dict + +from pyobs_cloudcover.measurement_log.logger_strategies.influx import Influx +from pyobs_cloudcover.measurement_log.logger_strategies.measurement_log import LoggerStrategy + + +class MeasurementLogFactory: + @staticmethod + def build(config: Dict[str, str]) -> LoggerStrategy: + if config["type"] == "influx": + return Influx( + url=config["url"], + bucket=config["bucket"], + org=config["org"], + token=config["token"] + ) + + raise NotImplementedError(config["type"]) \ No newline at end of file diff --git a/pyobs_cloudcover/measurement_log/measurement.py b/pyobs_cloudcover/measurement_log/measurement.py new file mode 100644 index 0000000..26b1f0b --- /dev/null +++ b/pyobs_cloudcover/measurement_log/measurement.py @@ -0,0 +1,14 @@ +from typing import Dict, Optional + +from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo +from pyobs_cloudcover.measurement_log.field_evaluators.field_evaluator import FieldEvaluator + + +class Measurement: + def __init__(self, field_evaluators: Dict[str, FieldEvaluator]): + self._field_evaluators = field_evaluators + + def __call__(self, cloud_info: CloudCoverageInfo) -> Dict[str, Optional[float]]: + return { + name: evaluator(cloud_info) for name, evaluator in self._field_evaluators.items() + } diff --git a/pyobs_cloudcover/measurement_log/measurement_log.py b/pyobs_cloudcover/measurement_log/measurement_log.py deleted file mode 100644 index 3b17335..0000000 --- a/pyobs_cloudcover/measurement_log/measurement_log.py +++ /dev/null @@ -1,10 +0,0 @@ -import abc -import datetime - -from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo - - -class MeasurementLog(object, metaclass=abc.ABCMeta): - @abc.abstractmethod - def __call__(self, measurement: CloudCoverageInfo) -> None: - ... diff --git a/pyobs_cloudcover/measurement_log/measurment_logger.py b/pyobs_cloudcover/measurement_log/measurment_logger.py new file mode 100644 index 0000000..494d062 --- /dev/null +++ b/pyobs_cloudcover/measurement_log/measurment_logger.py @@ -0,0 +1,20 @@ +from typing import Dict, Optional + +from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo +from pyobs_cloudcover.measurement_log.logger_strategies.measurement_log import LoggerStrategy +from pyobs_cloudcover.measurement_log.measurement import Measurement + + +class MeasurementLogger(object): + def __init__(self, logger: LoggerStrategy, measurements: Dict[str, Measurement]) -> None: + self._logger = logger + self._measurements = measurements + + def __call__(self, cloud_info: CloudCoverageInfo) -> None: + measurements = self._generate_measurements(cloud_info) + self._logger(measurements, cloud_info.obs_time) + + def _generate_measurements(self, cloud_info: CloudCoverageInfo) -> Dict[str, Dict[str, Optional[float]]]: + return { + name: measurement(cloud_info) for name, measurement in self._measurements.items() + } diff --git a/pyobs_cloudcover/measurement_log/measurment_logger_factory.py b/pyobs_cloudcover/measurement_log/measurment_logger_factory.py new file mode 100644 index 0000000..6e9f512 --- /dev/null +++ b/pyobs_cloudcover/measurement_log/measurment_logger_factory.py @@ -0,0 +1,26 @@ +from typing import Dict, Any + +from astroplan import Observer + +from pyobs_cloudcover.measurement_log.field_evaluators.field_evaluators_factory import FieldEvaluatorFactory +from pyobs_cloudcover.measurement_log.logger_strategies.measurment_log_factory import MeasurementLogFactory +from pyobs_cloudcover.measurement_log.measurement import Measurement +from pyobs_cloudcover.measurement_log.measurment_logger import MeasurementLogger + + +class MeasurementLoggerFactory: + def __init__(self, observer: Observer) -> None: + self._field_evaluator_factory = FieldEvaluatorFactory(observer) + + def __call__(self, config: Dict[str, Dict[str, Any]]) -> MeasurementLogger: + logger = MeasurementLogFactory.build(config["logger"]) + + measurements = { + name: Measurement( + { + field_config["name"]: self._field_evaluator_factory(field_config) for field_config in measurement + } + ) for name, measurement in config["measurements"].items() + } + + return MeasurementLogger(logger, measurements) diff --git a/pyobs_cloudcover/pipeline/day/pipeline_factory.py b/pyobs_cloudcover/pipeline/day/pipeline_factory.py index 4e5defb..08fe165 100644 --- a/pyobs_cloudcover/pipeline/day/pipeline_factory.py +++ b/pyobs_cloudcover/pipeline/day/pipeline_factory.py @@ -21,6 +21,6 @@ def __call__(self, options: DayPipelineOptions) -> DayPipeline: alt_az_generator = AltAzMapGenerator(model, 0) cloud_map_generator_factory = CloudMapGeneratorFactory(options.cloud_map) sun_masker = SunMasker(self._observer) - coverage_info_calculator_factory = CloudInfoCalculatorFactory(options.coverage_info) + coverage_info_calculator_factory = CloudInfoCalculatorFactory() return DayPipeline(mask, alt_az_generator, cloud_map_generator_factory(), sun_masker, coverage_info_calculator_factory()) diff --git a/pyobs_cloudcover/pipeline/day/pipeline_options.py b/pyobs_cloudcover/pipeline/day/pipeline_options.py index dcc25f6..e6fd3e6 100644 --- a/pyobs_cloudcover/pipeline/day/pipeline_options.py +++ b/pyobs_cloudcover/pipeline/day/pipeline_options.py @@ -2,14 +2,12 @@ from typing import Dict, Any, cast -from pyobs_cloudcover.cloud_info_calculator import CloudInfoCalculatorOptions from pyobs_cloudcover.pipeline.night.cloud_map_generator.cloud_map_generator_options import CloudMapGeneratorOptions class DayPipelineOptions: - def __init__(self, model_options: Dict[str, Any], mask_filepath: str, cloud_map: CloudMapGeneratorOptions, coverage_info: CloudInfoCalculatorOptions) -> None: + def __init__(self, model_options: Dict[str, Any], mask_filepath: str, cloud_map: CloudMapGeneratorOptions) -> None: self.model_options = model_options - self.coverage_info = coverage_info self.cloud_map = cloud_map self.mask_filepath = mask_filepath @@ -18,6 +16,5 @@ def from_dict(cls, options: Dict[str, Dict[str, Any]]) -> DayPipelineOptions: model_options = options.get("world_model", {}) mask_filepath = cast(str, options.get('mask_filepath')) cloud_map = CloudMapGeneratorOptions.from_dict(cast(Dict[str, Any], options.get('cloud_map'))) - coverage_info = CloudInfoCalculatorOptions.from_dict(cast(Dict[str, Any], options.get('coverage_info'))) - return DayPipelineOptions(model_options, mask_filepath, cloud_map, coverage_info) + return DayPipelineOptions(model_options, mask_filepath, cloud_map) diff --git a/pyobs_cloudcover/pipeline/night/pipeline_factory.py b/pyobs_cloudcover/pipeline/night/pipeline_factory.py index 3ae9337..09feedf 100644 --- a/pyobs_cloudcover/pipeline/night/pipeline_factory.py +++ b/pyobs_cloudcover/pipeline/night/pipeline_factory.py @@ -28,7 +28,7 @@ def __call__(self, options: NightPipelineOptions) -> NightPipeline: reverse_matcher_factory = StarReverseMatcherFactory(options.star_matcher_options) lim_mag_map_generator_factory = LimMagnitudeMapGeneratorFactory(options.lim_mag_map_generator_options) cloud_map_generator_factory = CloudMapGeneratorFactory(options.cloud_generator_options) - coverage_info_calculator_factory = CloudInfoCalculatorFactory(options.coverage_info_options) + coverage_info_calculator_factory = CloudInfoCalculatorFactory() preprocessor = preprocessor_factory() catalog_constructor = catalog_constructor_factory() diff --git a/pyobs_cloudcover/pipeline/night/pipeline_options.py b/pyobs_cloudcover/pipeline/night/pipeline_options.py index ca2d53a..fdd919c 100644 --- a/pyobs_cloudcover/pipeline/night/pipeline_options.py +++ b/pyobs_cloudcover/pipeline/night/pipeline_options.py @@ -4,8 +4,7 @@ from pyobs_cloudcover.pipeline.night.altaz_grid_generator.altaz_grid_options import AltAzGridOptions from pyobs_cloudcover.pipeline.night.catalog.catalog_constructor_options import CatalogConstructorOptions -from pyobs_cloudcover.cloud_info_calculator.cloud_info_calculator_options import \ - CloudInfoCalculatorOptions + from pyobs_cloudcover.pipeline.night.cloud_map_generator.cloud_map_generator_options import CloudMapGeneratorOptions from pyobs_cloudcover.pipeline.night.lim_magnitude_map_generator.lim_magnitude_map_generator_options import LimMagnitudeMapGeneratorOptions from pyobs_cloudcover.pipeline.night.preprocessor.preprocessor_options import PreprocessorOptions @@ -20,7 +19,6 @@ def __init__(self, star_matcher_options: StarReverseMatcherOptions, lim_mag_map_generator_options: LimMagnitudeMapGeneratorOptions, cloud_generator_options: CloudMapGeneratorOptions, - coverage_info_options: CloudInfoCalculatorOptions, altaz_grid_options: AltAzGridOptions ) -> None: @@ -31,7 +29,6 @@ def __init__(self, self.star_matcher_options = star_matcher_options self.lim_mag_map_generator_options = lim_mag_map_generator_options self.cloud_generator_options = cloud_generator_options - self.coverage_info_options = coverage_info_options @classmethod def from_dict(cls, options: Dict[str, Dict[str, Any]]) -> NightPipelineOptions: @@ -41,7 +38,6 @@ def from_dict(cls, options: Dict[str, Dict[str, Any]]) -> NightPipelineOptions: star_matcher_options = StarReverseMatcherOptions.from_dict(options.get("reverse_matcher", {})) lim_mag_map_generator_options = LimMagnitudeMapGeneratorOptions.from_dict(options.get("lim_mag_map", {})) cloud_map_generator_options = CloudMapGeneratorOptions.from_dict(options.get("cloud_map", {})) - coverage_info_options = CloudInfoCalculatorOptions.from_dict(options.get("coverage_info", {})) altaz_grid_generator = AltAzGridOptions.from_dict(options.get("altaz_grid", {})) return cls(model_options, @@ -50,6 +46,5 @@ def from_dict(cls, options: Dict[str, Dict[str, Any]]) -> NightPipelineOptions: star_matcher_options, lim_mag_map_generator_options, cloud_map_generator_options, - coverage_info_options, altaz_grid_generator ) diff --git a/pyobs_cloudcover/testing/module.py b/pyobs_cloudcover/testing/module.py index e9744ce..e5ba8bf 100644 --- a/pyobs_cloudcover/testing/module.py +++ b/pyobs_cloudcover/testing/module.py @@ -63,9 +63,9 @@ def _override_measurement_log(self, influx: InfluxDb2Container) -> None: Therefore, the InfluxDB parameters are updated here manually. """ cloud_cover: pyobs_cloudcover.application.Application = self.comm._network._clients["cloudcover"].module - cloud_cover._measurement_log._client, _ = influx.get_client(token=self._INFLUX_TOKEN) - cloud_cover._measurement_log._bucket = self._INFLUX_BUCKET - cloud_cover._measurement_log._org = self._INFLUX_ORG + cloud_cover._measurement_log._logger._client, _ = influx.get_client(token=self._INFLUX_TOKEN) # type: ignore + cloud_cover._measurement_log._logger._bucket = self._INFLUX_BUCKET # type: ignore + cloud_cover._measurement_log._logger._org = self._INFLUX_ORG # type: ignore async def _submit_image(self) -> None: image = Image.from_file(self._image_path) diff --git a/tests/integration/test_day_pipeline.py b/tests/integration/test_day_pipeline.py index 947c132..0123daa 100644 --- a/tests/integration/test_day_pipeline.py +++ b/tests/integration/test_day_pipeline.py @@ -30,13 +30,11 @@ def test_day_pipeline() -> None: sun_masker = SunMasker(observer) coverage_change_calculator = CoverageChangeCalculator() - zenith_masker = ZenithCloudCoverageCalculator(80) - cloud_coverage_info_calculator = CoverageInfoCalculator(coverage_change_calculator, zenith_masker) + cloud_coverage_info_calculator = CoverageInfoCalculator(coverage_change_calculator) pipeline = DayPipeline(mask, alt_az_generator, cloud_map_gen, sun_masker, cloud_coverage_info_calculator) result = pipeline(image, obs_time) - np.testing.assert_almost_equal(result.total_cover, 1.0, decimal=3) - np.testing.assert_almost_equal(result.zenith_cover, 1.0, decimal=3) + assert result.change is None diff --git a/tests/integration/test_day_pipeline_factory.py b/tests/integration/test_day_pipeline_factory.py index f0f525e..c05324e 100644 --- a/tests/integration/test_day_pipeline_factory.py +++ b/tests/integration/test_day_pipeline_factory.py @@ -33,9 +33,6 @@ def test_day_pipeline() -> None: "mask_filepath": "tests/integration/small_dummy_mask.npy", "cloud_map": { "threshold": 3.5 - }, - "coverage_info": { - "zenith_altitude": 80 } } diff --git a/tests/integration/test_night_pipeline.py b/tests/integration/test_night_pipeline.py index bc9a0ac..36f02ba 100644 --- a/tests/integration/test_night_pipeline.py +++ b/tests/integration/test_night_pipeline.py @@ -54,8 +54,7 @@ def test_night_pipeline() -> None: lim_mag_map_generator = LimMagnitudeMapGenerator(50.0) coverage_change_calculator = CoverageChangeCalculator() - zenith_masker = ZenithCloudCoverageCalculator(80) - cloud_coverage_info_calculator = CoverageInfoCalculator(coverage_change_calculator, zenith_masker) + cloud_coverage_info_calculator = CoverageInfoCalculator(coverage_change_calculator) pipeline = NightPipeline(preprocessor, catalog_constructor, altaz_list_generator, reverse_matcher, lim_mag_map_generator, cloud_map_gen, cloud_coverage_info_calculator) diff --git a/tests/integration/test_night_pipeline_factory.py b/tests/integration/test_night_pipeline_factory.py index 4b5edb6..2dc8471 100644 --- a/tests/integration/test_night_pipeline_factory.py +++ b/tests/integration/test_night_pipeline_factory.py @@ -55,9 +55,6 @@ def test_night_pipeline() -> None: }, "cloud_map": { "threshold": 0.5 - }, - "coverage_info": { - "zenith_altitude": 80 } } diff --git a/tests/integration/test_pipeline_controller.py b/tests/integration/test_pipeline_controller.py index 0f23a58..b4b6128 100644 --- a/tests/integration/test_pipeline_controller.py +++ b/tests/integration/test_pipeline_controller.py @@ -62,9 +62,6 @@ def test_night_pipeline() -> None: }, "cloud_map": { "threshold": 0.5 - }, - "coverage_info": { - "zenith_altitude": 80 } } } diff --git a/tests/unit/measurment_log/test_influx.py b/tests/unit/measurment_log/test_influx.py index 31a6ecb..8d88594 100644 --- a/tests/unit/measurment_log/test_influx.py +++ b/tests/unit/measurment_log/test_influx.py @@ -5,7 +5,7 @@ import numpy as np from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo -from pyobs_cloudcover.measurement_log.influx import Influx +from pyobs_cloudcover.measurement_log.logger_strategies.influx import Influx from influxdb_client.client.write_api import WriteApi @@ -14,11 +14,15 @@ def test_call(): influxdb_client.client.write_api.WriteApi.write = Mock() obs_time = datetime.datetime(2020, 1, 1, 0, 0, 0) - measurement = CloudCoverageInfo(np.array([]), 0, 1, 0.1, obs_time) + measurements = { + "measurement": { + "total-cover": 0.5 + } + } - measurement_log = Influx("", "bucket", "org", "token", "measurement") + measurement_log = Influx("", "bucket", "org", "token") - measurement_log(measurement) + measurement_log(measurements, obs_time) call_args = influxdb_client.client.write_api.WriteApi.write.call_args_list @@ -26,9 +30,8 @@ def test_call(): assert kwargs["bucket"] == "bucket" assert kwargs["org"] == "org" - data: influxdb_client.Point = kwargs["record"] + data: influxdb_client.Point = kwargs["record"][0] assert data._time == obs_time - assert data._fields["total-cover"] == 0 - assert data._fields["zenith-cover"] == 1 - assert data._fields["change"] == 0.1 + assert data._name == "measurement" + assert data._fields["total-cover"] == 0.5 diff --git a/tests/unit/cloud_info_calculator/test_zenith_mask.py b/tests/unit/measurment_log/test_zenith_mask.py similarity index 81% rename from tests/unit/cloud_info_calculator/test_zenith_mask.py rename to tests/unit/measurment_log/test_zenith_mask.py index 670974d..6d21a20 100644 --- a/tests/unit/cloud_info_calculator/test_zenith_mask.py +++ b/tests/unit/measurment_log/test_zenith_mask.py @@ -3,6 +3,7 @@ import numpy as np from cloudmap_rs import AltAzCoord, SkyPixelQuery +from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo from pyobs_cloudcover.cloud_info_calculator import \ ZenithCloudCoverageCalculator @@ -19,4 +20,5 @@ def test_zenith_mask() -> None: pixels = [False, True, False, True, True, True, False, True, False] sky_query = SkyPixelQuery(alt_az_list, pixels) - assert calculator(sky_query) == 1 + coverage_info = CloudCoverageInfo(sky_query, None, None) + assert calculator(coverage_info) == 1 diff --git a/tests/unit/pipeline/test_pipeline_controller.py b/tests/unit/pipeline/test_pipeline_controller.py index 8ec3531..09cbfd1 100644 --- a/tests/unit/pipeline/test_pipeline_controller.py +++ b/tests/unit/pipeline/test_pipeline_controller.py @@ -17,7 +17,7 @@ class MockPipeline(Pipeline): def __call__(self, image: npt.NDArray[np.float_], obs_time: datetime.datetime) -> CloudCoverageInfo: time = datetime.datetime(2024, 1, 1, 0, 0, 0) - return CloudCoverageInfo(np.array([]), 0.0, 1, 0.1, time) + return CloudCoverageInfo(np.array([]), 0.1, time) @pytest.fixture() diff --git a/tests/unit/web_api/test_coverage_query_executor.py b/tests/unit/web_api/test_coverage_query_executor.py index 9d24373..730ba72 100644 --- a/tests/unit/web_api/test_coverage_query_executor.py +++ b/tests/unit/web_api/test_coverage_query_executor.py @@ -27,7 +27,7 @@ def test_point_radec(observer, obs_time) -> None: executor._radec_to_altaz = Mock(return_value=(np.pi/2, 0)) # type: ignore cloud_query = SkyPixelQuery([AltAzCoord(0, 0)], [True]) - executor.set_measurement(CloudCoverageInfo(cloud_query, 1, 2, 0.1, obs_time)) + executor.set_measurement(CloudCoverageInfo(cloud_query, 0.1, obs_time)) assert executor.point_query_radec(10, 10) is True @@ -37,7 +37,7 @@ def test_point_radec_nan(observer, obs_time) -> None: executor._radec_to_altaz = Mock(return_value=(np.pi/2, 0)) # type: ignore cloud_query = SkyPixelQuery([AltAzCoord(0, 0)], [None]) - executor.set_measurement(CloudCoverageInfo(cloud_query, 1, 2, 0.1, obs_time)) + executor.set_measurement(CloudCoverageInfo(cloud_query, 0.1, obs_time)) assert executor.point_query_radec(10, 10) is None @@ -47,7 +47,7 @@ def test_area_radec(observer, obs_time) -> None: executor._radec_to_altaz = Mock(return_value=(np.pi/2, 0)) # type: ignore cloud_query = SkyPixelQuery([AltAzCoord(0, 0)], [True]) - executor.set_measurement(CloudCoverageInfo(cloud_query, 1, 2, 0.1, obs_time)) + executor.set_measurement(CloudCoverageInfo(cloud_query, 0.1, obs_time)) assert executor.area_query_radec(10, 10, 90) == 100.0 @@ -57,6 +57,6 @@ def test_area_radec_nan(observer, obs_time) -> None: executor._radec_to_altaz = Mock(return_value=(np.pi/2, 0)) # type: ignore cloud_query = SkyPixelQuery([AltAzCoord(0, 0)], [None]) - executor.set_measurement(CloudCoverageInfo(cloud_query, 1, 2, 0.1, obs_time)) + executor.set_measurement(CloudCoverageInfo(cloud_query, 0.1, obs_time)) assert executor.area_query_radec(10, 10, 90) is None \ No newline at end of file