diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 86d4083..5f63bb6 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -22,8 +22,7 @@ jobs: python-version: '3.11' cache: 'poetry' - - run: poetry install --all-extras - - run: poetry run maturin develop --release + - run: poetry run maturin develop --release --all-features - run: poetry run pytest -v mypy: @@ -46,6 +45,5 @@ jobs: python-version: '3.11' cache: 'poetry' - - run: poetry install --all-extras - - run: poetry run maturin develop --release + - run: poetry run maturin develop --release --all-features - run: poetry run mypy ./pyobs_cloudcover \ No newline at end of file diff --git a/example.yaml b/example.yaml new file mode 100644 index 0000000..b7f60b8 --- /dev/null +++ b/example.yaml @@ -0,0 +1,52 @@ +class: pyobs_cloudcover.application.Application + +image_sender: "allskycam" + +model: + class: pyobs_cloudcover.world_model.SimpleModel + a0: 4.81426598e-03 + F: 2.00000000e+00 + R: 1.06352627e+03 + c_x: 7.57115607e+02 + c_y: 5.11194838e+02 + +server: + url: "localhost" + port: 8080 + +measurement_log: + url: "" + bucket: "" + org: "" + token: "" + +pipelines: + night: + alt_interval: + start: None + end: -18 + + options: + preprocessor: + mask_filepath: "tests/integration/mask.npy" + bin_size: 2 + bkg_sigma_clip: 3.0 + bkg_box_size: (5, 5) + + catalog: + filepath: "tests/integration/catalog.csv" + filter: + alt: 30.0 + v_mag: 7.0 + distance: 0.0 + + reverse_matcher: + sigma_threshold: 3.0 + window_size: 6.0 + + cloud_map: + radius: 50.0 + + coverage_info: + cloud_threshold: 0.5 + zenith_radius: 20 diff --git a/poetry.lock b/poetry.lock index 28036bc..457940b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1132,6 +1132,30 @@ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.link perf = ["ipython"] testing = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] +[[package]] +name = "influxdb-client" +version = "1.42.0" +description = "InfluxDB 2.0 Python client library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "influxdb_client-1.42.0-py3-none-any.whl", hash = "sha256:0161b963f221d5c1769202f41ff55f5d79e00e6dc24e2a0729c82a1e131956ee"}, + {file = "influxdb_client-1.42.0.tar.gz", hash = "sha256:f5e877feb671eda41e2b5c98ed1dc8ec3327fd8991360dc614822119cda06491"}, +] + +[package.dependencies] +certifi = ">=14.05.14" +python-dateutil = ">=2.5.3" +reactivex = ">=4.0.4" +setuptools = ">=21.0.0" +urllib3 = ">=1.26.0" + +[package.extras] +async = ["aiocsv (>=1.2.2)", "aiohttp (>=3.8.1)"] +ciso = ["ciso8601 (>=2.1.1)"] +extra = ["numpy", "pandas (>=1.0.0)"] +test = ["aioresponses (>=0.7.3)", "coverage (>=4.0.3)", "flake8 (>=5.0.3)", "httpretty (==1.0.5)", "jinja2 (==3.1.3)", "nose (>=1.3.7)", "pluggy (>=0.3.1)", "psutil (>=5.6.3)", "py (>=1.4.31)", "pytest (>=5.0.0)", "pytest-cov (>=3.0.0)", "pytest-timeout (>=2.1.0)", "randomize (>=0.13)", "sphinx (==1.8.5)", "sphinx-rtd-theme"] + [[package]] name = "iniconfig" version = "2.0.0" @@ -2261,6 +2285,20 @@ files = [ {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] +[[package]] +name = "reactivex" +version = "4.0.4" +description = "ReactiveX (Rx) for Python" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "reactivex-4.0.4-py3-none-any.whl", hash = "sha256:0004796c420bd9e68aad8e65627d85a8e13f293de76656165dffbcb3a0e3fb6a"}, + {file = "reactivex-4.0.4.tar.gz", hash = "sha256:e912e6591022ab9176df8348a653fe8c8fa7a301f26f9931c9d8c78a650e04e8"}, +] + +[package.dependencies] +typing-extensions = ">=4.1.1,<5.0.0" + [[package]] name = "reproject" version = "0.12.0" @@ -2818,4 +2856,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.11,<3.12" -content-hash = "3ac3b9175f9ad410774c80214732b9ca8021f537b1ba6f543b3e240822b39c69" +content-hash = "ec5968cb6d3a47d6ee80b8887a2001d92a612b42ecf376db689c799064d4b2a7" diff --git a/pyobs_cloudcover/application.py b/pyobs_cloudcover/application.py new file mode 100644 index 0000000..3398680 --- /dev/null +++ b/pyobs_cloudcover/application.py @@ -0,0 +1,57 @@ +import datetime +from typing import Dict, Any + +from pyobs.events import NewImageEvent, Event +from pyobs.modules import Module +from pyobs.object import get_object + +from pyobs_cloudcover.measurement_log.influx import Influx +from pyobs_cloudcover.pipeline.pipeline_controller_factory import PipelineControllerFactory +from pyobs_cloudcover.web_api.server import Server +from pyobs_cloudcover.web_api.server_factory import ServerFactory +from pyobs_cloudcover.world_model import WorldModel + + +class Application(Module): + def __init__(self, + image_sender: str, + model: Dict[str, Any], + server: Dict[str, Any], + measurement_log: Dict[str, Any], + pipelines: Dict[str, Dict[str, Any]]) -> None: + + super(Application, self).__init__() + + self._image_sender = image_sender + + world_model: WorldModel = get_object(model, WorldModel) + + server_factory = ServerFactory(self.observer, world_model) + self._server = server_factory(server) + + self._measurement_log = Influx(**measurement_log) + + pipeline_controller_factory = PipelineControllerFactory(self.observer, world_model) + self._pipeline_controller = pipeline_controller_factory(pipelines) + + async def open(self) -> None: + await super(Application, self).open() + await self._server.start() + + await self.comm.register_event(NewImageEvent, self.process_new_image) + + async def process_new_image(self, event: Event, sender: str) -> None: + if not isinstance(event, NewImageEvent): + return + + if sender != self._image_sender: + return + + image = await self.vfs.read_image(event.filename) + + obs_time = datetime.datetime.strptime(image.header["DATE-OBS"], "%Y-%m-%dT%H:%M:%S.%f") + + measurement = self._pipeline_controller(image.data, obs_time) + + if measurement is not None: + self._server.set_measurement(measurement) \ No newline at end of file diff --git a/pyobs_cloudcover/application_factory.py b/pyobs_cloudcover/application_factory.py new file mode 100644 index 0000000..0ce3e3f --- /dev/null +++ b/pyobs_cloudcover/application_factory.py @@ -0,0 +1,6 @@ + + +class ApplicationFactory(object): + def __init__(self) -> None: + ... + diff --git a/pyobs_cloudcover/cloud_coverage_info.py b/pyobs_cloudcover/cloud_coverage_info.py index da35556..0587dc0 100644 --- a/pyobs_cloudcover/cloud_coverage_info.py +++ b/pyobs_cloudcover/cloud_coverage_info.py @@ -1,3 +1,4 @@ +import datetime from typing import Optional import numpy as np @@ -6,8 +7,9 @@ class CloudCoverageInfo(object): def __init__(self, cloud_cover_map: npt.NDArray[np.float_], - total_cover: float, zenith_cover: float, change: Optional[float]) -> None: + total_cover: float, zenith_cover: float, change: Optional[float], obs_time: datetime.datetime) -> None: self.cloud_cover_map = cloud_cover_map self.total_cover = total_cover self.zenith_cover = zenith_cover self.change = change + self.obs_time = obs_time diff --git a/pyobs_cloudcover/pipeline/night/world_model/__init__.py b/pyobs_cloudcover/measurement_log/__init__.py similarity index 100% rename from pyobs_cloudcover/pipeline/night/world_model/__init__.py rename to pyobs_cloudcover/measurement_log/__init__.py diff --git a/pyobs_cloudcover/measurement_log/influx.py b/pyobs_cloudcover/measurement_log/influx.py new file mode 100644 index 0000000..b8d3b2f --- /dev/null +++ b/pyobs_cloudcover/measurement_log/influx.py @@ -0,0 +1,31 @@ +import datetime + +import influxdb_client +from influxdb_client.client.write_api import SYNCHRONOUS + + +from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo +from pyobs_cloudcover.measurement_log.measurement_log import MeasurementLog + + +class Influx(MeasurementLog): + def __init__(self, url: str, bucket: str, org: str, token: str) -> None: + self._client = influxdb_client.InfluxDBClient( + url=url, + token=token, + org=org + ) + self._bucket = bucket + self._org = org + + def __call__(self, measurement: CloudCoverageInfo) -> None: + data = influxdb_client.Point("measurement") + + data.time(measurement.obs_time) + + data.field("total-cover", measurement.total_cover) + data.field("zenith-cover", measurement.zenith_cover) + data.field("change", measurement.change) + + with self._client.write_api(write_options=SYNCHRONOUS) as write_api: + write_api.write(bucket=self._bucket, org=self._org, record=data) diff --git a/pyobs_cloudcover/measurement_log/measurement_log.py b/pyobs_cloudcover/measurement_log/measurement_log.py new file mode 100644 index 0000000..3b17335 --- /dev/null +++ b/pyobs_cloudcover/measurement_log/measurement_log.py @@ -0,0 +1,10 @@ +import abc +import datetime + +from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo + + +class MeasurementLog(object, metaclass=abc.ABCMeta): + @abc.abstractmethod + def __call__(self, measurement: CloudCoverageInfo) -> None: + ... diff --git a/pyobs_cloudcover/pipeline/night/catalog/catalog_constructor.py b/pyobs_cloudcover/pipeline/night/catalog/catalog_constructor.py index 834996c..32472e0 100644 --- a/pyobs_cloudcover/pipeline/night/catalog/catalog_constructor.py +++ b/pyobs_cloudcover/pipeline/night/catalog/catalog_constructor.py @@ -4,7 +4,7 @@ from pyobs_cloudcover.pipeline.night.catalog.altaz_catalog_loader import AltAzCatalogLoader from pyobs_cloudcover.pipeline.night.catalog.pixel_catalog import PixelCatalog -from pyobs_cloudcover.pipeline.night.world_model.world_model import WorldModel +from pyobs_cloudcover.world_model import WorldModel class CatalogConstructor(object): diff --git a/pyobs_cloudcover/pipeline/night/catalog/catalog_constructor_factory.py b/pyobs_cloudcover/pipeline/night/catalog/catalog_constructor_factory.py new file mode 100644 index 0000000..f9ceb3e --- /dev/null +++ b/pyobs_cloudcover/pipeline/night/catalog/catalog_constructor_factory.py @@ -0,0 +1,22 @@ +from astroplan import Observer + +from pyobs_cloudcover.pipeline.night.catalog.altaz_catalog_loader import AltAzCatalogLoader +from pyobs_cloudcover.pipeline.night.catalog.catalog_constructor import CatalogConstructor +from pyobs_cloudcover.pipeline.night.catalog.catalog_constructor_options import CatalogConstructorOptions +from pyobs_cloudcover.world_model import WorldModel + + +class CatalogConstructorFactory(object): + def __init__(self, options: CatalogConstructorOptions, model: WorldModel, observer: Observer) -> None: + self._options = options + self._model = model + self._observer = observer + + def __call__(self) -> CatalogConstructor: + altaz_catalog_loader = AltAzCatalogLoader.from_csv(self._options.filepath) + catalog_constructor = CatalogConstructor( + altaz_catalog_loader, self._model, self._observer, + self._options.alt_filter, self._options.v_mag_filter, self._options.distance_filter + ) + + return catalog_constructor diff --git a/pyobs_cloudcover/pipeline/night/catalog/catalog_constructor_options.py b/pyobs_cloudcover/pipeline/night/catalog/catalog_constructor_options.py new file mode 100644 index 0000000..4b342d5 --- /dev/null +++ b/pyobs_cloudcover/pipeline/night/catalog/catalog_constructor_options.py @@ -0,0 +1,32 @@ +from __future__ import annotations +from typing import Dict, Any, Optional + + +class CatalogConstructorOptions(object): + def __init__(self, filepath: str, alt_filter: float = 20.0, v_mag_filter: float = 7.5, distance_filter: float = 0.0) -> None: + self.filepath = filepath + self.alt_filter = alt_filter + self.v_mag_filter = v_mag_filter + self.distance_filter = distance_filter + + @classmethod + def from_dict(cls, options: Dict[str, Any]) -> CatalogConstructorOptions: + filepath: str = str(options["filepath"]) + + if "filter" not in options: + return cls(filepath) + + filter_options: Dict[str, Any] = options["filter"] + + filter_kwargs: Dict[str, float] = {} + + if "alt" in filter_options: + filter_kwargs["alt_filter"] = filter_options["alt"] + + if "v_mag" in filter_options: + filter_kwargs["v_mag_filter"] = filter_options["v_mag"] + + if "distance" in filter_options: + filter_kwargs["distance_filter"] = filter_options["distance"] + + return cls(filepath, **filter_kwargs) diff --git a/pyobs_cloudcover/pipeline/night/catalog/pixel_catalog.py b/pyobs_cloudcover/pipeline/night/catalog/pixel_catalog.py index c8c1bee..7c023e9 100644 --- a/pyobs_cloudcover/pipeline/night/catalog/pixel_catalog.py +++ b/pyobs_cloudcover/pipeline/night/catalog/pixel_catalog.py @@ -1,13 +1,12 @@ from __future__ import annotations -import datetime from typing import cast import numpy as np import numpy.typing as npt from pyobs_cloudcover.pipeline.night.catalog.altaz_catalog import AltAzCatalog -from pyobs_cloudcover.pipeline.night.world_model.world_model import WorldModel +from pyobs_cloudcover.world_model import WorldModel class PixelCatalog(object): diff --git a/pyobs_cloudcover/pipeline/night/cloud_coverage_calculator/cloud_info_calculator_factory.py b/pyobs_cloudcover/pipeline/night/cloud_coverage_calculator/cloud_info_calculator_factory.py new file mode 100644 index 0000000..5bd1109 --- /dev/null +++ b/pyobs_cloudcover/pipeline/night/cloud_coverage_calculator/cloud_info_calculator_factory.py @@ -0,0 +1,23 @@ +from pyobs_cloudcover.pipeline.night.cloud_coverage_calculator.cloud_info_calculator_options import \ + CloudInfoCalculatorOptions +from pyobs_cloudcover.pipeline.night.cloud_coverage_calculator.coverage_calculator import CoverageCalculator +from pyobs_cloudcover.pipeline.night.cloud_coverage_calculator.coverage_change_calculator import \ + CoverageChangeCalculator +from pyobs_cloudcover.pipeline.night.cloud_coverage_calculator.coverage_info_calculator import CoverageInfoCalculator +from pyobs_cloudcover.pipeline.night.cloud_coverage_calculator.zenith_masker import ZenithMasker +from pyobs_cloudcover.world_model import WorldModel + + +class CloudInfoCalculatorFactory(object): + def __init__(self, options: CloudInfoCalculatorOptions, model: WorldModel): + self._options = options + self._model = model + + def __call__(self) -> CoverageInfoCalculator: + coverage_calculator = CoverageCalculator(self._options.cloud_threshold) + coverage_change_calculator = CoverageChangeCalculator() + zenith_masker = ZenithMasker(self._options.altitude_limit, self._model) + cloud_coverage_info_calculator = CoverageInfoCalculator(coverage_calculator, coverage_change_calculator, + zenith_masker) + + return cloud_coverage_info_calculator diff --git a/pyobs_cloudcover/pipeline/night/cloud_coverage_calculator/cloud_info_calculator_options.py b/pyobs_cloudcover/pipeline/night/cloud_coverage_calculator/cloud_info_calculator_options.py new file mode 100644 index 0000000..958a64d --- /dev/null +++ b/pyobs_cloudcover/pipeline/night/cloud_coverage_calculator/cloud_info_calculator_options.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from typing import Dict, Any + + +class CloudInfoCalculatorOptions(object): + def __init__(self, cloud_threshold: float, zenith_radius: float) -> None: + self.cloud_threshold = cloud_threshold + self.altitude_limit = 90 - zenith_radius + + @classmethod + def from_dict(cls, options: Dict[str, Any]) -> CloudInfoCalculatorOptions: + return CloudInfoCalculatorOptions(**options) diff --git a/pyobs_cloudcover/pipeline/night/cloud_coverage_calculator/coverage_info_calculator.py b/pyobs_cloudcover/pipeline/night/cloud_coverage_calculator/coverage_info_calculator.py index 6ad4746..5af9f7e 100644 --- a/pyobs_cloudcover/pipeline/night/cloud_coverage_calculator/coverage_info_calculator.py +++ b/pyobs_cloudcover/pipeline/night/cloud_coverage_calculator/coverage_info_calculator.py @@ -1,3 +1,4 @@ +import datetime from copy import copy import numpy as np @@ -16,9 +17,9 @@ def __init__(self, coverage_calculator: CoverageCalculator, coverage_change_calc self._coverage_change_calculator = coverage_change_calculator self._zenith_masker = zenith_masker - def __call__(self, cloud_map: npt.NDArray[np.float_]) -> CloudCoverageInfo: + def __call__(self, cloud_map: npt.NDArray[np.float_], obs_time: datetime.datetime) -> CloudCoverageInfo: change = self._coverage_change_calculator(cloud_map) coverage = self._coverage_calculator(cloud_map) zenith_coverage = self._coverage_calculator(self._zenith_masker(cloud_map)) - return CloudCoverageInfo(copy(cloud_map), coverage, zenith_coverage, change) \ No newline at end of file + return CloudCoverageInfo(copy(cloud_map), coverage, zenith_coverage, change, obs_time) \ No newline at end of file diff --git a/pyobs_cloudcover/pipeline/night/cloud_coverage_calculator/zenith_masker.py b/pyobs_cloudcover/pipeline/night/cloud_coverage_calculator/zenith_masker.py index e9e5ca6..19a4719 100644 --- a/pyobs_cloudcover/pipeline/night/cloud_coverage_calculator/zenith_masker.py +++ b/pyobs_cloudcover/pipeline/night/cloud_coverage_calculator/zenith_masker.py @@ -3,7 +3,7 @@ import numpy as np import numpy.typing as npt -from pyobs_cloudcover.pipeline.night.world_model.world_model import WorldModel +from pyobs_cloudcover.world_model import WorldModel class ZenithMasker(object): diff --git a/pyobs_cloudcover/pipeline/night/cloud_map_generator/__init__.py b/pyobs_cloudcover/pipeline/night/cloud_map_generator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyobs_cloudcover/pipeline/night/cloud_map_generator.py b/pyobs_cloudcover/pipeline/night/cloud_map_generator/cloud_map_generator.py similarity index 100% rename from pyobs_cloudcover/pipeline/night/cloud_map_generator.py rename to pyobs_cloudcover/pipeline/night/cloud_map_generator/cloud_map_generator.py diff --git a/pyobs_cloudcover/pipeline/night/cloud_map_generator/cloud_map_generator_factory.py b/pyobs_cloudcover/pipeline/night/cloud_map_generator/cloud_map_generator_factory.py new file mode 100644 index 0000000..82f4603 --- /dev/null +++ b/pyobs_cloudcover/pipeline/night/cloud_map_generator/cloud_map_generator_factory.py @@ -0,0 +1,10 @@ +from pyobs_cloudcover.pipeline.night.cloud_map_generator.cloud_map_generator import CloudMapGenerator +from pyobs_cloudcover.pipeline.night.cloud_map_generator.cloud_map_generator_options import CloudMapGeneratorOptions + + +class CloudMapGeneratorFactory(object): + def __init__(self, options: CloudMapGeneratorOptions) -> None: + self._options = options + + def __call__(self) -> CloudMapGenerator: + return CloudMapGenerator(self._options.radius) diff --git a/pyobs_cloudcover/pipeline/night/cloud_map_generator/cloud_map_generator_options.py b/pyobs_cloudcover/pipeline/night/cloud_map_generator/cloud_map_generator_options.py new file mode 100644 index 0000000..ec84bd5 --- /dev/null +++ b/pyobs_cloudcover/pipeline/night/cloud_map_generator/cloud_map_generator_options.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +from typing import Any, Dict + + +class CloudMapGeneratorOptions(object): + def __init__(self, radius: float = 5.0) -> None: + self.radius = radius + + @classmethod + def from_dict(cls, options: Dict[str, Any]) -> CloudMapGeneratorOptions: + return cls(**options) diff --git a/pyobs_cloudcover/pipeline/night/pipeline.py b/pyobs_cloudcover/pipeline/night/pipeline.py index c70b1f2..2f03b71 100644 --- a/pyobs_cloudcover/pipeline/night/pipeline.py +++ b/pyobs_cloudcover/pipeline/night/pipeline.py @@ -6,16 +6,17 @@ from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo from pyobs_cloudcover.pipeline.night.catalog.catalog_constructor import CatalogConstructor from pyobs_cloudcover.pipeline.night.cloud_coverage_calculator.coverage_info_calculator import CoverageInfoCalculator -from pyobs_cloudcover.pipeline.night.cloud_map_generator import CloudMapGenerator +from pyobs_cloudcover.pipeline.night.cloud_map_generator.cloud_map_generator import CloudMapGenerator from pyobs_cloudcover.pipeline.night.preprocessor.preprocessor import Preprocessor -from pyobs_cloudcover.pipeline.night.star_reverse_matcher.star_reverse_matcher import StareReverseMatcher +from pyobs_cloudcover.pipeline.night.star_reverse_matcher.star_reverse_matcher import StarReverseMatcher +from pyobs_cloudcover.pipeline.pipeline import Pipeline -class NightPipeline(object): +class NightPipeline(Pipeline): def __init__(self, preprocess: Preprocessor, catalog_constructor: CatalogConstructor, - star_reverse_matcher: StareReverseMatcher, + star_reverse_matcher: StarReverseMatcher, cloud_map_generator: CloudMapGenerator, coverage_info_calculator: CoverageInfoCalculator) -> None: @@ -33,4 +34,4 @@ def __call__(self, image: npt.NDArray[np.float_], obs_time: datetime.datetime) - matches = self._star_reverse_matcher(preprocessed_image, catalog) cloud_map = self._cloud_map_generator(catalog, matches, img_height, img_width) - return self._coverage_info_calculator(cloud_map) + return self._coverage_info_calculator(cloud_map, obs_time) diff --git a/pyobs_cloudcover/pipeline/night/pipeline_factory.py b/pyobs_cloudcover/pipeline/night/pipeline_factory.py new file mode 100644 index 0000000..3618694 --- /dev/null +++ b/pyobs_cloudcover/pipeline/night/pipeline_factory.py @@ -0,0 +1,40 @@ +from astroplan import Observer + +from pyobs_cloudcover.pipeline.night.catalog.catalog_constructor_factory import CatalogConstructorFactory +from pyobs_cloudcover.pipeline.night.cloud_coverage_calculator.cloud_info_calculator_factory import \ + CloudInfoCalculatorFactory +from pyobs_cloudcover.pipeline.night.cloud_map_generator.cloud_map_generator_factory import CloudMapGeneratorFactory +from pyobs_cloudcover.pipeline.night.pipeline import NightPipeline +from pyobs_cloudcover.pipeline.night.pipeline_options import NightPipelineOptions +from pyobs_cloudcover.pipeline.night.preprocessor.preprocessor_factory import PreprocessorFactory +from pyobs_cloudcover.pipeline.night.star_reverse_matcher.star_reverse_matcher_factory import StarReverseMatcherFactory +from pyobs_cloudcover.world_model import WorldModel + + +class NightPipelineFactory(object): + def __init__(self, observer: Observer, model: WorldModel): + self._observer = observer + self._model = model + + def __call__(self, options: NightPipelineOptions) -> NightPipeline: + preprocessor_factory = PreprocessorFactory(options.preprocessor_options) + catalog_constructor_factory = CatalogConstructorFactory(options.catalog_options, self._model, self._observer) + reverse_matcher_factory = StarReverseMatcherFactory(options.star_matcher_options) + cloud_map_generator_factory = CloudMapGeneratorFactory(options.cloud_generator_options) + coverage_info_calculator_factory = CloudInfoCalculatorFactory(options.coverage_info_options, self._model) + + preprocessor = preprocessor_factory() + catalog_constructor = catalog_constructor_factory() + star_reverse_matcher = reverse_matcher_factory() + cloud_map_generator = cloud_map_generator_factory() + coverage_info_calculator = coverage_info_calculator_factory() + + pipeline = NightPipeline( + preprocessor, + catalog_constructor, + star_reverse_matcher, + cloud_map_generator, + coverage_info_calculator + ) + + return pipeline diff --git a/pyobs_cloudcover/pipeline/night/pipeline_options.py b/pyobs_cloudcover/pipeline/night/pipeline_options.py new file mode 100644 index 0000000..fc9e291 --- /dev/null +++ b/pyobs_cloudcover/pipeline/night/pipeline_options.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from typing import Dict, Any + +from pyobs_cloudcover.pipeline.night.catalog.catalog_constructor_options import CatalogConstructorOptions +from pyobs_cloudcover.pipeline.night.cloud_coverage_calculator.cloud_info_calculator_options import \ + CloudInfoCalculatorOptions +from pyobs_cloudcover.pipeline.night.cloud_map_generator.cloud_map_generator_options import CloudMapGeneratorOptions +from pyobs_cloudcover.pipeline.night.preprocessor.preprocessor_options import PreprocessorOptions +from pyobs_cloudcover.pipeline.night.star_reverse_matcher.star_reverse_matcher_options import StarReverseMatcherOptions + + +class NightPipelineOptions(object): + def __init__(self, + preprocessor_options: PreprocessorOptions, + catalog_options: CatalogConstructorOptions, + star_matcher_options: StarReverseMatcherOptions, + cloud_generator_options: CloudMapGeneratorOptions, + coverage_info_options: CloudInfoCalculatorOptions + ) -> None: + + self.preprocessor_options = preprocessor_options + self.catalog_options = catalog_options + self.star_matcher_options = star_matcher_options + self.cloud_generator_options = cloud_generator_options + self.coverage_info_options = coverage_info_options + + @classmethod + def from_dict(cls, options: Dict[str, Any]) -> NightPipelineOptions: + preprocessor_options = PreprocessorOptions.from_dict(options.get("preprocessor", {})) + catalog_options = CatalogConstructorOptions.from_dict(options.get("catalog", {})) + star_matcher_options = StarReverseMatcherOptions.from_dict(options.get("reverse_matcher", {})) + cloud_generator_options = CloudMapGeneratorOptions.from_dict(options.get("cloud_map", {})) + coverage_info_options = CloudInfoCalculatorOptions.from_dict(options.get("coverage_info", {})) + + return cls(preprocessor_options, + catalog_options, + star_matcher_options, + cloud_generator_options, + coverage_info_options + ) diff --git a/pyobs_cloudcover/pipeline/night/preprocessor/background_remover.py b/pyobs_cloudcover/pipeline/night/preprocessor/background_remover.py index 11c2064..5be2e31 100644 --- a/pyobs_cloudcover/pipeline/night/preprocessor/background_remover.py +++ b/pyobs_cloudcover/pipeline/night/preprocessor/background_remover.py @@ -8,7 +8,7 @@ class BackgroundRemover(object): - def __init__(self, sigma_clip: float = 3.0, box_size: Tuple[int, int] = (5, 5)) -> None: + def __init__(self, sigma_clip: float, box_size: Tuple[int, int]) -> None: self._sigma_clip = sigma_clip self._box_size = box_size diff --git a/pyobs_cloudcover/pipeline/night/preprocessor/image_masker.py b/pyobs_cloudcover/pipeline/night/preprocessor/image_masker.py index f3dbd3b..9b26ac2 100644 --- a/pyobs_cloudcover/pipeline/night/preprocessor/image_masker.py +++ b/pyobs_cloudcover/pipeline/night/preprocessor/image_masker.py @@ -1,3 +1,4 @@ +from __future__ import annotations from copy import copy import numpy as np @@ -13,3 +14,8 @@ def __call__(self, image: npt.NDArray[np.float_]) -> npt.NDArray[np.float_]: masked_image *= self._mask return masked_image + + @classmethod + def from_npy_file(cls, file_path: str) -> ImageMasker: + mask = np.load(file_path) + return ImageMasker(mask) diff --git a/pyobs_cloudcover/pipeline/night/preprocessor/preprocessor_factory.py b/pyobs_cloudcover/pipeline/night/preprocessor/preprocessor_factory.py new file mode 100644 index 0000000..fdba2a9 --- /dev/null +++ b/pyobs_cloudcover/pipeline/night/preprocessor/preprocessor_factory.py @@ -0,0 +1,18 @@ +from pyobs_cloudcover.pipeline.night.preprocessor.background_remover import BackgroundRemover +from pyobs_cloudcover.pipeline.night.preprocessor.image_binner import ImageBinner +from pyobs_cloudcover.pipeline.night.preprocessor.image_masker import ImageMasker +from pyobs_cloudcover.pipeline.night.preprocessor.preprocessor import Preprocessor +from pyobs_cloudcover.pipeline.night.preprocessor.preprocessor_options import PreprocessorOptions + + +class PreprocessorFactory(object): + def __init__(self, options: PreprocessorOptions): + self._options = options + + def __call__(self) -> Preprocessor: + mask = ImageMasker.from_npy_file(self._options.mask_file_path) + binner = ImageBinner(self._options.bin_size) + background_remover = BackgroundRemover(self._options.bkg_sigma_clip, self._options.bkg_box_size) + preprocessor = Preprocessor(mask, binner, background_remover) + + return preprocessor diff --git a/pyobs_cloudcover/pipeline/night/preprocessor/preprocessor_options.py b/pyobs_cloudcover/pipeline/night/preprocessor/preprocessor_options.py new file mode 100644 index 0000000..cb13bb2 --- /dev/null +++ b/pyobs_cloudcover/pipeline/night/preprocessor/preprocessor_options.py @@ -0,0 +1,14 @@ +from __future__ import annotations +from typing import Tuple, Dict, Any + + +class PreprocessorOptions(object): + def __init__(self, mask_filepath: str, bin_size: int = 2, bkg_sigma_clip: float = 3.0, bkg_box_size: Tuple[int, int] = (5, 5)) -> None: + self.mask_file_path = mask_filepath + self.bin_size = bin_size + self.bkg_sigma_clip = bkg_sigma_clip + self.bkg_box_size = bkg_box_size + + @classmethod + def from_dict(cls, options: Dict[str, Any]) -> PreprocessorOptions: + return cls(**options) diff --git a/pyobs_cloudcover/pipeline/night/star_reverse_matcher/star_reverse_matcher.py b/pyobs_cloudcover/pipeline/night/star_reverse_matcher/star_reverse_matcher.py index 6f51429..72564ef 100644 --- a/pyobs_cloudcover/pipeline/night/star_reverse_matcher/star_reverse_matcher.py +++ b/pyobs_cloudcover/pipeline/night/star_reverse_matcher/star_reverse_matcher.py @@ -8,7 +8,7 @@ from pyobs_cloudcover.pipeline.night.star_reverse_matcher.window import ImageWindow -class StareReverseMatcher(object): +class StarReverseMatcher(object): def __init__(self, detector: StarDetector, window: ImageWindow): self._detector = detector self._window = window diff --git a/pyobs_cloudcover/pipeline/night/star_reverse_matcher/star_reverse_matcher_factory.py b/pyobs_cloudcover/pipeline/night/star_reverse_matcher/star_reverse_matcher_factory.py new file mode 100644 index 0000000..5747e24 --- /dev/null +++ b/pyobs_cloudcover/pipeline/night/star_reverse_matcher/star_reverse_matcher_factory.py @@ -0,0 +1,16 @@ +from pyobs_cloudcover.pipeline.night.star_reverse_matcher.detector.sigma_treshhold_detector import \ + SigmaThresholdDetector +from pyobs_cloudcover.pipeline.night.star_reverse_matcher.star_reverse_matcher import StarReverseMatcher +from pyobs_cloudcover.pipeline.night.star_reverse_matcher.star_reverse_matcher_options import StarReverseMatcherOptions +from pyobs_cloudcover.pipeline.night.star_reverse_matcher.window import ImageWindow + + +class StarReverseMatcherFactory(object): + def __init__(self, options: StarReverseMatcherOptions): + self._options = options + + def __call__(self) -> StarReverseMatcher: + detector = SigmaThresholdDetector(self._options.sigma_threshold) + window = ImageWindow(self._options.window_size) + reverse_matcher = StarReverseMatcher(detector, window) + return reverse_matcher diff --git a/pyobs_cloudcover/pipeline/night/star_reverse_matcher/star_reverse_matcher_options.py b/pyobs_cloudcover/pipeline/night/star_reverse_matcher/star_reverse_matcher_options.py new file mode 100644 index 0000000..401db99 --- /dev/null +++ b/pyobs_cloudcover/pipeline/night/star_reverse_matcher/star_reverse_matcher_options.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from typing import Dict, Any + + +class StarReverseMatcherOptions(object): + def __init__(self, sigma_threshold: float = 3.0, window_size: int = 6): + self.sigma_threshold = sigma_threshold + self.window_size = window_size + + @classmethod + def from_dict(cls, options: Dict[str, Any]) -> StarReverseMatcherOptions: + return StarReverseMatcherOptions(**options) diff --git a/pyobs_cloudcover/pipeline/pipeline_controller_factory.py b/pyobs_cloudcover/pipeline/pipeline_controller_factory.py new file mode 100644 index 0000000..0a4dfe7 --- /dev/null +++ b/pyobs_cloudcover/pipeline/pipeline_controller_factory.py @@ -0,0 +1,31 @@ +from typing import Dict, Any, List + +from astroplan import Observer + +from pyobs_cloudcover.pipeline.intervall import Interval +from pyobs_cloudcover.pipeline.night.pipeline_factory import NightPipelineFactory +from pyobs_cloudcover.pipeline.night.pipeline_options import NightPipelineOptions +from pyobs_cloudcover.pipeline.pipeline import Pipeline +from pyobs_cloudcover.pipeline.pipeline_controller import PipelineController +from pyobs_cloudcover.world_model import WorldModel + + +class PipelineControllerFactory(object): + def __init__(self, observer: Observer, model: WorldModel): + self._observer = observer + self._night_pipeline_factory = NightPipelineFactory(observer, model) + + def __call__(self, pipline_configs: Dict[str, Dict[str, Any]]) -> PipelineController: + pipelines: List[Pipeline] = [] + intervals: List[Interval] = [] + + for pipeline_type, pipeline_config in pipline_configs.items(): + if pipeline_type == 'night': + pipeline_options = NightPipelineOptions.from_dict(pipeline_config["options"]) + pipelines.append(self._night_pipeline_factory(pipeline_options)) + else: + raise ValueError(f"Pipeline type {pipeline_type} not implemented!") + + intervals.append(Interval(**pipeline_config["alt_interval"])) + + return PipelineController(pipelines, intervals, self._observer) diff --git a/pyobs_cloudcover/web_api/__init__.py b/pyobs_cloudcover/web_api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyobs_cloudcover/web_api/coverage_query_executor.py b/pyobs_cloudcover/web_api/coverage_query_executor.py new file mode 100644 index 0000000..4055d7a --- /dev/null +++ b/pyobs_cloudcover/web_api/coverage_query_executor.py @@ -0,0 +1,54 @@ +import datetime +from typing import Optional, Tuple, cast + +import numpy as np +import numpy.typing as npt +from astroplan import Observer +from astropy.coordinates import SkyCoord + +from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo +from pyobs_cloudcover.pipeline.night.star_reverse_matcher.window import ImageWindow +from pyobs_cloudcover.world_model import WorldModel + + +class CoverageQueryExecutor(object): + def __init__(self, model: WorldModel, observer: Observer, window: ImageWindow) -> None: + self._model = model + self._observer = observer + self._window = window + + self._cloud_map: Optional[Tuple[npt.NDArray[np.float_], datetime.datetime]] = None + + def set_measurement(self, measurement: CloudCoverageInfo) -> None: + self._cloud_map = (measurement.cloud_cover_map, measurement.obs_time) + + def get_obs_time(self) -> float: + if self._cloud_map is None: + raise ValueError("Measurement has not been set yet!") + + obs_time: datetime.datetime = self._cloud_map[1] + return obs_time.timestamp() + + def __call__(self, ra: float, dec: float) -> Optional[float]: + if self._cloud_map is None: + return None + + cloud_map, obs_time = self._cloud_map + self._window.set_image(cloud_map) + + alt, az = self._radec_to_altaz(ra, dec, obs_time) + px, py = self._model.altaz_to_pix(alt, az) + cloud_area = self._window(cast(float, px), cast(float, py)) + + average_cover = np.average(cloud_area[~np.isnan(cloud_area)]) + + if np.isnan(average_cover): + return None + else: + return float(average_cover) + + def _radec_to_altaz(self, ra: float, dec: float, obs_time: datetime.datetime) -> Tuple[float, float]: + coord = SkyCoord(ra, dec, unit='deg', frame="ircs", location=self._observer.location, obstime=obs_time) + coord = coord.altaz + + return coord.alt.rad, coord.az.rad diff --git a/pyobs_cloudcover/web_api/server.py b/pyobs_cloudcover/web_api/server.py new file mode 100644 index 0000000..a919159 --- /dev/null +++ b/pyobs_cloudcover/web_api/server.py @@ -0,0 +1,33 @@ +from aiohttp import web + +from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo +from pyobs_cloudcover.web_api.coverage_query_executor import CoverageQueryExecutor + + +class Server(object): + def __init__(self, query_executor: CoverageQueryExecutor, url: str = "localhost", port: int = 8080) -> None: + self._query_executor = query_executor + + self._url = url + self._port = port + + app = web.Application() + app.add_routes([web.get("/query", self.query)]) + self._runner = web.AppRunner(app) + + def set_measurement(self, measurement: CloudCoverageInfo) -> None: + self._query_executor.set_measurement(measurement) + + async def start(self) -> None: + await self._runner.setup() + site = web.TCPSite(self._runner, 'localhost', self._port) + await site.start() + + async def query(self, request: web.Request) -> web.Response: + ra = float(request.rel_url.query["ra"]) + dec = float(request.rel_url.query["dec"]) + + cover = self._query_executor(ra, dec) + obs_time = self._query_executor.get_obs_time() + + return web.json_response({'value': cover, 'obs_time': obs_time}) diff --git a/pyobs_cloudcover/web_api/server_factory.py b/pyobs_cloudcover/web_api/server_factory.py new file mode 100644 index 0000000..6ee0ee1 --- /dev/null +++ b/pyobs_cloudcover/web_api/server_factory.py @@ -0,0 +1,18 @@ +from typing import Dict, Any + +from astroplan import Observer + +from pyobs_cloudcover.pipeline.night.star_reverse_matcher.window import ImageWindow +from pyobs_cloudcover.web_api.coverage_query_executor import CoverageQueryExecutor +from pyobs_cloudcover.web_api.server import Server +from pyobs_cloudcover.world_model import WorldModel + + +class ServerFactory(object): + def __init__(self, observer: Observer, model: WorldModel) -> None: + window = ImageWindow(10) + self._executor = CoverageQueryExecutor(model, observer, window) + + def __call__(self, config: Dict[str, Any]) -> Server: + server = Server(query_executor=self._executor, **config) + return server diff --git a/pyobs_cloudcover/world_model/__init__.py b/pyobs_cloudcover/world_model/__init__.py new file mode 100644 index 0000000..ce9681f --- /dev/null +++ b/pyobs_cloudcover/world_model/__init__.py @@ -0,0 +1,4 @@ +from pyobs_cloudcover.world_model.world_model import WorldModel +from pyobs_cloudcover.world_model.simple_model import SimpleModel + +__all__ = ['WorldModel', 'SimpleModel'] diff --git a/pyobs_cloudcover/pipeline/night/world_model/simple_model.py b/pyobs_cloudcover/world_model/simple_model.py similarity index 93% rename from pyobs_cloudcover/pipeline/night/world_model/simple_model.py rename to pyobs_cloudcover/world_model/simple_model.py index 8773c9a..be5ceba 100644 --- a/pyobs_cloudcover/pipeline/night/world_model/simple_model.py +++ b/pyobs_cloudcover/world_model/simple_model.py @@ -3,7 +3,7 @@ import numpy as np import numpy.typing as npt -from pyobs_cloudcover.pipeline.night.world_model.world_model import WorldModel +from pyobs_cloudcover.world_model.world_model import WorldModel class SimpleModel(WorldModel): diff --git a/pyobs_cloudcover/pipeline/night/world_model/world_model.py b/pyobs_cloudcover/world_model/world_model.py similarity index 100% rename from pyobs_cloudcover/pipeline/night/world_model/world_model.py rename to pyobs_cloudcover/world_model/world_model.py diff --git a/pyproject.toml b/pyproject.toml index 20111e3..550755a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ packages = [{include = "pyobs_cloudcover"}] [tool.poetry.dependencies] python = ">=3.11,<3.12" pyobs-core = {version = ">=1.13.0", extras = ["full"]} +influxdb-client = "^1.42.0" [tool.poetry.group.dev.dependencies] pytest = "^7.4.0" @@ -20,6 +21,7 @@ mypy = "1.9.0" [tool.mypy] ignore_missing_imports = true strict = true +disallow_subclassing_any = false [build-system] requires = ["maturin>=1.5,<2.0"] @@ -34,5 +36,21 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] dynamic = ["version"] + +dependencies = [ + "pyobs-core[full]>=1.13.0", + "influxdb-client>=1.42.0" +] + +[project.optional-dependencies] +test = [ + "pytest>=7.4.0", + "pytest-cov>=4.1.0", + "pytest-mock>=3.11.1", + "pytest-asyncio>=0.21.1", + "mypy>=1.9.0" +] + [tool.maturin] features = ["pyo3/extension-module"] +python-packages = ["pyobs_cloudcover"] \ No newline at end of file diff --git a/tests/integration/mask.npy b/tests/integration/mask.npy new file mode 100644 index 0000000..ebe08ba Binary files /dev/null and b/tests/integration/mask.npy differ diff --git a/tests/integration/test_night_pipeline,.py b/tests/integration/test_night_pipeline.py similarity index 87% rename from tests/integration/test_night_pipeline,.py rename to tests/integration/test_night_pipeline.py index 6248e4c..e4f73b3 100644 --- a/tests/integration/test_night_pipeline,.py +++ b/tests/integration/test_night_pipeline.py @@ -12,7 +12,7 @@ CoverageChangeCalculator from pyobs_cloudcover.pipeline.night.cloud_coverage_calculator.coverage_info_calculator import CoverageInfoCalculator from pyobs_cloudcover.pipeline.night.cloud_coverage_calculator.zenith_masker import ZenithMasker -from pyobs_cloudcover.pipeline.night.cloud_map_generator import CloudMapGenerator +from pyobs_cloudcover.pipeline.night.cloud_map_generator.cloud_map_generator import CloudMapGenerator from pyobs_cloudcover.pipeline.night.pipeline import NightPipeline from pyobs_cloudcover.pipeline.night.preprocessor.image_masker import ImageMasker from pyobs_cloudcover.pipeline.night.preprocessor.preprocessor import Preprocessor @@ -20,12 +20,12 @@ from pyobs_cloudcover.pipeline.night.preprocessor.background_remover import BackgroundRemover from pyobs_cloudcover.pipeline.night.star_reverse_matcher.detector.sigma_treshhold_detector import \ SigmaThresholdDetector -from pyobs_cloudcover.pipeline.night.star_reverse_matcher.star_reverse_matcher import StareReverseMatcher +from pyobs_cloudcover.pipeline.night.star_reverse_matcher.star_reverse_matcher import StarReverseMatcher from pyobs_cloudcover.pipeline.night.star_reverse_matcher.window import ImageWindow -from pyobs_cloudcover.pipeline.night.world_model.simple_model import SimpleModel +from pyobs_cloudcover.world_model.simple_model import SimpleModel -def test_night_pipeline(): +def test_night_pipeline() -> None: observer = Observer(latitude=51.559299 * u.deg, longitude=9.945472 * u.deg, elevation=201 * u.m) obs_time = datetime.datetime(2024, 3, 9, 1, 48, 48, 297970) @@ -41,13 +41,13 @@ def test_night_pipeline(): mask = ImageMasker(np.ones((2*1040, 2*1548)).astype(np.bool_)) binner = ImageBinner(2) - background_remover = BackgroundRemover() + background_remover = BackgroundRemover(3.0, (5, 5)) preprocessor = Preprocessor(mask, binner, background_remover) altaz_catalog_loader = AltAzCatalogLoader.from_csv("tests/integration/catalog.csv") catalog_constructor = CatalogConstructor(altaz_catalog_loader, model, observer, 0.0, 3.0, 0.0) - reverse_matcher = StareReverseMatcher(SigmaThresholdDetector(3.0), ImageWindow(6.0)) + reverse_matcher = StarReverseMatcher(SigmaThresholdDetector(3.0), ImageWindow(6.0)) cloud_map_gem = CloudMapGenerator(50.0) diff --git a/tests/integration/test_night_pipeline_factory.py b/tests/integration/test_night_pipeline_factory.py new file mode 100644 index 0000000..fd9e661 --- /dev/null +++ b/tests/integration/test_night_pipeline_factory.py @@ -0,0 +1,59 @@ +import datetime + +import numpy as np +from astroplan import Observer + +import astropy.units as u + +from pyobs_cloudcover.pipeline.night.pipeline import NightPipeline +from pyobs_cloudcover.pipeline.night.pipeline_factory import NightPipelineFactory +from pyobs_cloudcover.pipeline.night.pipeline_options import NightPipelineOptions +from pyobs_cloudcover.world_model import SimpleModel + + +def test_night_pipeline() -> None: + observer = Observer(latitude=51.559299 * u.deg, longitude=9.945472 * u.deg, elevation=201 * u.m) + obs_time = datetime.datetime(2024, 3, 9, 1, 48, 48, 297970) + + model_parameters = [4.81426598e-03, 2.00000000e+00, 1.06352627e+03, 7.57115607e+02, 5.11194838e+02] + model = SimpleModel(*model_parameters) + + stars = np.loadtxt('tests/integration/matches_small_20240308.csv', delimiter=",") + + image = np.zeros((2*1040, 2*1548)) + + for star in stars: + image[int(star[2]), int(star[1])] = 10 + + pipeline_kwargs = { + "preprocessor": { + "mask_filepath": "tests/integration/mask.npy", + "bin_size": 2, + "bkg_sigma_clip": 3.0, + "bkg_box_size": (5, 5), + }, + "catalog": { + "filepath": "tests/integration/catalog.csv", + "filter": { + "alt": 30.0, + "v_mag": 7.0, + "distance": 0.0 + } + }, + "reverse_matcher": { + "sigma_threshold": 3.0, + "window_size": 6.0 + }, + "cloud_map": { + "radius": 50.0 + }, + "coverage_info": { + "cloud_threshold": 0.5, + "zenith_radius": 20 + } + } + + pipeline_options = NightPipelineOptions.from_dict(pipeline_kwargs) + pipeline_factory = NightPipelineFactory(observer, model) + pipeline = pipeline_factory(pipeline_options) + pipeline(image, obs_time) diff --git a/tests/integration/test_pipeline_controller.py b/tests/integration/test_pipeline_controller.py new file mode 100644 index 0000000..7b04708 --- /dev/null +++ b/tests/integration/test_pipeline_controller.py @@ -0,0 +1,66 @@ +import datetime + +import astropy.units as u +import numpy as np +from astroplan import Observer + +from pyobs_cloudcover.pipeline.pipeline_controller_factory import PipelineControllerFactory +from pyobs_cloudcover.world_model import SimpleModel + + +def test_night_pipeline() -> None: + observer = Observer(latitude=51.559299 * u.deg, longitude=9.945472 * u.deg, elevation=201 * u.m) + obs_time = datetime.datetime(2024, 3, 9, 12, 48, 48, 297970) + + model_parameters = [4.81426598e-03, 2.00000000e+00, 1.06352627e+03, 7.57115607e+02, 5.11194838e+02] + model = SimpleModel(*model_parameters) + + stars = np.loadtxt('tests/integration/matches_small_20240308.csv', delimiter=",") + + image = np.zeros((2*1040, 2*1548)) + + for star in stars: + image[int(star[2]), int(star[1])] = 10 + + kwargs = { + "night": { + "alt_interval": + { + "start": None, + "end": -18 + }, + "options": + { + "preprocessor": { + "mask_filepath": "tests/integration/mask.npy", + "bin_size": 2, + "bkg_sigma_clip": 3.0, + "bkg_box_size": (5, 5), + }, + "catalog": { + "filepath": "tests/integration/catalog.csv", + "filter": { + "alt": 30.0, + "v_mag": 7.0, + "distance": 0.0 + } + }, + "reverse_matcher": { + "sigma_threshold": 3.0, + "window_size": 6.0 + }, + "cloud_map": { + "radius": 50.0 + }, + "coverage_info": { + "cloud_threshold": 0.5, + "zenith_radius": 20 + } + } + } + + } + + factory = PipelineControllerFactory(observer, model) + controller = factory(kwargs) + assert controller(image, obs_time) is None diff --git a/tests/unit/measurment_log/__init__.py b/tests/unit/measurment_log/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/measurment_log/test_influx.py b/tests/unit/measurment_log/test_influx.py new file mode 100644 index 0000000..7163e4f --- /dev/null +++ b/tests/unit/measurment_log/test_influx.py @@ -0,0 +1,34 @@ +import datetime +from unittest.mock import Mock + +import influxdb_client +import numpy as np + +from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo +from pyobs_cloudcover.measurement_log.influx import Influx + +from influxdb_client.client.write_api import WriteApi + + +def test_call(): + influxdb_client.client.write_api.WriteApi.write = Mock() + + obs_time = datetime.datetime(2020, 1, 1, 0, 0, 0) + measurement = CloudCoverageInfo(np.array([]), 0, 1, 2, obs_time) + + measurement_log = Influx("", "bucket", "org", "token") + + measurement_log(measurement) + + call_args = influxdb_client.client.write_api.WriteApi.write.call_args_list + + kwargs = call_args[0].kwargs + assert kwargs["bucket"] == "bucket" + assert kwargs["org"] == "org" + + data: influxdb_client.Point = kwargs["record"] + assert data._time == obs_time + assert data._fields["total-cover"] == 0 + assert data._fields["zenith-cover"] == 1 + assert data._fields["change"] == 2 + diff --git a/tests/unit/pipeline/night/cloud_coverage_calculator/test_zenith_mask.py b/tests/unit/pipeline/night/cloud_coverage_calculator/test_zenith_mask.py index 5be4a54..ac47b42 100644 --- a/tests/unit/pipeline/night/cloud_coverage_calculator/test_zenith_mask.py +++ b/tests/unit/pipeline/night/cloud_coverage_calculator/test_zenith_mask.py @@ -4,7 +4,7 @@ from numpy import typing as npt from pyobs_cloudcover.pipeline.night.cloud_coverage_calculator.zenith_masker import ZenithMasker -from pyobs_cloudcover.pipeline.night.world_model.world_model import WorldModel +from pyobs_cloudcover.world_model import WorldModel class MockWorldModel(WorldModel): diff --git a/tests/unit/pipeline/night/preprocessor/test_background_remover.py b/tests/unit/pipeline/night/preprocessor/test_background_remover.py index 491d269..fe94dc7 100644 --- a/tests/unit/pipeline/night/preprocessor/test_background_remover.py +++ b/tests/unit/pipeline/night/preprocessor/test_background_remover.py @@ -4,7 +4,7 @@ def test_background_remover(): - remover = BackgroundRemover() + remover = BackgroundRemover(3.0, (5, 5)) image = np.ones((10, 10)) corr_image = remover(image) diff --git a/tests/unit/pipeline/night/test_cloud_map_generator.py b/tests/unit/pipeline/night/test_cloud_map_generator.py index 905d302..9546347 100644 --- a/tests/unit/pipeline/night/test_cloud_map_generator.py +++ b/tests/unit/pipeline/night/test_cloud_map_generator.py @@ -1,10 +1,10 @@ import numpy as np from pyobs_cloudcover.pipeline.night.catalog.pixel_catalog import PixelCatalog -from pyobs_cloudcover.pipeline.night.cloud_map_generator import CloudMapGenerator +from pyobs_cloudcover.pipeline.night.cloud_map_generator.cloud_map_generator import CloudMapGenerator -def test_call(): +def test_call() -> None: cloud_map_generator = CloudMapGenerator(1) catalog = PixelCatalog(sao=np.array([0, 1]), px=np.array([1, 1]), py=np.array([1, 1]), v_mag=np.array([0, 1])) diff --git a/tests/unit/pipeline/test_interval.py b/tests/unit/pipeline/test_interval.py index 25188d9..d3062ae 100644 --- a/tests/unit/pipeline/test_interval.py +++ b/tests/unit/pipeline/test_interval.py @@ -8,8 +8,8 @@ def test_upper_lower_bound(): def test_upper_bound(): - interval = Interval(start=None, end=10) - assert (-1 in interval) == True + interval = Interval(start=None, end=-18) + assert (-20 in interval) == True assert (11 in interval) == False diff --git a/tests/unit/pipeline/test_pipeline_controller.py b/tests/unit/pipeline/test_pipeline_controller.py index 2b68ed5..f3e9058 100644 --- a/tests/unit/pipeline/test_pipeline_controller.py +++ b/tests/unit/pipeline/test_pipeline_controller.py @@ -16,7 +16,8 @@ class MockPipeline(Pipeline): def __call__(self, image: npt.NDArray[np.float_], obs_time: datetime.datetime) -> CloudCoverageInfo: - return CloudCoverageInfo(np.array([]), 0, 0, 0) + time = datetime.datetime(2024, 1, 1, 0, 0, 0) + return CloudCoverageInfo(np.array([]), 0, 0, 0, time) @pytest.fixture() diff --git a/tests/unit/web_api/__init__.py b/tests/unit/web_api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/web_api/test_coverage_query_executor.py b/tests/unit/web_api/test_coverage_query_executor.py new file mode 100644 index 0000000..9ee1fb7 --- /dev/null +++ b/tests/unit/web_api/test_coverage_query_executor.py @@ -0,0 +1,77 @@ +import datetime +from unittest.mock import Mock + +import numpy as np +import pytest +from astroplan import Observer + +import astropy.units as u + +from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo +from pyobs_cloudcover.pipeline.night.star_reverse_matcher.window import ImageWindow +from pyobs_cloudcover.world_model.simple_model import SimpleModel +from pyobs_cloudcover.web_api.coverage_query_executor import CoverageQueryExecutor + + +@pytest.fixture() +def observer(): + return Observer(latitude=51.559299 * u.deg, longitude=9.945472 * u.deg, elevation=201 * u.m) + + +@pytest.fixture() +def obs_time(): + return datetime.datetime(2024, 3, 9, 1, 48, 48, 297970) + + +@pytest.fixture() +def model(): + model_parameters = [4.81426598e-03, 2.00000000e+00, 1.06352627e+03, 7.57115607e+02, 5.11194838e+02] + return SimpleModel(*model_parameters) + + +def test_call_no_map(observer, obs_time, model): + window = ImageWindow(2) + + executor = CoverageQueryExecutor(model, observer, window) + + assert executor(10, 10) is None + + +def test_call_map(observer, obs_time, model): + window = ImageWindow(1) + + executor = CoverageQueryExecutor(model, observer, window) + executor._radec_to_altaz = Mock(return_value=(np.pi/2, 0)) + executor._model.altaz_to_pix = Mock(return_value=(1, 1)) + + executor.set_measurement(CloudCoverageInfo(np.identity(3), 0, 0, 0, obs_time)) + + assert executor(10, 10) == 1/3 + executor._radec_to_altaz.assert_called_once_with(10, 10, obs_time) + executor._model.altaz_to_pix.assert_called_once_with(np.pi/2, 0) + + +def test_call_map_out_of_bounds(observer, obs_time, model): + window = ImageWindow(1) + + executor = CoverageQueryExecutor(model, observer, window) + executor._radec_to_altaz = Mock(return_value=(np.pi/2, 0)) + executor._model.altaz_to_pix = Mock(return_value=(5, 5)) + + executor.set_measurement(CloudCoverageInfo(np.identity(3), 0, 0, 0, obs_time)) + + assert executor(10, 10) is None + + +def test_call_map_nan(observer, obs_time, model): + window = ImageWindow(1) + + executor = CoverageQueryExecutor(model, observer, window) + executor._radec_to_altaz = Mock(return_value=(np.pi/2, 0)) + executor._model.altaz_to_pix = Mock(return_value=(1, 1)) + + cloud_map = np.identity(3) + cloud_map[0, 0] = np.nan + executor.set_measurement(CloudCoverageInfo(cloud_map, 0, 0, 0, obs_time)) + + assert executor(10, 10) == 2/8