Added pipeline and integration test

pyobs_cloudcover/pipeline/intervall.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+from typing import Optional
+
+
+class Interval(object):
+    def __init__(self, start: Optional[float] = None, end: Optional[float] = None):
+        self._start = start
+        self._end = end
+
+    def __contains__(self, value: float) -> bool:
+        in_interval = True
+
+        if self._start is not None:
+            in_interval &= self._start < value
+
+        if self._end is not None:
+            in_interval &= value < self._end
+
+        return in_interval
+
+    def does_intersect(self, other: Interval) -> bool:
+        if (other._start is None and other._end is None) or (self._start is None and self._end is None):
+            return True
+
+        if self == other:
+            return True
+
+        does_intersect = False
+
+        if other._start is not None:
+            does_intersect |= other._start in self
+
+        if other._end is not None:
+            does_intersect |= other._end in self
+
+        if self._start is not None:
+            does_intersect |= self._start in other
+
+        if self._end is not None:
+            does_intersect |= self._end in other
+
+        return does_intersect
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Interval):
+            return False
+
+        return self._start == other._start and self._end == other._end

pyobs_cloudcover/pipeline/night/catalog/catalog_constructor.py

@@ -17,12 +17,13 @@ def __init__(self, catalog_loader: AltAzCatalogLoader, model: WorldModel, observ
         self._v_mag_limit = v_mag_limit
         self._distance_limit = distance_limit
 
-    def __call__(self, time: datetime.datetime) -> PixelCatalog:
+    def __call__(self, time: datetime.datetime, height: int, width: int) -> PixelCatalog:
         altaz_catalog = self._catalog_loader(self._observer, time)
         altaz_catalog.filter_alt(self._alt_limit)
         altaz_catalog.filter_v_mag(self._v_mag_limit)
 
         pixel_catalog = PixelCatalog.from_altaz_catalog(altaz_catalog, self._model)
+        pixel_catalog.filter_window_size(height, width)
         pixel_catalog.filter_close(self._distance_limit)
 
         return pixel_catalog

pyobs_cloudcover/pipeline/night/cloud_coverage_calculator/zenith_masker.py

@@ -14,7 +14,7 @@ def __init__(self, altitude: float, model: WorldModel) -> None:
         self._radius = np.sqrt(np.sum(np.square(self._zenith - offset)))
 
     def __call__(self, image: npt.NDArray[np.float_]) -> npt.NDArray[np.float_]:
-        nx, ny = image.shape
+        ny, nx = image.shape
         x, y = np.arange(0, nx), np.arange(0, ny)
         x_coordinates, y_coordinates = np.meshgrid(x, y)
 

pyobs_cloudcover/pipeline/night/pipeline.py

@@ -2,11 +2,9 @@
 
 import numpy as np
 import numpy.typing as npt
-from cloudmap_rs.cloudmap_rs import Entry
 
 from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo
 from pyobs_cloudcover.pipeline.night.catalog.catalog_constructor import CatalogConstructor
-from pyobs_cloudcover.pipeline.night.catalog.pixel_catalog import PixelCatalog
 from pyobs_cloudcover.pipeline.night.cloud_coverage_calculator.coverage_info_calculator import CoverageInfoCalculator
 from pyobs_cloudcover.pipeline.night.cloud_map_generator import CloudMapGenerator
 from pyobs_cloudcover.pipeline.night.preprocessor.preprocessor import Preprocessor
@@ -27,9 +25,12 @@ def __init__(self,
         self._cloud_map_generator = cloud_map_generator
         self._coverage_info_calculator = coverage_info_calculator
 
-    def __call__(self, image: npt.NDArray[np.float_], obstime: datetime.datetime) -> CloudCoverageInfo:
+    def __call__(self, image: npt.NDArray[np.float_], obs_time: datetime.datetime) -> CloudCoverageInfo:
         preprocessed_image = self._preprocess(image)
-        catalog = self._catalog_constructor(obstime)
+        img_height, img_width = preprocessed_image.shape
+
+        catalog = self._catalog_constructor(obs_time, img_height, img_width)
         matches = self._star_reverse_matcher(preprocessed_image, catalog)
-        cloud_map = self._cloud_map_generator(catalog, matches, *image.shape)
+
+        cloud_map = self._cloud_map_generator(catalog, matches, img_height, img_width)
         return self._coverage_info_calculator(cloud_map)

pyobs_cloudcover/pipeline/night/preprocessor/preprocessor.py

@@ -13,8 +13,8 @@ def __init__(self, masker: ImageMasker, binner: ImageBinner, background_remover:
         self._background_remover = background_remover
 
     def __call__(self, image: npt.NDArray[np.float_]) -> npt.NDArray[np.float_]:
-        binned_image = self._binner(image)
-        masked_image = self._masker(binned_image)
-        processed_image = self._background_remover(masked_image)
+        masked_image = self._masker(image)
+        binned_image = self._binner(masked_image)
+        processed_image = self._background_remover(binned_image)
 
         return processed_image

pyobs_cloudcover/pipeline/night/world_model/simple_model.py

@@ -0,0 +1,32 @@
+from typing import Union, Tuple
+
+import numpy as np
+import numpy.typing as npt
+
+from pyobs_cloudcover.pipeline.night.world_model.world_model import WorldModel
+
+
+class SimpleModel(WorldModel):
+    def __init__(self, a0: float, F: float, R: float, c_x: float, c_y: float) -> None:
+        self._a0 = a0
+        self._F = F
+        self._R = R
+        self._cx, self._cy = c_x, c_y
+
+    def pix_to_altaz(self, x: Union[npt.NDArray[np.float_], float], y: Union[npt.NDArray[np.float_], float]) -> \
+            Union[Tuple[npt.NDArray[np.float_], npt.NDArray[np.float_]], Tuple[float, float]]:
+        az = self._a0 + np.arctan2((y - self._cy), (x - self._cx))
+        r = np.sqrt((x - self._cx) ** 2 + (y - self._cy) ** 2)
+        z = self._F * np.arcsin(r / self._R)
+        alt = np.pi / 2 - z
+
+        return alt, az
+
+    def altaz_to_pix(self, alt: Union[npt.NDArray[np.float_], float], az: Union[npt.NDArray[np.float_], float]) -> \
+            Union[Tuple[npt.NDArray[np.float_], npt.NDArray[np.float_]], Tuple[float, float]]:
+        z = np.pi / 2 - alt
+        r = self._R * np.sin(z / self._F)
+        x = self._cx + r * np.cos(az - self._a0)
+        y = self._cy + r * np.sin(az - self._a0)
+
+        return x, y

pyobs_cloudcover/pipeline/pipeline.py

@@ -0,0 +1,13 @@
+import abc
+import datetime
+
+import numpy as np
+import numpy.typing as npt
+
+from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo
+
+
+class Pipeline(object, metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def __call__(self, image: npt.NDArray[np.float_], obs_time: datetime.datetime) -> CloudCoverageInfo:
+        ...

pyobs_cloudcover/pipeline/pipeline_controller.py

@@ -0,0 +1,44 @@
+import datetime
+from typing import List, Optional
+
+import numpy as np
+import numpy.typing as npt
+from astroplan import Observer
+
+from pyobs_cloudcover.cloud_coverage_info import CloudCoverageInfo
+from pyobs_cloudcover.pipeline.intervall import Interval
+from pyobs_cloudcover.pipeline.pipeline import Pipeline
+
+
+class PipelineController(object):
+    def __init__(self, pipelines: List[Pipeline], sun_alt_intervals: List[Interval], observer: Observer) -> None:
+        self._pipelines = pipelines
+        self._sun_alt_intervals = sun_alt_intervals
+        self._observer = observer
+
+        self._check_arg_length()
+        self._check_interval_overlap()
+
+    def _check_arg_length(self) -> None:
+        if len(self._pipelines) != len(self._sun_alt_intervals):
+            raise ValueError("Number of pipelines must equal the intervals")
+
+    def _check_interval_overlap(self) -> None:
+        overlap = [
+            other.does_intersect(interval)
+            for interval in self._sun_alt_intervals
+            for other in self._sun_alt_intervals
+            if other is not interval
+        ]
+
+        if True in overlap:
+            raise ValueError("Sun altitude intervals can't overlap!")
+
+    def __call__(self, image: npt.NDArray[np.float_], obs_time: datetime.datetime) -> Optional[CloudCoverageInfo]:
+        sun_alt = self._observer.sun_altaz(obs_time).alt.deg
+
+        for pipeline, alt_interval in zip(self._pipelines, self._sun_alt_intervals):
+            if sun_alt in alt_interval:
+                return pipeline(image, obs_time)
+
+        return None

tests/integration/catalog.csv

@@ -0,0 +1,46 @@
+;SAO;_RAJ2000;_DEJ2000;Vmag
+468;232481;24.42875;-57.236666666666665;0.46
+1009;38787;51.08083333333333;49.86111111111112;1.79
+1448;94027;68.97999999999999;16.509166666666665;0.85
+1699;40186;79.17249999999999;45.99805555555556;0.08
+1704;131907;78.63458333333332;-8.201666666666666;0.12
+1781;112740;81.28291666666665;6.349722222222222;1.64
+1782;77168;81.57291666666667;28.6075;1.65
+1889;132346;84.05333333333333;-1.2019444444444445;1.7
+2046;113271;88.79291666666666;7.406944444444445;0.5
+2073;40750;89.88208333333333;44.9475;1.9
+2279;151428;95.67499999999998;-17.95583333333333;1.98
+2311;234480;95.98791666666666;-52.69583333333333;-0.72
+2405;95912;99.42791666666666;16.399166666666666;1.93
+2474;151881;101.28708333333331;-16.71611111111111;-1.46
+2600;172676;104.65625;-28.97222222222222;1.5
+2675;173047;107.09791666666663;-26.39333333333333;1.84
+2924;115756;114.82541666666664;5.2250000000000005;0.38
+2970;79666;116.32875;28.02611111111111;1.14
+3187;219504;122.38333333333333;-47.336666666666666;1.78
+3287;235932;125.62833333333332;-59.50972222222222;1.86
+3465;236232;131.1758333333333;-54.708333333333336;1.96
+3663;250495;138.29999999999998;-69.71722222222222;1.68
+3726;136871;141.89666666666665;-8.658611111111112;1.98
+3960;98967;152.09291666666664;11.967222222222222;1.35
+4279;15384;165.9320833333333;61.75083333333333;1.79
+4708;251904;186.6495833333333;-63.09916666666667;1.33
+4740;240019;187.79125;-57.11333333333334;1.63
+4830;240259;191.93;-59.68861111111111;1.25
+4882;28553;193.50708333333333;55.95972222222223;1.77
+5033;157923;201.2983333333333;-11.161388888888888;0.98
+5168;44752;206.885;49.31333333333333;1.86
+5244;252582;210.9558333333333;-60.37305555555555;0.61
+5317;100944;213.91541666666663;19.1825;-0.04
+5435;252838;219.8995833333333;-60.83527777777778;-0.01
+6105;184415;247.35166666666663;-26.431944444444447;0.96
+6187;253700;252.16625;-69.02777777777777;1.92
+6495;208954;263.40208333333334;-37.10388888888889;1.63
+6521;228201;264.33;-42.99777777777778;1.87
+6845;210091;276.04291666666666;-34.38472222222222;1.85
+6967;67174;279.2345833333333;38.78361111111111;0.03
+7520;125122;297.6958333333333;8.868333333333334;0.77
+7753;246574;306.4120833333333;-56.735;1.94
+7887;49941;310.35791666666665;45.280277777777776;1.25
+8386;230992;332.0583333333333;-46.96111111111112;1.74
+8689;191524;344.4129166666666;-29.622222222222224;1.16

tests/integration/matches_small_20240308.csv

@@ -0,0 +1,13 @@
+100944, 928.2203106172179, 1211.7426027580666
+67174, 1791.5219574240195, 1913.5232962913376
+40186, 2482.163274869461, 289.9592314901592
+40750, 2345.705917793547, 211.98359887535128
+25502, 2351.275605837462, 373.50455214888194
+44752, 1468.4800508769586, 1064.1458695943231
+28737, 1571.519279689383, 997.309613067345
+28553, 1599.3677199089575, 916.5491364305797
+28315, 1635.5706921944038, 827.434127727942
+28179, 1599.3677199089575, 755.0281831570489
+27876, 1702.406948721382, 651.9889543446243
+15384, 1785.952269380105, 710.4706788057304
+157923, 413.5459703229561, 970.4820932631524

tests/integration/test_night_pipeline,.py

@@ -0,0 +1,61 @@
+import datetime
+
+import numpy as np
+from astroplan import Observer
+
+import astropy.units as u
+
+from pyobs_cloudcover.pipeline.night.catalog.altaz_catalog_loader import AltAzCatalogLoader
+from pyobs_cloudcover.pipeline.night.catalog.catalog_constructor import CatalogConstructor
+from pyobs_cloudcover.pipeline.night.cloud_coverage_calculator.coverage_calculator import CoverageCalculator
+from pyobs_cloudcover.pipeline.night.cloud_coverage_calculator.coverage_change_calculator import \
+    CoverageChangeCalculator
+from pyobs_cloudcover.pipeline.night.cloud_coverage_calculator.coverage_info_calculator import CoverageInfoCalculator
+from pyobs_cloudcover.pipeline.night.cloud_coverage_calculator.zenith_masker import ZenithMasker
+from pyobs_cloudcover.pipeline.night.cloud_map_generator import CloudMapGenerator
+from pyobs_cloudcover.pipeline.night.pipeline import NightPipeline
+from pyobs_cloudcover.pipeline.night.preprocessor.image_masker import ImageMasker
+from pyobs_cloudcover.pipeline.night.preprocessor.preprocessor import Preprocessor
+from pyobs_cloudcover.pipeline.night.preprocessor.image_binner import ImageBinner
+from pyobs_cloudcover.pipeline.night.preprocessor.background_remover import BackgroundRemover
+from pyobs_cloudcover.pipeline.night.star_reverse_matcher.detector.sigma_treshhold_detector import \
+    SigmaThresholdDetector
+from pyobs_cloudcover.pipeline.night.star_reverse_matcher.star_reverse_matcher import StareReverseMatcher
+from pyobs_cloudcover.pipeline.night.star_reverse_matcher.window import ImageWindow
+from pyobs_cloudcover.pipeline.night.world_model.simple_model import SimpleModel
+
+
+def test_night_pipeline():
+    observer = Observer(latitude=51.559299 * u.deg, longitude=9.945472 * u.deg, elevation=201 * u.m)
+    obs_time = datetime.datetime(2024, 3, 9, 1, 48, 48, 297970)
+
+    model_parameters = [4.81426598e-03, 2.00000000e+00, 1.06352627e+03, 7.57115607e+02, 5.11194838e+02]
+    model = SimpleModel(*model_parameters)
+
+    stars = np.loadtxt('tests/integration/matches_small_20240308.csv', delimiter=",")
+
+    image = np.zeros((2*1040, 2*1548))
+
+    for star in stars:
+        image[int(star[2]), int(star[1])] = 10
+
+    mask = ImageMasker(np.ones((2*1040, 2*1548)).astype(np.bool_))
+    binner = ImageBinner(2)
+    background_remover = BackgroundRemover()
+    preprocessor = Preprocessor(mask, binner, background_remover)
+
+    altaz_catalog_loader = AltAzCatalogLoader.from_csv("tests/integration/catalog.csv")
+    catalog_constructor = CatalogConstructor(altaz_catalog_loader, model, observer, 0.0, 3.0, 0.0)
+
+    reverse_matcher = StareReverseMatcher(SigmaThresholdDetector(3.0), ImageWindow(6.0))
+
+    cloud_map_gem = CloudMapGenerator(50.0)
+
+    coverage_calculator = CoverageCalculator(0.5)
+    coverage_change_calculator = CoverageChangeCalculator()
+    zenith_masker = ZenithMasker(80, model)
+    cloud_coverage_info_calculator = CoverageInfoCalculator(coverage_calculator, coverage_change_calculator, zenith_masker)
+
+    pipeline = NightPipeline(preprocessor, catalog_constructor, reverse_matcher, cloud_map_gem, cloud_coverage_info_calculator)
+
+    pipeline(image, obs_time)