Feature/zeros error

CHANGES.md

@@ -1,3 +1,8 @@
+0.21.1 (2019-03-28)
+-------------------
+- Added a quality control stage that checks to ensure images are not
+  primarily comprised of 0s, which indicates a camera failure
+
 0.21.0 (2019-03-25)
 -------------------
 - Significant refactor to the pipeline context and settings files. We have now

banzai/qc/__init__.py

@@ -3,6 +3,7 @@
 from banzai.qc.sinistro_1000s import ThousandsTest
 from banzai.qc.pattern_noise import PatternNoiseDetector
 from banzai.qc.header_checker import HeaderSanity
+from banzai.qc.zeros import ZerosTest
 
 __all__ = ['SaturationTest', 'PointingTest', 'ThousandsTest',
-           'PatternNoiseDetector', 'HeaderSanity']
+           'PatternNoiseDetector', 'HeaderSanity', 'ZerosTest']

banzai/qc/zeros.py

@@ -0,0 +1,44 @@
+import logging
+
+import numpy as np
+
+from banzai.stages import Stage
+from banzai.utils import qc
+
+logger = logging.getLogger(__name__)
+
+
+class ZerosTest(Stage):
+    """
+    Reject any images that have ZEROS_THRESHOLD or more of their pixels in a single amp exactly equal to 0.
+
+    Notes
+    =====
+    Sometimes when a camera fails, all pixels have a value of 0.
+    """
+    ZEROS_THRESHOLD = 0.95
+
+    def __init__(self, runtime_context):
+        super(ZerosTest, self).__init__(runtime_context)
+
+    def do_stage(self, image):
+        fraction_0s_list = []
+        for i_amp in range(image.get_n_amps()):
+            npixels = np.product(image.data[i_amp].shape)
+            fraction_0s_list.append(float(np.sum(image.data[i_amp] == 0)) / npixels)
+
+        has_0s_error = any([fraction_0s > self.ZEROS_THRESHOLD for fraction_0s in fraction_0s_list])
+        logging_tags = {'FRAC0': fraction_0s_list,
+                        'threshold': self.ZEROS_THRESHOLD}
+        qc_results = {'zeros_test.failed': has_0s_error,
+                      'zeros_test.fraction': fraction_0s_list,
+                      'zeros_test.threshold': self.ZEROS_THRESHOLD}
+        if has_0s_error:
+            logger.error('Image is mostly 0s. Rejecting image', image=image, extra_tags=logging_tags)
+            qc_results['rejected'] = True
+            return None
+        else:
+            logger.info('Measuring fraction of 0s.', image=image, extra_tags=logging_tags)
+        qc.save_qc_results(self.runtime_context, qc_results, image)
+
+        return image

banzai/settings.py

@@ -24,6 +24,7 @@ def telescope_to_filename(image):
 ORDERED_STAGES = ['banzai.bpm.BPMUpdater',
                   'banzai.qc.HeaderSanity',
                   'banzai.qc.ThousandsTest',
+                  'banzai.qc.ZerosTest',
                   'banzai.qc.SaturationTest',
                   'banzai.bias.OverscanSubtractor',
                   'banzai.crosstalk.CrosstalkCorrector',

banzai/tests/test_zeros_qc.py

@@ -0,0 +1,71 @@
+import pytest
+import numpy as np
+
+from banzai.tests.utils import FakeImage
+from banzai.qc import ZerosTest
+
+
+@pytest.fixture(scope='module')
+def set_random_seed():
+    np.random.seed(81232385)
+
+
+def get_random_pixel_pairs(nx, ny, fraction):
+    return np.unravel_index(np.random.choice(range(nx * ny), size=int(fraction * nx * ny)), (ny, nx))
+
+
+def test_null_input_image():
+    tester = ZerosTest(None)
+    image = tester.run(None)
+    assert image is None
+
+
+def test_no_pixels_0_not_rejected():
+    tester = ZerosTest(None)
+    image = tester.do_stage(FakeImage(image_multiplier=5))
+    assert image is not None
+
+
+def test_image_all_0s_rejected():
+    tester = ZerosTest(None)
+    image = FakeImage()
+    image.data[:, :] = 0
+    image = tester.do_stage(image)
+    assert image is None
+
+
+def test_image_50_percent_0_not_rejected(set_random_seed):
+    tester = ZerosTest(None)
+    nx = 101
+    ny = 103
+    image = FakeImage(nx=nx, ny=ny)
+    random_pixels = get_random_pixel_pairs(nx, ny, 0.5)
+    for j,i in zip(random_pixels[0], random_pixels[1]):
+        image.data[i,i] = 0
+    image = tester.do_stage(image)
+    assert image is not None
+
+
+def test_image_99_percent_0_rejected(set_random_seed):
+    tester = ZerosTest(None)
+    nx = 101
+    ny = 103
+    image = FakeImage(nx=nx, ny=ny)
+    random_pixels = get_random_pixel_pairs(nx, ny, 0.99)
+    for j,i in zip(random_pixels[0], random_pixels[1]):
+        image.data[i] = 0
+    image = tester.do_stage(image)
+    assert image is None
+
+
+def test_single_amp_99_percent_0_rejected(set_random_seed):
+    tester = ZerosTest(None)
+    nx = 101
+    ny = 103
+    n_amps = 4
+    image = FakeImage(nx=nx, ny=ny, n_amps=n_amps)
+    random_pixels = get_random_pixel_pairs(nx, ny, 0.99)
+    for j,i in zip(random_pixels[0], random_pixels[1]):
+        image.data[0][i] = 0
+    image = tester.do_stage(image)
+    assert image is None

setup.cfg

@@ -77,7 +77,7 @@ edit_on_github = True
 github_project = lcogt/banzai
 
 # version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440)
-version = 0.21.0
+version = 0.21.1
 
 [entry_points]
 banzai = banzai.main:main