Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Run E2E Test w/ Github Actions #376

Open
wants to merge 20 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Added support for stacked images.
  • Loading branch information
cmccully committed Jul 24, 2024
commit 4489497a7b51f7abd36ddd311481bdaa584e8bba
14 changes: 13 additions & 1 deletion .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
@@ -43,10 +43,22 @@ jobs:
# Wait for banzai to be ready
kubectl wait --for=condition=Ready --timeout=60m pod/banzai-e2e-test

- name: Test Master Bias Creation
- name: Test Super Bias Creation
run: |
kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias

- name: Test Super Dark Creation
run: |
kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-dark.xml -m master_dark

- name: Test Super Flat Creation
run: |
kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-flat.xml -m master_flat

- name: Test Science Frame Creation
run: |
kubectl exec banzai-e2e-test -c banzai-listener -- pytest -o log_cli=true -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-science-files.xml -m science_files

- name: Cleanup
run: |
kubectl delete pod banzai-e2e-test
6 changes: 3 additions & 3 deletions banzai/bias.py
Original file line number Diff line number Diff line change
@@ -31,9 +31,9 @@ def calibration_type(self):
return 'bias'

def apply_master_calibration(self, image, master_calibration_image):
image -= master_calibration_image.bias_level
image -= master_calibration_image.bias_level * image.n_sub_exposures
image.meta['BIASLVL'] = master_calibration_image.bias_level, 'Bias level that was removed after overscan'
image -= master_calibration_image
image -= master_calibration_image * image.n_sub_exposures
image.meta['L1IDBIAS'] = master_calibration_image.filename, 'ID of bias frame'
image.meta['L1STATBI'] = 1, "Status flag for bias frame correction"
return image
@@ -65,7 +65,7 @@ def __init__(self, runtime_context):
def do_stage(self, image):
bias_level = stats.sigma_clipped_mean(image.data, 3.5, mask=image.mask)
image -= bias_level
image.meta['BIASLVL'] = bias_level, 'Bias level that was removed after overscan'
image.meta['BIASLVL'] = bias_level / image.n_sub_exposures, 'Bias level that was removed after overscan'
return image


6 changes: 5 additions & 1 deletion banzai/data.py
Original file line number Diff line number Diff line change
@@ -132,7 +132,7 @@ def __init__(self, data: Union[np.array, Table], meta: fits.Header,
mask: np.array = None, name: str = '', uncertainty: np.array = None, memmap=True):
super().__init__(data=data, meta=meta, mask=mask, name=name, memmap=memmap)
if uncertainty is None:
uncertainty = self.read_noise * np.ones(data.shape, dtype=data.dtype) / self.gain
uncertainty = self.read_noise * self.n_sub_exposures * np.ones(data.shape, dtype=data.dtype) / self.gain
self.uncertainty = self._init_array(uncertainty)
self._detector_section = Section.parse_region_keyword(self.meta.get('DETSEC'))
self._data_section = Section.parse_region_keyword(self.meta.get('DATASEC'))
@@ -302,6 +302,10 @@ def data_section(self, section):
self.meta['DATASEC'] = section.to_region_keyword()
self._data_section = section

@property
def n_sub_exposures(self):
return self.meta.get('NSUBREAD', 1)

def rebin(self, binning):
# TODO: Implement me
return self
2 changes: 1 addition & 1 deletion banzai/lco.py
Original file line number Diff line number Diff line change
@@ -450,7 +450,7 @@ def open(self, file_info, runtime_context) -> Optional[ObservationFrame]:
# update datasec/trimsec for fs01
if hdu.header.get('INSTRUME') == 'fs01':
self._update_fs01_sections(hdu)
if hdu.data.dtype == np.uint16:
if hdu.data.dtype == np.uint16 or hdu.data.dtype == np.uint32:
hdu.data = hdu.data.astype(np.float64)
# check if we need to propagate any header keywords from the primary header
if primary_hdu is not None:
3 changes: 2 additions & 1 deletion banzai/readnoise.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from banzai.calibrations import CalibrationUser
from banzai.logs import format_exception, get_logger
import numpy as np

logger = get_logger()

@@ -8,7 +9,7 @@ class ReadNoiseLoader(CalibrationUser):
def apply_master_calibration(self, image, master_calibration_image):
try:
for image_extension, readnoise_extension in zip(image.ccd_hdus, master_calibration_image.ccd_hdus):
image_extension.add_uncertainty(readnoise_extension.data)
image_extension.add_uncertainty(readnoise_extension.data * np.sqrt(image_extension.n_sub_exposures))
except:
logger.error(f"Can't add READNOISE to image, stopping reduction: {format_exception()}", image=image)
return None
4 changes: 2 additions & 2 deletions banzai/settings.py
Original file line number Diff line number Diff line change
@@ -54,8 +54,8 @@
EXTRA_STAGES = {'BIAS': ['banzai.bias.BiasMasterLevelSubtractor', 'banzai.bias.BiasComparer'],
'DARK': ['banzai.dark.DarkNormalizer', 'banzai.dark.DarkTemperatureChecker',
'banzai.dark.DarkComparer'],
'SKYFLAT': ['banzai.flats.FlatSNRChecker', 'banzai.flats.FlatNormalizer', 'banzai.qc.PatternNoiseDetector',
'banzai.flats.FlatComparer'],
'SKYFLAT': ['banzai.flats.FlatSNRChecker', 'banzai.flats.FlatNormalizer',
'banzai.qc.PatternNoiseDetector', 'banzai.flats.FlatComparer'],
'STANDARD': None,
'EXPOSE': None,
'EXPERIMENTAL': None}
43 changes: 17 additions & 26 deletions banzai/tests/data/test_data.dat
Original file line number Diff line number Diff line change
@@ -304,29 +304,20 @@ frameid filename filter site instrument
33347343 tst1m0XX-ep04-20200324-0018-d00.fits.fz ip*diffuser tst ep04
33347344 tst1m0XX-ep04-20200324-0019-d00.fits.fz ip*diffuser tst ep04
33347345 tst1m0XX-ep04-20200324-0020-d00.fits.fz ip*diffuser tst ep04
56016937 ogg0m404-sq30-20221123-0037-b00.fits.fz opaque ogg sq30
56016931 ogg0m404-sq30-20221123-0036-b00.fits.fz opaque ogg sq30
56016928 ogg0m404-sq30-20221123-0035-b00.fits.fz opaque ogg sq30
56016925 ogg0m404-sq30-20221123-0034-b00.fits.fz opaque ogg sq30
56016920 ogg0m404-sq30-20221123-0033-b00.fits.fz opaque ogg sq30
56016916 ogg0m404-sq30-20221123-0032-b00.fits.fz opaque ogg sq30
56016912 ogg0m404-sq30-20221123-0031-b00.fits.fz opaque ogg sq30
56016909 ogg0m404-sq30-20221123-0030-b00.fits.fz opaque ogg sq30
56016906 ogg0m404-sq30-20221123-0029-b00.fits.fz opaque ogg sq30
56016903 ogg0m404-sq30-20221123-0028-b00.fits.fz opaque ogg sq30
56016899 ogg0m404-sq30-20221123-0027-b00.fits.fz opaque ogg sq30
56016898 ogg0m404-sq30-20221123-0026-b00.fits.fz opaque ogg sq30
56016896 ogg0m404-sq30-20221123-0025-b00.fits.fz opaque ogg sq30
56016891 ogg0m404-sq30-20221123-0024-b00.fits.fz opaque ogg sq30
56016889 ogg0m404-sq30-20221123-0023-b00.fits.fz opaque ogg sq30
56017384 ogg0m404-sq30-20221123-0042-d00.fits.fz opaque ogg sq30
56017294 ogg0m404-sq30-20221123-0041-d00.fits.fz opaque ogg sq30
56017205 ogg0m404-sq30-20221123-0040-d00.fits.fz opaque ogg sq30
56017136 ogg0m404-sq30-20221123-0039-d00.fits.fz opaque ogg sq30
56017044 ogg0m404-sq30-20221123-0038-d00.fits.fz opaque ogg sq30
56018344 ogg0m404-sq30-20221123-0067-f00.fits.fz rp ogg sq30
56018340 ogg0m404-sq30-20221123-0066-f00.fits.fz rp ogg sq30
56018336 ogg0m404-sq30-20221123-0065-f00.fits.fz rp ogg sq30
56018331 ogg0m404-sq30-20221123-0064-f00.fits.fz rp ogg sq30
56018328 ogg0m404-sq30-20221123-0063-f00.fits.fz rp ogg sq30
56127852 ogg0m404-sq30-20221126-0098-e00.fits.fz rp ogg sq30
73414350 lsc0m476-sq34-20240723-0301-e00.fits.fz B lsc sq34
73422225 lsc0m476-sq34-20240723-0408-x00.fits.fz B lsc sq34
73409803 lsc0m476-sq34-20240723-0057-f00.fits.fz B lsc sq34
73409794 lsc0m476-sq34-20240723-0056-f00.fits.fz B lsc sq34
73409788 lsc0m476-sq34-20240723-0055-f00.fits.fz B lsc sq34
73409785 lsc0m476-sq34-20240723-0054-f00.fits.fz B lsc sq34
73215904 lsc0m476-sq34-20240723-0053-f00.fits.fz B lsc sq34
73429574 lsc0m476-sq34-20240723-0548-d00.fits.fz opaque lsc sq34
73429727 lsc0m476-sq34-20240723-0549-d00.fits.fz opaque lsc sq34
73429774 lsc0m476-sq34-20240723-0550-d00.fits.fz opaque lsc sq34
73429865 lsc0m476-sq34-20240723-0551-d00.fits.fz opaque lsc sq34
73429912 lsc0m476-sq34-20240723-0552-d00.fits.fz opaque lsc sq34
73429436 lsc0m476-sq34-20240723-0535-b00.fits.fz opaque lsc sq34
73429432 lsc0m476-sq34-20240723-0536-b00.fits.fz opaque lsc sq34
73429437 lsc0m476-sq34-20240723-0537-b00.fits.fz opaque lsc sq34
73429439 lsc0m476-sq34-20240723-0538-b00.fits.fz opaque lsc sq34
73429443 lsc0m476-sq34-20240723-0539-b00.fits.fz opaque lsc sq34
Loading