Skip to content

Commit

Permalink
Merge pull request #256 from roman-corgi/nonlin_kgain_e2e
Browse files Browse the repository at this point in the history
Nonlin kgain e2e test
  • Loading branch information
semaphoreP authored Nov 8, 2024
2 parents 27d4728 + 09506bd commit 2cd362f
Show file tree
Hide file tree
Showing 5 changed files with 141 additions and 3 deletions.
File renamed without changes.
4 changes: 2 additions & 2 deletions corgidrp/walker.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import corgidrp.calibrate_nonlin
import corgidrp.detector
import corgidrp.darks
import corgidrp.sort_pupilimg_frames
import corgidrp.sorting

all_steps = {
"prescan_biassub" : corgidrp.l1_to_l2a.prescan_biassub,
Expand All @@ -41,7 +41,7 @@
"create_onsky_flatfield" : corgidrp.detector.create_onsky_flatfield,
"combine_subexposures" : corgidrp.combine.combine_subexposures,
"build_trad_dark" : corgidrp.darks.build_trad_dark,
"sort_pupilimg_frames" : corgidrp.sort_pupilimg_frames.sort_pupilimg_frames
"sort_pupilimg_frames" : corgidrp.sorting.sort_pupilimg_frames
}

recipe_dir = os.path.join(os.path.dirname(__file__), "recipe_templates")
Expand Down
1 change: 1 addition & 0 deletions tests/e2e_tests/noisemap_cal_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,7 @@ def test_noisemap_calibration_from_l1(tvacdata_path, e2eoutput_path):
# iit_noisemap_fname = os.path.join(iit_noisemap_datadir,"iit_test_noisemaps.fits")

corgidrp_noisemap = data.autoload(corgidrp_noisemap_fname)
this_caldb.remove_entry(corgidrp_noisemap)

assert(np.nanmax(np.abs(corgidrp_noisemap.data[0]- F_map)) < 1e-11)
assert(np.nanmax(np.abs(corgidrp_noisemap.data[1]- C_map)) < 1e-11)
Expand Down
137 changes: 137 additions & 0 deletions tests/e2e_tests/nonlin_kgain_e2e.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
""" Module to test the generation of both nonlin and kgain from the same PUPILIMG dataset """
import os
import glob
import argparse
import pytest
import numpy as np
from astropy import time
from astropy.io import fits
import matplotlib.pyplot as plt

import corgidrp
from corgidrp import data
from corgidrp import mocks
from corgidrp import walker
from corgidrp import caldb

thisfile_dir = os.path.dirname(__file__) # this file's folder

def set_vistype_for_tvac(
list_of_fits,
):
""" Adds proper values to VISTYPE for non-linearity calibration.
This function is unnecessary with future data because data will have
the proper values in VISTYPE. Hence, the "tvac" string in its name.
Args:
list_of_fits (list): list of FITS files that need to be updated.
"""
print("Adding VISTYPE='PUPILIMG' to TVAC data")
for file in list_of_fits:
fits_file = fits.open(file)
prihdr = fits_file[0].header
# Adjust VISTYPE
prihdr['VISTYPE'] = 'PUPILIMG'
# Update FITS file
fits_file.writeto(file, overwrite=True)


@pytest.mark.e2e
def test_nonlin_and_kgain_e2e(
tvacdata_path,
e2eoutput_path,
):
"""
Performs the e2e test to generate both nonlin and kgain calibrations from the same
L1 pupilimg dataset
Args:
tvacdata_path (str): Location of L1 data. Folders for both kgain and nonlin
e2eoutput_path (str): Location of the output products: recipes, non-linearity
calibration FITS file, and kgain fits file
"""

# figure out paths, assuming everything is located in the same relative location
nonlin_l1_datadir = os.path.join(tvacdata_path,
'TV-20_EXCAM_noise_characterization', 'nonlin')
kgain_l1_datadir = os.path.join(tvacdata_path,
'TV-20_EXCAM_noise_characterization', 'kgain')

e2eoutput_path = os.path.join(e2eoutput_path, 'nonlin_and_kgain_output')

if not os.path.exists(nonlin_l1_datadir):
raise FileNotFoundError('Please store L1 data used to calibrate non-linearity',
f'in {nonlin_l1_datadir}')
if not os.path.exists(kgain_l1_datadir):
raise FileNotFoundError('Please store L1 data used to calibrate kgain',
f'in {kgain_l1_datadir}')

if not os.path.exists(e2eoutput_path):
os.mkdir(e2eoutput_path)

# Define the raw science data to process
nonlin_l1_list = glob.glob(os.path.join(nonlin_l1_datadir, "*.fits"))
nonlin_l1_list.sort()
kgain_l1_list = glob.glob(os.path.join(kgain_l1_datadir, "*.fits"))
kgain_l1_list.sort()

# both kgain and nonlin dirs have the same MNFRAME files
# only add the files from the kgain list that don't share the same filename
# grab filenames for l1
nonlin_l1_filenames = [filepath.split(os.path.sep)[-1] for filepath in nonlin_l1_list]
pupilimg_l1_list = nonlin_l1_list # start with the nonlin filelist
# iterate through kgain filelist to find ones that don't share the same filename
for filepath in kgain_l1_list:
filename = filepath.split(os.path.sep)[-1]
if filename not in nonlin_l1_filenames:
pupilimg_l1_list.append(filepath)


# Set TVAC data to have VISTYPE=PUPILIMG (flight data should have these values)
set_vistype_for_tvac(pupilimg_l1_list)


# Run the walker on some test_data
print('Running walker')
walker.walk_corgidrp(pupilimg_l1_list, '', e2eoutput_path)

# check that files can be loaded from disk successfully. no need to check correctness as done in other e2e tests
# NL from CORGIDRP
possible_nonlin_files = glob.glob(os.path.join(e2eoutput_path, '*_NonLinearityCalibration.fits'))
nonlin_drp_filepath = max(possible_nonlin_files, key=os.path.getmtime) # get the one most recently modified
nonlin = data.NonLinearityCalibration(nonlin_drp_filepath)

# kgain from corgidrp
possible_kgain_files = glob.glob(os.path.join(e2eoutput_path, '*_kgain.fits'))
kgain_filepath = max(possible_kgain_files, key=os.path.getmtime) # get the one most recently modified
kgain = data.KGain(kgain_filepath)

# remove entry from caldb
this_caldb = caldb.CalDB()
this_caldb.remove_entry(nonlin)
this_caldb.remove_entry(kgain)

# Print success message
print('e2e test for NL passed')

if __name__ == "__main__":

# Use arguments to run the test. Users can then write their own scripts
# that call this script with the correct arguments and they do not need
# to edit the file. The arguments use the variables in this file as their
# defaults allowing the use to edit the file if that is their preferred
# workflow.

TVACDATA_DIR = '/home/jwang/Desktop/CGI_TVAC_Data/'
OUTPUT_DIR = thisfile_dir

ap = argparse.ArgumentParser(description="run the non-linearity end-to-end test")
ap.add_argument("-tvac", "--tvacdata_dir", default=TVACDATA_DIR,
help="Path to CGI_TVAC_Data Folder [%(default)s]")
ap.add_argument("-o", "--output_dir", default=OUTPUT_DIR,
help="directory to write results to [%(default)s]")
args = ap.parse_args()
# Run the e2e test
test_nonlin_and_kgain_e2e(args.tvacdata_dir, args.output_dir)
2 changes: 1 addition & 1 deletion tests/test_sort_pupilimg_frames.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from pathlib import Path
import astropy.io.fits as fits

from corgidrp import sort_pupilimg_frames as sorting
from corgidrp import sorting as sorting
import corgidrp.data as data
from corgidrp.data import Image
from corgidrp.mocks import create_default_headers
Expand Down

0 comments on commit 2cd362f

Please sign in to comment.