From cc2fbc4beaecdbee22241bea89e65919f114ece3 Mon Sep 17 00:00:00 2001 From: "Tom J. Wilson" Date: Thu, 18 Apr 2024 13:44:21 +0100 Subject: [PATCH 1/5] Added saturation_magnitude as input parameter to CrossMatch and AstrometricCorrections, to be used in new two-parameter model normalisation fit. --- docs/inputs.rst | 6 ++- src/macauff/fit_astrometry.py | 44 ++++++++++++---- src/macauff/matching.py | 33 ++++++++---- src/macauff/misc_functions.py | 77 ++++++++++++++++++++++++++++ tests/macauff/test_fit_astrometry.py | 7 +-- tests/macauff/test_matching.py | 19 ++++--- 6 files changed, 154 insertions(+), 32 deletions(-) diff --git a/docs/inputs.rst b/docs/inputs.rst index b4ef0ee..c2fe525 100644 --- a/docs/inputs.rst +++ b/docs/inputs.rst @@ -184,7 +184,7 @@ the inputs required if either ``correct_astrometry`` or ``compute_snr_mag_relati and the inputs required if ``correct_astrometry`` is ``True``: -``best_mag_index``, ``nn_radius``, ``ref_csv_cat_file_string``, ``correct_mag_array``, ``correct_mag_slice``, ``correct_sig_slice``, ``chunk_overlap_col``, and ``best_mag_index_col``. +``best_mag_index``, ``nn_radius``, ``ref_csv_cat_file_string``, ``correct_mag_array``, ``correct_mag_slice``, ``correct_sig_slice``, ``chunk_overlap_col``, ``best_mag_index_col``, and ``saturation_magnitudes``. .. note:: ``run_fw_auf``, ``run_psf_auf``, ``psf_fwhms``, ``snr_mag_params_path``, ``download_tri``, ``tri_set_name``, ``tri_filt_names``, ``tri_filt_num``, ``tri_maglim_faint``, ``tri_num_faint``, ``dens_dist``, ``dd_params_path``, ``l_cut_path``, ``gal_wavs``, ``gal_zmax``, ``gal_nzs``, ``gal_aboffsets``, ``gal_filternames``, and ``gal_al_avs`` are all currently required if ``correct_astrometry`` is ``True``, bypassing the nested flags above. For example, ``dens_dist`` is required as an input if ``include_perturb_auf`` is ``True``, or if ``correct_astrometry`` is set. This means that ``AstrometricCorrections`` implicitly always runs and fits for a full Astrometric Uncertainty Function. @@ -426,6 +426,10 @@ The zero-indexed integer column number in the original input csv file used in `` Boolean flag indicating whether the astrometric or photometric uncertainties of the input catalogue should be used to derive the astrometric uncertainties from ensemble statistics in ``AstrometricCorrections``. +``saturation_magnitudes`` + +A list, one float per filter, of the magnitudes in the given filter at which the telescope or survey saturates, used in the filtering of source counts for model-fitting purposes in ``AstrometricCorrections``. + Parameter Dependency Graph ========================== diff --git a/src/macauff/fit_astrometry.py b/src/macauff/fit_astrometry.py index ab0de33..b3beb36 100644 --- a/src/macauff/fit_astrometry.py +++ b/src/macauff/fit_astrometry.py @@ -31,7 +31,7 @@ # pylint: disable=wrong-import-position,import-error,no-name-in-module from macauff.galaxy_counts import create_galaxy_counts from macauff.get_trilegal_wrapper import get_av_infinity -from macauff.misc_functions import min_max_lon +from macauff.misc_functions import find_model_counts_corrections, min_max_lon from macauff.misc_functions_fortran import misc_functions_fortran as mff from macauff.perturbation_auf import ( _calculate_magnitude_offsets, @@ -56,8 +56,8 @@ def __init__(self, psf_fwhm, numtrials, nn_radius, dens_search_radius, save_fold gal_wav_micron, gal_ab_offset, gal_filtname, gal_alav, dm, dd_params, l_cut, ax1_mids, ax2_mids, ax_dimension, mag_array, mag_slice, sig_slice, n_pool, npy_or_csv, coord_or_chunk, pos_and_err_indices, mag_indices, mag_unc_indices, - mag_names, best_mag_index, coord_system, pregenerate_cutouts, n_r, n_rho, max_rho, - trifolder=None, triname=None, maglim_f=None, magnum=None, tri_num_faint=None, + mag_names, best_mag_index, coord_system, saturation_magnitudes, pregenerate_cutouts, n_r, + n_rho, max_rho, trifolder=None, triname=None, maglim_f=None, magnum=None, tri_num_faint=None, trifilterset=None, trifiltname=None, tri_hist=None, tri_mags=None, dtri_mags=None, tri_uncert=None, use_photometric_uncertainties=False, cutout_area=None, cutout_height=None, single_sided_auf=True, chunks=None, return_nm=False): @@ -172,6 +172,9 @@ def __init__(self, psf_fwhm, numtrials, nn_radius, dens_search_radius, save_fold Identifier of which coordinate system the data are in. Both datasets must be in the same system, which can either be RA/Dec (equatorial) or l/b (galactic) coordinates. + saturation_magnitudes : list or numpy.ndarray + List of magnitudes brighter than which the given survey suffers + saturation, each element matching the filter of ``mag_indices``. pregenerate_cutouts : boolean or None Indicates whether sightline catalogues must have been pre-made, or whether they can be generated by ``AstrometricCorrections`` @@ -339,6 +342,8 @@ def __init__(self, psf_fwhm, numtrials, nn_radius, dens_search_radius, save_fold self.mag_slice = np.array(mag_slice) self.sig_slice = np.array(sig_slice) + self.saturation_magnitudes = np.array(saturation_magnitudes) + self.npy_or_csv = npy_or_csv self.coord_or_chunk = coord_or_chunk self.chunks = chunks @@ -1043,7 +1048,27 @@ def make_star_galaxy_counts(self): self.gal_wav_micron, self.gal_alpha0, self.gal_alpha1, self.gal_alphaweight, self.gal_ab_offset, self.gal_filtname, self.gal_alav*avs) - log10y = np.log10(tri_hist + gal_dns) + mag_ind = self.mag_indices[self.best_mag_index] + data_mags = self.b[~np.isnan(self.b[:, mag_ind]), mag_ind] + data_hist, data_bins = np.histogram(data_mags, bins='auto') + d_hc = np.where(data_hist > 3)[0] + data_hist = data_hist[d_hc] + data_dbins = np.diff(data_bins)[d_hc] + data_bins = data_bins[d_hc] + + rect_area = (ax1_max - (ax1_min)) * ( + np.sin(np.radians(ax2_max)) - np.sin(np.radians(ax2_min))) * 180/np.pi + + data_uncert = np.sqrt(data_hist) / data_dbins / rect_area + data_hist = data_hist / data_dbins / rect_area + data_loghist = np.log10(data_hist) + data_dloghist = 1/np.log(10) * data_uncert / data_hist + + q = (data_bins <= maxmag) & (data_bins >= self.saturation_magnitudes[self.best_mag_index]) + tri_corr, gal_corr = find_model_counts_corrections(data_loghist[q], data_dloghist[q], + data_bins[q]+data_dbins[q]/2, tri_hist, gal_dns, + tri_mags+dtri_mags/2) + log10y = np.log10(tri_hist * tri_corr + gal_dns * gal_corr) new_uncert = np.sqrt(tri_uncert**2 + (0.05*gal_dns)**2) dlog10y = 1/np.log(10) * new_uncert / (tri_hist + gal_dns) @@ -1053,6 +1078,7 @@ def make_star_galaxy_counts(self): self.tri_hist, self.tri_mags, self.dtri_mags = tri_hist, tri_mags, dtri_mags self.tri_uncert, self.gal_dns = tri_uncert, gal_dns self.minmag, self.maxmag, self.n_norm = minmag, maxmag, n_norm + self.tri_corr, self.gal_corr = tri_corr, gal_corr def plot_star_galaxy_counts(self): """ @@ -1075,16 +1101,12 @@ def plot_star_galaxy_counts(self): mag_ind = self.mag_indices[self.best_mag_index] data_mags = self.b[~np.isnan(self.b[:, mag_ind]), mag_ind] - # Correction to model is the ratio of data counts per unit area - # to model source density. - correction = np.sum((data_mags >= self.minmag) & - (data_mags <= self.maxmag)) / rect_area / self.n_norm ax = plt.subplot(gs[0]) ax1_name = 'l' if self.coord_system == 'galactic' else 'RA' ax2_name = 'b' if self.coord_system == 'galactic' else 'Dec' ax.set_title(f'{ax1_name} = {ax1_mid}, {ax2_name} = {ax2_mid}') - ax.errorbar(self.tri_mags+self.dtri_mags/2, self.log10y + np.log10(correction), + ax.errorbar(self.tri_mags+self.dtri_mags/2, self.log10y, yerr=self.dlog10y, c='k', marker='.', zorder=1, ls='None') data_hist, data_bins = np.histogram(data_mags, bins='auto') @@ -1103,10 +1125,10 @@ def plot_star_galaxy_counts(self): lims = ax.get_ylim() q = self.tri_hist > 0 ax.plot((self.tri_mags+self.dtri_mags/2)[q], np.log10(self.tri_hist[q]) + - np.log10(correction), 'b--') + np.log10(self.tri_corr), 'k--') q = self.gal_dns > 0 ax.plot((self.tri_mags+self.dtri_mags/2)[q], np.log10(self.gal_dns[q]) + - np.log10(correction), 'b:') + np.log10(self.gal_corr), 'k:') ax.set_ylim(*lims) ax.set_xlabel('Magnitude') diff --git a/src/macauff/matching.py b/src/macauff/matching.py index a37d1c2..a942acc 100644 --- a/src/macauff/matching.py +++ b/src/macauff/matching.py @@ -188,11 +188,11 @@ def _initialise_chunk(self, joint_file_path, cat_a_file_path, cat_b_file_path): self.a_correct_mag_slice, self.a_correct_sig_slice, self.n_pool, a_npy_or_csv, a_coord_or_chunk, self.a_pos_and_err_indices, self.a_mag_indices, self.a_mag_unc_indices, self.a_filt_names, self.a_best_mag_index, self.a_auf_region_frame, - trifolder=self.a_auf_folder_path, triname=a_correct_astro_tri_name, - maglim_f=self.a_tri_maglim_faint, magnum=self.a_tri_filt_num, - tri_num_faint=self.a_tri_num_faint, trifilterset=self.a_tri_set_name, - trifiltname=self.a_tri_filt_names[acbi], tri_hist=self.a_dens_hist_tri_list[acbi], - tri_mags=self.a_tri_model_mags_list[acbi], + self.a_saturation_magnitudes, trifolder=self.a_auf_folder_path, + triname=a_correct_astro_tri_name, maglim_f=self.a_tri_maglim_faint, + magnum=self.a_tri_filt_num, tri_num_faint=self.a_tri_num_faint, + trifilterset=self.a_tri_set_name, trifiltname=self.a_tri_filt_names[acbi], + tri_hist=self.a_dens_hist_tri_list[acbi], tri_mags=self.a_tri_model_mags_list[acbi], dtri_mags=self.a_tri_model_mags_interval_list[acbi], tri_uncert=self.a_tri_dens_uncert_list[acbi], use_photometric_uncertainties=self.a_use_photometric_uncertainties, pregenerate_cutouts=True, @@ -260,11 +260,11 @@ def _initialise_chunk(self, joint_file_path, cat_a_file_path, cat_b_file_path): self.b_correct_mag_slice, self.b_correct_sig_slice, self.n_pool, b_npy_or_csv, b_coord_or_chunk, self.b_pos_and_err_indices, self.b_mag_indices, self.b_mag_unc_indices, self.b_filt_names, self.b_best_mag_index, self.b_auf_region_frame, - trifolder=self.b_auf_folder_path, triname=b_correct_astro_tri_name, - maglim_f=self.b_tri_maglim_faint, magnum=self.b_tri_filt_num, - tri_num_faint=self.b_tri_num_faint, trifilterset=self.b_tri_set_name, - trifiltname=self.b_tri_filt_names[bcbi], tri_hist=self.b_dens_hist_tri_list[bcbi], - tri_mags=self.b_tri_model_mags_list[bcbi], + self.b_saturation_magnitudes, trifolder=self.b_auf_folder_path, + triname=b_correct_astro_tri_name, maglim_f=self.b_tri_maglim_faint, + magnum=self.b_tri_filt_num, tri_num_faint=self.b_tri_num_faint, + trifilterset=self.b_tri_set_name, trifiltname=self.b_tri_filt_names[bcbi], + tri_hist=self.b_dens_hist_tri_list[bcbi], tri_mags=self.b_tri_model_mags_list[bcbi], dtri_mags=self.b_tri_model_mags_interval_list[bcbi], tri_uncert=self.b_tri_dens_uncert_list[bcbi], use_photometric_uncertainties=self.b_use_photometric_uncertainties, @@ -1284,7 +1284,7 @@ def read_metadata(self): for check_flag in ['best_mag_index', 'nn_radius', 'ref_csv_cat_file_string', 'correct_mag_array', 'correct_mag_slice', 'correct_sig_slice', 'chunk_overlap_col', 'best_mag_index_col', - 'use_photometric_uncertainties']: + 'use_photometric_uncertainties', 'saturation_magnitudes']: if check_flag not in config: raise ValueError(f"Missing key {check_flag} from catalogue {catname} metadata file.") @@ -1369,6 +1369,17 @@ def read_metadata(self): 'the same number of entries.') setattr(self, f'{flag}correct_sig_slice', b) + a = config['saturation_magnitudes'].split() + try: + b = np.array([float(f) for f in a]) + except ValueError as exc: + raise ValueError('saturation_magnitudes should be a list of floats in the ' + f'catalogue {catname} metadata file.') from exc + if len(b) != len(getattr(self, f'{flag}mag_indices')): + raise ValueError(f'{flag}saturation_magnitudes and {flag}mag_indices should ' + 'contain the same number of entries.') + setattr(self, f'{flag}saturation_magnitudes', b) + def _read_metadata_csv(self, joint_config, cat_a_config, cat_b_config): """ Read metadata from input config files relating to outputting combined diff --git a/src/macauff/misc_functions.py b/src/macauff/misc_functions.py index 2c3e952..fdf58a4 100644 --- a/src/macauff/misc_functions.py +++ b/src/macauff/misc_functions.py @@ -5,6 +5,7 @@ ''' import numpy as np +from scipy.optimize import minimize __all__ = [] @@ -312,3 +313,79 @@ def min_max_lon(a): # Otherwise, the limits are inside [0, 360] and should be returned # as the "normal" minimum and maximum values. return min_lon, max_lon + + +def find_model_counts_corrections(data_loghist, data_dloghist, data_bin_mids, tri_hist, + gal_dns, tri_mag_mids): + ''' + Derivation of two-parameter corrections to differential source counts, fitting + both stellar and galaxy components with separate scaling factors. + + Parameters + ---------- + data_loghist : numpy.ndarray + Logarithmic differential source counts. + data_dloghist : numpy.ndarray + Uncertainties in log-counts from ``data_loghist``. + data_bin_mids : numpy.ndarray + Magnitudes of each bin corresponding to ``data_loghist``. + tri_hist : numpy.ndarray + Stellar model linear differential source counts. + gal_dns : numpy.ndarray + Galaxy model linear differential source counts. + tri_mag_mids : numpy.ndarray + Model magnitudes for each element of ``tri_hist`` and/or ``gal_dns``. + + Returns + ------- + res.x : numpy.ndarray + Value of least-squares fit scaling factors for stellar and galaxy + components of the differential source counts, as fit to the + input data. + ''' + def lst_sq(p, y, o, t, g): + ''' + Function evaluating the least-squares minimisation, and Jacobian, + of a fit to model and data differential source counts. + + Parameters + ---------- + p : list + ``a`` and ``b``, scaling factors for star and galaxy components + of the source counts. + y : numpy.ndarray + Data log-counts. + o : numpy.ndarray + Uncertainties corresponding to ``y``. + t : numpy.ndarray + Log-counts of stellar component of source counts. + g : numpy.ndarray + Log-counts of galaxy component of source counts. + + Returns + ------- + float + Chi-squared of the model source counts fit to the data. + list of floats + Gradient of the chi-squared, differentiated with respect to + the stellar and galaxy correction factors. + ''' + a, b = p + f = np.log10(10**t * a + 10**g * b) + dfda = 10**t / (np.log(10) * (10**t * a + 10**g * b)) + dfdb = 10**g / (np.log(10) * (10**t * a + 10**g * b)) + dchida = np.sum(-2 * (y - f) / o**2 * dfda) + dchidb = np.sum(-2 * (y - f) / o**2 * dfdb) + return np.sum((y - f)**2 / o**2), [dchida, dchidb] + + q = tri_hist > 0 + tri_log_hist_at_data = np.interp(data_bin_mids, tri_mag_mids[q], + np.log10(tri_hist[q]), left=np.nan, right=np.nan) + q = gal_dns > 0 + gal_log_hist_at_data = np.interp(data_bin_mids, tri_mag_mids[q], + np.log10(gal_dns[q]), left=np.nan, right=np.nan) + q = ~np.isnan(tri_log_hist_at_data) & ~np.isnan(gal_log_hist_at_data) + res = minimize(lst_sq, args=(data_loghist[q], np.ones_like(data_loghist[q]), + tri_log_hist_at_data[q], gal_log_hist_at_data[q]), x0=[1, 1], jac=True, + method='L-BFGS-B', options={'ftol': 1e-12}, bounds=[(0.01, None), (0.01, None)]) + return res.x diff --git a/tests/macauff/test_fit_astrometry.py b/tests/macauff/test_fit_astrometry.py index 91c5d55..461001f 100644 --- a/tests/macauff/test_fit_astrometry.py +++ b/tests/macauff/test_fit_astrometry.py @@ -110,7 +110,7 @@ def test_fit_astrometry_load_errors(self): # pylint: disable=too-many-statement 'mag_slice': magslice, 'sig_slice': sigslice, 'n_pool': 1, 'pos_and_err_indices': [[0, 1, 2], [0, 1, 2]], 'mag_indices': [3], 'mag_unc_indices': [4], 'mag_names': ['W1'], 'best_mag_index': 0, - 'n_r': 5000, 'n_rho': 5000, 'max_rho': 100} + 'n_r': 5000, 'n_rho': 5000, 'max_rho': 100, 'saturation_magnitudes': [15]} with pytest.raises(ValueError, match='single_sided_auf must be True.'): AstrometricCorrections( @@ -202,7 +202,8 @@ def test_fit_astrometry_load_errors(self): # pylint: disable=too-many-statement mag_array=magarray, mag_slice=magslice, sig_slice=sigslice, n_pool=1, npy_or_csv='npy', coord_or_chunk='coord', pos_and_err_indices=[[0, 1, 2], [0, 1, 2]], mag_indices=[3], mag_unc_indices=[4], mag_names=['W1'], best_mag_index=0, coord_system='equatorial', - chunks=chunks, pregenerate_cutouts=True, n_r=2000, n_rho=2000, max_rho=40) + chunks=chunks, pregenerate_cutouts=True, n_r=2000, n_rho=2000, max_rho=40, + saturation_magnitudes=[15]) with pytest.raises(ValueError, match="a_cat and b_cat must either both be None or "): ac(a_cat=None, b_cat=np.array([0]), a_cat_name=None, b_cat_name=None, a_cat_func=None, b_cat_func=None, tri_download=False, make_plots=True, make_summary_plot=True) @@ -295,7 +296,7 @@ def test_fit_astrometry(self, npy_or_csv, coord_or_chunk, coord_system, pregener pregenerate_cutouts=pregenerate_cutouts, cutout_area=60 if pregenerate_cutouts is False else None, cutout_height=6 if pregenerate_cutouts is False else None, n_r=2000, n_rho=2000, max_rho=40, - return_nm=return_nm) + return_nm=return_nm, saturation_magnitudes=[5]) if coord_or_chunk == 'coord': self.a_cat_name = 'store_data/a_cat{}{}' + ('.csv' if npy_or_csv == 'csv' else '.npy') diff --git a/tests/macauff/test_matching.py b/tests/macauff/test_matching.py index 818310a..1090e71 100644 --- a/tests/macauff/test_matching.py +++ b/tests/macauff/test_matching.py @@ -68,6 +68,7 @@ def setup_class(self): self.a_cat_folder_path = os.path.abspath(cat_a_config['cat_folder_path']) self.b_cat_folder_path = os.path.abspath(cat_b_config['cat_folder_path']) + os.makedirs(joint_config['joint_folder_path'], exist_ok=True) os.makedirs(self.a_cat_folder_path, exist_ok=True) os.makedirs(self.b_cat_folder_path, exist_ok=True) @@ -1620,13 +1621,14 @@ def test_crossmatch_correct_astrometry_inputs(self): 'best_mag_index = 0\n', 'nn_radius = 30\n', 'ref_csv_cat_file_string = ref_{}.csv\n', 'correct_mag_array = 14.07 14.17 14.27 14.37\n', 'correct_mag_slice = 0.05 0.05 0.05 0.05\n', 'correct_sig_slice = 0.1 0.1 0.1 0.1\n', 'chunk_overlap_col = None\n', - 'best_mag_index_col = 8\n', 'use_photometric_uncertainties = no\n'] + 'best_mag_index_col = 8\n', 'use_photometric_uncertainties = no\n', + 'saturation_magnitudes = 5 5 5\n'] for i, key in enumerate(['correct_astro_save_folder', 'csv_cat_file_string', 'pos_and_err_indices', 'mag_indices', 'mag_unc_indices', 'best_mag_index', 'nn_radius', 'ref_csv_cat_file_string', 'correct_mag_array', 'correct_mag_slice', 'correct_sig_slice', 'chunk_overlap_col', 'best_mag_index_col', - 'use_photometric_uncertainties']): + 'use_photometric_uncertainties', 'saturation_magnitudes']): new_line = '' for j in range(i+1): new_line = new_line + lines[j] @@ -1677,6 +1679,8 @@ def test_crossmatch_correct_astrometry_inputs(self): lines])[0][0]] = 'best_mag_index_col = 11\n' lines[np.where(['chunk_overlap_col' in y for y in lines])[0][0]] = 'chunk_overlap_col = 12\n' + lines[np.where(['saturation_magnitudes' in y for y in + lines])[0][0]] = 'saturation_magnitudes = 5 5 5 5\n' new_line = '' for line in lines: new_line = new_line + line @@ -1968,7 +1972,7 @@ def test_crossmatch_correct_astrometry_inputs(self): 'mag_unc_indices =', 'mag_unc_indices =', 'mag_unc_indices =', 'chunk_overlap_col = ', 'chunk_overlap_col = ', 'chunk_overlap_col = ', 'best_mag_index_col = ', 'best_mag_index_col = ', 'dd_params_path = ', - 'l_cut_path = '], + 'l_cut_path = ', 'saturation_magnitudes =', 'saturation_magnitudes ='], ['best_mag_index = A\n', 'best_mag_index = 2.5\n', 'best_mag_index = 7\n', 'nn_radius = A\n', 'nn_radius = 1 2\n', 'correct_mag_array = 1 2 A 4 5\n', 'correct_mag_slice = 0.1 0.1 0.1 A 0.1\n', 'correct_mag_slice = 0.1 0.1 0.1\n', @@ -1980,9 +1984,10 @@ def test_crossmatch_correct_astrometry_inputs(self): 'chunk_overlap_col = Non\n', 'chunk_overlap_col = A\n', 'chunk_overlap_col = 1.2\n', 'best_mag_index_col = A\n', 'best_mag_index_col = 1.2\n', 'dd_params_path = ./some_folder\n', - 'l_cut_path = ./l_cut_dummy_folder\n'], + 'l_cut_path = ./l_cut_dummy_folder\n', 'saturation_magnitudes = something else\n', + 'saturation_magnitudes = 1\n'], ['a', 'b', 'a', 'b', 'a', 'a', 'b', 'a', 'b', 'a', 'b', 'a', 'a', 'a', 'b', 'b', - 'b', 'a', 'a', 'a', 'b', 'a', 'a', 'b', 'b', 'a'], + 'b', 'a', 'a', 'a', 'b', 'a', 'a', 'b', 'b', 'a', 'a', 'b'], ['best_mag_index should be an integer in the catalogue "a"', 'best_mag_index should be an integer in the catalogue "b"', 'best_mag_index cannot be a larger index than the list of filters ' @@ -2008,7 +2013,9 @@ def test_crossmatch_correct_astrometry_inputs(self): 'best_mag_index_col should be an integer in the catalogue "a"', 'best_mag_index_col should be an integer in the catalogue "b"', 'b_dd_params_path does not exist. Please ensure that path for catalogue "b"', - 'l_cut file not found in catalogue "a" path. Please ensure PSF ']): + 'l_cut file not found in catalogue "a" path. Please ensure PSF ', + 'saturation_magnitudes should be a list of floats in the catalogue "a"', + 'b_saturation_magnitudes and b_mag_indices should contain']): with open(os.path.join(os.path.dirname(__file__), f'data/cat_{x}_params_2.txt'), encoding='utf-8') as file: f = file.readlines() From 3ecbf925ea7043bf3c5933480442db823cfd315b Mon Sep 17 00:00:00 2001 From: "Tom J. Wilson" Date: Thu, 18 Apr 2024 15:19:11 +0100 Subject: [PATCH 2/5] Called the same two-parameter corrections function in perturbation_auf when doing cross-match simulation AUF runs. --- src/macauff/matching.py | 17 ++----- src/macauff/perturbation_auf.py | 67 +++++++++++++++++++++----- tests/macauff/test_matching.py | 43 +++++++++-------- tests/macauff/test_perturbation_auf.py | 6 ++- 4 files changed, 85 insertions(+), 48 deletions(-) diff --git a/src/macauff/matching.py b/src/macauff/matching.py index a942acc..0aa7112 100644 --- a/src/macauff/matching.py +++ b/src/macauff/matching.py @@ -1113,12 +1113,12 @@ def read_metadata(self): fit_gal_flag = self.b_fit_gal_flag if correct_astro or fit_gal_flag: for check_flag in ['gal_wavs', 'gal_zmax', 'gal_nzs', - 'gal_aboffsets', 'gal_filternames']: + 'gal_aboffsets', 'gal_filternames', 'saturation_magnitudes']: if check_flag not in config: raise ValueError(f"Missing key {check_flag} from catalogue {catname} " "metadata file.") # Set all lists of floats - for var in ['gal_wavs', 'gal_zmax', 'gal_aboffsets']: + for var in ['gal_wavs', 'gal_zmax', 'gal_aboffsets', 'saturation_magnitudes']: a = config[var].split(' ') try: b = np.array([float(f) for f in a]) @@ -1284,7 +1284,7 @@ def read_metadata(self): for check_flag in ['best_mag_index', 'nn_radius', 'ref_csv_cat_file_string', 'correct_mag_array', 'correct_mag_slice', 'correct_sig_slice', 'chunk_overlap_col', 'best_mag_index_col', - 'use_photometric_uncertainties', 'saturation_magnitudes']: + 'use_photometric_uncertainties']: if check_flag not in config: raise ValueError(f"Missing key {check_flag} from catalogue {catname} metadata file.") @@ -1369,17 +1369,6 @@ def read_metadata(self): 'the same number of entries.') setattr(self, f'{flag}correct_sig_slice', b) - a = config['saturation_magnitudes'].split() - try: - b = np.array([float(f) for f in a]) - except ValueError as exc: - raise ValueError('saturation_magnitudes should be a list of floats in the ' - f'catalogue {catname} metadata file.') from exc - if len(b) != len(getattr(self, f'{flag}mag_indices')): - raise ValueError(f'{flag}saturation_magnitudes and {flag}mag_indices should ' - 'contain the same number of entries.') - setattr(self, f'{flag}saturation_magnitudes', b) - def _read_metadata_csv(self, joint_config, cat_a_config, cat_b_config): """ Read metadata from input config files relating to outputting combined diff --git a/src/macauff/perturbation_auf.py b/src/macauff/perturbation_auf.py index cffb7a8..dd677bd 100644 --- a/src/macauff/perturbation_auf.py +++ b/src/macauff/perturbation_auf.py @@ -20,7 +20,8 @@ # pylint: disable=import-error,no-name-in-module from macauff.galaxy_counts import create_galaxy_counts from macauff.get_trilegal_wrapper import get_av_infinity, get_trilegal -from macauff.misc_functions import _load_rectangular_slice, create_auf_params_grid, min_max_lon +from macauff.misc_functions import (_load_rectangular_slice, create_auf_params_grid, + find_model_counts_corrections, min_max_lon) from macauff.misc_functions_fortran import misc_functions_fortran as mff from macauff.perturbation_auf_fortran import perturbation_auf_fortran as paf @@ -85,6 +86,7 @@ def make_perturb_aufs(cm, which_cat): alpha_weight = cm.gal_alphaweight ab_offsets = getattr(cm, f'{which_cat}_gal_aboffsets') filter_names = getattr(cm, f'{which_cat}_gal_filternames') + saturation_magnitudes = getattr(cm, f'{which_cat}_saturation_magnitudes') # Extract either dummy or real TRILEGAL histogram lists. dens_hist_tri = getattr(cm, f'{which_cat}_dens_hist_tri_list') @@ -211,15 +213,19 @@ def make_perturb_aufs(cm, which_cat): index_slice = med_index_slice[good_mag_slice] for ii, ind_slice in enumerate(index_slice): local_n[ind_slice, j] = localn[ii] + if fit_gal_flag: + rect_area = (ax1_max - ax1_min) * ( + np.sin(np.radians(ax2_max)) - np.sin(np.radians(ax2_min))) * 180/np.pi ax1_list = np.linspace(ax1_min, ax1_max, 7) ax2_list = np.linspace(ax2_min, ax2_max, 7) if fit_gal_flag: single_perturb_auf_output = create_single_perturb_auf( auf_points[i], cm.r, cm.dr, cm.j0s, num_trials, psf_fwhms[j], dens_mags[j], a_photo, localn, d_mag, delta_mag_cuts, dd_params, l_cut, run_fw, run_psf, snr_mag_params[j], - al_avs[j], auf_region_frame, ax1_list, ax2_list, fit_gal_flag, cmau_array, wavs[j], - z_maxs[j], nzs[j], alpha0, alpha1, alpha_weight, ab_offsets[j], filter_names[j], - tri_folder=ax_folder, filt_header=tri_filt_names[j], dens_hist_tri=dens_hist_tri[j], + al_avs[j], auf_region_frame, ax1_list, ax2_list, fit_gal_flag, rect_area, + saturation_magnitudes[j], cmau_array, wavs[j], z_maxs[j], nzs[j], alpha0, alpha1, + alpha_weight, ab_offsets[j], filter_names[j], tri_folder=ax_folder, + filt_header=tri_filt_names[j], dens_hist_tri=dens_hist_tri[j], model_mags=tri_model_mags[j], model_mag_mids=tri_model_mag_mids[j], model_mags_interval=tri_model_mags_interval[j], n_bright_sources_star=tri_n_bright_sources_star[j]) @@ -590,13 +596,13 @@ def calculate_local_density(a_astro, a_tot_astro, a_tot_photo, density_radius, d # pylint: disable=too-many-locals,too-many-arguments -def create_single_perturb_auf(auf_point, r, dr, j0s, num_trials, psf_fwhm, - density_mag, a_photo, localn, d_mag, mag_cut, dd_params, l_cut, run_fw, run_psf, - snr_mag_params, al_av, region_frame, ax1s, ax2s, fit_gal_flag, cmau_array=None, - wav=None, z_max=None, nz=None, alpha0=None, alpha1=None, alpha_weight=None, - ab_offset=None, filter_name=None, tri_folder=None, filt_header=None, - dens_hist_tri=None, model_mags=None, model_mag_mids=None, - model_mags_interval=None, n_bright_sources_star=None): +def create_single_perturb_auf(auf_point, r, dr, j0s, num_trials, psf_fwhm, density_mag, a_photo, localn, + d_mag, mag_cut, dd_params, l_cut, run_fw, run_psf, snr_mag_params, al_av, + region_frame, ax1s, ax2s, fit_gal_flag, rect_area=None, + saturation_magnitude=None, cmau_array=None, wav=None, z_max=None, nz=None, + alpha0=None, alpha1=None, alpha_weight=None, ab_offset=None, filter_name=None, + tri_folder=None, filt_header=None, dens_hist_tri=None, model_mags=None, + model_mag_mids=None, model_mags_interval=None, n_bright_sources_star=None): r''' Creates the associated parameters for describing a single perturbation AUF component, for a single sky position. @@ -662,6 +668,12 @@ def create_single_perturb_auf(auf_point, r, dr, j0s, num_trials, psf_fwhm, fit_gal_flag : bool Flag to indicate whether to simulate galaxy counts for the purposes of simulating the perturbation component of the AUF. + rect_area : float, optional + Area of the region in question, in square degrees. Required if + ``fit_gal_flag`` is ``True``. + saturation_magnitude : float, optional + Magnitude at which the given filter experiences source dropout due to + saturation effects. Required if ``fit_gal_flag`` is ``True``. cmau_array : numpy.ndarray, optional Array holding the c/m/a/u values that describe the parameterisation of the Schechter functions with wavelength, following Wilson (2022, RNAAS, @@ -776,7 +788,36 @@ def create_single_perturb_auf(auf_point, r, dr, j0s, num_trials, psf_fwhm, model_mags_interval = model_mags_interval[hc] log10y_tri = log10y_tri[hc] - model_count = tri_count + gal_count + # If we have the two-component model for source counts, then we have to + # allow for their relative normalisation factors to not be right, and hence + # perform a quick least-squares fit to the ensemble counts now to re-weight + # the two components. If just one is used, then with the re-normalisation + # within paf.perturb_aufs this doesn't matter, so we set corrections to 1/0 + # respectively. + if fit_gal_flag: + data_hist, data_bins = np.histogram(a_photo, bins='auto') + d_hc = np.where(data_hist > 3)[0] + data_hist = data_hist[d_hc] + data_dbins = np.diff(data_bins)[d_hc] + data_bins = data_bins[d_hc] + + data_uncert = np.sqrt(data_hist) / data_dbins / rect_area + data_hist = data_hist / data_dbins / rect_area + data_loghist = np.log10(data_hist) + data_dloghist = 1/np.log(10) * data_uncert / data_hist + + # pylint: disable-next=fixme + # TODO: make the half-mag offset flexible, passing from CrossMatch and/or + # directly into AstrometricCorrections. + maxmag = data_bins[:-1][np.argmax(data_hist)] - 0.5 + + q = (data_bins <= maxmag) & (data_bins >= saturation_magnitude) + tri_corr, gal_corr = find_model_counts_corrections(data_loghist[q], data_dloghist[q], + data_bins[q]+data_dbins[q]/2, 10**log10y_tri, + 10**log10y_gal, model_mags_interval) + else: + tri_corr, gal_corr = 1, 0 + model_count = tri_count * tri_corr + gal_count * gal_corr if fit_gal_flag: # If we have both galaxies and stars to consider, both can be sufficiently @@ -795,7 +836,7 @@ def create_single_perturb_auf(auf_point, r, dr, j0s, num_trials, psf_fwhm, "reliably derive a model source density. Please include " "more simulated objects.") - log10y = np.log10(10**log10y_tri + 10**log10y_gal) + log10y = np.log10(10**log10y_tri * tri_corr + 10**log10y_gal * gal_corr) # Set a magnitude bin width of 0.25 mags, to avoid oversampling. dmag = 0.25 diff --git a/tests/macauff/test_matching.py b/tests/macauff/test_matching.py index 1090e71..52350ad 100644 --- a/tests/macauff/test_matching.py +++ b/tests/macauff/test_matching.py @@ -937,7 +937,8 @@ def test_crossmatch_perturb_auf_inputs(self): new_line = ('fit_gal_flag = yes\ngal_wavs = 0.513 0.641 0.778\n' 'gal_zmax = 4.5 4.5 5\ngal_nzs = 46 46 51\n' 'gal_aboffsets = 0.5 0.5 0.5\n' - 'gal_filternames = gaiadr2-BP gaiadr2-G gaiadr2-RP\n') + 'gal_filternames = gaiadr2-BP gaiadr2-G gaiadr2-RP\n' + 'saturation_magnitudes = 5 5 5\n') idx = np.where([old_line in line for line in f])[0][0] _replace_line(os.path.join(os.path.dirname(__file__), 'data/cat_a_params_new.txt'), idx, new_line, out_file=os.path.join(os.path.dirname(__file__), @@ -949,7 +950,8 @@ def test_crossmatch_perturb_auf_inputs(self): new_line = ('fit_gal_flag = yes\ngal_wavs = 3.37 4.62 12.08 22.19\n' 'gal_zmax = 3.2 4.0 1 4\ngal_nzs = 33 41 11 41\n' 'gal_aboffsets = 0.5 0.5 0.5 0.5\n' - 'gal_filternames = wise2010-W1 wise2010-W2 wise2010-W3 wise2010-W4\n') + 'gal_filternames = wise2010-W1 wise2010-W2 wise2010-W3 wise2010-W4\n' + 'saturation_magnitudes = 5 5 5 5\n') idx = np.where([old_line in line for line in f])[0][0] _replace_line(os.path.join(os.path.dirname(__file__), 'data/cat_b_params_new.txt'), idx, new_line, out_file=os.path.join(os.path.dirname(__file__), @@ -986,11 +988,13 @@ def test_crossmatch_perturb_auf_inputs(self): ['gal_wavs = 0.513 0.641 0.778', 'gal_aboffsets = 0.5 0.5 0.5 0.5', 'gal_nzs = 46 46 51', 'gal_nzs = 33 41 11 41', 'gal_nzs = 33 41 11 41', 'gal_filternames = gaiadr2-BP gaiadr2-G gaiadr2-RP', - 'gal_al_avs = 1.002 0.789 0.589', 'gal_al_avs = 1.002 0.789 0.589'], + 'gal_al_avs = 1.002 0.789 0.589', 'gal_al_avs = 1.002 0.789 0.589', + 'saturation_magnitudes = ', 'saturation_magnitudes = '], ['gal_wavs = 0.513 0.641\n', 'gal_aboffsets = a 0.5 0.5 0.5\n', 'gal_nzs = 46 a 51\n', 'gal_nzs = 33.1 41 11 41\n', 'gal_nzs = 33 41 11\n', 'gal_filternames = gaiadr2-BP gaiadr2-G gaiadr2-RP wise2010-W1\n', - 'gal_al_avs = words\n', 'gal_al_avs = 0.789 1.002\n'], + 'gal_al_avs = words\n', 'gal_al_avs = 0.789 1.002\n', + 'saturation_magnitudes = words\n', 'saturation_magnitudes = 4 4\n'], ['a_gal_wavs and a_filt_names should contain the same number', 'gal_aboffsets should be a list of floats in catalogue "b"', 'gal_nzs should be a list of integers in catalogue "a"', @@ -998,9 +1002,12 @@ def test_crossmatch_perturb_auf_inputs(self): 'b_gal_nzs and b_filt_names should contain the same number of entries.', 'a_gal_filternames and a_filt_names should contain the same number of entries.', 'gal_al_avs should be a list of floats in catalogue "a"', - 'a_gal_al_avs and a_filt_names should contain the same number of entries.'], + 'a_gal_al_avs and a_filt_names should contain the same number of entries.', + 'saturation_magnitudes should be a list of floats in catalogue "b"', + 'b_saturation_magnitudes and b_filt_names should contain the same number of entries.'], ['cat_a_params_new', 'cat_b_params_new', 'cat_a_params_new', 'cat_b_params_new', - 'cat_b_params_new', 'cat_a_params_new', 'cat_a_params_new', 'cat_a_params_new']): + 'cat_b_params_new', 'cat_a_params_new', 'cat_a_params_new', 'cat_a_params_new', + 'cat_b_params_new', 'cat_b_params_new']): with open(os.path.join(os.path.dirname(__file__), f'data/{in_file}_.txt'), encoding='utf-8') as file: f = file.readlines() @@ -1571,7 +1578,8 @@ def test_crossmatch_correct_astrometry_inputs(self): new_line = ('gal_wavs = 0.513 0.641 0.778\n' 'gal_zmax = 4.5 4.5 5\ngal_nzs = 46 46 51\n' 'gal_aboffsets = 0.5 0.5 0.5\n' - 'gal_filternames = gaiadr2-BP gaiadr2-G gaiadr2-RP\n') + 'gal_filternames = gaiadr2-BP gaiadr2-G gaiadr2-RP\n' + 'saturation_magnitudes = 5 5 5\n') idx = np.where([old_line in line for line in f])[0][0] _replace_line(os.path.join(os.path.dirname(__file__), 'data/cat_a_params_.txt'), idx, new_line, out_file=os.path.join(os.path.dirname(__file__), 'data/cat_a_params_.txt')) @@ -1581,7 +1589,8 @@ def test_crossmatch_correct_astrometry_inputs(self): new_line = ('gal_wavs = 3.37 4.62 12.08 22.19\n' 'gal_zmax = 3.2 4.0 1 4\ngal_nzs = 33 41 11 41\n' 'gal_aboffsets = 0.5 0.5 0.5 0.5\n' - 'gal_filternames = wise2010-W1 wise2010-W2 wise2010-W3 wise2010-W4\n') + 'gal_filternames = wise2010-W1 wise2010-W2 wise2010-W3 wise2010-W4\n' + 'saturation_magnitudes = 5 5 5 5\n') idx = np.where([old_line in line for line in f])[0][0] _replace_line(os.path.join(os.path.dirname(__file__), 'data/cat_b_params_.txt'), idx, new_line, out_file=os.path.join(os.path.dirname(__file__), @@ -1621,14 +1630,13 @@ def test_crossmatch_correct_astrometry_inputs(self): 'best_mag_index = 0\n', 'nn_radius = 30\n', 'ref_csv_cat_file_string = ref_{}.csv\n', 'correct_mag_array = 14.07 14.17 14.27 14.37\n', 'correct_mag_slice = 0.05 0.05 0.05 0.05\n', 'correct_sig_slice = 0.1 0.1 0.1 0.1\n', 'chunk_overlap_col = None\n', - 'best_mag_index_col = 8\n', 'use_photometric_uncertainties = no\n', - 'saturation_magnitudes = 5 5 5\n'] + 'best_mag_index_col = 8\n', 'use_photometric_uncertainties = no\n'] for i, key in enumerate(['correct_astro_save_folder', 'csv_cat_file_string', 'pos_and_err_indices', 'mag_indices', 'mag_unc_indices', 'best_mag_index', 'nn_radius', 'ref_csv_cat_file_string', 'correct_mag_array', 'correct_mag_slice', 'correct_sig_slice', 'chunk_overlap_col', 'best_mag_index_col', - 'use_photometric_uncertainties', 'saturation_magnitudes']): + 'use_photometric_uncertainties']): new_line = '' for j in range(i+1): new_line = new_line + lines[j] @@ -1679,8 +1687,6 @@ def test_crossmatch_correct_astrometry_inputs(self): lines])[0][0]] = 'best_mag_index_col = 11\n' lines[np.where(['chunk_overlap_col' in y for y in lines])[0][0]] = 'chunk_overlap_col = 12\n' - lines[np.where(['saturation_magnitudes' in y for y in - lines])[0][0]] = 'saturation_magnitudes = 5 5 5 5\n' new_line = '' for line in lines: new_line = new_line + line @@ -1972,7 +1978,7 @@ def test_crossmatch_correct_astrometry_inputs(self): 'mag_unc_indices =', 'mag_unc_indices =', 'mag_unc_indices =', 'chunk_overlap_col = ', 'chunk_overlap_col = ', 'chunk_overlap_col = ', 'best_mag_index_col = ', 'best_mag_index_col = ', 'dd_params_path = ', - 'l_cut_path = ', 'saturation_magnitudes =', 'saturation_magnitudes ='], + 'l_cut_path = '], ['best_mag_index = A\n', 'best_mag_index = 2.5\n', 'best_mag_index = 7\n', 'nn_radius = A\n', 'nn_radius = 1 2\n', 'correct_mag_array = 1 2 A 4 5\n', 'correct_mag_slice = 0.1 0.1 0.1 A 0.1\n', 'correct_mag_slice = 0.1 0.1 0.1\n', @@ -1984,10 +1990,9 @@ def test_crossmatch_correct_astrometry_inputs(self): 'chunk_overlap_col = Non\n', 'chunk_overlap_col = A\n', 'chunk_overlap_col = 1.2\n', 'best_mag_index_col = A\n', 'best_mag_index_col = 1.2\n', 'dd_params_path = ./some_folder\n', - 'l_cut_path = ./l_cut_dummy_folder\n', 'saturation_magnitudes = something else\n', - 'saturation_magnitudes = 1\n'], + 'l_cut_path = ./l_cut_dummy_folder\n'], ['a', 'b', 'a', 'b', 'a', 'a', 'b', 'a', 'b', 'a', 'b', 'a', 'a', 'a', 'b', 'b', - 'b', 'a', 'a', 'a', 'b', 'a', 'a', 'b', 'b', 'a', 'a', 'b'], + 'b', 'a', 'a', 'a', 'b', 'a', 'a', 'b', 'b', 'a'], ['best_mag_index should be an integer in the catalogue "a"', 'best_mag_index should be an integer in the catalogue "b"', 'best_mag_index cannot be a larger index than the list of filters ' @@ -2013,9 +2018,7 @@ def test_crossmatch_correct_astrometry_inputs(self): 'best_mag_index_col should be an integer in the catalogue "a"', 'best_mag_index_col should be an integer in the catalogue "b"', 'b_dd_params_path does not exist. Please ensure that path for catalogue "b"', - 'l_cut file not found in catalogue "a" path. Please ensure PSF ', - 'saturation_magnitudes should be a list of floats in the catalogue "a"', - 'b_saturation_magnitudes and b_mag_indices should contain']): + 'l_cut file not found in catalogue "a" path. Please ensure PSF ']): with open(os.path.join(os.path.dirname(__file__), f'data/cat_{x}_params_2.txt'), encoding='utf-8') as file: f = file.readlines() diff --git a/tests/macauff/test_perturbation_auf.py b/tests/macauff/test_perturbation_auf.py index ae80efc..69e00a1 100644 --- a/tests/macauff/test_perturbation_auf.py +++ b/tests/macauff/test_perturbation_auf.py @@ -900,8 +900,10 @@ def test_with_galaxy_counts(self): np.save(f'{self.cat_folder}/con_cat_astro.npy', np.concatenate(([0.3, 0.3, 0.1] * 101, [0.1, 0.1, 0.1], [0.9, 0.9, 0.1])).reshape(-1, 3)) + rng = np.random.default_rng(seed=83458923) + main_mags = rng.uniform(24.95, 25.05, size=100) np.save(f'{self.cat_folder}/con_cat_photo.npy', - np.array([np.concatenate(([14.99], [25]*100, [10], [10]))]).T) + np.array([np.concatenate(([14.99], main_mags, [10], [10]))]).T) np.save(f'{self.cat_folder}/magref.npy', np.array([0] * 103)) # Fake up a TRILEGAL simulation data file. @@ -1053,6 +1055,7 @@ def test_with_galaxy_counts(self): cm.a_gal_aboffsets = np.array([0.105]) cm.a_gal_filternames = np.array(['gaiadr2-G']) cm.a_gal_al_avs = np.array([0]) + cm.a_saturation_magnitudes = np.array([5]) cm.b_gal_wavs = np.array([3.4]) cm.b_gal_zmax = np.array([1]) @@ -1060,6 +1063,7 @@ def test_with_galaxy_counts(self): cm.b_gal_aboffsets = np.array([2.699]) cm.b_gal_filternames = ['wise2010-W1'] cm.b_gal_al_avs = np.array([0]) + cm.b_saturation_magnitudes = np.array([5]) cm.a_run_fw = True cm.a_run_psf = False From e9897df13fd6504a21915da3a0ea88e97d06fcb5 Mon Sep 17 00:00:00 2001 From: "Tom J. Wilson" Date: Thu, 18 Apr 2024 15:19:20 +0100 Subject: [PATCH 3/5] Changelog entry. --- CHANGES.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 27a5b3f..404c7fc 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -16,6 +16,10 @@ Bug Fixes API Changes ^^^^^^^^^^^ +- ``CrossMatch`` now expects ``saturation_magnitudes`` as an input parameter in + its input files, if ``fit_gal_flag`` or ``correct_astrometry`` are + ``True``. [#81] + - ``AstrometricCorrections`` accepts pre-computed TRILEGAL histograms, following the expansion of ``CrossMatch`` accepting them. [#79] From 51f87d1aee75e69e1f37696bf609489702b94174 Mon Sep 17 00:00:00 2001 From: "Tom J. Wilson" Date: Thu, 18 Apr 2024 15:26:05 +0100 Subject: [PATCH 4/5] isort changes. --- src/macauff/perturbation_auf.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/macauff/perturbation_auf.py b/src/macauff/perturbation_auf.py index dd677bd..ccba269 100644 --- a/src/macauff/perturbation_auf.py +++ b/src/macauff/perturbation_auf.py @@ -20,8 +20,12 @@ # pylint: disable=import-error,no-name-in-module from macauff.galaxy_counts import create_galaxy_counts from macauff.get_trilegal_wrapper import get_av_infinity, get_trilegal -from macauff.misc_functions import (_load_rectangular_slice, create_auf_params_grid, - find_model_counts_corrections, min_max_lon) +from macauff.misc_functions import ( + _load_rectangular_slice, + create_auf_params_grid, + find_model_counts_corrections, + min_max_lon, +) from macauff.misc_functions_fortran import misc_functions_fortran as mff from macauff.perturbation_auf_fortran import perturbation_auf_fortran as paf From 403f3f4615b8155f128b24c555247208654f2f10 Mon Sep 17 00:00:00 2001 From: "Tom J. Wilson" Date: Thu, 18 Apr 2024 15:49:57 +0100 Subject: [PATCH 5/5] Pylint fixups. --- src/.pylintrc | 2 +- src/macauff/derive_psf_auf_params.py | 2 +- src/macauff/fit_astrometry.py | 6 +++--- src/macauff/misc_functions.py | 1 + src/macauff/perturbation_auf.py | 2 +- tests/.pylintrc | 2 +- tests/macauff/test_galaxy_counts.py | 2 +- 7 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/.pylintrc b/src/.pylintrc index 1437cbc..ab4fd49 100644 --- a/src/.pylintrc +++ b/src/.pylintrc @@ -188,7 +188,7 @@ good-names=i, # Good variable names regexes, separated by a comma. If names match any regex, # they will always be accepted -good-names-rgxs= +good-names-rgxs=^[_a-zA-Z][_a-z0-9A-Z]?$ # Include a hint for the correct naming format with invalid-name. include-naming-hint=no diff --git a/src/macauff/derive_psf_auf_params.py b/src/macauff/derive_psf_auf_params.py index ea1f058..2d03403 100644 --- a/src/macauff/derive_psf_auf_params.py +++ b/src/macauff/derive_psf_auf_params.py @@ -20,7 +20,7 @@ # Assume that usetex = False only applies for tests where no TeX is installed # at all, instead of users having half-installed TeX, dvipng et al. somewhere. -usetex = not not shutil.which("tex") # pylint: disable=unnecessary-negation +usetex = not not shutil.which("tex") # pylint: disable=unneeded-not if usetex: plt.rcParams.update({"text.usetex": True, "text.latex.preamble": r"\usepackage{amsmath}"}) diff --git a/src/macauff/fit_astrometry.py b/src/macauff/fit_astrometry.py index b3beb36..7856d57 100644 --- a/src/macauff/fit_astrometry.py +++ b/src/macauff/fit_astrometry.py @@ -24,7 +24,7 @@ # Assume that usetex = False only applies for tests where no TeX is installed # at all, instead of users having half-installed TeX, dvipng et al. somewhere. -usetex = not not shutil.which("tex") # pylint: disable=unnecessary-negation +usetex = not not shutil.which("tex") # pylint: disable=unneeded-not if usetex: plt.rcParams.update({"text.usetex": True, "text.latex.preamble": r"\usepackage{amsmath}"}) @@ -949,7 +949,7 @@ def fit_snr_sqrt(p, x, y): return res, s_bins, s_d_snr_med, s_d_snr_dmed, snr_med, snr_dmed - def make_star_galaxy_counts(self): + def make_star_galaxy_counts(self): # pylint: disable=too-many-locals """ Generate differential source counts for each cutout region, simulating both stars and galaxies. @@ -1808,7 +1808,7 @@ def _get_cart_kdt(coord): mag_cut_ucoords = mag_cut_cat.realize_frame(mag_cut_urepr) mag_cut_kdt = _get_cart_kdt(mag_cut_ucoords) - r = (2 * np.sin(Angle(search_radius * u.degree) / 2.0)).value + r = (2 * np.sin(Angle(search_radius * u.degree) / 2.0)).value # pylint: disable=no-member overlap_number = np.empty(len(b), int) counter = np.arange(0, len(b)) diff --git a/src/macauff/misc_functions.py b/src/macauff/misc_functions.py index fdf58a4..9d35cbc 100644 --- a/src/macauff/misc_functions.py +++ b/src/macauff/misc_functions.py @@ -315,6 +315,7 @@ def min_max_lon(a): return min_lon, max_lon +# pylint: disable=unused-argument def find_model_counts_corrections(data_loghist, data_dloghist, data_bin_mids, tri_hist, gal_dns, tri_mag_mids): ''' diff --git a/src/macauff/perturbation_auf.py b/src/macauff/perturbation_auf.py index ccba269..e3aaecc 100644 --- a/src/macauff/perturbation_auf.py +++ b/src/macauff/perturbation_auf.py @@ -599,7 +599,7 @@ def calculate_local_density(a_astro, a_tot_astro, a_tot_photo, density_radius, d return count_density -# pylint: disable=too-many-locals,too-many-arguments +# pylint: disable=too-many-locals,too-many-arguments,too-many-statements def create_single_perturb_auf(auf_point, r, dr, j0s, num_trials, psf_fwhm, density_mag, a_photo, localn, d_mag, mag_cut, dd_params, l_cut, run_fw, run_psf, snr_mag_params, al_av, region_frame, ax1s, ax2s, fit_gal_flag, rect_area=None, diff --git a/tests/.pylintrc b/tests/.pylintrc index 9e39885..6eb7388 100644 --- a/tests/.pylintrc +++ b/tests/.pylintrc @@ -188,7 +188,7 @@ good-names=i, # Good variable names regexes, separated by a comma. If names match any regex, # they will always be accepted -good-names-rgxs= +good-names-rgxs=^[_a-zA-Z][_a-z0-9A-Z]?$ # Include a hint for the correct naming format with invalid-name. include-naming-hint=no diff --git a/tests/macauff/test_galaxy_counts.py b/tests/macauff/test_galaxy_counts.py index 15749bf..8fea4a0 100644 --- a/tests/macauff/test_galaxy_counts.py +++ b/tests/macauff/test_galaxy_counts.py @@ -84,7 +84,7 @@ def test_create_filter_and_galaxy_counts(self): filter_name = f'{f}-{n}' generate_speclite_filters(f, [n], [np.array([0.159, 0.16, 0.161])], [np.array([0, 1, 0])], - u.micron) + u.micron) # pylint: disable=no-member gal_dens = create_galaxy_counts(gal_values.cmau, self.mag_bins, self.z_array, wav, gal_values.alpha0, gal_values.alpha1,