From 1ff1f9d9185829d68c41c7679abc072db7c0701f Mon Sep 17 00:00:00 2001 From: Sandro Campos Date: Tue, 2 Jul 2024 17:42:48 -0400 Subject: [PATCH] Create classes for type of correlation (#11) * Create classes for type of correlation * Review PR comments * Fix method signatures * Raise exceptions on counting --- src/corrgi/correlation/__init__.py | 0 src/corrgi/correlation/angular_correlation.py | 41 +++++++ src/corrgi/correlation/correlation.py | 74 +++++++++++++ .../correlation/projected_correlation.py | 28 +++++ .../correlation/redshift_correlation.py | 28 +++++ src/corrgi/corrgi.py | 12 +- src/corrgi/dask.py | 104 +++--------------- tests/corrgi/conftest.py | 80 +++++++------- tests/corrgi/{test_corrgi.py => test_acf.py} | 7 +- tests/corrgi/test_counting.py | 8 +- .../{ => expected_results}/acf/dd_acf.npy | Bin .../acf/l_binedges_acf.npy | Bin .../acf/r_binedges_acf.npy | Bin .../{ => expected_results}/acf/rr_acf.npy | Bin .../{ => expected_results}/acf/w_acf_nat.npy | Bin tests/data/generate_data.ipynb | 2 +- .../DATA/Norder=0/Dir=0/Npix=1.parquet | Bin .../DATA/Norder=0/Dir=0/Npix=10.parquet | Bin .../DATA/Norder=0/Dir=0/Npix=2.parquet | Bin .../DATA/Norder=0/Dir=0/Npix=5.parquet | Bin .../DATA/Norder=0/Dir=0/Npix=6.parquet | Bin .../DATA/Norder=0/Dir=0/Npix=7.parquet | Bin .../DATA/Norder=0/Dir=0/Npix=9.parquet | Bin .../data/{ => hipscat}/DATA/_common_metadata | Bin tests/data/{ => hipscat}/DATA/_metadata | Bin .../data/{ => hipscat}/DATA/catalog_info.json | 0 .../{ => hipscat}/DATA/partition_info.csv | 0 .../{ => hipscat}/DATA/provenance_info.json | 0 .../Norder=0/Dir=0/Npix=0.parquet | Bin .../Norder=0/Dir=0/Npix=10.parquet | Bin .../Norder=0/Dir=0/Npix=11.parquet | Bin .../Norder=0/Dir=0/Npix=3.parquet | Bin .../Norder=0/Dir=0/Npix=4.parquet | Bin .../Norder=0/Dir=0/Npix=5.parquet | Bin .../Norder=0/Dir=0/Npix=7.parquet | Bin .../Norder=0/Dir=0/Npix=8.parquet | Bin .../Norder=0/Dir=0/Npix=9.parquet | Bin .../Norder=1/Dir=0/Npix=10.parquet | Bin .../Norder=1/Dir=0/Npix=11.parquet | Bin .../Norder=1/Dir=0/Npix=24.parquet | Bin .../Norder=1/Dir=0/Npix=25.parquet | Bin .../Norder=1/Dir=0/Npix=26.parquet | Bin .../Norder=1/Dir=0/Npix=27.parquet | Bin .../Norder=1/Dir=0/Npix=4.parquet | Bin .../Norder=1/Dir=0/Npix=5.parquet | Bin .../Norder=1/Dir=0/Npix=6.parquet | Bin .../Norder=1/Dir=0/Npix=7.parquet | Bin .../Norder=1/Dir=0/Npix=8.parquet | Bin .../Norder=1/Dir=0/Npix=9.parquet | Bin .../DR7-lrg-rand/_common_metadata | Bin .../data/{ => hipscat}/DR7-lrg-rand/_metadata | Bin .../DR7-lrg-rand/catalog_info.json | 0 .../DR7-lrg-rand/partition_info.csv | 0 .../DR7-lrg-rand/provenance_info.json | 0 .../DR7-lrg/Norder=0/Dir=0/Npix=0.parquet | Bin .../DR7-lrg/Norder=0/Dir=0/Npix=1.parquet | Bin .../DR7-lrg/Norder=0/Dir=0/Npix=10.parquet | Bin .../DR7-lrg/Norder=0/Dir=0/Npix=11.parquet | Bin .../DR7-lrg/Norder=0/Dir=0/Npix=2.parquet | Bin .../DR7-lrg/Norder=0/Dir=0/Npix=3.parquet | Bin .../DR7-lrg/Norder=0/Dir=0/Npix=4.parquet | Bin .../DR7-lrg/Norder=0/Dir=0/Npix=5.parquet | Bin .../DR7-lrg/Norder=0/Dir=0/Npix=6.parquet | Bin .../DR7-lrg/Norder=0/Dir=0/Npix=7.parquet | Bin .../DR7-lrg/Norder=0/Dir=0/Npix=8.parquet | Bin .../DR7-lrg/Norder=0/Dir=0/Npix=9.parquet | Bin .../{ => hipscat}/DR7-lrg/_common_metadata | Bin tests/data/{ => hipscat}/DR7-lrg/_metadata | Bin .../{ => hipscat}/DR7-lrg/catalog_info.json | 0 .../{ => hipscat}/DR7-lrg/partition_info.csv | 0 .../DR7-lrg/provenance_info.json | 0 .../RAND/Norder=0/Dir=0/Npix=1.parquet | Bin .../RAND/Norder=0/Dir=0/Npix=10.parquet | Bin .../RAND/Norder=0/Dir=0/Npix=2.parquet | Bin .../RAND/Norder=0/Dir=0/Npix=5.parquet | Bin .../RAND/Norder=0/Dir=0/Npix=6.parquet | Bin .../RAND/Norder=0/Dir=0/Npix=7.parquet | Bin .../RAND/Norder=0/Dir=0/Npix=9.parquet | Bin .../data/{ => hipscat}/RAND/_common_metadata | Bin tests/data/{ => hipscat}/RAND/_metadata | Bin .../data/{ => hipscat}/RAND/catalog_info.json | 0 .../{ => hipscat}/RAND/partition_info.csv | 0 .../{ => hipscat}/RAND/provenance_info.json | 0 83 files changed, 249 insertions(+), 135 deletions(-) create mode 100644 src/corrgi/correlation/__init__.py create mode 100644 src/corrgi/correlation/angular_correlation.py create mode 100644 src/corrgi/correlation/correlation.py create mode 100644 src/corrgi/correlation/projected_correlation.py create mode 100644 src/corrgi/correlation/redshift_correlation.py rename tests/corrgi/{test_corrgi.py => test_acf.py} (89%) rename tests/data/{ => expected_results}/acf/dd_acf.npy (100%) rename tests/data/{ => expected_results}/acf/l_binedges_acf.npy (100%) rename tests/data/{ => expected_results}/acf/r_binedges_acf.npy (100%) rename tests/data/{ => expected_results}/acf/rr_acf.npy (100%) rename tests/data/{ => expected_results}/acf/w_acf_nat.npy (100%) rename tests/data/{ => hipscat}/DATA/Norder=0/Dir=0/Npix=1.parquet (100%) rename tests/data/{ => hipscat}/DATA/Norder=0/Dir=0/Npix=10.parquet (100%) rename tests/data/{ => hipscat}/DATA/Norder=0/Dir=0/Npix=2.parquet (100%) rename tests/data/{ => hipscat}/DATA/Norder=0/Dir=0/Npix=5.parquet (100%) rename tests/data/{ => hipscat}/DATA/Norder=0/Dir=0/Npix=6.parquet (100%) rename tests/data/{ => hipscat}/DATA/Norder=0/Dir=0/Npix=7.parquet (100%) rename tests/data/{ => hipscat}/DATA/Norder=0/Dir=0/Npix=9.parquet (100%) rename tests/data/{ => hipscat}/DATA/_common_metadata (100%) rename tests/data/{ => hipscat}/DATA/_metadata (100%) rename tests/data/{ => hipscat}/DATA/catalog_info.json (100%) rename tests/data/{ => hipscat}/DATA/partition_info.csv (100%) rename tests/data/{ => hipscat}/DATA/provenance_info.json (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=0/Dir=0/Npix=0.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=0/Dir=0/Npix=10.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=0/Dir=0/Npix=11.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=0/Dir=0/Npix=3.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=0/Dir=0/Npix=4.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=0/Dir=0/Npix=5.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=0/Dir=0/Npix=7.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=0/Dir=0/Npix=8.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=0/Dir=0/Npix=9.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=1/Dir=0/Npix=10.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=1/Dir=0/Npix=11.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=1/Dir=0/Npix=24.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=1/Dir=0/Npix=25.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=1/Dir=0/Npix=26.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=1/Dir=0/Npix=27.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=1/Dir=0/Npix=4.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=1/Dir=0/Npix=5.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=1/Dir=0/Npix=6.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=1/Dir=0/Npix=7.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=1/Dir=0/Npix=8.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/Norder=1/Dir=0/Npix=9.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/_common_metadata (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/_metadata (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/catalog_info.json (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/partition_info.csv (100%) rename tests/data/{ => hipscat}/DR7-lrg-rand/provenance_info.json (100%) rename tests/data/{ => hipscat}/DR7-lrg/Norder=0/Dir=0/Npix=0.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg/Norder=0/Dir=0/Npix=1.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg/Norder=0/Dir=0/Npix=10.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg/Norder=0/Dir=0/Npix=11.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg/Norder=0/Dir=0/Npix=2.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg/Norder=0/Dir=0/Npix=3.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg/Norder=0/Dir=0/Npix=4.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg/Norder=0/Dir=0/Npix=5.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg/Norder=0/Dir=0/Npix=6.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg/Norder=0/Dir=0/Npix=7.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg/Norder=0/Dir=0/Npix=8.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg/Norder=0/Dir=0/Npix=9.parquet (100%) rename tests/data/{ => hipscat}/DR7-lrg/_common_metadata (100%) rename tests/data/{ => hipscat}/DR7-lrg/_metadata (100%) rename tests/data/{ => hipscat}/DR7-lrg/catalog_info.json (100%) rename tests/data/{ => hipscat}/DR7-lrg/partition_info.csv (100%) rename tests/data/{ => hipscat}/DR7-lrg/provenance_info.json (100%) rename tests/data/{ => hipscat}/RAND/Norder=0/Dir=0/Npix=1.parquet (100%) rename tests/data/{ => hipscat}/RAND/Norder=0/Dir=0/Npix=10.parquet (100%) rename tests/data/{ => hipscat}/RAND/Norder=0/Dir=0/Npix=2.parquet (100%) rename tests/data/{ => hipscat}/RAND/Norder=0/Dir=0/Npix=5.parquet (100%) rename tests/data/{ => hipscat}/RAND/Norder=0/Dir=0/Npix=6.parquet (100%) rename tests/data/{ => hipscat}/RAND/Norder=0/Dir=0/Npix=7.parquet (100%) rename tests/data/{ => hipscat}/RAND/Norder=0/Dir=0/Npix=9.parquet (100%) rename tests/data/{ => hipscat}/RAND/_common_metadata (100%) rename tests/data/{ => hipscat}/RAND/_metadata (100%) rename tests/data/{ => hipscat}/RAND/catalog_info.json (100%) rename tests/data/{ => hipscat}/RAND/partition_info.csv (100%) rename tests/data/{ => hipscat}/RAND/provenance_info.json (100%) diff --git a/src/corrgi/correlation/__init__.py b/src/corrgi/correlation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/corrgi/correlation/angular_correlation.py b/src/corrgi/correlation/angular_correlation.py new file mode 100644 index 0000000..70024fb --- /dev/null +++ b/src/corrgi/correlation/angular_correlation.py @@ -0,0 +1,41 @@ +from typing import Callable + +import gundam.cflibfor as cff +import pandas as pd +from hipscat.catalog.catalog_info import CatalogInfo + +from corrgi.correlation.correlation import Correlation + + +class AngularCorrelation(Correlation): + """The angular correlation utilities.""" + + def _get_auto_method(self): + return cff.mod.th_A_wg if self.use_weights else cff.mod.th_A_naiveway + + def _get_cross_method(self) -> Callable: + return cff.mod.th_C_wg if self.use_weights else cff.mod.th_C_naiveway + + def _construct_auto_args(self, df: pd.DataFrame, catalog_info: CatalogInfo) -> list: + return [ + len(df), # number of particles + *self.get_coords(df, catalog_info), # cartesian coordinates + self.params.nsept, # number of angular separation bins + self.bins, # bins in angular separation [deg] + ] + + def _construct_cross_args( + self, + left_df: pd.DataFrame, + right_df: pd.DataFrame, + left_catalog_info: CatalogInfo, + right_catalog_info: CatalogInfo, + ) -> list: + return [ + len(left_df), # number of particles of the left partition + *self.get_coords(left_df, left_catalog_info), # X,Y,Z coordinates of particles + len(right_df), # number of particles of the right partition + *self.get_coords(right_df, right_catalog_info), # X,Y,Z coordinates of particles + self.params.nsept, # number of angular separation bins + self.bins, # bins in angular separation [deg] + ] diff --git a/src/corrgi/correlation/correlation.py b/src/corrgi/correlation/correlation.py new file mode 100644 index 0000000..aba41fd --- /dev/null +++ b/src/corrgi/correlation/correlation.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Callable + +import numpy as np +import pandas as pd +from hipscat.catalog.catalog_info import CatalogInfo +from munch import Munch + +from corrgi.utils import project_coordinates + + +class Correlation(ABC): + """Correlation base class.""" + + def __init__( + self, + bins: np.ndarray, + params: Munch, + use_weights: bool = False, + ): + self.bins = bins + self.params = params + self.use_weights = use_weights + + def count_auto_pairs(self, df: pd.DataFrame, catalog_info: CatalogInfo) -> np.ndarray: + """Computes the counts for pairs of the same partition""" + args = self._construct_auto_args(df, catalog_info) + return self._get_auto_method()(*args) + + def count_cross_pairs( + self, + left_df: pd.DataFrame, + right_df: pd.DataFrame, + left_catalog_info: CatalogInfo, + right_catalog_info: CatalogInfo, + ) -> np.ndarray: + """Computes the counts for pairs of different partitions""" + args = self._construct_cross_args(left_df, right_df, left_catalog_info, right_catalog_info) + return self._get_cross_method()(*args) + + @abstractmethod + def _get_auto_method(self) -> Callable: + """Reference to Fortran routine to be called on auto pairing""" + raise NotImplementedError() + + @abstractmethod + def _construct_auto_args(self, df: pd.DataFrame, catalog_info: CatalogInfo) -> list: + """Generate the arguments required for the auto pairing method""" + raise NotImplementedError() + + @abstractmethod + def _get_cross_method(self) -> Callable: + """Reference to Fortran routine to be called on cross pairing""" + raise NotImplementedError() + + @abstractmethod + def _construct_cross_args( + self, + left_df: pd.DataFrame, + right_df: pd.DataFrame, + left_catalog_info: CatalogInfo, + right_catalog_info: CatalogInfo, + ) -> list: + """Generate the arguments required for the cross pairing method""" + raise NotImplementedError() + + @staticmethod + def get_coords(df: pd.DataFrame, catalog_info: CatalogInfo) -> tuple[float, float, float]: + """Calculate the cartesian coordinates for the points in the partition""" + return project_coordinates( + ra=df[catalog_info.ra_column].to_numpy(), dec=df[catalog_info.dec_column].to_numpy() + ) diff --git a/src/corrgi/correlation/projected_correlation.py b/src/corrgi/correlation/projected_correlation.py new file mode 100644 index 0000000..981863c --- /dev/null +++ b/src/corrgi/correlation/projected_correlation.py @@ -0,0 +1,28 @@ +from typing import Callable + +import pandas as pd +from hipscat.catalog.catalog_info import CatalogInfo + +from corrgi.correlation.correlation import Correlation + + +class ProjectedCorrelation(Correlation): + """The projected correlation utilities.""" + + def _get_auto_method(self) -> Callable: + raise NotImplementedError() + + def _construct_auto_args(self, df: pd.DataFrame, catalog_info: CatalogInfo) -> list: + raise NotImplementedError() + + def _get_cross_method(self) -> Callable: + raise NotImplementedError() + + def _construct_cross_args( + self, + left_df: pd.DataFrame, + right_df: pd.DataFrame, + left_catalog_info: CatalogInfo, + right_catalog_info: CatalogInfo, + ) -> list: + raise NotImplementedError() diff --git a/src/corrgi/correlation/redshift_correlation.py b/src/corrgi/correlation/redshift_correlation.py new file mode 100644 index 0000000..a92a37f --- /dev/null +++ b/src/corrgi/correlation/redshift_correlation.py @@ -0,0 +1,28 @@ +from typing import Callable + +import pandas as pd +from hipscat.catalog.catalog_info import CatalogInfo + +from corrgi.correlation.correlation import Correlation + + +class RedshiftCorrelation(Correlation): + """The redshift correlation utilities.""" + + def _get_auto_method(self) -> Callable: + raise NotImplementedError() + + def _construct_auto_args(self, df: pd.DataFrame, catalog_info: CatalogInfo) -> list: + raise NotImplementedError() + + def _get_cross_method(self) -> Callable: + raise NotImplementedError() + + def _construct_cross_args( + self, + left_df: pd.DataFrame, + right_df: pd.DataFrame, + left_catalog_info: CatalogInfo, + right_catalog_info: CatalogInfo, + ) -> list: + raise NotImplementedError() diff --git a/src/corrgi/corrgi.py b/src/corrgi/corrgi.py index ec86ee3..2dc62bd 100644 --- a/src/corrgi/corrgi.py +++ b/src/corrgi/corrgi.py @@ -2,14 +2,22 @@ from lsdb import Catalog from munch import Munch +from corrgi.correlation.correlation import Correlation from corrgi.dask import compute_autocorrelation_counts from corrgi.estimators import calculate_natural_estimate -def compute_autocorrelation(catalog: Catalog, random: Catalog, params: Munch) -> np.ndarray: +def compute_autocorrelation( + corr_type: type[Correlation], + catalog: Catalog, + random: Catalog, + params: Munch, +) -> np.ndarray: """Calculates the auto-correlation for a catalog. Args: + corr_type (type[Correlation]): The corrgi class corresponding to the type of + correlation (AngularCorrelation, RedshiftCorrelation, or ProjectedCorrelation). catalog (Catalog): The catalog. random (Catalog): A random samples catalog. params (Munch): The parameters dictionary to run gundam with. @@ -19,7 +27,7 @@ def compute_autocorrelation(catalog: Catalog, random: Catalog, params: Munch) -> """ num_galaxies = catalog.hc_structure.catalog_info.total_rows num_random = random.hc_structure.catalog_info.total_rows - counts_dd, counts_rr = compute_autocorrelation_counts(catalog, random, params) + counts_dd, counts_rr = compute_autocorrelation_counts(corr_type, catalog, random, params) return calculate_natural_estimate(counts_dd, counts_rr, num_galaxies, num_random) diff --git a/src/corrgi/dask.py b/src/corrgi/dask.py index c31bb0c..bf87851 100644 --- a/src/corrgi/dask.py +++ b/src/corrgi/dask.py @@ -1,5 +1,4 @@ import dask -import gundam.cflibfor as cff import numpy as np import pandas as pd from dask.distributed import print as dask_print @@ -11,14 +10,17 @@ from munch import Munch from corrgi.alignment import autocorrelation_alignment, crosscorrelation_alignment -from corrgi.parameters import generate_dd_rr_params -from corrgi.utils import join_count_histograms, project_coordinates +from corrgi.correlation.correlation import Correlation +from corrgi.utils import join_count_histograms -def compute_autocorrelation_counts(catalog: Catalog, random: Catalog, params: Munch) -> np.ndarray: +def compute_autocorrelation_counts( + corr_type: type[Correlation], catalog: Catalog, random: Catalog, params: Munch +) -> np.ndarray: """Computes the auto-correlation counts for a catalog. Args: + corr_type (type[Correlation]): The correlation class. catalog (Catalog): The catalog with galaxy samples. random (Catalog): The catalog with random samples. params (dict): The gundam parameters for the Fortran subroutine. @@ -28,11 +30,11 @@ def compute_autocorrelation_counts(catalog: Catalog, random: Catalog, params: Mu """ # Calculate the angular separation bins bins, _ = gundam.makebins(params.nsept, params.septmin, params.dsept, params.logsept) - params_dd, params_rr = generate_dd_rr_params(params) + # Create correlation with bins and params + correlation = corr_type(bins, params) # Generate the histograms with counts for each catalog - counts_dd = perform_auto_counts(catalog, bins, params_dd) - counts_rr = perform_auto_counts(random, bins, params_rr) - # Actually compute the results + counts_dd = perform_auto_counts(catalog, correlation) + counts_rr = perform_auto_counts(random, correlation) return dask.compute(*[counts_dd, counts_rr]) @@ -82,8 +84,7 @@ def perform_cross_counts(left: Catalog, right: Catalog, *args) -> np.ndarray: def count_auto_pairs( df: pd.DataFrame, catalog_info: CatalogInfo, - bins: np.ndarray, - params: Munch, + correlation: Correlation, ) -> np.ndarray: """Calls the fortran routine to compute the counts for pairs of partitions belonging to the same catalog. @@ -91,16 +92,16 @@ def count_auto_pairs( Args: df (pd.DataFrame): The partition dataframe. catalog_info (CatalogInfo): The catalog metadata. - bins (np.ndarray): The separation bins, in angular space. - params (Munch): The gundam subroutine parameters. + correlation (Correlation): The correlation instance. Returns: The count histogram for the partition pair. """ try: - return _count_auto_pairs(df, catalog_info, bins, params) + return correlation.count_auto_pairs(df, catalog_info) except Exception as exception: dask_print(exception) + raise exception @dask.delayed @@ -111,8 +112,7 @@ def count_cross_pairs( right_pix: HealpixPixel, left_catalog_info: CatalogInfo, right_catalog_info: CatalogInfo, - bins: np.ndarray, - params: Munch, + correlation: Correlation, ) -> np.ndarray: """Calls the fortran routine to compute the counts for pairs of partitions belonging to two different catalogs. @@ -124,86 +124,18 @@ def count_cross_pairs( right_pix (HealpixPixel): The pixel corresponding to `right_df`. left_catalog_info (CatalogInfo): The left catalog metadata. right_catalog_info (CatalogInfo): The right catalog metadata. - bins (np.ndarray): The separation bins, in angular space. - params (Munch): The gundam subroutine parameters. + correlation (Correlation): The correlation instance. Returns: The count histogram for the partition pair. """ try: - return _count_cross_pairs( + return correlation.count_cross_pairs( left_df, right_df, left_catalog_info, right_catalog_info, - bins, - params, ) except Exception as exception: dask_print(exception) - - -def _count_auto_pairs( - df: pd.DataFrame, - catalog_info: CatalogInfo, - bins: np.ndarray, - params: Munch, -) -> np.ndarray: - x, y, z = project_coordinates( - ra=df[catalog_info.ra_column].to_numpy(), - dec=df[catalog_info.dec_column].to_numpy(), - ) - args = [ - len(df), # number of particles - x, - y, - z, # X,Y,Z coordinates of particles - params.nsept, # number of angular separation bins - bins, # bins in angular separation [deg] - ] - counts = cff.mod.th_A_naiveway(*args) # fast unweighted counting - return counts - - -def _count_cross_pairs( - left_df: pd.DataFrame, - right_df: pd.DataFrame, - left_catalog_info: CatalogInfo, - right_catalog_info: CatalogInfo, - bins: np.ndarray, - params: Munch, -) -> np.ndarray: - left_x, left_y, left_z = project_coordinates( - ra=left_df[left_catalog_info.ra_column].to_numpy(), - dec=left_df[left_catalog_info.dec_column].to_numpy(), - ) - right_x, right_y, right_z = project_coordinates( - ra=right_df[right_catalog_info.ra_column].to_numpy(), - dec=right_df[right_catalog_info.dec_column].to_numpy(), - ) - args = [ - 1, # number of threads OpenMP - len(left_df), # number of particles of the left partition - left_df[left_catalog_info.ra_column].to_numpy(), # RA of particles [deg] - left_df[left_catalog_info.dec_column].to_numpy(), # DEC of particles [deg] - left_x, - left_y, - left_z, # X,Y,Z coordinates of particles - len(right_df), # number of particles of the right partition - right_x, - right_y, - right_z, # X,Y,Z coordinates of particles - params.nsept, # number of angular separation bins - bins, # bins in angular separation [deg] - params.sbound, - params.mxh1, - params.mxh2, - params.cntid, - params.logf, - params.sk1, - np.zeros(len(right_df)), - params.grid, - ] - # TODO: Create gundam th_C_naive_way that accepts only the necessary arguments - counts = cff.mod.th_C(*args) # fast unweighted counting - return counts + raise exception diff --git a/tests/corrgi/conftest.py b/tests/corrgi/conftest.py index 1543d16..7df4758 100644 --- a/tests/corrgi/conftest.py +++ b/tests/corrgi/conftest.py @@ -21,28 +21,58 @@ def test_data_dir(): @pytest.fixture -def data_catalog_dir(test_data_dir): - return test_data_dir / "DATA" +def hipscat_catalogs_dir(test_data_dir): + return test_data_dir / "hipscat" @pytest.fixture -def rand_catalog_dir(test_data_dir): - return test_data_dir / "RAND" +def acf_expected_results(test_data_dir): + return test_data_dir / "expected_results" / "acf" @pytest.fixture -def raw_catalog_dir(test_data_dir): - return test_data_dir / "RAW" +def data_catalog_dir(hipscat_catalogs_dir): + return hipscat_catalogs_dir / "DATA" @pytest.fixture -def dr7_lrg_catalog_dir(test_data_dir): - return test_data_dir / "DR7-lrg" +def dr7_lrg_catalog_dir(hipscat_catalogs_dir): + return hipscat_catalogs_dir / "DR7-lrg" @pytest.fixture -def dr7_lrg_rand_catalog_dir(test_data_dir): - return test_data_dir / "DR7-lrg-rand" +def dr7_lrg_rand_catalog_dir(hipscat_catalogs_dir): + return hipscat_catalogs_dir / "DR7-lrg-rand" + + +@pytest.fixture +def rand_catalog_dir(hipscat_catalogs_dir): + return hipscat_catalogs_dir / "RAND" + + +@pytest.fixture +def acf_bins_left_edges(acf_expected_results): + return np.load(acf_expected_results / "l_binedges_acf.npy") + + +@pytest.fixture +def acf_bins_right_edges(acf_expected_results): + return np.load(acf_expected_results / "r_binedges_acf.npy") + + +@pytest.fixture +def acf_dd_counts(acf_expected_results): + return np.load(acf_expected_results / "dd_acf.npy") + + +@pytest.fixture +def acf_rr_counts(acf_expected_results): + return np.load(acf_expected_results / "rr_acf.npy") + + +@pytest.fixture +def acf_nat_estimate(acf_expected_results): + return np.load(acf_expected_results / "w_acf_nat.npy") @pytest.fixture @@ -75,33 +105,3 @@ def autocorr_params(): params.mxh2 = 2 params.sk1 = [[1, 2], [1, 2]] return params - - -@pytest.fixture -def acf_results_dir(test_data_dir): - return test_data_dir / "acf" - - -@pytest.fixture -def acf_bins_left_edges(acf_results_dir): - return np.load(acf_results_dir / "l_binedges_acf.npy") - - -@pytest.fixture -def acf_bins_right_edges(acf_results_dir): - return np.load(acf_results_dir / "r_binedges_acf.npy") - - -@pytest.fixture -def acf_dd_counts(acf_results_dir): - return np.load(acf_results_dir / "dd_acf.npy") - - -@pytest.fixture -def acf_rr_counts(acf_results_dir): - return np.load(acf_results_dir / "rr_acf.npy") - - -@pytest.fixture -def acf_nat_estimate(acf_results_dir): - return np.load(acf_results_dir / "w_acf_nat.npy") diff --git a/tests/corrgi/test_corrgi.py b/tests/corrgi/test_acf.py similarity index 89% rename from tests/corrgi/test_corrgi.py rename to tests/corrgi/test_acf.py index 035503f..8f4249c 100644 --- a/tests/corrgi/test_corrgi.py +++ b/tests/corrgi/test_acf.py @@ -4,6 +4,7 @@ import numpy.testing as npt from gundam import gundam +from corrgi.correlation.angular_correlation import AngularCorrelation from corrgi.corrgi import compute_autocorrelation from corrgi.dask import compute_autocorrelation_counts from corrgi.estimators import calculate_natural_estimate @@ -35,7 +36,7 @@ def test_acf_counts_are_correct( assert isinstance(galaxy_catalog, lsdb.Catalog) assert isinstance(random_catalog, lsdb.Catalog) counts_dd, counts_rr = compute_autocorrelation_counts( - galaxy_catalog, random_catalog, autocorr_params + AngularCorrelation, galaxy_catalog, random_catalog, autocorr_params ) npt.assert_allclose(counts_dd, acf_dd_counts, rtol=1e-3) npt.assert_allclose(counts_rr, acf_rr_counts, rtol=2e-3) @@ -65,5 +66,7 @@ def test_acf_e2e( random_catalog = lsdb.read_hipscat(rand_catalog_dir) assert isinstance(galaxy_catalog, lsdb.Catalog) assert isinstance(random_catalog, lsdb.Catalog) - estimate = compute_autocorrelation(galaxy_catalog, random_catalog, autocorr_params) + estimate = compute_autocorrelation( + AngularCorrelation, galaxy_catalog, random_catalog, autocorr_params + ) npt.assert_allclose(estimate, acf_nat_estimate, rtol=1e-7) diff --git a/tests/corrgi/test_counting.py b/tests/corrgi/test_counting.py index b6a73e3..fc5d842 100644 --- a/tests/corrgi/test_counting.py +++ b/tests/corrgi/test_counting.py @@ -1,6 +1,6 @@ import hipscat -from corrgi.dask import _count_auto_pairs +from corrgi.correlation.angular_correlation import AngularCorrelation def test_count_auto_pairs( @@ -10,7 +10,7 @@ def test_count_auto_pairs( autocorr_params, ): data_catalog = hipscat.read_from_hipscat(data_catalog_dir) - partial = _count_auto_pairs( - single_data_partition, data_catalog.catalog_info, corr_bins, autocorr_params - ) + partial = AngularCorrelation( + corr_bins, autocorr_params, use_weights=False + ).count_auto_pairs(single_data_partition, data_catalog.catalog_info) assert len(partial) == len(corr_bins) - 1 diff --git a/tests/data/acf/dd_acf.npy b/tests/data/expected_results/acf/dd_acf.npy similarity index 100% rename from tests/data/acf/dd_acf.npy rename to tests/data/expected_results/acf/dd_acf.npy diff --git a/tests/data/acf/l_binedges_acf.npy b/tests/data/expected_results/acf/l_binedges_acf.npy similarity index 100% rename from tests/data/acf/l_binedges_acf.npy rename to tests/data/expected_results/acf/l_binedges_acf.npy diff --git a/tests/data/acf/r_binedges_acf.npy b/tests/data/expected_results/acf/r_binedges_acf.npy similarity index 100% rename from tests/data/acf/r_binedges_acf.npy rename to tests/data/expected_results/acf/r_binedges_acf.npy diff --git a/tests/data/acf/rr_acf.npy b/tests/data/expected_results/acf/rr_acf.npy similarity index 100% rename from tests/data/acf/rr_acf.npy rename to tests/data/expected_results/acf/rr_acf.npy diff --git a/tests/data/acf/w_acf_nat.npy b/tests/data/expected_results/acf/w_acf_nat.npy similarity index 100% rename from tests/data/acf/w_acf_nat.npy rename to tests/data/expected_results/acf/w_acf_nat.npy diff --git a/tests/data/generate_data.ipynb b/tests/data/generate_data.ipynb index 54fcf1f..e59165d 100644 --- a/tests/data/generate_data.ipynb +++ b/tests/data/generate_data.ipynb @@ -31,7 +31,7 @@ " catalog_name=catalog_name,\n", " catalog_type=\"object\",\n", " )\n", - " catalog.to_hipscat(catalog_name)" + " catalog.to_hipscat(f\"hipscat/{catalog_name}\")" ] }, { diff --git a/tests/data/DATA/Norder=0/Dir=0/Npix=1.parquet b/tests/data/hipscat/DATA/Norder=0/Dir=0/Npix=1.parquet similarity index 100% rename from tests/data/DATA/Norder=0/Dir=0/Npix=1.parquet rename to tests/data/hipscat/DATA/Norder=0/Dir=0/Npix=1.parquet diff --git a/tests/data/DATA/Norder=0/Dir=0/Npix=10.parquet b/tests/data/hipscat/DATA/Norder=0/Dir=0/Npix=10.parquet similarity index 100% rename from tests/data/DATA/Norder=0/Dir=0/Npix=10.parquet rename to tests/data/hipscat/DATA/Norder=0/Dir=0/Npix=10.parquet diff --git a/tests/data/DATA/Norder=0/Dir=0/Npix=2.parquet b/tests/data/hipscat/DATA/Norder=0/Dir=0/Npix=2.parquet similarity index 100% rename from tests/data/DATA/Norder=0/Dir=0/Npix=2.parquet rename to tests/data/hipscat/DATA/Norder=0/Dir=0/Npix=2.parquet diff --git a/tests/data/DATA/Norder=0/Dir=0/Npix=5.parquet b/tests/data/hipscat/DATA/Norder=0/Dir=0/Npix=5.parquet similarity index 100% rename from tests/data/DATA/Norder=0/Dir=0/Npix=5.parquet rename to tests/data/hipscat/DATA/Norder=0/Dir=0/Npix=5.parquet diff --git a/tests/data/DATA/Norder=0/Dir=0/Npix=6.parquet b/tests/data/hipscat/DATA/Norder=0/Dir=0/Npix=6.parquet similarity index 100% rename from tests/data/DATA/Norder=0/Dir=0/Npix=6.parquet rename to tests/data/hipscat/DATA/Norder=0/Dir=0/Npix=6.parquet diff --git a/tests/data/DATA/Norder=0/Dir=0/Npix=7.parquet b/tests/data/hipscat/DATA/Norder=0/Dir=0/Npix=7.parquet similarity index 100% rename from tests/data/DATA/Norder=0/Dir=0/Npix=7.parquet rename to tests/data/hipscat/DATA/Norder=0/Dir=0/Npix=7.parquet diff --git a/tests/data/DATA/Norder=0/Dir=0/Npix=9.parquet b/tests/data/hipscat/DATA/Norder=0/Dir=0/Npix=9.parquet similarity index 100% rename from tests/data/DATA/Norder=0/Dir=0/Npix=9.parquet rename to tests/data/hipscat/DATA/Norder=0/Dir=0/Npix=9.parquet diff --git a/tests/data/DATA/_common_metadata b/tests/data/hipscat/DATA/_common_metadata similarity index 100% rename from tests/data/DATA/_common_metadata rename to tests/data/hipscat/DATA/_common_metadata diff --git a/tests/data/DATA/_metadata b/tests/data/hipscat/DATA/_metadata similarity index 100% rename from tests/data/DATA/_metadata rename to tests/data/hipscat/DATA/_metadata diff --git a/tests/data/DATA/catalog_info.json b/tests/data/hipscat/DATA/catalog_info.json similarity index 100% rename from tests/data/DATA/catalog_info.json rename to tests/data/hipscat/DATA/catalog_info.json diff --git a/tests/data/DATA/partition_info.csv b/tests/data/hipscat/DATA/partition_info.csv similarity index 100% rename from tests/data/DATA/partition_info.csv rename to tests/data/hipscat/DATA/partition_info.csv diff --git a/tests/data/DATA/provenance_info.json b/tests/data/hipscat/DATA/provenance_info.json similarity index 100% rename from tests/data/DATA/provenance_info.json rename to tests/data/hipscat/DATA/provenance_info.json diff --git a/tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=0.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=0.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=0.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=0.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=10.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=10.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=10.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=10.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=11.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=11.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=11.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=11.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=3.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=3.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=3.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=3.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=4.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=4.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=4.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=4.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=5.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=5.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=5.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=5.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=7.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=7.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=7.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=7.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=8.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=8.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=8.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=8.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=9.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=9.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=0/Dir=0/Npix=9.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=0/Dir=0/Npix=9.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=10.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=10.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=10.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=10.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=11.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=11.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=11.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=11.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=24.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=24.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=24.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=24.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=25.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=25.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=25.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=25.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=26.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=26.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=26.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=26.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=27.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=27.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=27.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=27.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=4.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=4.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=4.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=4.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=5.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=5.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=5.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=5.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=6.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=6.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=6.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=6.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=7.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=7.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=7.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=7.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=8.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=8.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=8.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=8.parquet diff --git a/tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=9.parquet b/tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=9.parquet similarity index 100% rename from tests/data/DR7-lrg-rand/Norder=1/Dir=0/Npix=9.parquet rename to tests/data/hipscat/DR7-lrg-rand/Norder=1/Dir=0/Npix=9.parquet diff --git a/tests/data/DR7-lrg-rand/_common_metadata b/tests/data/hipscat/DR7-lrg-rand/_common_metadata similarity index 100% rename from tests/data/DR7-lrg-rand/_common_metadata rename to tests/data/hipscat/DR7-lrg-rand/_common_metadata diff --git a/tests/data/DR7-lrg-rand/_metadata b/tests/data/hipscat/DR7-lrg-rand/_metadata similarity index 100% rename from tests/data/DR7-lrg-rand/_metadata rename to tests/data/hipscat/DR7-lrg-rand/_metadata diff --git a/tests/data/DR7-lrg-rand/catalog_info.json b/tests/data/hipscat/DR7-lrg-rand/catalog_info.json similarity index 100% rename from tests/data/DR7-lrg-rand/catalog_info.json rename to tests/data/hipscat/DR7-lrg-rand/catalog_info.json diff --git a/tests/data/DR7-lrg-rand/partition_info.csv b/tests/data/hipscat/DR7-lrg-rand/partition_info.csv similarity index 100% rename from tests/data/DR7-lrg-rand/partition_info.csv rename to tests/data/hipscat/DR7-lrg-rand/partition_info.csv diff --git a/tests/data/DR7-lrg-rand/provenance_info.json b/tests/data/hipscat/DR7-lrg-rand/provenance_info.json similarity index 100% rename from tests/data/DR7-lrg-rand/provenance_info.json rename to tests/data/hipscat/DR7-lrg-rand/provenance_info.json diff --git a/tests/data/DR7-lrg/Norder=0/Dir=0/Npix=0.parquet b/tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=0.parquet similarity index 100% rename from tests/data/DR7-lrg/Norder=0/Dir=0/Npix=0.parquet rename to tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=0.parquet diff --git a/tests/data/DR7-lrg/Norder=0/Dir=0/Npix=1.parquet b/tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=1.parquet similarity index 100% rename from tests/data/DR7-lrg/Norder=0/Dir=0/Npix=1.parquet rename to tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=1.parquet diff --git a/tests/data/DR7-lrg/Norder=0/Dir=0/Npix=10.parquet b/tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=10.parquet similarity index 100% rename from tests/data/DR7-lrg/Norder=0/Dir=0/Npix=10.parquet rename to tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=10.parquet diff --git a/tests/data/DR7-lrg/Norder=0/Dir=0/Npix=11.parquet b/tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=11.parquet similarity index 100% rename from tests/data/DR7-lrg/Norder=0/Dir=0/Npix=11.parquet rename to tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=11.parquet diff --git a/tests/data/DR7-lrg/Norder=0/Dir=0/Npix=2.parquet b/tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=2.parquet similarity index 100% rename from tests/data/DR7-lrg/Norder=0/Dir=0/Npix=2.parquet rename to tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=2.parquet diff --git a/tests/data/DR7-lrg/Norder=0/Dir=0/Npix=3.parquet b/tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=3.parquet similarity index 100% rename from tests/data/DR7-lrg/Norder=0/Dir=0/Npix=3.parquet rename to tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=3.parquet diff --git a/tests/data/DR7-lrg/Norder=0/Dir=0/Npix=4.parquet b/tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=4.parquet similarity index 100% rename from tests/data/DR7-lrg/Norder=0/Dir=0/Npix=4.parquet rename to tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=4.parquet diff --git a/tests/data/DR7-lrg/Norder=0/Dir=0/Npix=5.parquet b/tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=5.parquet similarity index 100% rename from tests/data/DR7-lrg/Norder=0/Dir=0/Npix=5.parquet rename to tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=5.parquet diff --git a/tests/data/DR7-lrg/Norder=0/Dir=0/Npix=6.parquet b/tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=6.parquet similarity index 100% rename from tests/data/DR7-lrg/Norder=0/Dir=0/Npix=6.parquet rename to tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=6.parquet diff --git a/tests/data/DR7-lrg/Norder=0/Dir=0/Npix=7.parquet b/tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=7.parquet similarity index 100% rename from tests/data/DR7-lrg/Norder=0/Dir=0/Npix=7.parquet rename to tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=7.parquet diff --git a/tests/data/DR7-lrg/Norder=0/Dir=0/Npix=8.parquet b/tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=8.parquet similarity index 100% rename from tests/data/DR7-lrg/Norder=0/Dir=0/Npix=8.parquet rename to tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=8.parquet diff --git a/tests/data/DR7-lrg/Norder=0/Dir=0/Npix=9.parquet b/tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=9.parquet similarity index 100% rename from tests/data/DR7-lrg/Norder=0/Dir=0/Npix=9.parquet rename to tests/data/hipscat/DR7-lrg/Norder=0/Dir=0/Npix=9.parquet diff --git a/tests/data/DR7-lrg/_common_metadata b/tests/data/hipscat/DR7-lrg/_common_metadata similarity index 100% rename from tests/data/DR7-lrg/_common_metadata rename to tests/data/hipscat/DR7-lrg/_common_metadata diff --git a/tests/data/DR7-lrg/_metadata b/tests/data/hipscat/DR7-lrg/_metadata similarity index 100% rename from tests/data/DR7-lrg/_metadata rename to tests/data/hipscat/DR7-lrg/_metadata diff --git a/tests/data/DR7-lrg/catalog_info.json b/tests/data/hipscat/DR7-lrg/catalog_info.json similarity index 100% rename from tests/data/DR7-lrg/catalog_info.json rename to tests/data/hipscat/DR7-lrg/catalog_info.json diff --git a/tests/data/DR7-lrg/partition_info.csv b/tests/data/hipscat/DR7-lrg/partition_info.csv similarity index 100% rename from tests/data/DR7-lrg/partition_info.csv rename to tests/data/hipscat/DR7-lrg/partition_info.csv diff --git a/tests/data/DR7-lrg/provenance_info.json b/tests/data/hipscat/DR7-lrg/provenance_info.json similarity index 100% rename from tests/data/DR7-lrg/provenance_info.json rename to tests/data/hipscat/DR7-lrg/provenance_info.json diff --git a/tests/data/RAND/Norder=0/Dir=0/Npix=1.parquet b/tests/data/hipscat/RAND/Norder=0/Dir=0/Npix=1.parquet similarity index 100% rename from tests/data/RAND/Norder=0/Dir=0/Npix=1.parquet rename to tests/data/hipscat/RAND/Norder=0/Dir=0/Npix=1.parquet diff --git a/tests/data/RAND/Norder=0/Dir=0/Npix=10.parquet b/tests/data/hipscat/RAND/Norder=0/Dir=0/Npix=10.parquet similarity index 100% rename from tests/data/RAND/Norder=0/Dir=0/Npix=10.parquet rename to tests/data/hipscat/RAND/Norder=0/Dir=0/Npix=10.parquet diff --git a/tests/data/RAND/Norder=0/Dir=0/Npix=2.parquet b/tests/data/hipscat/RAND/Norder=0/Dir=0/Npix=2.parquet similarity index 100% rename from tests/data/RAND/Norder=0/Dir=0/Npix=2.parquet rename to tests/data/hipscat/RAND/Norder=0/Dir=0/Npix=2.parquet diff --git a/tests/data/RAND/Norder=0/Dir=0/Npix=5.parquet b/tests/data/hipscat/RAND/Norder=0/Dir=0/Npix=5.parquet similarity index 100% rename from tests/data/RAND/Norder=0/Dir=0/Npix=5.parquet rename to tests/data/hipscat/RAND/Norder=0/Dir=0/Npix=5.parquet diff --git a/tests/data/RAND/Norder=0/Dir=0/Npix=6.parquet b/tests/data/hipscat/RAND/Norder=0/Dir=0/Npix=6.parquet similarity index 100% rename from tests/data/RAND/Norder=0/Dir=0/Npix=6.parquet rename to tests/data/hipscat/RAND/Norder=0/Dir=0/Npix=6.parquet diff --git a/tests/data/RAND/Norder=0/Dir=0/Npix=7.parquet b/tests/data/hipscat/RAND/Norder=0/Dir=0/Npix=7.parquet similarity index 100% rename from tests/data/RAND/Norder=0/Dir=0/Npix=7.parquet rename to tests/data/hipscat/RAND/Norder=0/Dir=0/Npix=7.parquet diff --git a/tests/data/RAND/Norder=0/Dir=0/Npix=9.parquet b/tests/data/hipscat/RAND/Norder=0/Dir=0/Npix=9.parquet similarity index 100% rename from tests/data/RAND/Norder=0/Dir=0/Npix=9.parquet rename to tests/data/hipscat/RAND/Norder=0/Dir=0/Npix=9.parquet diff --git a/tests/data/RAND/_common_metadata b/tests/data/hipscat/RAND/_common_metadata similarity index 100% rename from tests/data/RAND/_common_metadata rename to tests/data/hipscat/RAND/_common_metadata diff --git a/tests/data/RAND/_metadata b/tests/data/hipscat/RAND/_metadata similarity index 100% rename from tests/data/RAND/_metadata rename to tests/data/hipscat/RAND/_metadata diff --git a/tests/data/RAND/catalog_info.json b/tests/data/hipscat/RAND/catalog_info.json similarity index 100% rename from tests/data/RAND/catalog_info.json rename to tests/data/hipscat/RAND/catalog_info.json diff --git a/tests/data/RAND/partition_info.csv b/tests/data/hipscat/RAND/partition_info.csv similarity index 100% rename from tests/data/RAND/partition_info.csv rename to tests/data/hipscat/RAND/partition_info.csv diff --git a/tests/data/RAND/provenance_info.json b/tests/data/hipscat/RAND/provenance_info.json similarity index 100% rename from tests/data/RAND/provenance_info.json rename to tests/data/hipscat/RAND/provenance_info.json