Skip to content
This repository has been archived by the owner on Jan 14, 2025. It is now read-only.

Commit

Permalink
Reduced shared parquet file contention in testing (#440)
Browse files Browse the repository at this point in the history
* Hotfix for from_parquet contention

* Fix typo
  • Loading branch information
wilsonbb authored Apr 29, 2024
1 parent 40236a3 commit 8f33ee8
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 1 deletion.
Binary file added tests/tape_tests/data/object/test_object2.parquet
Binary file not shown.
Binary file added tests/tape_tests/data/source/test_source2.parquet
Binary file not shown.
15 changes: 14 additions & 1 deletion tests/tape_tests/test_ensemble.py
Original file line number Diff line number Diff line change
Expand Up @@ -2054,11 +2054,24 @@ def my_mean(flux, time, dup_time, band):

@pytest.mark.parametrize("on", [None, ["ps1_objid", "filterName"], ["filterName", "ps1_objid"]])
@pytest.mark.parametrize("func_label", ["mean", "bounds"])
def test_batch_by_band(parquet_ensemble, func_label, on):
def test_batch_by_band(func_label, on):
"""
Test that ensemble.batch(by_band=True) works as intended.
"""

# TODO(wbeebe): This is a temporary workaround for https://github.com/lincc-frameworks/tape/issues/434 to avoid contention when reading data
# from the test files in parallel. This is the same as `parquet_ensemble` but uses copies of the parquet files referenced there.
parquet_ensemble = Ensemble(client=False)
parquet_ensemble.from_parquet(
"tests/tape_tests/data/source/test_source2.parquet",
"tests/tape_tests/data/object/test_object2.parquet",
id_col="ps1_objid",
time_col="midPointTai",
band_col="filterName",
flux_col="psFlux",
err_col="psFluxErr",
)

if func_label == "mean":

def my_mean(flux):
Expand Down

0 comments on commit 8f33ee8

Please sign in to comment.