Merge pull request #106 from LSSTDESC/tqz/utils_refactor

WIP: Utils name changing

src/rail/cli/scripts.py

@@ -4,7 +4,7 @@
 import rail.stages
 from rail.core import RailEnv
 from rail.cli.options import GitMode
-from rail.core.utils import RAILDIR
+from rail.utils.path_utils import RAILDIR
 
 
 def render_nb(outdir, clear_output, dry_run, inputs, skip, **_kwargs):

src/rail/core/__init__.py

@@ -7,7 +7,7 @@
 from .stage import RailPipeline, RailStage
 
 # from .utilPhotometry import PhotormetryManipulator, HyperbolicSmoothing, HyperbolicMagnitudes
-from .util_stages import ColumnMapper, RowSelector, TableConverter
+# from .util_stages import ColumnMapper, RowSelector, TableConverter
 from .introspection import RailEnv
 from .point_estimation import PointEstimationMixin
 

src/rail/core/common_params.py

@@ -3,6 +3,9 @@
 from ceci.config import StageParameter as Param
 from ceci.config import StageConfig
 
+
+
+
 lsst_bands = "ugrizy"
 lsst_mag_cols = [f"mag_{band}_lsst" for band in lsst_bands]
 lsst_mag_err_cols = [f"mag_err_{band}_lsst" for band in lsst_bands]

src/rail/core/utils.py

@@ -1,20 +1,7 @@
-""" Utility functions """
 
-import os
-import rail
-import rail.core
+# This is a deprecative path for importing RAILDIR, it is only for the transition period
+# to move RAILDIR and find_rail_file to utils.path_utils
 
-RAILDIR = os.path.abspath(os.path.join(os.path.dirname(rail.core.__file__), "..", ".."))
+from rail.utils.path_utils import find_rail_file, RAILDIR
 
-
-def find_rail_file(relpath):
-    """Find a file somewhere in rail by searching the namespace path
-
-    This lets us avoid issues that the paths can be different depending
-    on if we have installed things from source or not
-    """
-    for path_ in rail.__path__:
-        fullpath = os.path.abspath(os.path.join(path_, relpath))
-        if os.path.exists(fullpath):
-            return fullpath
-    raise ValueError(f"Could not file {relpath} in {rail.__path__}")
+find_rail_file = find_rail_file

src/rail/stages/__init__.py

@@ -14,9 +14,9 @@
 
 
 from rail.creation.degrader import *
-#from rail.creation.degradation.spectroscopic_degraders import *
-# from rail.creation.degradation.spectroscopic_selections import *
-from rail.creation.degradation.quantityCut import *
+#from rail.creation.degraders.spectroscopic_degraders import *
+# from rail.creation.degraders.spectroscopic_selections import *
+from rail.creation.degraders.quantityCut import *
 
 from rail.creation.engine import *
 
@@ -31,6 +31,7 @@
 from rail.evaluation.point_to_point_evaluator import PointToPointEvaluator
 from rail.evaluation.single_evaluator import SingleEvaluator
 
+from rail.tools.table_tools import ColumnMapper, RowSelector, TableConverter
 
 def import_and_attach_all():
     """Import all the packages in the rail ecosystem and attach them to this module"""

src/rail/utils/path_utils.py

@@ -0,0 +1,19 @@
+""" Utility functions """
+
+import os
+import rail
+import rail.core
+
+RAILDIR = os.path.abspath(os.path.join(os.path.dirname(rail.core.__file__), "..", ".."))
+
+def find_rail_file(relpath):
+    """Find a file somewhere in rail by searching the namespace path
+
+    This lets us avoid issues that the paths can be different depending
+    on if we have installed things from source or not
+    """
+    for path_ in rail.__path__:
+        fullpath = os.path.abspath(os.path.join(path_, relpath))
+        if os.path.exists(fullpath):
+            return fullpath
+    raise ValueError(f"Could not file {relpath} in {rail.__path__}")

src/rail/core/algo_utils.py → src/rail/utils/testing_utils.py

@@ -2,7 +2,7 @@
 import os
 import scipy.special
 from rail.core.stage import RailStage
-from rail.core.utils import RAILDIR
+from rail.utils.path_utils import RAILDIR
 from rail.core.data import TableHandle
 
 

tests/core/test_core.py

@@ -18,55 +18,13 @@
     TableHandle,
 )
 from rail.core.stage import RailStage
-from rail.core.utils import RAILDIR, find_rail_file
-from rail.core.util_stages import (
-    ColumnMapper,
-    RowSelector,
-    TableConverter,
-)
+from rail.utils.path_utils import RAILDIR
 
 
 # def test_data_file():
 #    with pytest.raises(ValueError) as errinfo:
 #        df = DataFile('dummy', 'x')
 
-def test_find_rail_file():
-    afile = find_rail_file(os.path.join("examples_data", "testdata", "test_dc2_training_9816.pq"))
-    assert afile
-    with pytest.raises(ValueError):
-        _not_a_file = find_rail_file("not_a_file")
-
-
-def test_util_stages():
-    DS = RailStage.data_store
-    DS.clear()
-    datapath = os.path.join(
-        RAILDIR, "rail", "examples_data", "testdata", "test_dc2_training_9816.pq"
-    )
-
-    data = DS.read_file("data", TableHandle, datapath)
-
-    table_conv = TableConverter.make_stage(name="conv", output_format="numpyDict")
-    col_map = ColumnMapper.make_stage(name="col_map", columns={})
-    row_sel = RowSelector.make_stage(name="row_sel", start=1, stop=15)
-
-    with pytest.raises(KeyError) as _errinfo:
-        table_conv.get_handle("nope", allow_missing=False)
-
-    _conv_data = table_conv(data)
-    mapped_data = col_map(data)
-    _sel_data = row_sel(mapped_data)
-
-    row_sel_2 = RowSelector.make_stage(name="row_sel_2", start=1, stop=15)
-    row_sel_2.set_data("input", mapped_data.data)
-    handle = row_sel_2.get_handle("input")
-
-    row_sel_3 = RowSelector.make_stage(
-        name="row_sel_3", input=handle.path, start=1, stop=15
-    )
-    row_sel_3.set_data("input", None, do_read=True)
-
-
 def do_data_handle(datapath, handle_class):
     _DS = RailStage.data_store
 
@@ -291,40 +249,6 @@ def test_model_handle():
         pickle.dump(obj=mh3.data, file=fout, protocol=pickle.HIGHEST_PROTOCOL)
     os.remove(model_path_copy)
 
-
-def test_data_hdf5_iter():
-    DS = RailStage.data_store
-    DS.clear()
-
-    datapath = os.path.join(
-        RAILDIR, "rail", "examples_data", "testdata", "test_dc2_training_9816.hdf5"
-    )
-
-    # data = DS.read_file('data', TableHandle, datapath)
-    th = Hdf5Handle("data", path=datapath)
-    x = th.iterator(groupname="photometry", chunk_size=1000)
-
-    assert isinstance(x, GeneratorType)
-    for i, xx in enumerate(x):
-        assert xx[0] == i * 1000
-        assert xx[1] - xx[0] <= 1000
-
-    _data = DS.read_file("input", TableHandle, datapath)
-    cm = ColumnMapper.make_stage(
-        input=datapath,
-        chunk_size=1000,
-        hdf5_groupname="photometry",
-        columns=dict(id="bob"),
-    )
-    x = cm.input_iterator("input")
-
-    assert isinstance(x, GeneratorType)
-
-    for i, xx in enumerate(x):
-        assert xx[0] == i * 1000
-        assert xx[1] - xx[0] <= 1000
-
-
 def test_data_store():
     DS = RailStage.data_store
     DS.clear()
@@ -403,37 +327,3 @@ def test_common_params():
     assert par.value == 0.1
     assert par.dtype == float
 
-
-def test_set_data_nonexistent_file():
-    """Create an instance of a child class of RailStage. Exercise the `set_data`
-    method and pass in a path to a nonexistent file. A `FileNotFound` exception
-    should be raised.
-    """
-
-    col_map = ColumnMapper.make_stage(name="col_map", columns={})
-    with pytest.raises(FileNotFoundError) as err:
-        col_map.set_data("model", None, path="./bad_directory/no_file.py")
-        assert "Unable to find file" in err.context
-
-
-def test_set_data_real_file():
-    """Create an instance of a child class of RailStage. Exercise the `set_data`
-    method and pass in a path to model. The output of set_data should be `None`.
-    """
-    DS = RailStage.data_store
-    DS.clear()
-    model_path = os.path.join(
-        RAILDIR,
-        "rail",
-        "examples_data",
-        "estimation_data",
-        "data",
-        "CWW_HDFN_prior.pkl",
-    )
-    DS.add_data("model", None, ModelHandle, path=model_path)
-
-    col_map = ColumnMapper.make_stage(name="col_map", columns={})
-
-    ret_val = col_map.set_data("model", None, path=model_path, do_read=False)
-
-    assert ret_val is None

tests/core/test_pipeline.py

@@ -4,8 +4,8 @@
 import numpy as np
 
 from rail.core.stage import RailPipeline, RailStage
-from rail.core.utils import RAILDIR
-from rail.core.util_stages import ColumnMapper, TableConverter
+from rail.utils.path_utils import RAILDIR
+from rail.tools.table_tools import ColumnMapper, TableConverter
 
 
 def test_pipeline():

tests/creation/test_degraders.py

@@ -6,9 +6,9 @@
 import pytest
 
 from rail.core.data import DATA_STORE, TableHandle
-from rail.core.util_stages import ColumnMapper
-from rail.creation.degradation.quantityCut import QuantityCut
-from rail.creation.degradation.addRandom import AddColumnOfRandom
+from rail.tools.table_tools import ColumnMapper
+from rail.creation.degraders.quantityCut import QuantityCut
+from rail.creation.degraders.addRandom import AddColumnOfRandom
 
 
 @pytest.fixture

tests/estimation/test_algos.py

@@ -1,7 +1,7 @@
 import numpy as np
 import scipy.special
 
-from rail.core.algo_utils import one_algo
+from rail.utils.testing_utils import one_algo
 from rail.core.stage import RailStage
 from rail.estimation.algos import random_gauss, train_z
 

tests/estimation/test_classifier.py

@@ -2,7 +2,7 @@
 import numpy as np
 import pytest
 
-from rail.core.utils import RAILDIR
+from rail.utils.path_utils import RAILDIR
 from rail.core.stage import RailStage
 from rail.core.data import QPHandle
 from rail.estimation.algos.uniform_binning import UniformBinningClassifier

tests/estimation/test_summarizers.py

@@ -2,7 +2,7 @@
 
 from rail.core.data import QPHandle
 from rail.core.stage import RailStage
-from rail.core.utils import RAILDIR
+from rail.utils.path_utils import RAILDIR
 from rail.estimation.algos import naive_stack, point_est_hist, var_inf
 
 testdata = os.path.join(RAILDIR, "rail/examples_data/testdata/output_BPZ_lite.hdf5")

tests/evaluation/test_evaluation.py

@@ -8,7 +8,7 @@
 import rail.evaluation.metrics.pointestimates as pe
 from rail.core.data import QPHandle, TableHandle,  QPOrTableHandle
 from rail.core.stage import RailStage
-from rail.core.utils import find_rail_file
+from rail.utils.path_utils import find_rail_file
 from rail.evaluation.evaluator import OldEvaluator
 from rail.evaluation.dist_to_dist_evaluator import DistToDistEvaluator
 from rail.evaluation.dist_to_point_evaluator import DistToPointEvaluator