Merge branch 'master' into dependency

.github/workflows/python-plain-run-test.yml

@@ -46,9 +46,10 @@ jobs:
     - name: Upload coverage to Codecov
       uses: Wandalen/wretry.action@v1
       with:
-        action: codecov/codecov-action@v3
+        action: codecov/codecov-action@v4
         with: |
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true
         attempt_limit: 5
         attempt_delay: 10000

dynamo/vectorfield/utils.py

@@ -916,7 +916,7 @@ def compute_curl(f_jac, X):
 
 # ---------------------------------------------------------------------------------------------------
 # ranking related utilies
-def get_metric_gene_in_rank(mat: np.mat, genes: list, neg: bool = False) -> Tuple[np.ndarray, np.ndarray]:
+def get_metric_gene_in_rank(mat: np.asmatrix, genes: list, neg: bool = False) -> Tuple[np.ndarray, np.ndarray]:
     """Calculate ranking of genes based on mean value of matrix.
 
     Args:
@@ -936,7 +936,7 @@ def get_metric_gene_in_rank(mat: np.mat, genes: list, neg: bool = False) -> Tupl
 
 
 def get_metric_gene_in_rank_by_group(
-    mat: np.mat, genes: list, groups: np.array, selected_group, neg: bool = False
+    mat: np.asmatrix, genes: list, groups: np.array, selected_group, neg: bool = False
 ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
     """Calculate ranking of genes based on mean value of matrix, grouped by selected group.
 
@@ -993,7 +993,7 @@ def get_sorted_metric_genes_df(df: pd.DataFrame, genes: list, neg: bool = False)
     return sorted_metric, sorted_genes
 
 
-def rank_vector_calculus_metrics(mat: np.mat, genes: list, group, groups: list, uniq_group: list) -> Tuple:
+def rank_vector_calculus_metrics(mat: np.asmatrix, genes: list, group, groups: list, uniq_group: list) -> Tuple:
     """Calculate ranking of genes based on vector calculus metric.
 
     Args:

pytest.ini

@@ -1,4 +1,3 @@
 [pytest]
-python_files = *.py
 testpaths = tests
 xfail_strict = true

requirements.txt

@@ -1,10 +1,10 @@
-numpy>=1.20.0
+numpy>=1.20.0,<2.0.0
 pandas>=1.3.5
 scipy>=1.4.1
-scikit-learn>=0.19.1
+scikit-learn>=0.19.1,<1.5.0
 anndata>=0.8.0
 loompy>=3.0.5
-matplotlib>=3.5.3
+matplotlib>=3.5.3,<3.9.0
 setuptools
 numdifftools>=0.9.39
 umap-learn>=0.5.1

tests/conftest.py

@@ -42,15 +42,24 @@ def mkdirs_wrapper(path: Union[str, Path], abort=True):
 
     def gen_zebrafish_test_data():
         raw_adata = dyn.sample_data.zebrafish()
-        adata = raw_adata[:, :5000].copy()
+        adata = raw_adata[:300, :1000].copy()
         del raw_adata
 
         preprocessor = dyn.pp.Preprocessor(cell_cycle_score_enable=True)
-        preprocessor.config_monocle_recipe(adata, n_top_genes=100)
+        preprocessor.config_monocle_recipe(adata, n_top_genes=40)
         preprocessor.filter_genes_by_outliers_kwargs["inplace"] = True
         preprocessor.select_genes_kwargs["keep_filtered"] = False
+        preprocessor.pca_kwargs["n_pca_components"] = 5
         preprocessor.preprocess_adata_monocle(adata)
 
+        dyn.tl.dynamics(adata, model="stochastic")
+        dyn.tl.reduceDimension(adata)
+        dyn.tl.cell_velocities(adata)
+        dyn.vf.VectorField(adata, basis="umap")
+
+        dyn.tl.cell_velocities(adata, basis="pca")
+        dyn.vf.VectorField(adata, basis="pca")
+
         TestUtils.mkdirs_wrapper(test_data_dir, abort=False)
         adata.write_h5ad(test_zebrafish_data_path)
 

tests/test_config.py

@@ -36,7 +36,3 @@ def test_config_change():
     import dynamo.configuration as imported_config
 
     assert imported_config.DynamoAdataConfig.data_store_mode == "succinct"
-
-
-if __name__ == "__main__":
-    test_config_change()

tests/test_data_io.py

@@ -111,11 +111,3 @@ def alpha_minus_gamma_s(new, gamma, t, M_s):
 
     dyn.vf.rank_jacobian_genes(adata, groups="leiden")
     adata.write_h5ad("debug11.h5ad")
-
-
-if __name__ == "__main__":
-    # test_scEU_seq()
-    # adata = utils.gen_or_read_zebrafish_data()
-    # test_save_rank_info(adata)
-    # test_save_adata()
-    pass

tests/test_logger.py

@@ -111,17 +111,3 @@ def test_cell_cycle_score_logger_pancreatic_endocrinogenesis():
         # genes_to_exclude=["Sbspon", "Adgrb3", "Eif2s3y"],
     )
     dyn.pp.cell_cycle_scores(adata)
-
-
-if __name__ == "__main__":
-    test_tqdm_style_loops()
-
-    test_logger_simple_output_1(LoggerManager.get_main_logger())
-    test_logger_simple_progress_naive(LoggerManager.get_main_logger())
-    test_logger_simple_progress_logger(LoggerManager.get_main_logger())
-    test_logger_simple_progress_logger(LoggerManager.get_temp_timer_logger())
-
-    test_vectorField_logger()
-    test_zebrafish_topography_tutorial_logger()
-    test_cell_cycle_score_logger_pancreatic_endocrinogenesis()
-    test_sparseVFC_logger()

tests/test_pipeline.py

@@ -1,9 +1,99 @@
 import dynamo as dyn
-
+import pytest
 
 def test_dynamcis(adata):
-    adata.uns["pp"]["tkey"] = None
-    dyn.tl.dynamics(adata, model="stochastic")
-    dyn.tl.reduceDimension(adata)
-    dyn.tl.cell_velocities(adata)
-    dyn.vf.VectorField(adata, basis="umap", M=100)
+    raw_adata = dyn.sample_data.zebrafish()
+    adata = raw_adata[:300, :1000].copy()
+    del raw_adata
+
+    preprocessor = dyn.pp.Preprocessor(cell_cycle_score_enable=True)
+    preprocessor.config_monocle_recipe(adata, n_top_genes=100)
+    preprocessor.filter_genes_by_outliers_kwargs["inplace"] = True
+    preprocessor.select_genes_kwargs["keep_filtered"] = False
+    preprocessor.preprocess_adata_monocle(adata)
+
+    dyn.tl.dynamics(adata, model="deterministic")
+
+
+@pytest.mark.skip(reason="extra dependency requests-cache not installed")
+def test_run_rpe1_tutorial():
+    import numpy as np
+
+    raw_adata = dyn.sample_data.scEU_seq_rpe1()
+    rpe1 = raw_adata[5000:, :500].copy()
+    del raw_adata
+
+    # create rpe1 kinectics
+    rpe1_kinetics = rpe1[rpe1.obs.exp_type == "Pulse", :]
+    rpe1_kinetics.obs["time"] = rpe1_kinetics.obs["time"].astype(str)
+    rpe1_kinetics.obs.loc[rpe1_kinetics.obs["time"] == "dmso", "time"] = -1
+    rpe1_kinetics.obs["time"] = rpe1_kinetics.obs["time"].astype(float)
+    rpe1_kinetics = rpe1_kinetics[rpe1_kinetics.obs.time != -1, :]
+
+    rpe1_kinetics.layers["new"], rpe1_kinetics.layers["total"] = (
+        rpe1_kinetics.layers["ul"] + rpe1_kinetics.layers["sl"],
+        rpe1_kinetics.layers["su"]
+        + rpe1_kinetics.layers["sl"]
+        + rpe1_kinetics.layers["uu"]
+        + rpe1_kinetics.layers["ul"],
+    )
+
+    del rpe1_kinetics.layers["uu"], rpe1_kinetics.layers["ul"], rpe1_kinetics.layers["su"], rpe1_kinetics.layers["sl"]
+    dyn.pl.basic_stats(rpe1_kinetics, save_show_or_return="return")
+    rpe1_genes = ["UNG", "PCNA", "PLK1", "HPRT1"]
+
+    assert np.sum(rpe1_kinetics.var_names.isnull()) == 0
+
+    rpe1_kinetics.obs.time = rpe1_kinetics.obs.time.astype("float")
+    rpe1_kinetics.obs.time = rpe1_kinetics.obs.time / 60
+    rpe1_kinetics.obs.time.value_counts()
+    # rpe1_kinetics = dyn.pp.recipe_monocle(rpe1_kinetics, n_top_genes=1000, total_layers=False, copy=True)
+    dyn.pp.recipe_monocle(rpe1_kinetics, n_top_genes=100, total_layers=False)
+
+    dyn.tl.dynamics(rpe1_kinetics, model="deterministic", tkey="time", est_method="twostep", cores=16)
+    dyn.tl.reduceDimension(rpe1_kinetics, reduction_method="umap")
+    dyn.tl.cell_velocities(rpe1_kinetics, enforce=True, vkey="velocity_T", ekey="M_t")
+
+    rpe1_kinetics.obsm["X_RFP_GFP"] = rpe1_kinetics.obs.loc[
+        :, ["RFP_log10_corrected", "GFP_log10_corrected"]
+    ].values.astype("float")
+    rpe1_kinetics.layers["velocity_S"] = rpe1_kinetics.layers["velocity_T"].copy()
+    dyn.tl.cell_velocities(rpe1_kinetics, enforce=True, vkey="velocity_S", ekey="M_t", basis="RFP_GFP")
+
+    rpe1_kinetics.obs.Cell_cycle_relativePos = rpe1_kinetics.obs.Cell_cycle_relativePos.astype(float)
+    rpe1_kinetics.obs.Cell_cycle_possition = rpe1_kinetics.obs.Cell_cycle_possition.astype(float)
+
+    dyn.pl.streamline_plot(
+        rpe1_kinetics,
+        color=["Cell_cycle_possition", "Cell_cycle_relativePos"],
+        basis="RFP_GFP",
+        save_show_or_return="return",
+    )
+    dyn.pl.streamline_plot(rpe1_kinetics, color=["cell_cycle_phase"], basis="RFP_GFP", save_show_or_return="return")
+    dyn.vf.VectorField(rpe1_kinetics, basis="RFP_GFP")
+    progenitor = rpe1_kinetics.obs_names[rpe1_kinetics.obs.Cell_cycle_relativePos < 0.1]
+
+    np.random.seed(19491001)
+
+    from matplotlib import animation
+
+    info_genes = rpe1_kinetics.var_names[rpe1_kinetics.var.use_for_transition]
+    dyn.pd.fate(
+        rpe1_kinetics,
+        basis="RFP_GFP",
+        init_cells=progenitor,
+        interpolation_num=100,
+        direction="forward",
+        inverse_transform=False,
+        average=False,
+    )
+
+    import matplotlib.pyplot as plt
+
+    fig, ax = plt.subplots()
+    ax = dyn.pl.topography(
+        rpe1_kinetics, basis="RFP_GFP", color="Cell_cycle_relativePos", ax=ax, save_show_or_return="return"
+    )
+    ax.set_aspect(0.8)
+
+    instance = dyn.mv.StreamFuncAnim(adata=rpe1_kinetics, basis="RFP_GFP", color="Cell_cycle_relativePos", ax=ax)