Merge pull request #9 from GPZ-Bioinfo/beta-lth

tmap V1.2

.gitignore

@@ -12,6 +12,7 @@ __pycache__/
 # FGFP_metadata.tsv
 # FGFP_metadata_ready.tsv
 check_doc_code.py
+temp-plot.html
 
 # C extensions
 *.so

README.md

@@ -22,11 +22,11 @@ pip install tmap
 
 
 After install the tmap, for avoid other dependency problems. Please install ``scikit-bio`` and ``vegan`` in R.
-
-run::
-
-    pip install scikit-bio
-    R -e "install.packages('vegan',repo='http://cran.rstudio.com/')"
+run:
+```
+pip install scikit-bio
+R -e "install.packages('vegan',repo='http://cran.rstudio.com/')"
+```
 
 If you encounter any error like `Import error: tkinter`, you need to run `sudo apt install python-tk` or `sudo apt install python3-tk`.
 

setup.py

@@ -42,7 +42,7 @@ def run(self):
 cmdclass = {'clean': CleanCommand}
 
 setup(name='tmap',
-      version='1.1.3',
+      version='1.2',
       description='A topological data analysis framework implementing the TDA Mapper algorithm for population-scale microbiome data analysis ',
       author='Haokui Zhou, Tianhua Liao',
       author_email='[email protected]',
@@ -62,7 +62,7 @@ def run(self):
                         'tqdm',
                         'scikit-learn>=0.19.1',
                         'matplotlib>=2.2.2',
-                        'networkx>=2.1',
+                        'networkx==2.2',
                         'pandas>=0.23.0',
                         'scipy',
                         'matplotlib!=3.0.0rc2',

tmap/api/Network_generator.py

@@ -8,10 +8,11 @@
 from tmap.tda.Filter import _filter_dict
 from tmap.tda.cover import Cover
 from tmap.tda.metric import Metric
-from tmap.tda.utils import optimize_dbscan_eps, cover_ratio, dump_graph
+from tmap.tda.utils import optimize_dbscan_eps
 
 pd.options.display.expand_frame_repr = False
 pd.options.display.max_rows = 1000
+
 import time
 import argparse
 import warnings
@@ -49,14 +50,8 @@ def generate_graph(input_data, dis=None, _eu_dm=None, eps_threshold=95, overlap=
     clusterer = DBSCAN(eps=eps, min_samples=min_samples)
     cover = Cover(projected_data=MinMaxScaler().fit_transform(projected_X), resolution=r, overlap=overlap)
     graph = tm.map(data=input_data, cover=cover, clusterer=clusterer)
-    logger('Graph covers %.2f percentage of samples.' % cover_ratio(graph, input_data), verbose=verbose)
+    logger(graph.info(), verbose=verbose)
     logger("graph generator take: ", time.time() - t1, verbose=verbose)
-
-    accessory_obj = {'raw_X': projected_X,
-                     'cover': cover,
-                     'clusterer': clusterer,
-                     'input_data': input_data}
-    graph['accessory_obj'] = accessory_obj
     return graph
 
 
@@ -87,7 +82,7 @@ def main(input, output, dis=None, _eu_dm=None, metric="braycurtis", eps=95, over
         logger("Empty graph generated... ERROR occur", verbose=1)
     else:
         # todo : implement a jsonable ndarray.
-        dump_graph(graph, output, method=method)
+        graph.write(output)
         logger("Graph has been generated and stored into ", output,verbose=1)
 
 

tmap/api/SAFE_analysis.py

@@ -1,15 +1,11 @@
 #! /usr/bin/python3
 import argparse
-import pickle
 import copy
-import time
-
-import pandas as pd
-from pandas.api.types import is_categorical_dtype
+import pickle
 
 from tmap.api.general import *
 from tmap.netx.SAFE import SAFE_batch, get_SAFE_summary
-from tmap.tda.utils import read_graph
+from tmap.tda.Graph import Graph
 
 _pickle_safe_format = {'data': '',
                        'params': {"n_iter": 0,
@@ -39,23 +35,41 @@ def generate_SAFE_score(graph, metadata, n_iter=1000, pval=0.05, nr_threshold=0.
     _pickle_safe_format['params']["nr_threshold"] = nr_threshold
 
     if _mode == 'both':
-        enriched_SAFE, declined_SAFE = SAFE_batch(graph, meta_data=metadata, n_iter=n_iter,
-                                                  nr_threshold=nr_threshold, _cal_type=_cal_type, _mode=_mode, verbose=verbose)
-        enriched_SAFE_summary = get_SAFE_summary(graph=graph, meta_data=metadata, safe_scores=enriched_SAFE,
-                                                 n_iter_value=n_iter, p_value=pval)
-        declined_SAFE_summary = get_SAFE_summary(graph=graph, meta_data=metadata, safe_scores=declined_SAFE,
-                                                 n_iter_value=n_iter, p_value=pval)
+        safe_scores = SAFE_batch(graph,
+                                 metadata=metadata,
+                                 n_iter=n_iter,
+                                 nr_threshold=nr_threshold,
+                                 _mode=_mode,
+                                 verbose=verbose)
+        enriched_SAFE, declined_SAFE = safe_scores['enrich'], safe_scores['decline']
+        enriched_SAFE_summary = get_SAFE_summary(graph=graph,
+                                                 metadata=metadata,
+                                                 safe_scores=enriched_SAFE,
+                                                 n_iter=n_iter,
+                                                 p_value=pval)
+        declined_SAFE_summary = get_SAFE_summary(graph=graph,
+                                                 metadata=metadata,
+                                                 safe_scores=declined_SAFE,
+                                                 n_iter=n_iter,
+                                                 p_value=pval)
         _pickle_safe_format['data'] = enriched_SAFE
         collect_result['raw']['enrich'] = copy.deepcopy(_pickle_safe_format)
         _pickle_safe_format['data'] = declined_SAFE
         collect_result['raw']['decline'] = copy.deepcopy(_pickle_safe_format)
         collect_result['enrich'] = enriched_SAFE_summary.sort_values('SAFE enriched score', ascending=False)
         collect_result['decline'] = declined_SAFE_summary.sort_values('SAFE enriched score', ascending=False)
     else:
-        SAFE_data = SAFE_batch(graph, meta_data=metadata, n_iter=n_iter,
-                               nr_threshold=nr_threshold, _cal_type=_cal_type, _mode=_mode, verbose=verbose)
-        SAFE_summary = get_SAFE_summary(graph=graph, meta_data=metadata, safe_scores=SAFE_data,
-                                        n_iter_value=n_iter, p_value=pval)
+        SAFE_data = SAFE_batch(graph,
+                               metadata=metadata,
+                               n_iter=n_iter,
+                               nr_threshold=nr_threshold,
+                               _mode=_mode,
+                               verbose=verbose)
+        SAFE_summary = get_SAFE_summary(graph=graph,
+                                        metadata=metadata,
+                                        safe_scores=SAFE_data,
+                                        n_iter=n_iter,
+                                        p_value=pval)
         _pickle_safe_format['data'] = SAFE_data
         collect_result['raw']['enrich'] = copy.deepcopy(_pickle_safe_format)
         collect_result['raw']['decline'] = copy.deepcopy(_pickle_safe_format)
@@ -75,10 +89,9 @@ def main(args):
     nr_threshold = args.nr_threshold
     _mode = args.mode
     _cal_type = args.cal_type
-    method = args.method
 
     logger("Loading precomputed graph from \"{}\" ".format(graph), verbose=1)
-    graph = read_graph(graph, method=method)
+    graph = Graph().read(graph)
     logger("Start performing SAFE analysis", verbose=1)
     result = generate_SAFE_score(graph, metadata, n_iter=n_iter,
                                  pval=pval, nr_threshold=nr_threshold,
@@ -147,7 +160,7 @@ def main(args):
     n_iter = args.iter
     pval = args.pvalue
 
-    metadata, cols_dict = data_parser(metadata,ft='metadatas')
+    metadata, cols_dict = data_parser(metadata, ft='metadatas')
     process_output(output=prefix)
     args.metadata = metadata
     main(args)

tmap/api/SAFE_visualization.py

@@ -4,14 +4,16 @@
 from collections import Counter
 
 import plotly
-import plotly.io as pio
+
 from plotly import graph_objs as go
 from plotly import tools
 from sklearn.decomposition import PCA
 from sklearn.preprocessing import MinMaxScaler
 
 from tmap.api.general import *
 from tmap.tda.plot import vis_progressX, Color
+from tmap.tda.Graph import Graph
+from tmap.tda.utils import output_fig
 
 
 def draw_PCOA(rawdatas, summary_datas, output, mode='html', width=1500, height=1000, sort_col='SAFE enriched score'):
@@ -67,22 +69,23 @@ def draw_PCOA(rawdatas, summary_datas, output, mode='html', width=1500, height=1
                            height=height,
                            font=dict(size=15),
                            hovermode='closest', ))
-    if mode != 'html' or not output.endswith('html'):
-        pio.write_image(fig, output, format=mode)
-    else:
-        plotly.offline.plot(fig, filename=output, auto_open=False)
+
+    output_fig(fig,output,mode)
     logger("Ordination graph has been output to", output, verbose=1)
 
 
 def draw_stratification(graph, SAFE_dict, cols, output, mode='html', n_iter=1000, p_val=0.05, width=1000, height=1000, allnodes=False):
     # Enterotyping-like stratification map based on SAFE score
-    node_pos = graph["node_positions"]
-    sizes = graph["node_sizes"][:, 0]
-    transformed_sizes = MinMaxScaler(feature_range=(10, 40)).fit_transform(sizes.reshape(-1, 1)).ravel()
-    projected_X = graph.get('accessory_obj', dict()).get('raw_X', '')
+
+    node_pos = graph.nodePos
+    sizes = graph.size
+    nodes = graph.nodes
+    sizes = np.array([sizes[_] for _ in range(len(nodes))]).reshape(-1, 1)
+
+    transformed_sizes = MinMaxScaler(feature_range=(10, 40)).fit_transform(sizes).ravel()
     xs = []
     ys = []
-    for edge in graph["edges"]:
+    for edge in graph.edges:
         xs += [node_pos[edge[0], 0],
                node_pos[edge[1], 0],
                None]
@@ -118,16 +121,15 @@ def draw_stratification(graph, SAFE_dict, cols, output, mode='html', n_iter=1000
             if allnodes:
                 color = Color(SAFE_dict[fea], target_by='node', dtype='numerical')
                 subfig = vis_progressX(graph,
-                                       projected_X=projected_X,
                                        simple=True,
                                        mode='obj',
                                        color=color
                                        )
                 subfig.data[1]['name'] = fea
                 fig.append_trace(subfig.data[1], 1, 1)
             else:
-                get_nodes_bool = safe_score_df.loc[:, fea] >= SAFE_pvalue
-                if not all(get_nodes_bool):
+                get_nodes_bool = (safe_score_df.loc[:, fea] >= SAFE_pvalue).all()
+                if not get_nodes_bool:
                     # if all False....
                     logger("fea: %s get all False bool indicated there are not enriched nodes showed at the graph" % fea, verbose=1)
                 else:
@@ -138,7 +140,7 @@ def draw_stratification(graph, SAFE_dict, cols, output, mode='html', n_iter=1000
                         y=node_pos[get_nodes_bool, 1],
                         hoverinfo="text",
                         marker=dict(  # color=node_colors,
-                            size=[5 + sizes[_] for _ in np.arange(node_pos.shape[0])[get_nodes_bool]],
+                            size=[sizes[_,0] for _ in np.arange(node_pos.shape[0])[get_nodes_bool]],
                             opacity=0.9),
                         showlegend=True,
                         name=str(fea) + ' (%s)' % str(t.get(fea, 0)),
@@ -154,7 +156,7 @@ def draw_stratification(graph, SAFE_dict, cols, output, mode='html', n_iter=1000
                 y=node_pos[np.array(tmp) == fea, 1],
                 hoverinfo="text",
                 marker=dict(  # color=node_colors,
-                    size=[5 + transformed_sizes[_] for _ in np.arange(node_pos.shape[0])[np.array(tmp) == fea]],
+                    size=[transformed_sizes[_] for _ in np.arange(node_pos.shape[0])[np.array(tmp) == fea]],
                     opacity=0.9),
                 showlegend=True,
                 name=str(fea) + ' (%s)' % str(t[fea]),
@@ -165,10 +167,7 @@ def draw_stratification(graph, SAFE_dict, cols, output, mode='html', n_iter=1000
     fig.layout.font.size = 15
     fig.layout.hovermode = 'closest'
 
-    if mode != 'html' or not output.endswith('html'):
-        pio.write_image(fig, output, format=mode)
-    else:
-        plotly.offline.plot(fig, filename=output, auto_open=False)
+    output_fig(fig,output,mode)
     logger("Stratification graph has been output to", output, verbose=1)
 
 
@@ -229,10 +228,7 @@ def _add_trace(name, col):
     fig.layout.width = width
     fig.layout.height = height
 
-    if mode == 'html' or output.endswith('html'):
-        plotly.offline.plot(fig, filename=output, auto_open=False)
-    else:
-        pio.write_image(fig, output, format=mode)
+    output_fig(fig,output,mode)
     logger("Ranking graph has been output to", output, verbose=1)
 
 
@@ -250,7 +246,7 @@ def main(args):
         dict_data = pickle.load(open(args.SAFE[0], 'rb'))
         safe_dict = dict_data['data']
         n_iter = dict_data['params']['n_iter']
-        graph = pickle.load(open(args.graph, 'rb'))
+        graph = Graph().read(args.graph)
         draw_stratification(graph=graph,
                             SAFE_dict=safe_dict,
                             cols=args.col,

tmap/api/quick_vis.py

@@ -1,18 +1,15 @@
 #! /usr/bin/python3
 import argparse
-import pickle
 import warnings
 
 from tmap.api.general import *
 from tmap.tda.plot import vis_progressX, Color
-
+from tmap.tda.Graph import Graph
 warnings.filterwarnings('ignore')
 
 
 def main(args):
-    graph = pickle.load(open(args.graph, 'rb'))
-    accessory_objs = graph['accessory_obj']
-    projected_X = accessory_objs['raw_X']
+    graph = Graph().read(args.graph)
     color = None
     if args.metadata:
         metadata = data_parser(args.metadata)
@@ -25,7 +22,6 @@ def main(args):
             color = Color(col_data, dtype=args.dtype, target_by='sample')
 
     vis_progressX(graph,
-                  projected_X=projected_X,
                   mode='file',
                   simple=False if args.complex else True,
                   color=color,