Merge pull request #97 from Teichlab/develop

Develop

.dockerignore

@@ -3,4 +3,6 @@
 ./docker/
 .dockerignore
 .git
-./venv
+./venv
+./dist
+./cpdb-venv/

cellphonedb/cellphonedb_cli.py

@@ -3,11 +3,9 @@
 
 import click
 
-import cellphonedb.src.api_endpoints.terminal_api.database_terminal_api_endpoints.database_terminal_commands
-from cellphonedb.src.api_endpoints.terminal_api import database_terminal_api_endpoints
+from cellphonedb.src.api_endpoints.terminal_api.database_terminal_api_endpoints import database_terminal_commands
 from cellphonedb.src.api_endpoints.terminal_api.method_terminal_api_endpoints import method_terminal_commands
 from cellphonedb.src.api_endpoints.terminal_api.plot_terminal_api_endpoints import plot_terminal_commands
-
 from cellphonedb.src.api_endpoints.terminal_api.query_terminal_api_endpoints import query_terminal_commands
 from cellphonedb.src.api_endpoints.terminal_api.tools_terminal_api_endpoints import tools_terminal_commands
 
@@ -42,10 +40,10 @@ def plot():
 query.add_command(query_terminal_commands.find_interactions_by_element)
 query.add_command(query_terminal_commands.get_interaction_gene)
 
-database.add_command(database_terminal_api_endpoints.database_terminal_commands.download)
-database.add_command(database_terminal_api_endpoints.database_terminal_commands.list_remote)
-database.add_command(database_terminal_api_endpoints.database_terminal_commands.list_local)
-database.add_command(database_terminal_api_endpoints.database_terminal_commands.generate)
+database.add_command(database_terminal_commands.download)
+database.add_command(database_terminal_commands.list_remote)
+database.add_command(database_terminal_commands.list_local)
+database.add_command(database_terminal_commands.generate)
 
 plot.add_command(plot_terminal_commands.dot_plot)
 plot.add_command(plot_terminal_commands.heatmap_plot)
@@ -58,7 +56,8 @@ def tools():
         pass
 
 
-    database.add_command(database_terminal_api_endpoints.database_terminal_commands.collect)
+    query.add_command(query_terminal_commands.autocomplete)
+    database.add_command(database_terminal_commands.collect)
     tools.add_command(tools_terminal_commands.generate_genes)
     tools.add_command(tools_terminal_commands.generate_proteins)
     tools.add_command(tools_terminal_commands.generate_complex)

cellphonedb/src/api_endpoints/terminal_api/method_terminal_api_endpoints/method_terminal_commands.py

@@ -5,13 +5,13 @@
 import click
 from click import Context, Argument
 
+from cellphonedb.src.api_endpoints.terminal_api.util.choose_database import choose_database
 from cellphonedb.src.app import cpdb_app
 from cellphonedb.src.app.app_logger import app_logger
 from cellphonedb.src.core.exceptions.AllCountsFilteredException import AllCountsFilteredException
 from cellphonedb.src.core.exceptions.EmptyResultException import EmptyResultException
 from cellphonedb.src.core.exceptions.ThresholdValueException import ThresholdValueException
 from cellphonedb.src.core.utils.subsampler import Subsampler
-from cellphonedb.src.database.manager import DatabaseVersionManager
 from cellphonedb.src.exceptions.ParseCountsException import ParseCountsException
 from cellphonedb.src.exceptions.ParseMetaException import ParseMetaException
 from cellphonedb.src.exceptions.ReadFileException import ReadFileException
@@ -59,10 +59,6 @@ def subsampling_options(f: Callable) -> Callable:
     return f
 
 
-def choose_database(ctx: Context, argument: Argument, value: str) -> Optional[str]:
-    return DatabaseVersionManager.find_database_for(value)
-
-
 def common_options(f: Callable) -> Callable:
     options = [
         click.argument('meta-filename'),

cellphonedb/src/api_endpoints/terminal_api/query_terminal_api_endpoints/query_terminal_commands.py

@@ -1,22 +1,40 @@
-from click._unicodefun import click
+from typing import Callable
 
+import click
+
+from cellphonedb.src.api_endpoints.terminal_api.util.choose_database import choose_database
+from cellphonedb.src.app import cpdb_app
 from cellphonedb.src.local_launchers.local_query_launcher import LocalQueryLauncher
-from cellphonedb.src.app.cellphonedb_app import cellphonedb_app
+
+
+def common_options(f: Callable) -> Callable:
+    options = [
+        click.option('--verbose/--quiet', default=True, help='Print or hide cellphonedb logs [verbose]'),
+        click.option('--database', default='latest', callback=choose_database),
+    ]
+
+    for option in reversed(options):
+        f = option(f)
+
+    return f
 
 
 @click.command()
 @click.argument('element')
-def find_interactions_by_element(element: str):
-    LocalQueryLauncher(cellphonedb_app).find_interactions_by_element(element)
+@common_options
+def find_interactions_by_element(element: str, verbose: bool, database: str):
+    LocalQueryLauncher(cpdb_app.create_app(verbose, database)).find_interactions_by_element(element)
 
 
 @click.command()
 @click.option('--columns', default=None, help='Columns to set in the result')
-def get_interaction_gene(columns: str):
-    LocalQueryLauncher(cellphonedb_app).get_interaction_gene(columns)
+@common_options
+def get_interaction_gene(columns: str, verbose: bool, database: str):
+    LocalQueryLauncher(cpdb_app.create_app(verbose, database)).get_interaction_gene(columns)
 
 
 @click.command()
 @click.argument('partial_element')
-def autocomplete(partial_element: str) -> None:
-    LocalQueryLauncher(cellphonedb_app).autocomplete_element(partial_element)
+@common_options
+def autocomplete(partial_element: str, verbose: bool, database: str) -> None:
+    LocalQueryLauncher(cpdb_app.create_app(verbose, database)).autocomplete_element(partial_element)

cellphonedb/src/api_endpoints/terminal_api/util/choose_database.py

@@ -0,0 +1,9 @@
+from typing import Optional
+
+from click import Context, Argument
+
+from cellphonedb.src.database.manager import DatabaseVersionManager
+
+
+def choose_database(ctx: Context, argument: Argument, value: str) -> Optional[str]:
+    return DatabaseVersionManager.find_database_for(value)

cellphonedb/src/core/queries/autocomplete_queries.py

@@ -1,21 +1,29 @@
-import pandas as pd
 import re
 
+import pandas as pd
+
 
 def autocomplete_query(genes: pd.DataFrame, multidatas: pd.DataFrame, partial_element: pd.DataFrame) -> pd.DataFrame:
-    values = genes[genes['ensembl'].str.contains(partial_element, flags=re.IGNORECASE)]['ensembl']
-    values = values.append(
-        genes[genes['protein_name'].str.contains(partial_element, flags=re.IGNORECASE)]['protein_name'],
-        ignore_index=True)
-    values = values.append(
-        genes[genes['gene_name'].str.contains(partial_element, flags=re.IGNORECASE)]['gene_name'],
-        ignore_index=True)
-    values = values.append(
-        genes[genes['hgnc_symbol'].str.contains(partial_element, flags=re.IGNORECASE)]['hgnc_symbol'],
-        ignore_index=True)
-    values = values.append(
-        multidatas[multidatas['name'].str.contains(partial_element, flags=re.IGNORECASE)]['name'],
-        ignore_index=True)
-    result = pd.DataFrame(data=values, columns=['value'])
+    values = _partial_filter(genes, 'ensembl', partial_element)
+
+    by_protein_name = _partial_filter(genes, 'protein_name', partial_element)
+    by_gene_name = _partial_filter(genes, 'gene_name', partial_element)
+
+    with_hgnc_symbol = genes.dropna(subset=['hgnc_symbol'])
+    by_hgnc_symbol = _partial_filter(with_hgnc_symbol, 'hgnc_symbol', partial_element)
+
+    by_name = _partial_filter(multidatas, 'name', partial_element)
+
+    values = values.append(by_protein_name, ignore_index=True)
+    values = values.append(by_gene_name, ignore_index=True)
+    values = values.append(by_hgnc_symbol, ignore_index=True)
+    values = values.append(by_name, ignore_index=True)
+
+    result = pd.DataFrame(data=values, columns=['value']).drop_duplicates()
 
     return result
+
+
+def _partial_filter(input_data, name, partial_element):
+    matching = input_data[input_data[name].str.contains(partial_element, flags=re.IGNORECASE)][name]
+    return matching

cellphonedb/src/core/tests/queries/test_autocomplete_queries.py

@@ -0,0 +1,56 @@
+import random
+from unittest import TestCase
+
+from cellphonedb.src.app.app_config import AppConfig
+from cellphonedb.src.core.CellphonedbSqlalchemy import CellphonedbSqlalchemy
+
+
+class TestAutocompleteQueries(TestCase):
+
+    def setUp(self) -> None:
+        self.cellphone = CellphonedbSqlalchemy(AppConfig().get_cellphone_core_config())
+        gene_repository = self.cellphone.database_manager.get_repository('gene')
+        self.all_genes = gene_repository.get_all_expanded().to_dict(orient='records')
+
+    def test_find_elements_by_gene_name(self):
+        self._test_find_elements_by_('gene_name')
+
+    def test_find_elements_by_protein_name(self):
+        self._test_find_elements_by_('protein_name')
+
+    def test_find_elements_by_hgnc_symbol(self):
+        self._test_find_elements_by_('hgnc_symbol')
+
+    def test_find_elements_by_ensembl(self):
+        self._test_find_elements_by_('ensembl')
+
+    def test_find_elements_by_name(self):
+        self._test_find_elements_by_('name')
+
+    def _test_find_elements_by_(self, field):
+        random_gene = random.choice(self.all_genes)
+        whole_input = random_gene[field]
+
+        whole_query_result = self.cellphone.query.autocomplete_launcher(whole_input)
+
+        whole_results = whole_query_result['value'].tolist()
+        self.assertIn(whole_input, whole_results)
+
+        partial_input = self._random_substring(whole_input)
+
+        partial_query_result = self.cellphone.query.autocomplete_launcher(partial_input)
+
+        partial_results = partial_query_result['value'].tolist()
+        self.assertIn(whole_input, partial_results)
+
+        self.assertGreaterEqual(len(partial_results), len(whole_results))
+
+    def _random_substring(self, whole_input):
+        start_index = self._random_position_to_half(whole_input)
+        end_index = self._random_position_to_half(whole_input)
+
+        return whole_input[start_index:-end_index if end_index else None]
+
+    @staticmethod
+    def _random_position_to_half(string):
+        return random.randint(0, int((len(string)) / 2))

cellphonedb/src/local_launchers/local_query_launcher.py

@@ -14,13 +14,13 @@ def __init__(self, cellphonedb_app):
         self.cellphonedb_app = cellphonedb_app
 
     def find_interactions_by_element(self, element: str) -> None:
-        print(self.cellphonedb_app.cellphonedb.query.find_interactions_by_element(element).to_csv(index=False))
+        print(self.cellphonedb_app.query.find_interactions_by_element(element).to_csv(index=False))
 
     def get_interaction_gene(self, columns: str) -> None:
         if columns:
             columns = columns.split(',')
 
-        print(self.cellphonedb_app.cellphonedb.query.get_all_genes(columns).to_csv(index=False))
+        print(self.cellphonedb_app.query.get_all_genes(columns).to_csv(index=False))
 
     def autocomplete_element(self, partial_element: str) -> None:
-        print(self.cellphonedb_app.cellphonedb.query.autocomplete_launcher(partial_element).to_csv(index=False))
+        print(self.cellphonedb_app.query.autocomplete_launcher(partial_element).to_csv(index=False))

cellphonedb/src/plotters/R/plot_dot_by_column_name.R

@@ -14,23 +14,22 @@ dot_plot = function(selected_rows = NULL,
   all_pval = read.table(pvalues_path, header=T, stringsAsFactors = F, sep=means_separator, comment.char = '', check.names=F)
   all_means = read.table(means_path, header=T, stringsAsFactors = F, sep=pvalues_separator, comment.char = '', check.names=F)
 
-  rownames(all_pval) = all_pval$interacting_pair
-  rownames(all_means) = all_means$interacting_pair
+  intr_pairs = all_pval$interacting_pair
   all_pval = all_pval[,-c(1:11)]
   all_means = all_means[,-c(1:11)]
 
   if(is.null(selected_rows)){
-    selected_rows = rownames(all_pval)
+    selected_rows = intr_pairs
   }
 
   if(is.null(selected_columns)){
     selected_columns = colnames(all_pval)
   }
 
-  sel_pval = all_pval[selected_rows, selected_columns]
-  sel_means = all_means[selected_rows, selected_columns]
+  sel_pval = all_pval[match(selected_rows, intr_pairs), selected_columns]
+  sel_means = all_means[match(selected_rows, intr_pairs), selected_columns]
 
-  df_names = expand.grid(rownames(sel_pval), colnames(sel_pval))
+  df_names = expand.grid(selected_rows, selected_columns)
   pval = unlist(sel_pval)
   pval[pval==0] = 0.0009
   plot.data = cbind(df_names,pval)

cellphonedb/tools/generate_data/getters/get_imex.py

@@ -155,7 +155,7 @@ def _get_chunked_api_results(carry, columns_to_save, proteins, source):
 
 def _get_single_api_results(carry, columns_to_save, source):
     url = source['base_url']
-    print('Fetching {}'.format(source['name']))
+    tqdm.tqdm.write('Fetching {}'.format(source['name']))
     try:
         response = requests.get(url)
         if response.text:

rabbit_logger.py

@@ -0,0 +1,24 @@
+import logging
+import sys
+
+
+class RabbitLogger:
+    def __init__(self):
+        self._logger = logging.getLogger(__name__)
+        formatter = logging.Formatter('[ ][QUEUE][%(asctime)s][%(levelname)s] %(message)s', "%d/%m/%y-%H:%M:%S")
+        handler = logging.StreamHandler(sys.stdout)
+        handler.setFormatter(formatter)
+        self._logger.addHandler(handler)
+        self._logger.setLevel(logging.INFO)
+
+    def __getattr__(self, item):
+        return getattr(self._logger, item)
+
+
+class RabbitAdapter(logging.LoggerAdapter):
+    def process(self, msg, kwargs):
+        return '[{}] {}'.format(self.extra['job_id'], msg), kwargs
+
+    @classmethod
+    def logger_for(cls, logger, job_id):
+        return cls(logger, {'job_id': job_id})