diff --git a/config.py.example b/config.py.example index 10c31fa4..4883d8e7 100644 --- a/config.py.example +++ b/config.py.example @@ -58,6 +58,8 @@ RATELIMIT_WINDOW = 10 DATASET_DIR = "/data/datasets" FILE_STORAGE_DIR = "/data/files" SIMILARITY_INDEX_DIR = "/data/annoy_indices" +# How many threads to use when building an annoy index +SIMILARITY_BUILD_NUM_JOBS = 1 #Feature Flags # Choose a server to perform the evaluation on diff --git a/consul_config.py.ctmpl b/consul_config.py.ctmpl index 38e4bea6..ef16b619 100644 --- a/consul_config.py.ctmpl +++ b/consul_config.py.ctmpl @@ -48,7 +48,9 @@ LOG_SENTRY = { DATASET_DIR = '''{{template "KEY" "dataset_dir"}}''' FILE_STORAGE_DIR = '''{{template "KEY" "file_storage_dir"}}''' -SIMILARITY_INDEX_DIR = '''{{template "KEY" "similarity_index_dir"}}''' +SIMILARITY_INDEX_DIR = '''{{template "KEY" "similarity/index_dir"}}''' +# How many threads to use when building an annoy index +SIMILARITY_BUILD_NUM_JOBS = {{template "KEY" "similarity/build_n_jobs"}} #Feature Flags # Choose a server to perform the evaluation on diff --git a/similarity/index_model.py b/similarity/index_model.py index 3f57f0d2..6ec9f947 100644 --- a/similarity/index_model.py +++ b/similarity/index_model.py @@ -54,7 +54,8 @@ def parse_initial_params(self, metric_name, n_trees, distance_type): def build(self): """Build and load the index using the specified number of trees. An index must be built before it can be queried.""" - self.index.build(self.n_trees) + n_jobs = current_app.config['SIMILARITY_BUILD_NUM_JOBS'] + self.index.build(n_trees=self.n_trees, n_jobs=n_jobs) self.in_loaded_state = True def save(self, location=None, name=None):