Merge pull request #1 from CashStar/elasticsearch5_support

improved elasticsearch5 support

haystack/backends/elasticsearch5_backend.py

@@ -138,12 +138,6 @@ def build_search_kwargs(
 
         filters = []
 
-        if fields:
-            if isinstance(fields, (list, set)):
-                fields = " ".join(fields)
-
-            kwargs["stored_fields"] = fields
-
         if sort_by is not None:
             order_list = []
             for field, direction in sort_by:
@@ -272,6 +266,23 @@ def build_search_kwargs(
         if dwithin is not None:
             filters.append(self._build_search_query_dwithin(dwithin))
 
+        if limit_to_registered_models is None:
+            limit_to_registered_models = getattr(
+                settings, "HAYSTACK_LIMIT_TO_REGISTERED_MODELS", True
+            )
+
+        if models and len(models):
+            model_choices = sorted(get_model_ct(model) for model in models)
+        elif limit_to_registered_models:
+            # Using narrow queries, limit the results to only models handled
+            # with the current routers.
+            model_choices = self.build_models_list()
+        else:
+            model_choices = []
+
+        if len(model_choices) > 0:
+            filters.append({"terms": {DJANGO_CT: model_choices}})
+
         # if we want to filter, change the query type to bool
         if filters:
             kwargs["query"] = {"bool": {"must": kwargs.pop("query")}}

haystack/backends/elasticsearch_backend.py

@@ -153,7 +153,17 @@ def setup(self):
         # during the ``update`` & if it doesn't match, we'll put the new
         # mapping.
         try:
-            self.existing_mapping = self.conn.indices.get_mapping(index=self.index_name)
+            mapping_from_server = self.conn.indices.get_mapping(index=self.index_name)
+
+            # The mapping that comes back from elasticsearch is nested 2 levels deeper than the
+            # mapping we create for comparison. Strip those levels off here (the topmost level is the ACTUAL name of
+            # the index, even if the request was made using an alias, so we have to get the first value using iter/next
+            # instead of doing something like self.existing_mapping[self.index_name]
+            self.existing_mapping = next(iter(mapping_from_server.values()))['mappings']
+
+            # elasticsearch adds this to the response but haystack never includes it, so remove for comparison
+            if 'modelresult' in self.existing_mapping:
+                self.existing_mapping['modelresult']['properties'].pop('id', None)
         except NotFoundError:
             pass
         except Exception:
@@ -166,6 +176,10 @@ def setup(self):
         )
         current_mapping = {"modelresult": {"properties": field_mapping}}
 
+        explicit_dynamic_setting = haystack.connections[self.connection_alias].options.get('DYNAMIC', None)
+        if explicit_dynamic_setting is not None:
+            current_mapping['modelresult']['dynamic'] = explicit_dynamic_setting
+
         if current_mapping != self.existing_mapping:
             try:
                 # Make sure the index is there first.
@@ -558,12 +572,33 @@ def search(self, query_string, **kwargs):
         if end_offset is not None and end_offset > start_offset:
             search_kwargs["size"] = end_offset - start_offset
 
+        extra_source_kwargs = {}
+        fields = kwargs.get('fields', [])
+
+        if fields:
+            include_fields = []
+            exclude_fields = []
+            for field in fields:
+                if field.startswith('-'):
+                    exclude_fields.append(field.replace('-', '', 1))
+                else:
+                    include_fields.append(field)
+
+            if include_fields:
+                # process_results always needs these fields to be present
+                include_fields.extend(["id", "django_ct", "django_id", "score"])
+                extra_source_kwargs['_source_include'] = ','.join(include_fields)
+
+            if exclude_fields:
+                extra_source_kwargs['_source_exclude'] = ','.join(exclude_fields)
+
         try:
             raw_results = self.conn.search(
                 body=search_kwargs,
                 index=self.index_name,
                 doc_type="modelresult",
                 _source=True,
+                **extra_source_kwargs,
             )
         except elasticsearch.TransportError as e:
             if not self.silently_fail: