crate · apatrida · Nov 11, 2013
diff --git a/pom.xml b/pom.xml
@@ -5,7 +5,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>crate</groupId>
 	<artifactId>elasticsearch-inout-plugin</artifactId>
-	<version>0.5.0</version>
+	<version>0.6.0</version>
     <packaging>jar</packaging>
     <description>An Elasticsearch plugin which provides the ability to export and import data by query on server side.</description>
     <scm>
@@ -14,7 +14,7 @@
         <url>https://github.com/crate/elasticsearch-inout-plugin</url>
     </scm>
 	<properties>
-		<elasticsearch.version>0.90.3</elasticsearch.version>
+		<elasticsearch.version>1.0.0.beta1</elasticsearch.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     </properties>
 	<build>

diff --git a/src/main/java/crate/elasticsearch/action/export/AbstractTransportExportAction.java b/src/main/java/crate/elasticsearch/action/export/AbstractTransportExportAction.java
@@ -149,7 +149,7 @@ protected ShardExportResponse shardOperation(ShardExportRequest request) throws
         SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().id(), request.index(), request.shardId());
         ExportContext context = new ExportContext(0,
             new ShardSearchRequest().types(request.types()).filteringAliases(request.filteringAliases()),
-            shardTarget, indexShard.searcher(), indexService, indexShard, scriptService, cacheRecycler, nodePath);
+            shardTarget, indexShard.acquireSearcher("inout-plugin"), indexService, indexShard, scriptService, cacheRecycler, nodePath);
         ExportContext.setCurrent(context);
 
         try {

diff --git a/src/main/java/crate/elasticsearch/action/export/ExportContext.java b/src/main/java/crate/elasticsearch/action/export/ExportContext.java
@@ -10,6 +10,7 @@
 import org.elasticsearch.index.shard.service.IndexShard;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.internal.DefaultSearchContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 
@@ -20,7 +21,7 @@
 /**
  * Container class for export specific informations.
  */
-public class ExportContext extends SearchContext {
+public class ExportContext extends DefaultSearchContext {
 
     private static final String VAR_SHARD = "${shard}";
     private static final String VAR_INDEX = "${index}";

diff --git a/src/main/java/crate/elasticsearch/action/export/ExportRequest.java b/src/main/java/crate/elasticsearch/action/export/ExportRequest.java
@@ -3,7 +3,6 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.broadcast.BroadcastOperationRequest;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.Required;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -60,13 +59,11 @@ public BytesReference source() {
         return source;
     }
 
-    @Required
     public ExportRequest source(String source) {
         return this.source(new BytesArray(source), false);
     }
 
 
-    @Required
     public ExportRequest source(BytesReference source, boolean unsafe) {
         this.source = source;
         this.querySourceUnsafe = unsafe;

diff --git a/src/main/java/crate/elasticsearch/action/import_/ImportRequest.java b/src/main/java/crate/elasticsearch/action/import_/ImportRequest.java
@@ -1,7 +1,6 @@
 package crate.elasticsearch.action.import_;
 
 import org.elasticsearch.action.support.nodes.NodesOperationRequest;
-import org.elasticsearch.common.Required;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -32,12 +31,10 @@ public BytesReference source() {
         return source;
     }
 
-    @Required
     public ImportRequest source(String source) {
         return this.source(new BytesArray(source), false);
     }
 
-    @Required
     public ImportRequest source(BytesReference source, boolean unsafe) {
         this.source = source;
         return this;

diff --git a/src/main/java/crate/elasticsearch/action/searchinto/AbstractTransportSearchIntoAction.java b/src/main/java/crate/elasticsearch/action/searchinto/AbstractTransportSearchIntoAction.java
@@ -168,7 +168,7 @@ protected ShardSearchIntoResponse shardOperation(ShardSearchIntoRequest
                 request.shardId());
         SearchIntoContext context = new SearchIntoContext(0,
             new ShardSearchRequest().types(request.types()).filteringAliases(request.filteringAliases()),
-            shardTarget, indexShard.searcher(), indexService, indexShard, scriptService, cacheRecycler
+            shardTarget, indexShard.acquireSearcher("inout-plugin"), indexService, indexShard, scriptService, cacheRecycler
         );
         SearchIntoContext.setCurrent(context);
 

diff --git a/src/main/java/crate/elasticsearch/action/searchinto/SearchIntoContext.java b/src/main/java/crate/elasticsearch/action/searchinto/SearchIntoContext.java
@@ -9,7 +9,7 @@
 import org.elasticsearch.index.shard.service.IndexShard;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.DefaultSearchContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 
 import java.util.HashMap;
@@ -19,7 +19,7 @@
 /**
  * Container class for inout specific informations.
  */
-public class SearchIntoContext extends SearchContext {
+public class SearchIntoContext extends DefaultSearchContext {
 
     // currently we only support index targets
     private String targetType = "index";

diff --git a/src/main/java/crate/elasticsearch/action/searchinto/SearchIntoRequest.java b/src/main/java/crate/elasticsearch/action/searchinto/SearchIntoRequest.java
@@ -3,7 +3,6 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.broadcast.BroadcastOperationRequest;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.Required;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -63,13 +62,11 @@ public BytesReference source() {
         return source;
     }
 
-    @Required
     public SearchIntoRequest source(String source) {
         return this.source(new BytesArray(source), false);
     }
 
 
-    @Required
     public SearchIntoRequest source(BytesReference source, boolean unsafe) {
         this.source = source;
         this.querySourceUnsafe = unsafe;

diff --git a/src/main/java/crate/elasticsearch/export/ExportCollector.java b/src/main/java/crate/elasticsearch/export/ExportCollector.java
@@ -23,15 +23,13 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor;
-import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
-import org.elasticsearch.index.fieldvisitor.JustUidFieldsVisitor;
-import org.elasticsearch.index.fieldvisitor.UidAndSourceFieldsVisitor;
+import org.elasticsearch.index.fieldvisitor.*;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMappers;
 import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.fetch.FetchSubPhase;
+import org.elasticsearch.search.fetch.source.FetchSourceContext;
 import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.InternalSearchHitField;
 
@@ -62,17 +60,21 @@ public ExportCollector(ExportContext context,
 
         if (!context.hasFieldNames()) {
             if (context.hasPartialFields()) {
-                // partial fields need the source, so fetch it, but don't return it
+                // partial fields need the source, so fetch it
                 fieldsVisitor = new UidAndSourceFieldsVisitor();
-            } else if (context.hasScriptFields()) {
-                // we ask for script fields, and no field names, don't load the source
-                fieldsVisitor = new JustUidFieldsVisitor();
             } else {
-                sourceRequested = true;
-                fieldsVisitor = new UidAndSourceFieldsVisitor();
+                // no fields specified, default to return source if no explicit indication
+                if (!context.hasScriptFields() && !context.hasFetchSourceContext()) {
+                    context.fetchSourceContext(new FetchSourceContext(true));
+                }
+                fieldsVisitor = context.sourceRequested() ? new UidAndSourceFieldsVisitor() : new JustUidFieldsVisitor();
             }
         } else if (context.fieldNames().isEmpty()) {
-            fieldsVisitor = new JustUidFieldsVisitor();
+            if (context.sourceRequested()) {
+                fieldsVisitor = new UidAndSourceFieldsVisitor();
+            } else {
+                fieldsVisitor = new JustUidFieldsVisitor();
+            }
         } else {
             boolean loadAllStored = false;
             Set<String> fieldNames = null;
@@ -82,7 +84,11 @@ public ExportCollector(ExportContext context,
                     continue;
                 }
                 if (fieldName.equals(SourceFieldMapper.NAME)) {
-                    sourceRequested = true;
+                    if (context.hasFetchSourceContext()) {
+                        context.fetchSourceContext().fetchSource(true);
+                    } else {
+                        context.fetchSourceContext(new FetchSourceContext(true));
+                    }
                     continue;
                 }
                 FieldMappers x = context.smartNameFieldMappers
@@ -100,15 +106,11 @@ public ExportCollector(ExportContext context,
                 }
             }
             if (loadAllStored) {
-                if (sourceRequested || extractFieldNames != null) {
-                    fieldsVisitor = new CustomFieldsVisitor(true, true); // load everything, including _source
-                } else {
-                    fieldsVisitor = new CustomFieldsVisitor(true, false);
-                }
+                fieldsVisitor = new AllFieldsVisitor(); // load everything, including _source
             } else if (fieldNames != null) {
-                boolean loadSource = extractFieldNames != null || sourceRequested;
+                boolean loadSource = extractFieldNames != null || context.sourceRequested();
                 fieldsVisitor = new CustomFieldsVisitor(fieldNames, loadSource);
-            } else if (extractFieldNames != null || sourceRequested) {
+            } else if (extractFieldNames != null || context.sourceRequested()) {
                 fieldsVisitor = new UidAndSourceFieldsVisitor();
             } else {
                 fieldsVisitor = new JustUidFieldsVisitor();
@@ -159,8 +161,8 @@ public void collect(int doc) throws IOException {
 
         InternalSearchHit searchHit = new InternalSearchHit(doc,
                 fieldsVisitor.uid().id(), typeText,
-                sourceRequested ? fieldsVisitor.source() : null,
-                searchFields);
+                searchFields).sourceRef(fieldsVisitor.source());
+
 
         for (FetchSubPhase fetchSubPhase : fetchSubPhases) {
             FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext();

diff --git a/src/main/java/crate/elasticsearch/rest/action/admin/export/RestExportAction.java b/src/main/java/crate/elasticsearch/rest/action/admin/export/RestExportAction.java
@@ -3,7 +3,6 @@
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
 import static org.elasticsearch.rest.RestStatus.OK;
-import static org.elasticsearch.rest.action.support.RestActions.splitTypes;
 
 import java.io.IOException;
 
@@ -12,6 +11,7 @@
 import org.elasticsearch.action.support.IgnoreIndices;
 import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -52,7 +52,7 @@ protected Action<ExportRequest, ExportResponse, ExportRequestBuilder> action() {
     }
 
     public void handleRequest(final RestRequest request, final RestChannel channel) {
-        ExportRequest exportRequest = new ExportRequest(RestActions.splitIndices(request.param("index")));
+        ExportRequest exportRequest = new ExportRequest(Strings.splitStringByCommaToArray(request.param("index")));
 
         if (request.hasParam("ignore_indices")) {
             exportRequest.ignoreIndices(IgnoreIndices.fromString(request.param("ignore_indices")));
@@ -79,7 +79,7 @@ public void handleRequest(final RestRequest request, final RestChannel channel)
                 }
             }
             exportRequest.routing(request.param("routing"));
-            exportRequest.types(splitTypes(request.param("type")));
+            exportRequest.types(Strings.splitStringByCommaToArray(request.param("type")));
             exportRequest.preference(request.param("preference", "_primary"));
         } catch (Exception e) {
             try {

diff --git a/src/main/java/crate/elasticsearch/rest/action/admin/searchinto/RestSearchIntoAction.java b/src/main/java/crate/elasticsearch/rest/action/admin/searchinto/RestSearchIntoAction.java
@@ -3,7 +3,6 @@
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
 import static org.elasticsearch.rest.RestStatus.OK;
-import static org.elasticsearch.rest.action.support.RestActions.splitTypes;
 
 import java.io.IOException;
 
@@ -12,6 +11,7 @@
 import org.elasticsearch.action.support.IgnoreIndices;
 import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -55,7 +55,7 @@ protected Action<SearchIntoRequest, SearchIntoResponse, SearchIntoRequestBuilder
     public void handleRequest(final RestRequest request,
             final RestChannel channel) {
         SearchIntoRequest searchIntoRequest = new SearchIntoRequest(
-                RestActions.splitIndices(request.param("index")));
+                Strings.splitStringByCommaToArray(request.param("index")));
 
         if (request.hasParam("ignore_indices")) {
             searchIntoRequest.ignoreIndices(IgnoreIndices.fromString(
@@ -90,7 +90,7 @@ public void handleRequest(final RestRequest request,
                 }
             }
             searchIntoRequest.routing(request.param("routing"));
-            searchIntoRequest.types(splitTypes(request.param("type")));
+            searchIntoRequest.types(Strings.splitStringByCommaToArray(request.param("type")));
             searchIntoRequest.preference(request.param("preference",
                     "_primary"));
         } catch (Exception e) {

diff --git a/src/main/java/crate/elasticsearch/searchinto/WriterCollector.java b/src/main/java/crate/elasticsearch/searchinto/WriterCollector.java
@@ -9,16 +9,14 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.text.StringAndBytesText;
 import org.elasticsearch.common.text.Text;
-import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor;
-import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
-import org.elasticsearch.index.fieldvisitor.JustUidFieldsVisitor;
-import org.elasticsearch.index.fieldvisitor.UidAndSourceFieldsVisitor;
+import org.elasticsearch.index.fieldvisitor.*;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMappers;
 import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.fetch.FetchSubPhase;
+import org.elasticsearch.search.fetch.source.FetchSourceContext;
 import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.InternalSearchHitField;
 
@@ -79,7 +77,7 @@ public void collect(int doc) throws IOException {
 
         InternalSearchHit searchHit = new InternalSearchHit(doc,
                 fieldsVisitor.uid().id(), typeText,
-                sourceRequested ? fieldsVisitor.source() : null, searchFields);
+                searchFields).sourceRef(fieldsVisitor.source());
 
         // it looks like it is safe to reuse the HitContext,
         // the cache is only used by the highlighter which we do not use.
@@ -106,19 +104,21 @@ public WriterCollector(SearchIntoContext context,
         this.mappedFields = new MappedFields(context);
         if (!context.hasFieldNames()) {
             if (context.hasPartialFields()) {
-                // partial fields need the source, so fetch it,
-                // but don't return it
+                // partial fields need the source, so fetch it
                 fieldsVisitor = new UidAndSourceFieldsVisitor();
-            } else if (context.hasScriptFields()) {
-                // we ask for script fields, and no field names,
-                // don't load the source
-                fieldsVisitor = new JustUidFieldsVisitor();
             } else {
-                sourceRequested = true;
-                fieldsVisitor = new UidAndSourceFieldsVisitor();
+                // no fields specified, default to return source if no explicit indication
+                if (!context.hasScriptFields() && !context.hasFetchSourceContext()) {
+                    context.fetchSourceContext(new FetchSourceContext(true));
+                }
+                fieldsVisitor = context.sourceRequested() ? new UidAndSourceFieldsVisitor() : new JustUidFieldsVisitor();
             }
         } else if (context.fieldNames().isEmpty()) {
-            fieldsVisitor = new JustUidFieldsVisitor();
+            if (context.sourceRequested()) {
+                fieldsVisitor = new UidAndSourceFieldsVisitor();
+            } else {
+                fieldsVisitor = new JustUidFieldsVisitor();
+            }
         } else {
             boolean loadAllStored = false;
             Set<String> fieldNames = null;
@@ -128,7 +128,11 @@ public WriterCollector(SearchIntoContext context,
                     continue;
                 }
                 if (fieldName.equals(SourceFieldMapper.NAME)) {
-                    sourceRequested = true;
+                    if (context.hasFetchSourceContext()) {
+                        context.fetchSourceContext().fetchSource(true);
+                    } else {
+                        context.fetchSourceContext(new FetchSourceContext(true));
+                    }
                     continue;
                 }
                 FieldMappers x = context.smartNameFieldMappers(fieldName);
@@ -145,21 +149,11 @@ public WriterCollector(SearchIntoContext context,
                 }
             }
             if (loadAllStored) {
-                if (sourceRequested || extractFieldNames != null) {
-                    fieldsVisitor = new CustomFieldsVisitor(true,
-                            true); // load
-                    // everything,
-                    // including
-                    // _source
-                } else {
-                    fieldsVisitor = new CustomFieldsVisitor(true, false);
-                }
+                fieldsVisitor = new AllFieldsVisitor(); // load everything, including _source
             } else if (fieldNames != null) {
-                boolean loadSource = extractFieldNames != null ||
-                        sourceRequested;
-                fieldsVisitor = new CustomFieldsVisitor(fieldNames,
-                        loadSource);
-            } else if (extractFieldNames != null || sourceRequested) {
+                boolean loadSource = extractFieldNames != null || context.sourceRequested();
+                fieldsVisitor = new CustomFieldsVisitor(fieldNames, loadSource);
+            } else if (extractFieldNames != null || context.sourceRequested()) {
                 fieldsVisitor = new UidAndSourceFieldsVisitor();
             } else {
                 fieldsVisitor = new JustUidFieldsVisitor();