Skip to content

Commit

Permalink
Merge branch 'bulkReadWrite'
Browse files Browse the repository at this point in the history
  • Loading branch information
@sammefford
sammefford committed Jun 18, 2014
2 parents 0842bce + a02e8c2 commit b532dc3
Show file tree
Hide file tree
Showing 4 changed files with 115 additions and 43 deletions.
14 changes: 4 additions & 10 deletions src/main/java/com/marklogic/client/impl/DocumentManagerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,20 +417,14 @@ public void write(DocumentWriteSet writeSet, Transaction transaction) {
}

public void write(DocumentWriteSet writeSet, ServerTransform transform, Transaction transaction) {
JacksonHandle jacksonHandle = new JacksonHandle();
jacksonHandle = services.postBulkDocuments(
Format defaultFormat = contentFormat;
services.postBulkDocuments(
requestLogger,
writeSet,
(transform != null) ? transform : getWriteTransform(),
(transaction == null) ? null : transaction.getTransactionId(),
jacksonHandle);
JsonNode root = jacksonHandle.get();
for (JsonNode item: root.get("documents")) {
String uri = item.get("uri").asText();
String mimetype = item.get("mime-type").asText();
String category = item.get("category").get(0).asText();
}

defaultFormat,
null);
}

// shortcut writers
Expand Down
63 changes: 37 additions & 26 deletions src/main/java/com/marklogic/client/impl/JerseyServices.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLException;
import javax.ws.rs.core.Cookie;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.StreamingOutput;
Expand Down Expand Up @@ -1722,9 +1723,9 @@ private void updateMimetype(ContentDescriptor descriptor, String mimetype) {
}
}

private String getHeaderMimetype(MultivaluedMap<String, String> headers) {
if (headers.containsKey("Content-Type")) {
List<String> values = headers.get("Content-Type");
private String getHeaderMimetype(Map<String, List<String>> headers) {
if (headers.containsKey(HttpHeaders.CONTENT_TYPE)) {
List<String> values = headers.get(HttpHeaders.CONTENT_TYPE);
if (values != null) {
String contentType = values.get(0);
String mimetype = contentType.contains(";") ? contentType
Expand All @@ -1748,8 +1749,8 @@ private void updateLength(ContentDescriptor descriptor, long length) {
}

private long getHeaderLength(MultivaluedMap<String, String> headers) {
if (headers.containsKey("Content-Length")) {
List<String> values = headers.get("Content-Length");
if (headers.containsKey(HttpHeaders.CONTENT_LENGTH)) {
List<String> values = headers.get(HttpHeaders.CONTENT_LENGTH);
if (values != null) {
return Long.valueOf(values.get(0));
}
Expand Down Expand Up @@ -3297,14 +3298,13 @@ public <R extends AbstractReadHandle, W extends AbstractWriteHandle> R postResou
@Override
public <R extends AbstractReadHandle, W extends AbstractWriteHandle> R postResource(
RequestLogger reqlog, String path, RequestParameters params,
W[] input, Map<String, List>[] headers, R output) throws ResourceNotFoundException,
W[] input, Map<String, List<String>>[] headers, R output) throws ResourceNotFoundException,
ResourceNotResendableException, ForbiddenUserException,
FailedRequestException {
HandleImplementation outputBase = HandleAccessor.checkHandle(output,
"read");
HandleImplementation outputBase = HandleAccessor.checkHandle(output, "read");

String outputMimetype = outputBase.getMimetype();
Class as = outputBase.receiveAs();
String outputMimetype = outputBase != null ? outputBase.getMimetype() : null;
Class as = outputBase != null ? outputBase.receiveAs() : null;

ClientResponse response = null;
ClientResponse.Status status = null;
Expand Down Expand Up @@ -3371,45 +3371,49 @@ public <R extends AbstractReadHandle, W extends AbstractWriteHandle> R postResou
@Override
public void postBulkDocuments(
RequestLogger reqlog, DocumentWriteSet writeSet,
ServerTransform transform, String transactionId)
ServerTransform transform, Format defaultFormat, String transactionId)
throws ForbiddenUserException, FailedRequestException
{
postBulkDocuments(reqlog, writeSet, transform, transactionId, null);
postBulkDocuments(reqlog, writeSet, transform, transactionId, defaultFormat, null);
}

@Override
public <R extends AbstractReadHandle> R postBulkDocuments(
RequestLogger reqlog, DocumentWriteSet writeSet,
ServerTransform transform, String transactionId, R output)
ServerTransform transform, String transactionId, Format defaultFormat, R output)
throws ForbiddenUserException, FailedRequestException
{
ArrayList<AbstractWriteHandle> writeHandles = new ArrayList<AbstractWriteHandle>();
ArrayList<Map<String, List>> headerList = new ArrayList<Map<String, List>>();
ArrayList<Map<String, List<String>>> headerList = new ArrayList<Map<String, List<String>>>();
for ( DocumentWriteOperation write: writeSet ) {
HandleImplementation metadata =
HandleAccessor.checkHandle(write.getMetadata(), "write");
HandleImplementation content =
HandleAccessor.checkHandle(write.getContent(), "write");
MultivaluedMap headers = new MultivaluedMapImpl();
if ( metadata != null ) {
headers.add("Content-Type", metadata.getMimetype());
MultivaluedMap headers = new MultivaluedMapImpl();
headers.add(HttpHeaders.CONTENT_TYPE, metadata.getMimetype());
if ( write.getOperationType() == DocumentWriteOperation.OperationType.METADATA_DEFAULT ) {
headers.add("Content-Disposition",
ContentDisposition.type("inline").build().toString()
);
headers.add("Content-Disposition", "inline; category=metadata");
} else {
headers.add("Content-Disposition",
ContentDisposition
.type("attachment")
.fileName(write.getUri())
.build().toString()
.build().toString() +
"; category=metadata"
);
}
headerList.add(headers);
writeHandles.add(write.getMetadata());
}
if ( content != null ) {
headers.add("Content-Type", content.getMimetype());
MultivaluedMap headers = new MultivaluedMapImpl();
String mimeType = content.getMimetype();
if ( mimeType == null && defaultFormat != null ) {
mimeType = defaultFormat.getDefaultMimetype();
}
headers.add(HttpHeaders.CONTENT_TYPE, mimeType);
headers.add("Content-Disposition",
ContentDisposition
.type("attachment")
Expand All @@ -3427,7 +3431,7 @@ public <R extends AbstractReadHandle> R postBulkDocuments(
"documents",
params,
(AbstractWriteHandle[]) writeHandles.toArray(new AbstractWriteHandle[0]),
(Map<String, List>[]) headerList.toArray(new HashMap[0]),
(Map<String, List<String>>[]) headerList.toArray(new HashMap[0]),
output);
}

Expand Down Expand Up @@ -3929,7 +3933,7 @@ private <W extends AbstractWriteHandle> boolean addParts(

private <W extends AbstractWriteHandle> boolean addParts(
MultiPart multiPart, RequestLogger reqlog, String[] mimetypes,
W[] input, Map<String, List>[] headers) {
W[] input, Map<String, List<String>>[] headers) {
if (mimetypes != null && mimetypes.length != input.length)
throw new IllegalArgumentException(
"Mismatch between count of mimetypes and input");
Expand All @@ -3944,13 +3948,19 @@ private <W extends AbstractWriteHandle> boolean addParts(
AbstractWriteHandle handle = input[i];
HandleImplementation handleBase = HandleAccessor.checkHandle(
handle, "write");
Object value = handleBase.sendContent();
String inputMimetype = (mimetypes != null) ? mimetypes[i]
: handleBase.getMimetype();

if (!hasStreamingPart)
hasStreamingPart = !handleBase.isResendable();

Object value = handleBase.sendContent();

String inputMimetype = null;
if ( mimetypes != null ) inputMimetype = mimetypes[i];
if ( inputMimetype == null && headers != null ) {
inputMimetype = getHeaderMimetype(headers[i]);
}
if ( inputMimetype == null ) inputMimetype = handleBase.getMimetype();

String[] typeParts = (inputMimetype != null && inputMimetype
.contains("/")) ? inputMimetype.split("/", 2) : null;

Expand Down Expand Up @@ -4737,4 +4747,5 @@ public InputStream match(String[] docIds, String[] candidateRules, ServerTransfo

return entity;
}

}
6 changes: 3 additions & 3 deletions src/main/java/com/marklogic/client/impl/RESTServices.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,10 @@ public DocumentPage getBulkDocuments(RequestLogger logger, QueryDefinition query
throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException;

public void postBulkDocuments(RequestLogger logger, DocumentWriteSet writeSet,
ServerTransform transform, String transactionId)
ServerTransform transform, Format defaultFormat, String transactionId)
throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException;
public <T extends AbstractReadHandle> T postBulkDocuments(RequestLogger logger, DocumentWriteSet writeSet,
ServerTransform transform, String transactionId, T output)
ServerTransform transform, String transactionId, Format defaultFormat, T output)
throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException;

public void putDocument(RequestLogger logger, DocumentDescriptor desc, String transactionId,
Expand Down Expand Up @@ -186,7 +186,7 @@ public <R extends AbstractReadHandle, W extends AbstractWriteHandle> R postResou
ResourceNotResendableException, ForbiddenUserException, FailedRequestException;
public <R extends AbstractReadHandle, W extends AbstractWriteHandle> R postResource(
RequestLogger reqlog, String path, RequestParameters params,
W[] input, Map<String, List>[] headers, R output)
W[] input, Map<String, List<String>>[] headers, R output)
throws ResourceNotFoundException, ResourceNotResendableException,
ResourceNotResendableException, ForbiddenUserException, FailedRequestException;
public ServiceResultIterator postIteratedResource(
Expand Down
75 changes: 71 additions & 4 deletions src/test/java/com/marklogic/client/test/BulkReadWriteTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.FileHandler;
import java.util.logging.Handler;
import java.util.logging.Logger;
import java.util.logging.Level;

import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
Expand All @@ -39,10 +43,15 @@
import com.marklogic.client.document.DocumentPage;
import com.marklogic.client.document.DocumentRecord;
import com.marklogic.client.document.DocumentWriteSet;
import com.marklogic.client.document.JSONDocumentManager;
import com.marklogic.client.document.TextDocumentManager;
import com.marklogic.client.document.XMLDocumentManager;
import com.marklogic.client.io.DocumentMetadataHandle;
import com.marklogic.client.io.Format;
import com.marklogic.client.io.JacksonHandle;
import com.marklogic.client.io.JAXBHandle;
import com.marklogic.client.io.SearchHandle;
import com.marklogic.client.io.StringHandle;
import com.marklogic.client.query.DeleteQueryDefinition;
import com.marklogic.client.query.QueryManager;
import com.marklogic.client.query.StructuredQueryBuilder;
Expand All @@ -63,9 +72,12 @@ public class BulkReadWriteTest {
public static void beforeClass() throws JAXBException {
Common.connect();
context = JAXBContext.newInstance(City.class);
//System.setProperty("org.apache.commons.logging.simplelog.log.org.apache.http.wire", "debug");
//System.setProperty("org.apache.commons.logging.simplelog.log.org.apache.http", "debug");
}
@AfterClass
public static void afterClass() {
cleanUp();
Common.release();
}

Expand Down Expand Up @@ -306,7 +318,7 @@ public void testBulkLoad() throws IOException, JAXBException {
}

@Test
public void testBulkRead() throws IOException, JAXBException {
public void testBulkRead() {
XMLDocumentManager docMgr = Common.client.newXMLDocumentManager();

DocumentPage page = docMgr.read(DIRECTORY + "1016670.xml", DIRECTORY + "108410.xml", DIRECTORY + "1205733.xml");
Expand All @@ -320,7 +332,7 @@ public void testBulkRead() throws IOException, JAXBException {
}

@Test
public void testBulkSearch() throws IOException, JAXBException {
public void testBulkSearch() {
XMLDocumentManager docMgr = Common.client.newXMLDocumentManager();

SearchHandle searchHandle = new SearchHandle();
Expand All @@ -335,7 +347,37 @@ public void testBulkSearch() throws IOException, JAXBException {
assertEquals("Failed to find number of records expected", RECORDS_EXPECTED, page.getTotalSize());
assertEquals("SearchHandle failed to report number of records expected", RECORDS_EXPECTED, searchHandle.getTotalResults());
assertEquals("SearchHandle failed to report pageLength expected", pageLength, searchHandle.getPageLength());
cleanUp();
}

//public void testMixedLoad() {
@Test
public void testJsonLoad() {
JSONDocumentManager docMgr = Common.client.newJSONDocumentManager();

StringHandle doc1 =
new StringHandle("{\"animal\": \"dog\", \"says\": \"woof\"}").withFormat(Format.JSON);

StringHandle doc2 =
new StringHandle("{\"animal\": \"cat\", \"says\": \"meow\"}").withFormat(Format.JSON);

StringHandle doc2Metadata =
new StringHandle("{\"quality\" : 2.0}").withFormat(Format.JSON);

DocumentWriteSet writeSet = docMgr.newWriteSet();
writeSet.add("doc1.json", doc1);
writeSet.add("doc2.json", doc2Metadata, doc2);

docMgr.write(writeSet);

JacksonHandle content1 = new JacksonHandle();
docMgr.read("doc1.json", content1);
JacksonHandle content2 = new JacksonHandle();
DocumentMetadataHandle metadata2 = new DocumentMetadataHandle();
docMgr.read("doc2.json", metadata2, content2);

assertEquals("Failed to read document 1", "dog", content1.get().get("animal").textValue());
assertEquals("Failed to read expected quality", 2, metadata2.getQuality());
assertEquals("Failed to read document 2", "cat", content2.get().get("animal").textValue());
}

public void validateRecord(DocumentRecord record) {
Expand All @@ -360,6 +402,28 @@ public void validateRecord(DocumentRecord record) {
}
}

@Test
public void testTextLoad() {
String docId[] = {"/foo/test/myFoo1.xml","/foo/test/myFoo2.xml","/foo/test/myFoo3.xml"};
TextDocumentManager docMgr = Common.client.newTextDocumentManager();
DocumentWriteSet writeset =docMgr.newWriteSet();

writeset.add(docId[0], new StringHandle().with("This is so foo1"));
writeset.add(docId[1], new StringHandle().with("This is so foo2"));
writeset.add(docId[2], new StringHandle().with("This is so foo3"));
docMgr.write(writeset);

assertEquals("Text document write difference", "This is so foo1", docMgr.read(docId[0], new StringHandle()).get());
assertEquals("Text document write difference", "This is so foo2", docMgr.read(docId[1], new StringHandle()).get());
assertEquals("Text document write difference", "This is so foo3", docMgr.read(docId[2], new StringHandle()).get());

docMgr.delete(docId[0]);
docMgr.delete(docId[1]);
docMgr.delete(docId[2]);
}



private static void addCountry(String line, Map<String, Country> countries) {
// skip comment lines
if ( line.startsWith("#") ) return;
Expand Down Expand Up @@ -408,10 +472,13 @@ private static City newCity(String line, Map<String, Country> countries) {
}
}

private void cleanUp() {
private static void cleanUp() {
QueryManager queryMgr = Common.client.newQueryManager();
DeleteQueryDefinition deleteQuery = queryMgr.newDeleteDefinition();
deleteQuery.setDirectory("/cities/");
queryMgr.delete(deleteQuery);
JSONDocumentManager docMgr = Common.client.newJSONDocumentManager();
docMgr.delete("doc1.json");
docMgr.delete("doc2.json");
}
}

0 comments on commit b532dc3

Please sign in to comment.