diff --git a/build.gradle b/build.gradle index 30e3e16..b07f141 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,6 @@ buildscript { ext { - elasticsearchVersion = '7.7.0' + elasticsearchVersion = '7.8.0' } repositories { @@ -63,5 +63,7 @@ dependencies { testCompile group: 'org.elasticsearch.test', name: 'framework', version: elasticsearchVersion } +validateNebulaPom.enabled = false dependencyLicenses.enabled = false thirdPartyAudit.enabled = false +licenseHeaders.enabled = false diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 1353677..62d4c05 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 844554a..622ab64 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,5 @@ -#Fri Sep 28 10:36:22 IDT 2018 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https://services.gradle.org/distributions/gradle-6.4.1-all.zip diff --git a/gradlew b/gradlew index cccdd3d..fbd7c51 100755 --- a/gradlew +++ b/gradlew @@ -1,5 +1,21 @@ #!/usr/bin/env sh +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + ############################################################################## ## ## Gradle start up script for UN*X @@ -28,7 +44,7 @@ APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" @@ -66,6 +82,7 @@ esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then @@ -109,10 +126,11 @@ if $darwin; then GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" fi -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` # We build the pattern for arguments to be converted via cygpath @@ -138,19 +156,19 @@ if $cygwin ; then else eval `echo args$i`="\"$arg\"" fi - i=$((i+1)) + i=`expr $i + 1` done case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; esac fi @@ -159,14 +177,9 @@ save () { for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done echo " " } -APP_ARGS=$(save "$@") +APP_ARGS=`save "$@"` # Collect all arguments for the java command, following the shell quoting and substitution rules eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" -fi - exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat index e95643d..a9f778a 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -1,3 +1,19 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + @if "%DEBUG%" == "" @echo off @rem ########################################################################## @rem @@ -13,8 +29,11 @@ if "%DIRNAME%" == "" set DIRNAME=. set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome @@ -65,6 +84,7 @@ set CMD_LINE_ARGS=%* set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + @rem Execute Gradle "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% diff --git a/src/main/java/org/wikimedia/elasticsearch/swift/repositories/SwiftRepository.java b/src/main/java/org/wikimedia/elasticsearch/swift/repositories/SwiftRepository.java index 47659fe..b6d44e0 100644 --- a/src/main/java/org/wikimedia/elasticsearch/swift/repositories/SwiftRepository.java +++ b/src/main/java/org/wikimedia/elasticsearch/swift/repositories/SwiftRepository.java @@ -16,7 +16,7 @@ package org.wikimedia.elasticsearch.swift.repositories; -import org.elasticsearch.cluster.metadata.RepositoryMetaData; +import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; @@ -91,7 +91,7 @@ public interface Swift { * an instance of ClusterService */ @Inject - public SwiftRepository(RepositoryMetaData metadata, Settings settings, + public SwiftRepository(RepositoryMetadata metadata, Settings settings, NamedXContentRegistry namedXContentRegistry, SwiftService swiftService, ClusterService clusterService) { super(metadata, Swift.COMPRESS_SETTING.get(metadata.settings()), namedXContentRegistry, clusterService); diff --git a/src/main/java/org/wikimedia/elasticsearch/swift/repositories/blobstore/SwiftBlobContainer.java b/src/main/java/org/wikimedia/elasticsearch/swift/repositories/blobstore/SwiftBlobContainer.java index 05e4904..d74e227 100644 --- a/src/main/java/org/wikimedia/elasticsearch/swift/repositories/blobstore/SwiftBlobContainer.java +++ b/src/main/java/org/wikimedia/elasticsearch/swift/repositories/blobstore/SwiftBlobContainer.java @@ -17,18 +17,21 @@ package org.wikimedia.elasticsearch.swift.repositories.blobstore; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.blobstore.BlobMetaData; +import org.elasticsearch.common.blobstore.BlobMetadata; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.DeleteResult; import org.elasticsearch.common.blobstore.support.AbstractBlobContainer; -import org.elasticsearch.common.blobstore.support.PlainBlobMetaData; +import org.elasticsearch.common.blobstore.support.PlainBlobMetadata; import org.elasticsearch.common.collect.MapBuilder; import org.javaswift.joss.exception.CommandException; import org.javaswift.joss.exception.NotFoundException; import org.javaswift.joss.model.Directory; import org.javaswift.joss.model.DirectoryOrObject; import org.javaswift.joss.model.StoredObject; +import org.javaswift.joss.instructions.DownloadInstructions; +import org.javaswift.joss.headers.object.range.AbstractRange; +import org.javaswift.joss.headers.object.range.MidPartRange; import org.wikimedia.elasticsearch.swift.SwiftPerms; import org.wikimedia.elasticsearch.swift.repositories.SwiftRepository; @@ -133,9 +136,9 @@ public void deleteBlobsIgnoringIfNotExists(List blobNames) throws IOExce * @return blobs metadata */ @Override - public Map listBlobsByPrefix(@Nullable final String blobNamePrefix) { + public Map listBlobsByPrefix(@Nullable final String blobNamePrefix) { return SwiftPerms.exec(() -> { - MapBuilder blobsBuilder = MapBuilder.newMapBuilder(); + MapBuilder blobsBuilder = MapBuilder.newMapBuilder(); Collection files; if (blobNamePrefix != null) { files = blobStore.swift().listDirectory(new Directory(buildKey(blobNamePrefix), '/')); @@ -147,7 +150,7 @@ public Map listBlobsByPrefix(@Nullable final String blobNa for (DirectoryOrObject object : files) { if (object.isObject()) { String name = object.getName().substring(keyPath.length()); - blobsBuilder.put(name, new PlainBlobMetaData(name, object.getAsObject().getContentLength())); + blobsBuilder.put(name, new PlainBlobMetadata(name, object.getAsObject().getContentLength())); } } } @@ -184,7 +187,7 @@ public Map children() throws IOException { * Get all the blobs */ @Override - public Map listBlobs() { + public Map listBlobs() { return listBlobsByPrefix(null); } @@ -222,6 +225,37 @@ public InputStream readBlob(final String blobName) throws IOException { } } + /** + * Fetch a given blob into a BufferedInputStream + * @param blobName The blob name to read + * @param position The position in the blob where the next byte will be read + * @param length An indication of the number of bytes to be read + * @return a stream + */ + @Override + public InputStream readBlob(final String blobName, long position, long length) throws IOException { + DownloadInstructions downloadInstructions = new DownloadInstructions(); + AbstractRange range = new MidPartRange((int)position, (int)(position + length)); + downloadInstructions.setRange(range); + + try { + final InputStream is = SwiftPerms.exec( + (PrivilegedAction) () -> new BufferedInputStream( + blobStore.swift().getObject(buildKey(blobName)).downloadObjectAsInputStream(downloadInstructions), + blobStore.bufferSizeInBytes())); + + if (null == is) { + throw new NoSuchFileException("Blob object [" + blobName + "] not found."); + } + + return is; + } catch (NotFoundException e){ + NoSuchFileException e2 = new NoSuchFileException("Blob object [" + blobName + "] not found."); + e2.initCause(e); + throw e2; + } + } + @Override public void writeBlob(final String blobName, final InputStream in, final long blobSize, boolean failIfAlreadyExists) throws IOException {