diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..fa6d17b --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright 2016-2018 Jayson Minard (jayson.minard@gmail.com) and Kohesive + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md index c60c570..5413df3 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +[![GitHub release](https://img.shields.io/github/release/kohesive/elasticsearch-data-import-handler.svg)](https://github.com/kohesive/elasticsearch-data-import-handler/releases) [![CircleCI branch](https://img.shields.io/circleci/project/kohesive/elasticsearch-data-import-handler/master.svg)](https://circleci.com/gh/kohesive/elasticsearch-data-import-handler/tree/master) [![License](https://img.shields.io/badge/license-Apache%202-blue.svg)](https://github.com/kohesive/elasticsearch-data-import-handler/blob/master/LICENSE) [![ES](https://img.shields.io/badge/ES-5.x-orange.svg)](https://github.com/elastic/elasticsearch) [![ES](https://img.shields.io/badge/ES-6.x-orange.svg)](https://github.com/elastic/elasticsearch) + + +# Elasticsearch Data Import Handler # Elasticsearch/Algolia Data Import Handler A data import handler for Elasticsearch/Algolia diff --git a/build.gradle b/build.gradle index 2d74e3c..998d954 100644 --- a/build.gradle +++ b/build.gradle @@ -1,14 +1,10 @@ plugins { - id 'nebula.kotlin' version '1.1.2' apply false - id 'nebula.facet' version '3.3.0' apply false - id 'nebula.dependency-lock' version '4.9.4' apply false - id 'nebula.resolution-rules' version '2.5.10' apply false + id 'nebula.kotlin' version '1.1.61' apply false + id 'nebula.resolution-rules' version '5.1.1' apply false } apply plugin: 'nebula.kotlin' apply plugin: 'nebula.resolution-rules' -apply plugin: 'nebula.dependency-lock' -apply plugin: 'nebula.facet' apply plugin: 'java' apply plugin: 'idea' @@ -21,10 +17,10 @@ sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = JavaVersion.VERSION_1_8 dependencies { - compile group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-jre8' - compile group: 'org.jetbrains.kotlin', name: 'kotlin-reflect' + compile group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib-jre8', version: version_kotlin + compile group: 'org.jetbrains.kotlin', name: 'kotlin-reflect', version: version_kotlin - compile group: 'com.fasterxml.jackson.module', name: 'jackson-module-kotlin', version: version_jackson_compatible_with_spark + compile group: 'com.fasterxml.jackson.module', name: 'jackson-module-kotlin', version: version_jackson_kotlin_module compile group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jdk8', version: version_jackson_compatible_with_spark compile group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310', version: version_jackson_compatible_with_spark @@ -33,7 +29,7 @@ dependencies { compile group: 'org.elasticsearch', name: "elasticsearch-spark-20_${version_scala}", version: version_elasticsearch compile group: 'com.squareup.okhttp3', name: 'okhttp', version: version_okhttp - compile group: 'org.apache.spark', name: "spark-sql_${version_scala}", version: '2.1.0' + compile group: 'org.apache.spark', name: "spark-sql_${version_scala}", version: version_spark compile group: 'org.jsoup', name: 'jsoup', version: version_jsoup @@ -54,6 +50,7 @@ dependencies { testCompile group: 'junit', name: 'junit', version: version_junit testCompile group: 'org.jetbrains.kotlin', name: 'kotlin-test-junit' + } nebulaResolutionRules { @@ -62,9 +59,6 @@ nebulaResolutionRules { repositories { mavenCentral() - maven { - url "http://dl.bintray.com/kotlin/kotlin-eap-1.1" - } } compileKotlin { diff --git a/circle.yml b/circle.yml new file mode 100644 index 0000000..fc492ea --- /dev/null +++ b/circle.yml @@ -0,0 +1,42 @@ +# Java Gradle CircleCI 2.0 configuration file +# +# Check https://circleci.com/docs/2.0/language-java/ for more details +# +version: 2 +jobs: + build: + docker: + # specify the version you desire here + - image: circleci/openjdk:8-jdk + + # Specify service dependencies here if necessary + # CircleCI maintains a library of pre-built images + # documented at https://circleci.com/docs/2.0/circleci-images/ + # - image: circleci/postgres:9.4 + + working_directory: ~/repo + + environment: + # Customize the JVM maximum heap limit + JVM_OPTS: -Xmx3200m + TERM: dumb + + steps: + - checkout + + # Download and cache dependencies + - restore_cache: + keys: + - v1-dependencies-{{ checksum "build.gradle" }} + # fallback to using the latest cache if no exact match is found + - v1-dependencies- + + - run: gradle dependencies + + - save_cache: + paths: + - ~/.gradle + key: v1-dependencies-{{ checksum "build.gradle" }} + + # run tests! + - run: ./gradlew clean build test -x signArchives \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index c7f76c7..d438c5a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,24 +1,25 @@ group = uy.kohesive.elasticsearch version = 0.20.3-ALPHA -version_gradle=3.5 -version_kotlin=1.1.2 +version_gradle=4.5.1 +version_kotlin=1.1.61 -version_nebula_resolution=0.46.0 +version_nebula_resolution=0.52.0 version_jackson_compatible_with_spark=2.6.7 -version_spark=2.1.1 +version_jackson_kotlin_module=2.6.7 +version_spark=2.2.1 version_typesafe_config=1.3.1 -version_elasticsearch=5.4.0 -version_okhttp=3.6.0 +version_elasticsearch=5.6.7 +version_okhttp=3.10.0 version_scala=2.11 version_jsoup=1.10.2 -version_slf4j=1.+ -version_logback=1.+ -version_junit=4.+ +version_slf4j=1.7.+ +version_logback=1.2.+ +version_junit=4.12 version_jdbc_mysql=5.1.41 version_jdbc_postresql=42.0.0 diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 51288f9..636d914 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ffbfea3..96528bc 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Sun May 14 14:02:57 UYT 2017 +#Sun Feb 25 15:56:41 CST 2018 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-3.5-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-4.5.1-bin.zip diff --git a/settings.gradle b/settings.gradle index f0fd3f1..c3dc939 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,8 +1 @@ -pluginRepositories { - maven { - url 'https://dl.bintray.com/kotlin/kotlin-eap-1.1' - } - gradlePluginPortal() -} - rootProject.name = 'kohesive-elasticsearch-data-import-handler' \ No newline at end of file diff --git a/src/main/kotlin/uy/kohesive/elasticsearch/dataimport/MicroEsClient.kt b/src/main/kotlin/uy/kohesive/elasticsearch/dataimport/MicroEsClient.kt index d0999d8..846b983 100644 --- a/src/main/kotlin/uy/kohesive/elasticsearch/dataimport/MicroEsClient.kt +++ b/src/main/kotlin/uy/kohesive/elasticsearch/dataimport/MicroEsClient.kt @@ -2,7 +2,6 @@ package uy.kohesive.elasticsearch.dataimport import com.fasterxml.jackson.module.kotlin.readValue import okhttp3.* -import okhttp3.internal.http.HttpHeaders /** * TODO: Change to use RestClient from ES / Spark integration @@ -30,13 +29,13 @@ class MicroEsClient(nodes: List, port: Int = 9200, enableSsl: Boolean = private fun OkHttpClient.get(url: String): CallResponse { val request = Request.Builder().url(url).build() val response = http.newCall(request).execute() - return CallResponse(response.code(), response.use { it.body().string() }) + return CallResponse(response.code(), response.use { it.body()?.string() ?: "" }) } private fun OkHttpClient.delete(url: String): CallResponse { val request = Request.Builder().url(url).delete().build() val response = http.newCall(request).execute() - return CallResponse(response.code(), response.use { it.body().string() }) + return CallResponse(response.code(), response.use { it.body()?.string() ?: "" }) } private fun OkHttpClient.delete(url: String, jsonBody: String): CallResponse { @@ -44,7 +43,7 @@ class MicroEsClient(nodes: List, port: Int = 9200, enableSsl: Boolean = val body = RequestBody.create(jsonMediaType, jsonBody) val request = Request.Builder().url(url).delete(body).build() val response = http.newCall(request).execute() - return CallResponse(response.code(), response.use { it.body().string() }) + return CallResponse(response.code(), response.use { it.body()?.string() ?: "" }) } private fun OkHttpClient.post(url: String, jsonBody: String): CallResponse { @@ -52,7 +51,7 @@ class MicroEsClient(nodes: List, port: Int = 9200, enableSsl: Boolean = val body = RequestBody.create(jsonMediaType, jsonBody) val request = Request.Builder().url(url).post(body).build() val response = http.newCall(request).execute() - return CallResponse(response.code(), response.use { it.body().string() }) + return CallResponse(response.code(), response.use { it.body()?.string() ?: "" }) } private fun OkHttpClient.put(url: String, jsonBody: String): CallResponse { @@ -60,7 +59,7 @@ class MicroEsClient(nodes: List, port: Int = 9200, enableSsl: Boolean = val body = RequestBody.create(jsonMediaType, jsonBody) val request = Request.Builder().url(url).put(body).build() val response = http.newCall(request).execute() - return CallResponse(response.code(), response.use { it.body().string() }) + return CallResponse(response.code(), response.use { it.body()?.string() ?: "" }) } fun indexTypePOST(indexName: String, indexType: String, restOfUrl: String, postJson: String): CallResponse { diff --git a/src/test/kotlin/uy/kohesive/elasticsearch/dataimport/ManualTestOfDataImport.kt b/src/test/kotlin/uy/kohesive/elasticsearch/dataimport/ManualTestOfDataImport.kt index 34b3c3b..3babf96 100644 --- a/src/test/kotlin/uy/kohesive/elasticsearch/dataimport/ManualTestOfDataImport.kt +++ b/src/test/kotlin/uy/kohesive/elasticsearch/dataimport/ManualTestOfDataImport.kt @@ -11,4 +11,4 @@ class ManualTestOfDataImport { App().run(confFile.inputStream(), confFile.parentFile) } } -} \ No newline at end of file +}