diff --git a/web/src/main/java/com/rudderstack/web/WebService.kt b/web/src/main/java/com/rudderstack/web/WebService.kt index ca4c5fbfd..cb152102a 100644 --- a/web/src/main/java/com/rudderstack/web/WebService.kt +++ b/web/src/main/java/com/rudderstack/web/WebService.kt @@ -112,6 +112,7 @@ interface WebService { body: String?, endpoint: String, responseClass: Class, + isGzipEnabled: Boolean = false, ): Future> /** @@ -132,6 +133,7 @@ interface WebService { body: String?, endpoint: String, responseTypeAdapter: RudderTypeAdapter, + isGzipEnabled: Boolean = false, ): Future> /** @@ -151,6 +153,7 @@ interface WebService { body: String?, endpoint: String, responseClass: Class, + isGzipEnabled: Boolean = false, callback: (HttpResponse) -> Unit, ) @@ -171,6 +174,7 @@ interface WebService { body: String?, endpoint: String, responseTypeAdapter: RudderTypeAdapter, + isGzipEnabled: Boolean = false, callback: (HttpResponse) -> Unit, ) @@ -181,4 +185,11 @@ interface WebService { * @see HttpInterceptor */ fun setInterceptor(httpInterceptor: HttpInterceptor) + + /** + * Performs Cleanup. Also shuts down the executor service. Do not call this method if the + * same executor service is being used by other objects. In that case handle the executor + * shutdown yourself. + */ + fun shutdown(shutdownExecutor: Boolean = true) } diff --git a/web/src/main/java/com/rudderstack/web/internal/WebServiceImpl.kt b/web/src/main/java/com/rudderstack/web/internal/WebServiceImpl.kt index 0fa0a697e..1664191a4 100644 --- a/web/src/main/java/com/rudderstack/web/internal/WebServiceImpl.kt +++ b/web/src/main/java/com/rudderstack/web/internal/WebServiceImpl.kt @@ -19,6 +19,7 @@ import com.rudderstack.rudderjsonadapter.RudderTypeAdapter import com.rudderstack.web.HttpInterceptor import com.rudderstack.web.HttpResponse import com.rudderstack.web.WebService +import com.rudderstack.web.utils.GzipUtils import java.io.BufferedInputStream import java.io.ByteArrayOutputStream import java.io.IOException @@ -38,8 +39,7 @@ class WebServiceImpl internal constructor( private val baseUrl: String private enum class HttpMethod { - POST, - GET + POST, GET } init { @@ -53,7 +53,7 @@ class WebServiceImpl internal constructor( responseClass: Class ): Future> { return executor.submit(Callable { - httpCall(headers, query, null, endpoint, HttpMethod.GET, responseClass) + httpCall(headers, query, null, endpoint, HttpMethod.GET, responseClass, false) }) } @@ -64,7 +64,7 @@ class WebServiceImpl internal constructor( responseTypeAdapter: RudderTypeAdapter ): Future> { return executor.submit(Callable { - httpCall(headers, query, null, endpoint, HttpMethod.GET, responseTypeAdapter) + httpCall(headers, query, null, endpoint, HttpMethod.GET, responseTypeAdapter, false) }) } @@ -77,7 +77,7 @@ class WebServiceImpl internal constructor( ) { executor.execute { callback.invoke( - httpCall(headers, query, null, endpoint, HttpMethod.GET, responseTypeAdapter) + httpCall(headers, query, null, endpoint, HttpMethod.GET, responseTypeAdapter, false) ) } } @@ -92,7 +92,7 @@ class WebServiceImpl internal constructor( executor.execute { callback.invoke( - httpCall(headers, query, null, endpoint, HttpMethod.GET, responseClass) + httpCall(headers, query, null, endpoint, HttpMethod.GET, responseClass, false) ) } } @@ -102,10 +102,11 @@ class WebServiceImpl internal constructor( query: Map?, body: String?, endpoint: String, - responseClass: Class + responseClass: Class, + isGzipEnabled: Boolean ): Future> { return executor.submit(Callable { - httpCall(headers, query, body, endpoint, HttpMethod.POST, responseClass) + httpCall(headers, query, body, endpoint, HttpMethod.POST, responseClass, isGzipEnabled) }) } @@ -114,10 +115,19 @@ class WebServiceImpl internal constructor( query: Map?, body: String?, endpoint: String, - responseTypeAdapter: RudderTypeAdapter + responseTypeAdapter: RudderTypeAdapter, + isGzipEnabled: Boolean ): Future> { return executor.submit(Callable { - httpCall(headers, query, body, endpoint, HttpMethod.POST, responseTypeAdapter) + httpCall( + headers, + query, + body, + endpoint, + HttpMethod.POST, + responseTypeAdapter, + isGzipEnabled + ) }) } @@ -127,11 +137,20 @@ class WebServiceImpl internal constructor( body: String?, endpoint: String, responseClass: Class, + isGzipEnabled: Boolean, callback: (HttpResponse) -> Unit ) { executor.execute { callback.invoke( - httpCall(headers, query, body, endpoint, HttpMethod.POST, responseClass) + httpCall( + headers, + query, + body, + endpoint, + HttpMethod.POST, + responseClass, + isGzipEnabled + ) ) } } @@ -142,6 +161,7 @@ class WebServiceImpl internal constructor( body: String?, endpoint: String, responseTypeAdapter: RudderTypeAdapter, + isGzipEnabled: Boolean, callback: (HttpResponse) -> Unit ) { executor.execute { @@ -152,7 +172,8 @@ class WebServiceImpl internal constructor( body, endpoint, HttpMethod.POST, - responseTypeAdapter + responseTypeAdapter, + isGzipEnabled ) ) } @@ -163,6 +184,12 @@ class WebServiceImpl internal constructor( _interceptor = httpInterceptor } + override fun shutdown(shutdownExecutor: Boolean) { + if(shutdownExecutor) + executor.shutdown() + _interceptor = null + } + // @Throws(Throwable::class) private fun httpCall( headers: Map?, @@ -170,13 +197,13 @@ class WebServiceImpl internal constructor( body: String?, endpoint: String, type: HttpMethod, - - responseClass: Class + responseClass: Class, + isGzipEncoded: Boolean ): HttpResponse { return rawHttpCall(headers, query, body, endpoint, type, deserializer = { json -> jsonAdapter.readJson(json, responseClass) - ?: throw IllegalArgumentException("Json adapter not able to parse response body") - }) + ?: throw IllegalArgumentException("Json adapter not able to parse response body") + }, isGzipEncoded = isGzipEncoded) } @Throws(IOException::class) @@ -186,17 +213,17 @@ class WebServiceImpl internal constructor( body: String?, endpoint: String, type: HttpMethod, - typeAdapter: RudderTypeAdapter + typeAdapter: RudderTypeAdapter, + isGzipEncoded: Boolean ): HttpResponse { return rawHttpCall(headers, query, body, endpoint, type, deserializer = { json -> if (json.isEmpty()) { //TODO add logger // logger.debug("Empty response body") null - } else - jsonAdapter.readJson(json, typeAdapter) - ?: throw IllegalArgumentException("Json adapter not able to parse response body") - }) + } else jsonAdapter.readJson(json, typeAdapter) + ?: throw IllegalArgumentException("Json adapter not able to parse response body") + }, isGzipEncoded = isGzipEncoded) } @@ -206,15 +233,23 @@ class WebServiceImpl internal constructor( body: String?, endpoint: String, type: HttpMethod, + isGzipEncoded: Boolean, deserializer: (String) -> T? ): HttpResponse { try { - val httpConnection = - createHttpConnection(endpoint, headers, type, query, body, defaultTimeout) { - //call interceptor if any changes to HttpConnection required - _interceptor?.intercept(it) ?: it - } + val httpConnection = createHttpConnection( + endpoint, + headers, + type, + query, + body, + defaultTimeout, + isGzipEncoded + ) { + //call interceptor if any changes to HttpConnection required + _interceptor?.intercept(it) ?: it + } // create connection httpConnection.connect() @@ -233,9 +268,7 @@ class WebServiceImpl internal constructor( return Utils.NetworkResponses.SUCCESS }*/ HttpResponse( - httpConnection.responseCode, - deserializer.invoke(baos.toString()), - null + httpConnection.responseCode, deserializer.invoke(baos.toString()), null ) } else { @@ -256,10 +289,7 @@ class WebServiceImpl internal constructor( // RudderLogger.logError(ex) ex.printStackTrace() return HttpResponse( - status = HttpResponse.HTTP_STATUS_NONE, - body = null, - errorBody = null, - error = ex + status = HttpResponse.HTTP_STATUS_NONE, body = null, errorBody = null, error = ex ) } @@ -269,9 +299,12 @@ class WebServiceImpl internal constructor( @Throws(IOException::class) private fun createHttpConnection( endpoint: String, - headers: Map?, type: HttpMethod, + headers: Map?, + type: HttpMethod, query: Map?, - body: String?, defaultTimeout: Int, + body: String?, + defaultTimeout: Int, + isGzipEncoded: Boolean, onHttpConnectionCreated: (HttpURLConnection) -> HttpURLConnection ): HttpURLConnection { //the url to hit @@ -290,15 +323,10 @@ class WebServiceImpl internal constructor( } // set content type for network request if not present - if (headers?.containsKey("Content-Type") == false) - httpConn.setRequestProperty("Content-Type", "application/json") - // set authorization header - /*httpConnection.setRequestProperty( - "Authorization", - String.format(Locale.US, "Basic %s", this.authHeaderString) - )*/ - // set anonymousId header for definitive routing -// httpConnection.setRequestProperty("AnonymousId", this.anonymousIdHeaderString) + if (headers?.containsKey("Content-Type") == false) httpConn.setRequestProperty( + "Content-Type", "application/json" + ) + // set request method httpConn.requestMethod = when (type) { HttpMethod.GET -> "GET" @@ -310,7 +338,10 @@ class WebServiceImpl internal constructor( if (type == HttpMethod.POST) { // set connection object to return output httpConn.doOutput = true - val os = httpConn.outputStream + val os = if (isGzipEncoded) GzipUtils.getGzipOutputStream(httpConn.outputStream) + ?: //TODO add logger.debug("Gzip compression not supported") + httpConn.outputStream + else httpConn.outputStream val osw = OutputStreamWriter(os, "UTF-8") body?.apply { @@ -323,10 +354,9 @@ class WebServiceImpl internal constructor( return httpConn } - private fun Map.createQueryString() = - takeIf { isNotEmpty() }?.map { - "${it.key}=${it.value}" - }?.reduce { acc, s -> "$acc&$s" } ?: "" + private fun Map.createQueryString() = takeIf { isNotEmpty() }?.map { + "${it.key}=${it.value}" + }?.reduce { acc, s -> "$acc&$s" } ?: "" private val String.validatedBaseUrl get() = if (this.endsWith('/')) this else "$this/" diff --git a/web/src/main/java/com/rudderstack/web/utils/GzipUtils.kt b/web/src/main/java/com/rudderstack/web/utils/GzipUtils.kt new file mode 100644 index 000000000..2947b30ec --- /dev/null +++ b/web/src/main/java/com/rudderstack/web/utils/GzipUtils.kt @@ -0,0 +1,41 @@ +/* + * Creator: Debanjan Chatterjee on 13/09/23, 12:11 pm Last modified: 13/09/23, 12:11 pm + * Copyright: All rights reserved Ⓒ 2023 http://rudderstack.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain a + * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.rudderstack.web.utils + +import java.io.IOException +import java.io.InputStream +import java.io.OutputStream +import java.util.zip.GZIPInputStream +import java.util.zip.GZIPOutputStream + +object GzipUtils { + fun getGzipOutputStream(outputStream: OutputStream?): OutputStream? { + try { + return GZIPOutputStream(outputStream) + } catch (e: IOException) { + e.printStackTrace() + } + return null + } + fun getGzipInputStream(inputStream: InputStream?): InputStream? { + try { + return GZIPInputStream(inputStream) + } catch (e: IOException) { + e.printStackTrace() + } + return null + } + +} \ No newline at end of file diff --git a/web/src/test/java/com/rudderstack/web/WebApiTest.kt b/web/src/test/java/com/rudderstack/web/WebApiTest.kt index 33a68c7f0..708fa83c8 100644 --- a/web/src/test/java/com/rudderstack/web/WebApiTest.kt +++ b/web/src/test/java/com/rudderstack/web/WebApiTest.kt @@ -25,14 +25,32 @@ import org.awaitility.Awaitility import org.hamcrest.MatcherAssert import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers -import org.hamcrest.Matchers.* +import org.hamcrest.Matchers.aMapWithSize +import org.hamcrest.Matchers.allOf +import org.hamcrest.Matchers.emptyString +import org.hamcrest.Matchers.equalTo +import org.hamcrest.Matchers.greaterThan +import org.hamcrest.Matchers.hasEntry +import org.hamcrest.Matchers.not +import org.hamcrest.Matchers.notNullValue +import org.junit.After import org.junit.Before import org.junit.Test +import java.io.BufferedReader +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.io.InputStream +import java.io.InputStreamReader +import java.io.OutputStream +import java.net.HttpURLConnection +import java.net.URL import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean +import java.util.stream.Collectors +import java.util.zip.GZIPInputStream open class WebApiTest { - protected var jsonAdapter: JsonAdapter = MoshiAdapter() + protected var jsonAdapter: JsonAdapter = GsonAdapter() private lateinit var webService: WebService private lateinit var agifyWebService: WebService @@ -46,31 +64,29 @@ open class WebApiTest { //https://jsonplaceholder.typicode.com/guide/ - @Before fun init() { webService = WebServiceFactory.getWebService( - "https://api.artic.edu/api/v1/", - jsonAdapter + "https://api.artic.edu/api/v1/", jsonAdapter ) agifyWebService = WebServiceFactory.getWebService( - "https://api.agify.io/", - jsonAdapter + "https://api.agify.io/", jsonAdapter ) jsonPlaceHolderWebService = WebServiceFactory.getWebService( - "https://jsonplaceholder.typicode.com/", - jsonAdapter + "https://jsonplaceholder.typicode.com/", jsonAdapter ) } + @After + fun destroy() { + webService.shutdown() + } + @Test fun testSimpleGetSync() { val response = webService.get( - null, - mapOf("fields" to "id,title"), - "artworks/200154", - ArtDataResponse::class.java + null, mapOf("fields" to "id,title"), "artworks/200154", ArtDataResponse::class.java ).get().body MatcherAssert.assertThat( @@ -86,20 +102,17 @@ open class WebApiTest { ) ) ) - assertThat(response?.info?.licenseText, allOf( - notNullValue(), - `is`("The data in this response is licensed under a Creative Commons Zero (CC0) " + - "1.0 designation and the Terms and Conditions of artic.edu.") - )) + assertThat( + response?.info?.licenseText, allOf( + notNullValue(), not(emptyString()) + ) + ) } @Test fun testParameterizedGetSync() { val response = webService.get( - null, - mapOf("fields" to "id,title"), - "artworks", - ArtDataListResponse::class.java + null, mapOf("fields" to "id,title"), "artworks", ArtDataListResponse::class.java ).get().body MatcherAssert.assertThat( @@ -130,12 +143,10 @@ open class WebApiTest { @Test fun testTypeAdaptedGetSync() { - val response = agifyWebService.get( - null, + val response = agifyWebService.get(null, mapOf("name" to "bella"), "", - object : RudderTypeAdapter>() {} - ).get().body + object : RudderTypeAdapter>() {}).get().body MatcherAssert.assertThat( response, Matchers.allOf( @@ -154,10 +165,7 @@ open class WebApiTest { val isComplete = AtomicBoolean(false) var response: ArtDataResponse? = null webService.get( - null, - mapOf("fields" to "id,title"), - "artworks/200154", - ArtDataResponse::class.java + null, mapOf("fields" to "id,title"), "artworks/200154", ArtDataResponse::class.java ) { response = it.body MatcherAssert.assertThat( @@ -198,8 +206,7 @@ open class WebApiTest { MatcherAssert.assertThat( response, Matchers.allOf( - Matchers.notNullValue(), - Matchers.aMapWithSize(4) //a field "id" is sent alongside + Matchers.notNullValue(), Matchers.aMapWithSize(4) //a field "id" is sent alongside ) ) val title = response?.get("title") @@ -223,7 +230,10 @@ open class WebApiTest { val postStringedBody = jsonAdapter.writeToJson(postBody, object : RudderTypeAdapter>() {}) jsonPlaceHolderWebService.post(mapOf("Content-Type" to "application/json; charset=UTF-8"), - null, postStringedBody, "posts", object : RudderTypeAdapter>() {}) { + null, + postStringedBody, + "posts", + object : RudderTypeAdapter>() {}) { val response = it.body MatcherAssert.assertThat( response, Matchers.allOf( @@ -242,17 +252,76 @@ open class WebApiTest { Awaitility.await().atMost(1, TimeUnit.MINUTES).untilTrue(isComplete) } + @Test + fun `test post call with gzip enabled`() { + val testingPayload = + "{\"test\":\"test\", \"test2\":\"test2\", \"test3\":{ \"test4\":\"test4\"}}" + val baos = ByteArrayOutputStream() + webService.setInterceptor { _ -> + object : HttpURLConnection(URL("http://www.dummyurl.com")) { + override fun getOutputStream(): OutputStream { + //return simple output stream, which will be gzipped + return baos + } + + override fun connect() { + //no-op + } + + override fun disconnect() { + } + + override fun usingProxy(): Boolean { + return false + } + + override fun getResponseCode(): Int { + return 200 + } + + override fun getInputStream(): InputStream { + val bios = ByteArrayInputStream(baos.toByteArray()) + //use Gzip input stream to decode, remove this, and the test will fail + val result = BufferedReader(InputStreamReader(GZIPInputStream(bios))).lines() + .collect(Collectors.joining("\n")) +// val response = HttpResponse(200, result, null) + return ByteArrayInputStream( result.toByteArray()) + } + } + } + val output = webService.post( + null, + null, + testingPayload, + "test", + object : RudderTypeAdapter>() {}, + true + ).get() + assertThat(output.body, allOf(notNullValue(), aMapWithSize(3), + hasEntry("test", "test"), hasEntry("test2", "test2"), + /*hasEntry("test3", allOf(*//*aMapWithSize(1)*//*, *//*hasEntry("test4", "test4")*//*))*/)) + println(output.body!!["test3"]?.javaClass) + assertThat(output.body!!["test3"] as Map, allOf(notNullValue(), + aMapWithSize + (1), hasEntry + ("test4", "test4"))) + } + } + class WebApiTestJackson : WebApiTest() { init { jsonAdapter = GsonAdapter() } } + class WebApiTestGson : WebApiTest() { init { jsonAdapter = GsonAdapter() } } + class WebApiTestMoshi : WebApiTest() { init { jsonAdapter = MoshiAdapter()