always decode response body if content encoding is on the response he…

…ader;
bogdanfinn · Jun 13, 2023 · fb533ec · fb533ec
1 parent 7bb004a
commit fb533ec
Show file tree

Hide file tree

Showing 15 changed files with 23 additions and 36 deletions.
diff --git a/cffi_dist/example_csharp/Requester.cs b/cffi_dist/example_csharp/Requester.cs
@@ -26,7 +26,6 @@ class RequestPayload
     public bool WithoutCookieJar { get; set; } = false;
     public bool WithDefaultCookieJar { get; set; } = false;
     public bool IsByteRequest { get; set; } = false;
-    public string AdditionalDecode { get; set; } = "";
     public bool ForceHttp1 { get; set; } = false;
     public bool WithDebug { get; set; } = false;
     public bool CatchPanics { get; set; } = false;
@@ -46,11 +45,11 @@ class RequestPayload
 
 class TLSSession
 {
-    [DllImport("../dist/tls-client-windows-64-1.3.13.dll", CallingConvention = CallingConvention.Cdecl)]
+    [DllImport("../dist/tls-client-windows-64-1.4.0.dll", CallingConvention = CallingConvention.Cdecl)]
     public static extern IntPtr request(byte[] requestPayload, string sessionID);
 
 
-    [DllImport("../dist/tls-client-windows-64-1.3.13.dll", CallingConvention = CallingConvention.Cdecl)]
+    [DllImport("../dist/tls-client-windows-64-1.4.0.dll", CallingConvention = CallingConvention.Cdecl)]
     public static extern void freeMemory(string sessionID);
 
     private string sessionID;
@@ -68,7 +67,6 @@ public TLSSession(Dictionary<string, string> headers = null, string TlsClientIde
             FollowRedirects = FollowRedirects,
             InsecureSkipVerify = false,
             IsByteRequest = false,
-            AdditionalDecode = "",
             ForceHttp1 = false,
             WithDebug = false,
             CatchPanics = false,

diff --git a/cffi_dist/example_node/index.js b/cffi_dist/example_node/index.js
@@ -1,7 +1,7 @@
 const ffi = require('ffi-napi');
 
 // load the tls-client shared package for your OS you are currently running your nodejs script (i'm running on mac)
-const tlsClientLibrary = ffi.Library('./../dist/tls-client-darwin-amd64-1.3.13.dylib', {
+const tlsClientLibrary = ffi.Library('./../dist/tls-client-darwin-amd64-1.4.0.dylib', {
     'request': ['string', ['string']],
     'getCookiesFromSession': ['string', ['string']],
     'addCookiesToSession': ['string', ['string']],
@@ -18,7 +18,6 @@ const requestPayload = {
     "withDefaultCookieJar": false,
     "isByteRequest": false,
     "catchPanics": false,
-    "additionalDecode": null,
     "withDebug": false,
     "forceHttp1": false,
     "withRandomTLSExtensionOrder": false,

diff --git a/cffi_dist/example_node/index_async.js b/cffi_dist/example_node/index_async.js
@@ -1,7 +1,7 @@
 const ffi = require('ffi-napi');
 
 // load the tls-client shared package for your OS you are currently running your nodejs script (i'm running on mac)
-const tlsClientLibrary = ffi.Library('./../dist/tls-client-darwin-amd64-1.3.13.dylib', {
+const tlsClientLibrary = ffi.Library('./../dist/tls-client-darwin-amd64-1.4.0.dylib', {
     'request': ['string', ['string']],
     'getCookiesFromSession': ['string', ['string']],
     'addCookiesToSession': ['string', ['string']],
@@ -17,7 +17,6 @@ const requestPayload = {
     "withoutCookieJar": false,
     "withDefaultCookieJar": false,
     "isByteRequest": false,
-    "additionalDecode": null,
     "forceHttp1": false,
     "withRandomTLSExtensionOrder": false,
     "timeoutSeconds": 30,

diff --git a/cffi_dist/example_node/index_custom_client.js b/cffi_dist/example_node/index_custom_client.js
@@ -1,7 +1,7 @@
 const ffi = require('ffi-napi');
 
 // load the tls-client shared package for your OS you are currently running your nodejs script (i'm running on mac)
-const tlsClientLibrary = ffi.Library('./../dist/tls-client-darwin-amd64-1.3.13.dylib', {
+const tlsClientLibrary = ffi.Library('./../dist/tls-client-darwin-amd64-1.4.0.dylib', {
     'request': ['string', ['string']],
     'getCookiesFromSession': ['string', ['string']],
     'addCookiesToSession': ['string', ['string']],
@@ -16,7 +16,6 @@ const requestPayload = {
     "withoutCookieJar": false,
     "withDefaultCookieJar": false,
     "isByteRequest": false,
-    "additionalDecode": null,
     "catchPanics": false,
     "forceHttp1": false,
     "withDebug": false,

diff --git a/cffi_dist/example_node/index_image.js b/cffi_dist/example_node/index_image.js
@@ -2,7 +2,7 @@ const ffi = require('ffi-napi');
 const fs = require("fs")
 
 // load the tls-client shared package for your OS you are currently running your nodejs script (i'm running on mac)
-const tlsClientLibrary = ffi.Library('./../dist/tls-client-darwin-amd64-1.3.13.dylib', {
+const tlsClientLibrary = ffi.Library('./../dist/tls-client-darwin-amd64-1.4.0.dylib', {
     'request': ['string', ['string']],
     'getCookiesFromSession': ['string', ['string']],
     'addCookiesToSession': ['string', ['string']],
@@ -22,7 +22,6 @@ const requestPayload = {
     "withRandomTLSExtensionOrder": false,
     "isByteResponse": true,
     "isByteRequest": false,
-    "additionalDecode": null,
     "catchPanics": false,
     "timeoutSeconds": 30,
     "timeoutMilliseconds": 0,

diff --git a/cffi_dist/example_node/index_image_upload.js b/cffi_dist/example_node/index_image_upload.js
@@ -2,7 +2,7 @@ const ffi = require('ffi-napi');
 const fs = require("fs")
 
 // load the tls-client shared package for your OS you are currently running your nodejs script (i'm running on mac)
-const tlsClientLibrary = ffi.Library('./../dist/tls-client-darwin-amd64-1.3.13.dylib', {
+const tlsClientLibrary = ffi.Library('./../dist/tls-client-darwin-amd64-1.4.0.dylib', {
     'request': ['string', ['string']],
     'getCookiesFromSession': ['string', ['string']],
     'addCookiesToSession': ['string', ['string']],
@@ -25,7 +25,6 @@ const requestPayload = {
     "isByteResponse": false,
     "isByteRequest": true,
     "catchPanics": false,
-    "additionalDecode": null,
     "timeoutSeconds": 30,
     "timeoutMilliseconds": 0,
     "certificatePinningHosts": {},

diff --git a/cffi_dist/example_node/index_post.js b/cffi_dist/example_node/index_post.js
@@ -1,7 +1,7 @@
 const ffi = require('ffi-napi');
 
 // load the tls-client shared package for your OS you are currently running your nodejs script (i'm running on mac)
-const tlsClientLibrary = ffi.Library('./../dist/tls-client-darwin-amd64-1.3.13.dylib', {
+const tlsClientLibrary = ffi.Library('./../dist/tls-client-darwin-amd64-1.4.0.dylib', {
     'request': ['string', ['string']],
     'getCookiesFromSession': ['string', ['string']],
     'addCookiesToSession': ['string', ['string']],
@@ -21,7 +21,6 @@ const requestPayload = {
     "withRandomTLSExtensionOrder": false,
     "isByteResponse": false,
     "isByteRequest": false,
-    "additionalDecode": null,
     "catchPanics": false,
     "timeoutSeconds": 30,
     "timeoutMilliseconds": 0,

diff --git a/cffi_dist/example_python/example.py b/cffi_dist/example_python/example.py
@@ -2,7 +2,7 @@
 import json
 
 # load the tls-client shared package for your OS you are currently running your python script (i'm running on mac)
-library = ctypes.cdll.LoadLibrary('./../dist/tls-client-darwin-amd64-1.3.13.dylib')
+library = ctypes.cdll.LoadLibrary('./../dist/tls-client-darwin-amd64-1.4.0.dylib')
 
 # extract the exposed request function from the shared package
 request = library.request
@@ -34,7 +34,6 @@
     "withoutCookieJar": False,
     "withDefaultCookieJar": False,
     "isByteRequest": False,
-    "additionalDecode": "",
     "forceHttp1": False,
     "withDebug": False,
     "catchPanics": False,

diff --git a/cffi_dist/example_python/example_custom_client.py b/cffi_dist/example_python/example_custom_client.py
@@ -2,7 +2,7 @@
 import json
 
 # load the tls-client shared package for your OS you are currently running your python script (i'm running on mac)
-library = ctypes.cdll.LoadLibrary('./../dist/tls-client-darwin-amd64-1.3.13.dylib')
+library = ctypes.cdll.LoadLibrary('./../dist/tls-client-darwin-amd64-1.4.0.dylib')
 
 # extract the exposed request function from the shared package
 request = library.request
@@ -33,7 +33,6 @@
     "withoutCookieJar": False,
     "withDefaultCookieJar": False,
     "isByteRequest": False,
-    "additionalDecode": "",
     "forceHttp1": False,
     "catchPanics": False,
     "withDebug": False,

diff --git a/cffi_dist/example_python/example_image.py b/cffi_dist/example_python/example_image.py
@@ -4,7 +4,7 @@
 import re
 
 # load the tls-client shared package for your OS you are currently running your python script (i'm running on mac)
-library = ctypes.cdll.LoadLibrary('./../dist/tls-client-darwin-amd64-1.3.13.dylib')
+library = ctypes.cdll.LoadLibrary('./../dist/tls-client-darwin-amd64-1.4.0.dylib')
 
 # extract the exposed request function from the shared package
 request = library.request
@@ -36,7 +36,6 @@
     "withoutCookieJar": False,
     "withDefaultCookieJar": False,
     "isByteRequest": False,
-    "additionalDecode": "",
     "forceHttp1": False,
     "withDebug": False,
     "catchPanics": False,

diff --git a/cffi_dist/example_python/example_image_upload.py b/cffi_dist/example_python/example_image_upload.py
@@ -3,7 +3,7 @@
 import base64
 
 # load the tls-client shared package for your OS you are currently running your python script (i'm running on mac)
-library = ctypes.cdll.LoadLibrary('./../dist/tls-client-darwin-amd64-1.3.13.dylib')
+library = ctypes.cdll.LoadLibrary('./../dist/tls-client-darwin-amd64-1.4.0.dylib')
 
 # extract the exposed request function from the shared package
 request = library.request
@@ -39,7 +39,6 @@
     "withoutCookieJar": False,
     "withDefaultCookieJar": False,
     "isByteRequest": True,
-    "additionalDecode": "",
     "forceHttp1": False,
     "withDebug": False,
     "catchPanics": False,

diff --git a/cffi_dist/example_python/example_post.py b/cffi_dist/example_python/example_post.py
@@ -4,7 +4,7 @@
 import re
 
 # load the tls-client shared package for your OS you are currently running your python script (i'm running on mac)
-library = ctypes.cdll.LoadLibrary('./../dist/tls-client-darwin-amd64-1.3.13.dylib')
+library = ctypes.cdll.LoadLibrary('./../dist/tls-client-darwin-amd64-1.4.0.dylib')
 
 # extract the exposed request function from the shared package
 request = library.request
@@ -41,7 +41,6 @@
     "isByteResponse": False,
     "isByteRequest": False,
     "catchPanics": False,
-    "additionalDecode": "",
     "timeoutSeconds": 30,
     "timeoutMilliseconds": 0,
     "certificatePinningHosts": {},

diff --git a/cffi_dist/example_typescript/src/client.ts b/cffi_dist/example_typescript/src/client.ts
@@ -66,10 +66,10 @@ export class TLSClient implements TLSClientInstance {
 const createWrapper = (): LibraryObject<never> => {
     const sharedLibraryPath = join(__dirname, './../../dist/');
     const sharedLibraryFilename = platform() === 'win32'
-        ? `tls-client-windows-64-1.3.13.dll`
+        ? `tls-client-windows-64-1.4.0.dll`
         : arch() === 'arm64'
-            ? `tls-client-darwin-arm64-1.3.13.dylib`
-            : `tls-client-darwin-amd64-1.3.13.dylib`;
+            ? `tls-client-darwin-arm64-1.4.0.dylib`
+            : `tls-client-darwin-amd64-1.4.0.dylib`;
 
     return Library(join(sharedLibraryPath, sharedLibraryFilename), {
         request: ['string', ['string']],

diff --git a/cffi_src/factory.go b/cffi_src/factory.go
@@ -181,19 +181,20 @@ func BuildResponse(sessionId string, withSession bool, resp *http.Response, cook
 	defer resp.Body.Close()
 
 	isByteResponse := input.IsByteResponse
-	additionalDecodeAlgo := ""
 
-	if input.AdditionalDecode != nil {
-		additionalDecodeAlgo = *input.AdditionalDecode
-	}
+	ce := resp.Header.Get("Content-Encoding")
 
 	var respBodyBytes []byte
 	var err error
 
+	if !resp.Uncompressed {
+		resp.Body = http.DecompressBodyByType(resp.Body, ce)
+	}
+
 	if input.StreamOutputPath != nil {
-		respBodyBytes, err = readAllBodyWithStreamToFile(http.DecompressBodyByType(resp.Body, additionalDecodeAlgo), input)
+		respBodyBytes, err = readAllBodyWithStreamToFile(resp.Body, input)
 	} else {
-		respBodyBytes, err = io.ReadAll(http.DecompressBodyByType(resp.Body, additionalDecodeAlgo))
+		respBodyBytes, err = io.ReadAll(resp.Body)
 	}
 
 	if err != nil {

diff --git a/cffi_src/types.go b/cffi_src/types.go
@@ -47,7 +47,6 @@ type CookiesFromSessionOutput struct {
 
 // RequestInput is the data a Python client can construct a client and request from.
 type RequestInput struct {
-	AdditionalDecode            *string             `json:"additionalDecode"`
 	CatchPanics                 bool                `json:"catchPanics"`
 	CertificatePinningHosts     map[string][]string `json:"certificatePinningHosts"`
 	CustomTlsClient             *CustomTlsClient    `json:"customTlsClient"`