draft: implemented hawk aio cf api;

antibot.go

@@ -0,0 +1,9 @@
+package tls_client
+
+import (
+	http "github.com/bogdanfinn/fhttp"
+)
+
+type CFSolvingHandler interface {
+	Solve(logger Logger, client HttpClient, response *http.Response) (*http.Response, error)
+}

client.go

@@ -40,9 +40,10 @@ var _ HttpClient = (*httpClient)(nil)
 
 type httpClient struct {
 	http.Client
-	headerLck sync.Mutex
-	logger    Logger
-	config    *httpClientConfig
+	headerLck        sync.Mutex
+	logger           Logger
+	config           *httpClientConfig
+	cfSolvingHandler CFSolvingHandler
 }
 
 var DefaultTimeoutSeconds = 30
@@ -98,10 +99,11 @@ func NewHttpClient(logger Logger, options ...HttpClientOption) (HttpClient, erro
 	}
 
 	return &httpClient{
-		Client:    *client,
-		logger:    logger,
-		config:    config,
-		headerLck: sync.Mutex{},
+		Client:           *client,
+		logger:           logger,
+		config:           config,
+		headerLck:        sync.Mutex{},
+		cfSolvingHandler: config.cfSolvingHandler,
 	}, nil
 }
 
@@ -310,7 +312,7 @@ func (c *httpClient) Do(req *http.Request) (*http.Response, error) {
 		c.logger.Debug("raw request bytes sent over wire: %d (%d kb)", len(requestBytes), len(requestBytes)/1024)
 	}
 
-	resp, err := c.Client.Do(req)
+	resp, err := c.requestWithAntibotSolution(req)
 	if err != nil {
 		c.logger.Debug("failed to do request: %s", err.Error())
 		return nil, err
@@ -348,6 +350,20 @@ func (c *httpClient) Do(req *http.Request) (*http.Response, error) {
 	return resp, nil
 }
 
+func (c *httpClient) requestWithAntibotSolution(request *http.Request) (*http.Response, error) {
+	resp, err := c.Client.Do(request)
+	if err != nil {
+		return nil, err
+	}
+
+	if c.cfSolvingHandler != nil {
+		// we are passing the whole client to the solver as we have to do various sub requests for solving the anti bot challenge
+		return c.cfSolvingHandler.Solve(c.logger, c, resp)
+	}
+
+	return resp, nil
+}
+
 func (c *httpClient) Get(url string) (resp *http.Response, err error) {
 	req, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {

client_options.go

@@ -39,6 +39,7 @@ type httpClientConfig struct {
 	badPinHandler               BadPinHandlerFunc
 	proxyUrl                    string
 	serverNameOverwrite         string
+	cfSolvingHandler            CFSolvingHandler
 	transportOptions            *TransportOptions
 	cookieJar                   http.CookieJar
 	clientProfile               ClientProfile
@@ -199,3 +200,11 @@ func WithServerNameOverwrite(serverName string) HttpClientOption {
 		config.serverNameOverwrite = serverName
 	}
 }
+
+// WithCFSolvingHandler configures a TLS client to use a CF solving handler.
+// You can instantiate the built-in HawkAIO CF integration.
+func WithCFSolvingHandler(cfSolvingHandler CFSolvingHandler) HttpClientOption {
+	return func(config *httpClientConfig) {
+		config.cfSolvingHandler = cfSolvingHandler
+	}
+}

example/main.go

@@ -86,7 +86,6 @@ func sslPinning() {
 	}
 
 	resp, err := client.Do(req)
-
 	if err != nil {
 		log.Println(err)
 		return
@@ -109,10 +108,17 @@ func requestToppsAsChrome107Client() {
 		tls_client.WithTimeoutSeconds(30),
 		tls_client.WithClientProfile(tls_client.Chrome_107),
 		tls_client.WithDebug(),
-		//tls_client.WithProxyUrl("http://user:pass@host:port"),
-		//tls_client.WithNotFollowRedirects(),
-		//tls_client.WithInsecureSkipVerify(),
-		tls_client.WithCookieJar(jar), // create cookieJar instance and pass it as argument
+		tls_client.WithTransportOptions(&tls_client.TransportOptions{
+			DisableKeepAlives:      false,
+			DisableCompression:     false,
+			MaxIdleConns:           0,
+			MaxIdleConnsPerHost:    0,
+			MaxConnsPerHost:        0,
+			MaxResponseHeaderBytes: 0,
+			WriteBufferSize:        0,
+			ReadBufferSize:         0,
+		}),
+		tls_client.WithCookieJar(jar),
 	}
 
 	client, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(), options...)
@@ -161,7 +167,6 @@ func requestToppsAsChrome107Client() {
 	}
 
 	resp, err := client.Do(req)
-
 	if err != nil {
 		log.Println(err)
 		return
@@ -172,7 +177,6 @@ func requestToppsAsChrome107Client() {
 	log.Printf("requesting topps as chrome107 => status code: %d\n", resp.StatusCode)
 
 	u, err := url.Parse("https://www.topps.com/")
-
 	if err != nil {
 		log.Println(err)
 		return
@@ -219,7 +223,6 @@ func postAsTlsClient() {
 	}
 
 	resp, err := client.Do(req)
-
 	if err != nil {
 		log.Println(err)
 		return
@@ -284,7 +287,6 @@ func requestWithFollowRedirectSwitch() {
 	}
 
 	resp, err := client.Do(req)
-
 	if err != nil {
 		log.Println(err)
 		return
@@ -327,7 +329,6 @@ func downloadImageWithTlsClient() {
 	}
 
 	resp, err := client.Do(req)
-
 	if err != nil {
 		log.Println(err)
 		return
@@ -340,7 +341,6 @@ func downloadImageWithTlsClient() {
 	log.Printf("requesting image => status code: %d\n", resp.StatusCode)
 
 	ex, err := os.Executable()
-
 	if err != nil {
 		log.Println(err)
 		return
@@ -420,7 +420,6 @@ func rotateProxiesOnClient() {
 	}
 
 	resp, err := client.Do(req)
-
 	if err != nil {
 		log.Println(err)
 		return
@@ -627,7 +626,6 @@ func requestWithCustomClient() {
 	}
 
 	resp, err := client.Do(req)
-
 	if err != nil {
 		log.Println(err)
 		return

go.mod

@@ -3,6 +3,8 @@ module github.com/bogdanfinn/tls-client
 go 1.20
 
 require (
+	github.com/Lazarus/lz-string-go v0.0.0-20220923232958-c256c46c2022
+	github.com/anaskhan96/soup v1.2.5
 	github.com/bogdanfinn/fhttp v0.5.22
 	github.com/bogdanfinn/utls v1.5.16
 	github.com/google/uuid v1.3.0
@@ -25,3 +27,4 @@ require (
 // replace github.com/bogdanfinn/utls => ../utls
 
 // replace github.com/bogdanfinn/fhttp => ../fhttp
+replace github.com/juiced-aio/hawk-go => ../hawk-go

go.sum

@@ -1,3 +1,7 @@
+github.com/Lazarus/lz-string-go v0.0.0-20220923232958-c256c46c2022 h1:hKm9YZMsKTfEoWzsHFEiaS3hMUacsjxlJZl8FOFN9fw=
+github.com/Lazarus/lz-string-go v0.0.0-20220923232958-c256c46c2022/go.mod h1:zhKAxszD+dMWmLOH6zlemBxkK+agsDEO2zabxpfgTik=
+github.com/anaskhan96/soup v1.2.5 h1:V/FHiusdTrPrdF4iA1YkVxsOpdNcgvqT1hG+YtcZ5hM=
+github.com/anaskhan96/soup v1.2.5/go.mod h1:6YnEp9A2yywlYdM4EgDz9NEHclocMepEtku7wg6Cq3s=
 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/bogdanfinn/fhttp v0.5.22 h1:U1jhZRtuaOanWWcm1WdMFnwMvSxUQgvO6berqAVTc5o=
@@ -15,17 +19,22 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 h1:YqAladjX7xpA6BM04leXMWAEjS0mTZ5kUU9KRBriQJc=
 github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5/go.mod h1:2JjD2zLQYH5HO74y5+aE3remJQvl6q4Sn6aWA2wD1Ng=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
 golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
 golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
 golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=