From d7dd7c890fe7433d451a3a560e03088a2d200c39 Mon Sep 17 00:00:00 2001 From: Bogdan Finn Date: Sun, 8 Jan 2023 19:56:05 +0100 Subject: [PATCH] draft: implemented hawk aio cf api; --- antibot.go | 9 + client.go | 32 +++- client_options.go | 9 + example/main.go | 24 ++- go.mod | 3 + go.sum | 9 + hawk/captcha_solver.go | 204 ++++++++++++++++++++++ hawk/cf.go | 148 ++++++++++++++++ hawk/challenge_solver.go | 338 +++++++++++++++++++++++++++++++++++++ hawk/fingerprint_solver.go | 146 ++++++++++++++++ hawk/headers.go | 53 ++++++ hawk/notes.txt | 1 + hawk/solver_factory.go | 155 +++++++++++++++++ hawk/types.go | 76 +++++++++ hawk/utils.go | 213 +++++++++++++++++++++++ profiles.txt | 335 ++++++++++++++++++++++++++++++++++++ 16 files changed, 1734 insertions(+), 21 deletions(-) create mode 100644 antibot.go create mode 100644 hawk/captcha_solver.go create mode 100644 hawk/cf.go create mode 100644 hawk/challenge_solver.go create mode 100644 hawk/fingerprint_solver.go create mode 100644 hawk/headers.go create mode 100644 hawk/notes.txt create mode 100644 hawk/solver_factory.go create mode 100644 hawk/types.go create mode 100644 hawk/utils.go create mode 100644 profiles.txt diff --git a/antibot.go b/antibot.go new file mode 100644 index 0000000..73b508b --- /dev/null +++ b/antibot.go @@ -0,0 +1,9 @@ +package tls_client + +import ( + http "github.com/bogdanfinn/fhttp" +) + +type CFSolvingHandler interface { + Solve(logger Logger, client HttpClient, response *http.Response) (*http.Response, error) +} diff --git a/client.go b/client.go index db0eb0c..3d9d063 100644 --- a/client.go +++ b/client.go @@ -40,9 +40,10 @@ var _ HttpClient = (*httpClient)(nil) type httpClient struct { http.Client - headerLck sync.Mutex - logger Logger - config *httpClientConfig + headerLck sync.Mutex + logger Logger + config *httpClientConfig + cfSolvingHandler CFSolvingHandler } var DefaultTimeoutSeconds = 30 @@ -98,10 +99,11 @@ func NewHttpClient(logger Logger, options ...HttpClientOption) (HttpClient, erro } return &httpClient{ - Client: *client, - logger: logger, - config: config, - headerLck: sync.Mutex{}, + Client: *client, + logger: logger, + config: config, + headerLck: sync.Mutex{}, + cfSolvingHandler: config.cfSolvingHandler, }, nil } @@ -310,7 +312,7 @@ func (c *httpClient) Do(req *http.Request) (*http.Response, error) { c.logger.Debug("raw request bytes sent over wire: %d (%d kb)", len(requestBytes), len(requestBytes)/1024) } - resp, err := c.Client.Do(req) + resp, err := c.requestWithAntibotSolution(req) if err != nil { c.logger.Debug("failed to do request: %s", err.Error()) return nil, err @@ -348,6 +350,20 @@ func (c *httpClient) Do(req *http.Request) (*http.Response, error) { return resp, nil } +func (c *httpClient) requestWithAntibotSolution(request *http.Request) (*http.Response, error) { + resp, err := c.Client.Do(request) + if err != nil { + return nil, err + } + + if c.cfSolvingHandler != nil { + // we are passing the whole client to the solver as we have to do various sub requests for solving the anti bot challenge + return c.cfSolvingHandler.Solve(c.logger, c, resp) + } + + return resp, nil +} + func (c *httpClient) Get(url string) (resp *http.Response, err error) { req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { diff --git a/client_options.go b/client_options.go index 5d6ca13..7339f77 100644 --- a/client_options.go +++ b/client_options.go @@ -39,6 +39,7 @@ type httpClientConfig struct { badPinHandler BadPinHandlerFunc proxyUrl string serverNameOverwrite string + cfSolvingHandler CFSolvingHandler transportOptions *TransportOptions cookieJar http.CookieJar clientProfile ClientProfile @@ -199,3 +200,11 @@ func WithServerNameOverwrite(serverName string) HttpClientOption { config.serverNameOverwrite = serverName } } + +// WithCFSolvingHandler configures a TLS client to use a CF solving handler. +// You can instantiate the built-in HawkAIO CF integration. +func WithCFSolvingHandler(cfSolvingHandler CFSolvingHandler) HttpClientOption { + return func(config *httpClientConfig) { + config.cfSolvingHandler = cfSolvingHandler + } +} diff --git a/example/main.go b/example/main.go index 010fe4b..e92185b 100644 --- a/example/main.go +++ b/example/main.go @@ -86,7 +86,6 @@ func sslPinning() { } resp, err := client.Do(req) - if err != nil { log.Println(err) return @@ -109,10 +108,17 @@ func requestToppsAsChrome107Client() { tls_client.WithTimeoutSeconds(30), tls_client.WithClientProfile(tls_client.Chrome_107), tls_client.WithDebug(), - //tls_client.WithProxyUrl("http://user:pass@host:port"), - //tls_client.WithNotFollowRedirects(), - //tls_client.WithInsecureSkipVerify(), - tls_client.WithCookieJar(jar), // create cookieJar instance and pass it as argument + tls_client.WithTransportOptions(&tls_client.TransportOptions{ + DisableKeepAlives: false, + DisableCompression: false, + MaxIdleConns: 0, + MaxIdleConnsPerHost: 0, + MaxConnsPerHost: 0, + MaxResponseHeaderBytes: 0, + WriteBufferSize: 0, + ReadBufferSize: 0, + }), + tls_client.WithCookieJar(jar), } client, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(), options...) @@ -161,7 +167,6 @@ func requestToppsAsChrome107Client() { } resp, err := client.Do(req) - if err != nil { log.Println(err) return @@ -172,7 +177,6 @@ func requestToppsAsChrome107Client() { log.Printf("requesting topps as chrome107 => status code: %d\n", resp.StatusCode) u, err := url.Parse("https://www.topps.com/") - if err != nil { log.Println(err) return @@ -219,7 +223,6 @@ func postAsTlsClient() { } resp, err := client.Do(req) - if err != nil { log.Println(err) return @@ -284,7 +287,6 @@ func requestWithFollowRedirectSwitch() { } resp, err := client.Do(req) - if err != nil { log.Println(err) return @@ -327,7 +329,6 @@ func downloadImageWithTlsClient() { } resp, err := client.Do(req) - if err != nil { log.Println(err) return @@ -340,7 +341,6 @@ func downloadImageWithTlsClient() { log.Printf("requesting image => status code: %d\n", resp.StatusCode) ex, err := os.Executable() - if err != nil { log.Println(err) return @@ -420,7 +420,6 @@ func rotateProxiesOnClient() { } resp, err := client.Do(req) - if err != nil { log.Println(err) return @@ -627,7 +626,6 @@ func requestWithCustomClient() { } resp, err := client.Do(req) - if err != nil { log.Println(err) return diff --git a/go.mod b/go.mod index f4ff88b..215d9eb 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/bogdanfinn/tls-client go 1.20 require ( + github.com/Lazarus/lz-string-go v0.0.0-20220923232958-c256c46c2022 + github.com/anaskhan96/soup v1.2.5 github.com/bogdanfinn/fhttp v0.5.22 github.com/bogdanfinn/utls v1.5.16 github.com/google/uuid v1.3.0 @@ -25,3 +27,4 @@ require ( // replace github.com/bogdanfinn/utls => ../utls // replace github.com/bogdanfinn/fhttp => ../fhttp +replace github.com/juiced-aio/hawk-go => ../hawk-go diff --git a/go.sum b/go.sum index 0c4f634..cf47133 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/Lazarus/lz-string-go v0.0.0-20220923232958-c256c46c2022 h1:hKm9YZMsKTfEoWzsHFEiaS3hMUacsjxlJZl8FOFN9fw= +github.com/Lazarus/lz-string-go v0.0.0-20220923232958-c256c46c2022/go.mod h1:zhKAxszD+dMWmLOH6zlemBxkK+agsDEO2zabxpfgTik= +github.com/anaskhan96/soup v1.2.5 h1:V/FHiusdTrPrdF4iA1YkVxsOpdNcgvqT1hG+YtcZ5hM= +github.com/anaskhan96/soup v1.2.5/go.mod h1:6YnEp9A2yywlYdM4EgDz9NEHclocMepEtku7wg6Cq3s= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/bogdanfinn/fhttp v0.5.22 h1:U1jhZRtuaOanWWcm1WdMFnwMvSxUQgvO6berqAVTc5o= @@ -15,17 +19,22 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 h1:YqAladjX7xpA6BM04leXMWAEjS0mTZ5kUU9KRBriQJc= github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5/go.mod h1:2JjD2zLQYH5HO74y5+aE3remJQvl6q4Sn6aWA2wD1Ng= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/hawk/captcha_solver.go b/hawk/captcha_solver.go new file mode 100644 index 0000000..7e211a4 --- /dev/null +++ b/hawk/captcha_solver.go @@ -0,0 +1,204 @@ +package hawk + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "net/url" + "strings" + "time" + + lz "github.com/Lazarus/lz-string-go" + http "github.com/bogdanfinn/fhttp" + tls_client "github.com/bogdanfinn/tls-client" +) + +// In Progress +func handleFirstCaptchaFactory(logger tls_client.Logger, client tls_client.HttpClient, config cfConfig) solver { + return func(s cfChallengeState) (cfChallengeState, error) { + /* Handling captcha + Note that this function is designed to work with cloudscraper, + if you are building your own flow you will need to rework this part a bit. + */ + + var token string + var err error + if s.finalApi.Click { + token = "click" + } else { + logger.Debug("Captcha needed, requesting token.") + + token, err = config.CaptchaFunc(s.originalResponse.Request.URL.String(), s.finalApi.SiteKey) + if err != nil { + return s, err + } + } + + payload, err := json.Marshal(map[string]interface{}{ + "result": s.result, + "token": token, + "h-captcha-response": token, + "data": s.finalApi.Result, + }) + if err != nil { + return s, err + } + + requestUrl, err := url.Parse(fmt.Sprintf("https://%v/cf-a/ov1/cap1", config.ApiDomain)) + if err != nil { + return s, err + } + + for key, value := range config.AuthParams { + requestUrl.Query().Set(key, value) + } + + ff, err := client.Post(requestUrl.String(), "application/json", bytes.NewBuffer(payload)) + if err != nil { + return s, err + } + defer ff.Body.Close() + + var handleCaptchaResponse apiResponse + if err := readAndUnmarshalBody(ff.Body, &handleCaptchaResponse); err != nil { + return s, err + } + s.firstCaptchaResult = handleCaptchaResponse + + return s, nil + } + +} + +func submitCaptchaFactory(logger tls_client.Logger, client tls_client.HttpClient, config cfConfig) solver { + return func(s cfChallengeState) (cfChallengeState, error) { + // Submits the challenge + captcha and trys to access target url + if !s.captchaResponseAPI.Valid { + return s, nil + } + + payloadMap := map[string]string{ + "r": s.requestR, + "cf_captcha_kind": "h", + "vc": s.requestPass, + "captcha_vc": s.captchaResponseAPI.JschlVc, + "captcha_answer": s.captchaResponseAPI.JschlAnswer, + "cf_ch_verify": "plat", + } + + if s.captchaResponseAPI.CfChCpReturn != "" { + payloadMap["cf_ch_cp_return"] = s.captchaResponseAPI.CfChCpReturn + } + + if s.md != "" { + payloadMap["md"] = s.md + } + + // "captchka" Spelling mistake? + payloadMap["h-captcha-response"] = "captchka" + + payload := createParams(payloadMap) + + req, err := http.NewRequest(http.MethodPost, s.requestURL, bytes.NewBufferString(payload)) + if err != nil { + return s, err + } + + req.Header = submitHeaders + req.Header["referer"] = []string{s.originalResponse.Request.URL.String()} + req.Header["origin"] = []string{"https://" + s.domain} + req.Header["user-agent"] = s.originalResponse.Request.Header["user-agent"] + + if (time.Now().Unix() - s.startTime.Unix()) < 5 { + // Waiting X amount of sec for CF delay + logger.Debug("sleeping %d sec for cf delay", 5-(time.Now().Unix()-s.startTime.Unix())) + + time.Sleep(time.Duration(5-(time.Now().Unix()-s.startTime.Unix())) * time.Second) + } + + final, err := client.Do(req) + if err != nil { + return s, err + } + defer final.Body.Close() + + s.finalResponse = final + + logger.Debug("submitted captcha challange") + + return s, nil + } +} + +func handleSecondCaptchaFactory(logger tls_client.Logger, client tls_client.HttpClient, config cfConfig) solver { + return func(s cfChallengeState) (cfChallengeState, error) { + /* Handling captcha + Note that this function is designed to work with cloudscraper, + if you are building your own flow you will need to rework this part a bit. + */ + resultDecoded, err := base64.StdEncoding.DecodeString(s.firstCaptchaResult.Result) + if err != nil { + return s, fmt.Errorf("posting to cloudflare challenge endpoint error: %w", err) + } + + payload := []byte(createParams(map[string]string{ + s.name: lz.Compress(string(resultDecoded), s.keyStrUriSafe), + })) + + req, err := http.NewRequest(http.MethodPost, s.initURL, bytes.NewBuffer(payload)) + if err != nil { + return s, err + } + + req.Header = challengeHeaders + initURLSplit := strings.Split(s.initURL, "/") + req.Header["cf-challenge"] = []string{initURLSplit[len(initURLSplit)-1]} + req.Header["referer"] = []string{strings.Split(s.originalResponse.Request.URL.String(), "?")[0]} + req.Header["origin"] = []string{"https://" + s.domain} + req.Header["user-agent"] = s.originalResponse.Request.Header["user-agent"] + + gg, err := client.Do(req) + if err != nil { + return s, err + } + defer gg.Body.Close() + + body, err := readAndCloseBody(gg.Body) + if err != nil { + return s, err + } + + payload, err = json.Marshal(map[string]interface{}{ + "body_sensor": base64.StdEncoding.EncodeToString(body), + "result": s.baseObj, + }) + if err != nil { + return s, err + } + + requestUrl, err := url.Parse(fmt.Sprintf("https://%v/cf-a/ov1/cap2", config.ApiDomain)) + if err != nil { + return s, err + } + + for key, value := range config.AuthParams { + requestUrl.Query().Set(key, value) + } + + hh, err := client.Post(requestUrl.String(), "application/json", bytes.NewBuffer(payload)) + if err != nil { + return s, err + } + defer hh.Body.Close() + + handleCaptchaResponse := apiResponse{} + err = readAndUnmarshalBody(hh.Body, &handleCaptchaResponse) + if err != nil { + return s, err + } + s.captchaResponseAPI = handleCaptchaResponse + + return s, fmt.Errorf("captcha was not accepted - most likly wrong token") + } +} diff --git a/hawk/cf.go b/hawk/cf.go new file mode 100644 index 0000000..69036b7 --- /dev/null +++ b/hawk/cf.go @@ -0,0 +1,148 @@ +package hawk + +import ( + "fmt" + "time" + + http "github.com/bogdanfinn/fhttp" + tls_client "github.com/bogdanfinn/tls-client" +) + +type hawkCF struct { + apiKey string + config cfConfig +} + +func NewHawkCF(apiKey string) (tls_client.CFSolvingHandler, error) { + /*jar := tls_client.NewCookieJar() + + options := []tls_client.HttpClientOption{ + tls_client.WithTimeoutSeconds(60), + tls_client.WithClientProfile(tls_client.CloudflareCustom), + tls_client.WithCookieJar(jar), + } + + httpClient, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(), options...) + + if err != nil { + return nil, err + }*/ + + cfConfig := cfConfig{ + ApiDomain: "cf-v2.hwkapi.com", + AuthParams: map[string]string{ + "auth": apiKey, + }, + ErrorDelay: 30, + MaxRetries: 5, + } + + return &hawkCF{ + apiKey: apiKey, + config: cfConfig, + }, nil +} + +func (h *hawkCF) Solve(logger tls_client.Logger, client tls_client.HttpClient, response *http.Response) (*http.Response, error) { + originalResponseBody, err := readAndCopyBody(response) + if err != nil { + return response, err + } + + challengeState := cfChallengeState{ + originalResponseBody: originalResponseBody, + originalResponse: response, + startTime: time.Now(), + domain: response.Request.URL.Host, + } + + if isNewIUAMChallenge(response) { + resp, err := h.run(logger, client, challengeState) + if err != nil { + return response, err + } + + return resp, nil + } + + if isNewCaptchaChallenge(response) { + h.config.Captcha = true + + resp, err := h.run(logger, client, challengeState) + if err != nil { + return response, err + } + + return resp, nil + } + + if isFingerprintChallenge(response) { + h.config.FingerPrint = true + + resp, err := h.fingerprint(logger, client, challengeState) + if err != nil { + return response, err + } + + return resp, nil + } + + return response, nil +} + +func (h *hawkCF) run(logger tls_client.Logger, client tls_client.HttpClient, challengeState cfChallengeState) (*http.Response, error) { + for retries := 0; challengeState.finalResponse == nil || h.config.MaxRetries >= retries; retries++ { + if h.config.Captcha && h.config.CaptchaFunc == nil { + return nil, fmt.Errorf("captcha is present with nil CaptchaFunction") + } + + var err error + for _, f := range getChallengeSolver(logger, client, h.config) { + challengeState, err = executeWithRetries(f, h.config, challengeState) + if err != nil { + return nil, err + } + } + + if challengeState.finalApi.Status == "rerun" { + continue + } + + if !h.config.Captcha && challengeState.finalApi.Captcha { + return nil, fmt.Errorf("cf returned captcha and captcha handling is disabled") + } + + if challengeState.finalApi.Captcha { + for _, f := range getCaptchaSolver(logger, client, h.config) { + challengeState, err = executeWithRetries(f, h.config, challengeState) + if err != nil { + return nil, err + } + } + + continue + } + + submitChallenge := submitChallengeFactory(logger, client, h.config) + + challengeState, err = executeWithRetries(submitChallenge, h.config, challengeState) + if err != nil { + return nil, err + } + } + + return challengeState.finalResponse, nil +} + +func (h *hawkCF) fingerprint(logger tls_client.Logger, client tls_client.HttpClient, challengeState cfChallengeState) (*http.Response, error) { + var err error + + for _, f := range getFingerprintSolver(logger, client, h.config) { + challengeState, err = executeWithRetries(f, h.config, challengeState) + if err != nil { + return nil, err + } + } + + return challengeState.finalResponse, nil +} diff --git a/hawk/challenge_solver.go b/hawk/challenge_solver.go new file mode 100644 index 0000000..b0e5f3e --- /dev/null +++ b/hawk/challenge_solver.go @@ -0,0 +1,338 @@ +package hawk + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "net/url" + "strings" + + lz "github.com/Lazarus/lz-string-go" + http "github.com/bogdanfinn/fhttp" + tls_client "github.com/bogdanfinn/tls-client" +) + +func solveFactory(logger tls_client.Logger, client tls_client.HttpClient, config cfConfig) solver { + return func(s cfChallengeState) (cfChallengeState, error) { + // get initial response body + if len(s.initialResponseBody) != 0 && len(s.urlPart) != 0 && len(s.keyStrUriSafe) != 0 { + return s, nil + } + + extractedScript, err := extractChallengeScript(string(s.originalResponseBody)) + if err != nil { + return s, err + } + + // Fetching CF script + script := fmt.Sprintf("https://%s%s", s.domain, *extractedScript) + + req, err := http.NewRequest(http.MethodGet, script, nil) + if err != nil { + return s, fmt.Errorf("could no create request for challenging solving: %w", err) + } + + req.Header = initHeaders + req.Header["user-agent"] = s.originalResponse.Request.Header["user-agent"] + resp, err := client.Do(req) + if err != nil { + return s, fmt.Errorf("could not request: %w", err) + } + + defer resp.Body.Close() + + initialResponseBody, err := readAndCloseBody(resp.Body) + if err != nil { + return s, fmt.Errorf("could not request: %w", err) + } + + urlPartP, err := extractUrlPartRegex(string(s.initialResponseBody)) + if err != nil { + return s, err + } + urlPart := *urlPartP + + keyStringUri, err := extractKeyStringUri(string(s.initialResponseBody)) + if err != nil { + return s, err + } + + s.urlPart = urlPart + s.keyStrUriSafe = *keyStringUri + + s.initialResponseBody = initialResponseBody + + logger.Debug("Loaded init script.") + + return s, nil + } +} + +// initUrl +// initURL +// requestURL +// result +// name +// baseObj +// requestPass +// requestR +// ts +// md +func challengeInitiationPayloadFactory(logger tls_client.Logger, client tls_client.HttpClient, config cfConfig) solver { + return func(s cfChallengeState) (cfChallengeState, error) { + if len(s.initURL) != 0 && len(s.requestURL) != 0 && len(s.result) != 0 && len(s.name) != 0 && len(s.baseObj) != 0 && len(s.requestPass) != 0 && len(s.requestR) != 0 && s.ts != 0 && len(s.md) != 0 { + return s, nil + } + + payload, err := json.Marshal(map[string]interface{}{ + "body": base64.StdEncoding.EncodeToString(s.originalResponseBody), + "url": s.urlPart, + "domain": s.domain, + "captcha": config.Captcha, + "key": s.keyStrUriSafe, + }) + + if err != nil { + return s, fmt.Errorf("could not marshal body: %w", err) + } + + requestUrl, err := url.Parse(fmt.Sprintf("https://%s/cf-a/ov1/p1", config.ApiDomain)) + if err != nil { + return s, fmt.Errorf("could not create requestUrl: %w", err) + } + + for key, value := range config.AuthParams { + requestUrl.Query().Set(key, value) + } + + challengePayload, err := client.Post(requestUrl.String(), "application/json", bytes.NewBuffer(payload)) + if err != nil { + return s, err + } + defer challengePayload.Body.Close() + + var challengePayloadResponse apiResponse + err = readAndUnmarshalBody(challengePayload.Body, &challengePayloadResponse) + if err != nil { + return s, err + } + + s.initURL = challengePayloadResponse.URL + s.requestURL = challengePayloadResponse.ResultURL + s.result = challengePayloadResponse.Result + s.name = challengePayloadResponse.Name + s.baseObj = challengePayloadResponse.BaseObj + s.requestPass = challengePayloadResponse.Pass + s.requestR = challengePayloadResponse.R + s.ts = challengePayloadResponse.TS + s.md = challengePayloadResponse.Md + + logger.Debug("Submitted init payload to the api.") + + return s, nil + } +} + +func initiateCloudflareFactory(logger tls_client.Logger, client tls_client.HttpClient, config cfConfig) solver { + return func(s cfChallengeState) (cfChallengeState, error) { + // challengePayloadBody + if len(s.challengePayloadBody) != 0 { + return s, nil + } + + resultDecoded, err := base64.StdEncoding.DecodeString(s.result) + if err != nil { + return s, err + } + + payload := createParams(map[string]string{ + s.name: lz.Compress(string(resultDecoded), s.keyStrUriSafe), + }) + + if err != nil { + return s, err + } + + req, err := http.NewRequest(http.MethodPost, s.initURL, bytes.NewBufferString(payload)) + if err != nil { + return s, err + } + + req.Header = challengeHeaders + initURLSplit := strings.Split(s.initURL, "/") + req.Header["cf-challenge"] = []string{initURLSplit[len(initURLSplit)-1]} + req.Header["referer"] = []string{strings.Split(s.originalResponse.Request.URL.String(), "?")[0]} + req.Header["origin"] = []string{"https://" + s.domain} + req.Header["user-agent"] = s.originalResponse.Request.Header["user-agent"] + resp, err := client.Do(req) + if err != nil { + return s, err + } + + challengePayloadBody, err := readAndCopyBody(resp) + if err != nil { + return s, err + } + defer resp.Body.Close() + + s.challengePayloadBody = string(challengePayloadBody) + s.challengePayloadBody = string(challengePayloadBody) + + logger.Debug("Initiated challenge.") + + return s, nil + } +} + +func solvePayloadFactory(logger tls_client.Logger, client tls_client.HttpClient, config cfConfig) solver { + return func(s cfChallengeState) (cfChallengeState, error) { + // result + // Fetches main challenge payload from hawk api + + body := map[string]interface{}{ + "body_home": base64.RawURLEncoding.EncodeToString(s.originalResponseBody), + "body_sensor": base64.RawURLEncoding.EncodeToString([]byte(s.challengePayloadBody)), + "result": s.baseObj, + "ts": s.ts, + "url": s.initURL, + } + + if len(s.result) != 0 { + body["rerun_base"] = s.result + body["rerun"] = true + } else { + body["ua"] = s.originalResponse.Request.Header["user-agent"][0] + } + + payload, err := json.Marshal(map[string]interface{}{ + "body_home": base64.RawURLEncoding.EncodeToString(s.originalResponseBody), + "body_sensor": base64.RawURLEncoding.EncodeToString([]byte(s.challengePayloadBody)), + "result": s.baseObj, + "ts": s.ts, + "url": s.initURL, + "ua": s.originalResponse.Request.Header["user-agent"][0], + }) + if err != nil { + return s, err + } + + requestUrl, err := url.Parse(fmt.Sprintf("https://%s/cf-a/ov1/p2", config.ApiDomain)) + if err != nil { + return s, err + } + + for key, value := range config.AuthParams { + requestUrl.Query().Set(key, value) + } + + cc, err := client.Post(requestUrl.String(), "application/json", bytes.NewBuffer(payload)) + if err != nil { + return s, err + } + defer cc.Body.Close() + + var solvePayloadResponse apiResponse + if err := readAndUnmarshalBody(cc.Body, &solvePayloadResponse); err != nil { + return s, err + } + s.result = solvePayloadResponse.Result + s.result = solvePayloadResponse.Result + + logger.Debug("Fetched challenge payload.") + + return s, nil + } +} + +func sendMainPayloadFactory(logger tls_client.Logger, client tls_client.HttpClient, config cfConfig) solver { + return func(s cfChallengeState) (cfChallengeState, error) { + // mainPayloadResponseBody + // Sends the main payload to cf + if len(s.mainPayloadResponseBody) != 0 { + return s, nil + } + + resultDecoded, err := base64.StdEncoding.DecodeString(s.result) + if err != nil { + return s, err + } + payload := createParams(map[string]string{ + s.name: lz.Compress(string(resultDecoded), s.keyStrUriSafe), + }) + + if err != nil { + return s, err + } + + req, err := http.NewRequest(http.MethodPost, s.initURL, bytes.NewBufferString(payload)) + if err != nil { + return s, err + } + + req.Header = challengeHeaders + + initURLSplit := strings.Split(s.initURL, "/") + req.Header["cf-challenge"] = []string{initURLSplit[len(initURLSplit)-1]} + req.Header["referer"] = []string{strings.Split(s.originalResponse.Request.URL.String(), "?")[0]} + req.Header["origin"] = []string{"https://" + s.domain} + req.Header["user-agent"] = s.originalResponse.Request.Header["user-agent"] + + mainPayloadResponse, err := client.Do(req) + if err != nil { + return s, err + } + + body, err := readAndCopyBody(mainPayloadResponse) + s.mainPayloadResponseBody = string(body) + + defer mainPayloadResponse.Body.Close() + + logger.Debug("Submitted challenge.") + + return s, nil + } +} + +func getChallengeResultFactory(logger tls_client.Logger, client tls_client.HttpClient, config cfConfig) solver { + return func(s cfChallengeState) (cfChallengeState, error) { + // get finalApi + if s.finalApi != nil { + return s, nil + } + + payload, err := json.Marshal(map[string]interface{}{ + "body_sensor": base64.StdEncoding.EncodeToString([]byte(s.mainPayloadResponseBody)), + "result": s.baseObj, + }) + if err != nil { + return s, err + } + + requestUrl, err := url.Parse(fmt.Sprintf("https://%s/cf-a/ov1/p3", config.ApiDomain)) + if err != nil { + return s, err + } + + for key, value := range config.AuthParams { + requestUrl.Query().Set(key, value) + } + + ee, err := client.Post(requestUrl.String(), "application/json", bytes.NewBuffer(payload)) + if err != nil { + return s, err + } + defer ee.Body.Close() + + response := &apiResponse{} + if err = readAndUnmarshalBody(ee.Body, response); err != nil { + return s, err + } + + s.finalApi = response + + logger.Debug("Fetched challenge response.") + + return s, nil + } +} diff --git a/hawk/fingerprint_solver.go b/hawk/fingerprint_solver.go new file mode 100644 index 0000000..a3d98cf --- /dev/null +++ b/hawk/fingerprint_solver.go @@ -0,0 +1,146 @@ +package hawk + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/url" + "strings" + + http "github.com/bogdanfinn/fhttp" + tls_client "github.com/bogdanfinn/tls-client" +) + +// get initialResponseBody +func initiateScriptFactory(logger tls_client.Logger, client tls_client.HttpClient, config cfConfig) solver { + return func(s cfChallengeState) (cfChallengeState, error) { + if len(s.initialResponseBody) != 0 { + return s, nil + } + + urlPath := strings.Split(strings.Split(string(s.originalResponseBody), `