Add support for uTLS (fingerprinting resistance)

[nclv]

utls is a fork of the Go standard TLS library, providing low-level access to the ClientHello. I am particularly interested in generating randomized fingerprints.

This would require to modify the addTLS method of persistConn in transport.go. For instance :

// Initiate TLS and check remote host name against certificate.
cfg := cloneTLSConfig(pc.t.TLSClientConfig)
// if cfg.ServerName == "" {
// 	cfg.ServerName = name
// }
// if pc.cacheKey.onlyH1 {
// 	cfg.NextProtos = nil
// }
utlsConfig := &utls.Config{ServerName: name, NextProtos: cfg.NextProtos}

plainConn := pc.conn
tlsConn := utls.UClient(plainConn, utlsConfig, utls.HelloFirefox_Auto)
...
cs := tlsConn.ConnectionState()
// Partially convert tls.ConnectionState to utls.ConnectionState
// ekm cannot be set so a call to ExportKeyingMaterial will fail
utlsCs := tls.ConnectionState{
    Version:                     cs.Version,
    HandshakeComplete:           cs.HandshakeComplete,
    DidResume:                   cs.DidResume,
    CipherSuite:                 cs.CipherSuite,
    NegotiatedProtocol:          cs.NegotiatedProtocol,
    NegotiatedProtocolIsMutual:  cs.NegotiatedProtocolIsMutual,
    ServerName:                  cs.ServerName,
    SignedCertificateTimestamps: cs.SignedCertificateTimestamps,
    PeerCertificates:            cs.PeerCertificates,
    VerifiedChains:              cs.VerifiedChains,
    OCSPResponse:                cs.OCSPResponse,
    TLSUnique:                   cs.TLSUnique,
}
if trace != nil && trace.TLSHandshakeDone != nil {
    trace.TLSHandshakeDone(tls.ConnectionState{}, nil)
}
pc.tlsState = &utlsCs

However, I was unable to negotiate the TLS connection with only these modifications.

[imroc]

You can use SetDialTLS to override the tls dial function, replace with utls to dial tls. And if you don't want to break the protocol negotiation, the returned connection should implement req's Conn interface, here is an example:

package main

import (
	"context"
	"crypto/tls"
	"github.com/imroc/req/v3"
	utls "github.com/refraction-networking/utls"
	"net"
)

type TLSConn struct {
	*utls.Conn
}

func (conn *TLSConn) ConnectionState() tls.ConnectionState {
	cs := conn.Conn.ConnectionState()
	return tls.ConnectionState{
		Version:                     cs.Version,
		HandshakeComplete:           cs.HandshakeComplete,
		DidResume:                   cs.DidResume,
		CipherSuite:                 cs.CipherSuite,
		NegotiatedProtocol:          cs.NegotiatedProtocol,
		NegotiatedProtocolIsMutual:  cs.NegotiatedProtocolIsMutual,
		ServerName:                  cs.ServerName,
		PeerCertificates:            cs.PeerCertificates,
		VerifiedChains:              cs.VerifiedChains,
		SignedCertificateTimestamps: cs.SignedCertificateTimestamps,
		OCSPResponse:                cs.OCSPResponse,
		TLSUnique:                   cs.TLSUnique,
	}
}

func main() {
	c := req.C().DevMode()
	utlsConfig := &utls.Config{NextProtos: c.GetTLSClientConfig().NextProtos}
	c.SetDialTLS(func(ctx context.Context, network, addr string) (net.Conn, error) {
		conn, err := utls.Dial(network, addr, utlsConfig)
		if err != nil {
			return nil, err
		}
		return &TLSConn{conn}, nil
	})
	c.R().MustGet("https://httpbin.org/get")
}

[imroc]

I can add a document explaining how to use utls to support tls ClientHello fingerprinting resistance in req, but I don't want to integrate it into req, which introduces too many unnecessary dependencies for most users

[imroc]

Try this:

package main

import (
	"context"
	"crypto/tls"
	"github.com/imroc/req/v3"
	utls "github.com/refraction-networking/utls"
	"net"
	"strings"
)

type TLSConn struct {
	*utls.UConn
}

func (conn *TLSConn) ConnectionState() tls.ConnectionState {
	cs := conn.UConn.ConnectionState()
	return tls.ConnectionState{
		Version:                     cs.Version,
		HandshakeComplete:           cs.HandshakeComplete,
		DidResume:                   cs.DidResume,
		CipherSuite:                 cs.CipherSuite,
		NegotiatedProtocol:          cs.NegotiatedProtocol,
		NegotiatedProtocolIsMutual:  cs.NegotiatedProtocolIsMutual,
		ServerName:                  cs.ServerName,
		PeerCertificates:            cs.PeerCertificates,
		VerifiedChains:              cs.VerifiedChains,
		SignedCertificateTimestamps: cs.SignedCertificateTimestamps,
		OCSPResponse:                cs.OCSPResponse,
		TLSUnique:                   cs.TLSUnique,
	}
}

func main() {
	c := req.C().DevMode()
	c.SetDialTLS(func(ctx context.Context, network, addr string) (net.Conn, error) {
		plainConn, err := net.Dial(network, addr)
		if err != nil {
			return nil, err
		}
		colonPos := strings.LastIndex(addr, ":")
		if colonPos == -1 {
			colonPos = len(addr)
		}
		hostname := addr[:colonPos]
		utlsConfig := &utls.Config{ServerName: hostname, NextProtos: c.GetTLSClientConfig().NextProtos}
		conn := utls.UClient(plainConn, utlsConfig, utls.HelloFirefox_Auto)
		return &TLSConn{conn}, nil
	})
	c.R().MustGet("https://httpbin.org/get")
}

[nclv]

It was exactly what I was looking for thank you.
Below an updated version with a one-liner to check the fingerprint : curl https://tls.peet.ws/api/search-ja3?by=$(go run utls.go | jq -r ."tls"."ja3")

package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"io"
	"net"
	"strings"

	"github.com/imroc/req/v3"
	utls "github.com/refraction-networking/utls"
)

type TLSConn struct {
	*utls.UConn
}

func (conn *TLSConn) ConnectionState() tls.ConnectionState {
	cs := conn.UConn.ConnectionState()
	return tls.ConnectionState{
		Version:                     cs.Version,
		HandshakeComplete:           cs.HandshakeComplete,
		DidResume:                   cs.DidResume,
		CipherSuite:                 cs.CipherSuite,
		NegotiatedProtocol:          cs.NegotiatedProtocol,
		NegotiatedProtocolIsMutual:  cs.NegotiatedProtocolIsMutual,
		ServerName:                  cs.ServerName,
		PeerCertificates:            cs.PeerCertificates,
		VerifiedChains:              cs.VerifiedChains,
		SignedCertificateTimestamps: cs.SignedCertificateTimestamps,
		OCSPResponse:                cs.OCSPResponse,
		TLSUnique:                   cs.TLSUnique,
	}
}

// curl https://tls.peet.ws/api/search-ja3?by=$(go run main.go | jq -r ."tls"."ja3")
func main() {
	c := req.C()
	c.SetDialTLS(func(ctx context.Context, network, addr string) (net.Conn, error) {
		plainConn, err := net.Dial(network, addr)
		if err != nil {
			return nil, err
		}
		colonPos := strings.LastIndex(addr, ":")
		if colonPos == -1 {
			colonPos = len(addr)
		}
		hostname := addr[:colonPos]
		utlsConfig := &utls.Config{ServerName: hostname, NextProtos: c.GetTLSClientConfig().NextProtos}
		conn := utls.UClient(plainConn, utlsConfig, utls.HelloFirefox_Auto)
		return &TLSConn{conn}, nil
	})
	resp := c.R().MustGet("https://tls.peet.ws/api/all")
	defer resp.Body.Close()

	bodyBytes, err := io.ReadAll(resp.Body)
	if err != nil {
		fmt.Errorf("%w", err)
	}
	fmt.Println(string(bodyBytes))
}

[imroc]

You're welcome. BTW, you can call resp.String() or resp.ToString() to get response body as string, no need to use io.ReadAll(resp.Body)

[0xc1f]

Hello, @imroc! Will you be able to expose more h2 transport properties for the users? It's not JA3 related question but still about fingerprinting.

[imroc]

Hello, @imroc! Will you be able to expose more h2 transport properties for the users? It's not JA3 related question but still about fingerprinting.

The latest version exposed almost all h2 settings

[vellrya]

@imroc, thanks for the example, but I can't get it work with SetProxyURL. If I add a proxy (http or socks5) via SetProxyURL function, the SetDialTLS function is not even called, so the fingerprint does not change. Can you give an example of how to use this with a proxy server?

[imroc]

It may be supported in the future, and it may be a bit difficult. The standard net/http also does not support proxy after the custom dial function is specified.

[vellrya]

I found this solution, maybe there is a way to adapt it for req?
https://stackoverflow.com/questions/60624954/connect-via-proxy-while-using-utls-and-http-1-1-request

[imroc]

Nice catch! If that works, you only change the dial function from func(network, addr string, _ *tls.Config) (net.Conn, error) to func(_ context.Context, network, addr string) (net.Conn, error), and pass to client.SetDialTLS

[vellrya]

It seems to be working)

package main

import (
	"context"
	"crypto/tls"
	"github.com/imroc/req/v3"
	utls "github.com/refraction-networking/utls"
	"golang.org/x/net/proxy"
	"net"
	"strings"
)

type TLSConn struct {
	*utls.UConn
}

func (conn *TLSConn) ConnectionState() tls.ConnectionState {
	cs := conn.UConn.ConnectionState()
	return tls.ConnectionState{
		Version:                     cs.Version,
		HandshakeComplete:           cs.HandshakeComplete,
		DidResume:                   cs.DidResume,
		CipherSuite:                 cs.CipherSuite,
		NegotiatedProtocol:          cs.NegotiatedProtocol,
		NegotiatedProtocolIsMutual:  cs.NegotiatedProtocolIsMutual,
		ServerName:                  cs.ServerName,
		PeerCertificates:            cs.PeerCertificates,
		VerifiedChains:              cs.VerifiedChains,
		SignedCertificateTimestamps: cs.SignedCertificateTimestamps,
		OCSPResponse:                cs.OCSPResponse,
		TLSUnique:                   cs.TLSUnique,
	}
}

var proxyString = "x.x.x.x:1080" //socks5

func main() {
	c := req.C().DevMode()
	c.SetDialTLS(func(_ context.Context, network, addr string) (net.Conn, error) {

		proxyDialer, err := proxy.SOCKS5("tcp", proxyString, nil, proxy.Direct)

		if err != nil {
			return nil, err
		}

		plainConn, err := proxyDialer.Dial("tcp", addr)

		if err != nil {
			return nil, err
		}
		colonPos := strings.LastIndex(addr, ":")
		if colonPos == -1 {
			colonPos = len(addr)
		}
		hostname := addr[:colonPos]
		utlsConfig := &utls.Config{ServerName: hostname, NextProtos: c.GetTLSClientConfig().NextProtos}
		conn := utls.UClient(plainConn, utlsConfig, utls.HelloFirefox_Auto)
		return &TLSConn{conn}, nil
	})
	c.R().MustGet("https://tls.peet.ws/api/all")
}

[imroc]

Great!

[imroc]

Now utls is integrated into req, see: https://req.cool/docs/tutorial/tls-fingerprinting/

[3ldar]

@imroc It is so good to hear that req supports TLS fingerprinting, but now I am failing to find any documentation about the header organization. Is it allows re-ordering headers?. It seems there is a type for this: type headerSorter struct.

[imroc]

Custom header sorting is currently not supported, but since it is proposed, we can consider supporting it, added it into TODO.