起因
抓取某个HTTPS网站的时候
开启charles代理能够抓取成功,关闭被风控
通过检测,怀疑可能是tls的时候有区别
尝试
golang的http中,Transport.TLSClientConfig是可以自定义设置的
但起初通过随意设置并不能绕过风控
困难
- 使用golang的http客户端,修改DialTLSContext函数的方式是可以实绕过风控,但使用proxy的时候,代码会使用pconn.addTLS(ctx, cm.tlsHost(), trace) 重新以普通方式进行握手,导致JA3修改失败
- 因为golang强关联,第三方库并不能完美的集成到现有代码中,都需要重构代码
- 某些网站会对于新建链接进行ClientSession检测,因此需要 KeepAlive+ClientSessionCache,这样通过复用连接减少风控概率
最终实现
- 只需要拿到合法的参数,并且配置到TLSClientConfig里即可
- 使用github.com/refraction-networking/utls中的UTLSIdToSpec拿到CipherSuites并传入
go
package main
import (
"bytes"
"crypto/tls"
tlsx "github.com/refraction-networking/utls"
"net/http"
)
func main() {
c, _ := tlsx.UTLSIdToSpec(tlsx.HelloRandomized)
a := &http.Client{
Transport: &http.Transport{
DisableKeepAlives: false,
Proxy: proxy,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
MinVersion: c.TLSVersMin,
MaxVersion: c.TLSVersMax,
CipherSuites: c.CipherSuites,
ClientSessionCache: tls.NewLRUClientSessionCache(32),
},
},
}
aw, bw := a.Get("https://tls.browserleaks.com/json")
defer aw.Body.Close()
var buf bytes.Buffer
aw.Write(&buf)
println(string(buf.String()), bw)
}
参考文章
- https://github.com/baixudong007/gospider
- https://juejin.cn/post/7073264626506399751 用Go构建你专属的JA3指纹
- https://blog.csdn.net/qq523176585/article/details/127116542 好库推荐|强烈推荐,支持Ja3指纹修改的golang请求库
- https://github.com/wangluozhe/requests
- https://github.com/refraction-networking/utls
- http://www.ctfiot.com/64337.html 如何绕过 JA3 指纹校验?
- https://www.coder.work/article/7192419 http - 发送请求时如何使用uTLS连接?
- https://segmentfault.com/a/1190000041699815/en Build your own JA3 fingerprint with Go
- https://zhuanlan.zhihu.com/p/601474166 curl_cffi: 支持原生模拟浏览器 TLS/JA3 指纹的 Python 库
- https://tools.scrapfly.io/api/fp/ja3
历史编写的代码
这些代码都不太好用
go
package main
import (
"context"
"fmt"
tls "github.com/refraction-networking/utls"
xtls "github.com/refraction-networking/utls"
"net"
"net/http"
)
func GetTransport(helloID *xtls.ClientHelloID) *http.Transport {
if helloID == nil {
helloID = &xtls.HelloChrome_83
}
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.DialTLSContext = func(ctx context.Context, network, addr string) (_ net.Conn, err error) {
dialer := net.Dialer{}
con, err := dialer.DialContext(ctx, network, addr)
if err != nil {
return nil, err
}
// 根据地址获取host信息
host, _, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
c := transport.TLSClientConfig
// 并且不验证host信息
xtlsConf := &xtls.Config{
ServerName: host,
//Renegotiation: xtls.RenegotiateNever,
ClientSessionCache: xtls.NewLRUClientSessionCache(32),
NextProtos: []string{"h2", "http/1.1"},
Rand: c.Rand,
Time: c.Time,
VerifyPeerCertificate: c.VerifyPeerCertificate,
RootCAs: c.RootCAs,
ClientCAs: c.ClientCAs,
InsecureSkipVerify: c.InsecureSkipVerify,
CipherSuites: c.CipherSuites,
PreferServerCipherSuites: c.PreferServerCipherSuites,
SessionTicketsDisabled: c.SessionTicketsDisabled,
SessionTicketKey: c.SessionTicketKey,
MinVersion: c.MinVersion,
MaxVersion: c.MaxVersion,
DynamicRecordSizingDisabled: c.DynamicRecordSizingDisabled,
KeyLogWriter: c.KeyLogWriter,
}
// 构建tls.UConn
xtlsConn := xtls.UClient(con, xtlsConf, *helloID)
// 握手
err = xtlsConn.HandshakeContext(ctx)
if err != nil {
return nil, err
}
fmt.Println("当前请求使用协议:", xtlsConn.HandshakeState.ServerHello.AlpnProtocol)
return xtlsConn, err
}
//transport.DialContext = transport.DialTLSContext
transport.DisableKeepAlives = true
return transport
}
func CreateHTTPClient() *http.Transport {
return &http.Transport{
DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
//initialize the tcp connection
tcpConn, err := (&net.Dialer{}).DialContext(ctx, network, addr)
if err != nil {
return nil, err
}
host, _, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
//initialize the conifg for tls
config := tls.Config{
ServerName: host, //set the server name with the provided addr
ClientSessionCache: xtls.NewLRUClientSessionCache(0),
}
//initialize a tls connection with the underlying tcop connection and config
tlsConn := tls.UClient(tcpConn, &config, tls.HelloRandomized)
//start the tls handshake between servers
err = tlsConn.Handshake()
if err != nil {
return nil, fmt.Errorf("uTlsConn.Handshake() error: %w", err)
}
return tlsConn, nil
},
ForceAttemptHTTP2: false,
}
}