Xray-core: More robust browser header masquerading (chrome, firefox, edge) (#5802)

Fixes https://github.com/XTLS/Xray-core/issues/5800

app/dns/nameserver_doh.go

@@ -214,7 +214,7 @@ func (s *DoHNameServer) dohHTTPSContext(ctx context.Context, b []byte) ([]byte,
 
 	req.Header.Add("Accept", "application/dns-message")
 	req.Header.Add("Content-Type", "application/dns-message")
-	req.Header.Set("User-Agent", utils.ChromeUA)
+	utils.TryDefaultHeadersWith(req.Header, "fetch")
 	req.Header.Set("X-Padding", utils.H2Base62Pad(crypto.RandBetween(100, 1000)))
 
 	hc := s.httpClient

app/observatory/burst/ping.go

@@ -62,7 +62,7 @@ func (s *pingClient) MeasureDelay(httpMethod string) (time.Duration, error) {
 	if err != nil {
 		return rttFailed, err
 	}
-	req.Header.Set("User-Agent", utils.ChromeUA)
+	utils.TryDefaultHeadersWith(req.Header, "nav")
 
 	start := time.Now()
 	resp, err := s.httpClient.Do(req)

app/observatory/observer.go

@@ -164,7 +164,7 @@ func (o *Observer) probe(outbound string) ProbeResult {
 			probeURL = o.config.ProbeUrl
 		}
 		req, _ := http.NewRequest(http.MethodGet, probeURL, nil)
-		req.Header.Set("User-Agent", utils.ChromeUA)
+		utils.TryDefaultHeadersWith(req.Header, "nav")
 		response, err := httpClient.Do(req)
 		if err != nil {
 			return errors.New("outbound failed to relay connection").Base(err)

common/utils/browser.go

@@ -4,6 +4,8 @@ import (
 	"math/rand"
 	"strconv"
 	"time"
+	"net/http"
+	"strings"
 
 	"github.com/klauspost/cpuid/v2"
 )
@@ -24,5 +26,166 @@ func ChromeVersion() int {
 	return version - 1
 }
 
-// ChromeUA provides default browser User-Agent based on CPU-seeded PRNG.
-var ChromeUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/" + strconv.Itoa(ChromeVersion()) + ".0.0.0 Safari/537.36"
+// The full Chromium brand GREASE implementation
+var clientHintGreaseNA = []string{" ", "(", ":", "-", ".", "/", ")", ";", "=", "?", "_"}
+var clientHintVersionNA = []string{"8", "99", "24"}
+var clientHintShuffle3 = [][3]int{{0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}}
+var clientHintShuffle4 = [][4]int{
+	{0, 1, 2, 3}, {0, 1, 3, 2}, {0, 2, 1, 3}, {0, 2, 3, 1}, {0, 3, 1, 2}, {0, 3, 2, 1},
+	{1, 0, 2, 3}, {1, 0, 3, 2}, {1, 2, 0, 3}, {1, 2, 3, 0}, {1, 3, 0, 2}, {1, 3, 2, 0},
+	{2, 0, 1, 3}, {2, 0, 3, 1}, {2, 1, 0, 3}, {2, 1, 3, 0}, {2, 3, 0, 1}, {2, 3, 1, 0},
+	{3, 0, 1, 2}, {3, 0, 2, 1}, {3, 1, 0, 2}, {3, 1, 2, 0}, {3, 2, 0, 1}, {3, 2, 1, 0}}
+func getGreasedChInvalidBrand(seed int) string {
+	return "\"Not" + clientHintGreaseNA[seed % len(clientHintGreaseNA)] + "A" + clientHintGreaseNA[(seed + 1) % len(clientHintGreaseNA)] + "Brand\";v=\"" + clientHintVersionNA[seed % len(clientHintVersionNA)] + "\"";
+}
+func getGreasedChOrder(brandLength int, seed int) []int {
+	switch brandLength {
+		case 1:
+			return []int{0}
+		case 2:
+			return []int{seed % brandLength, (seed + 1) % brandLength}
+		case 3:
+			return clientHintShuffle3[seed % len(clientHintShuffle3)][:]
+		default:
+			return clientHintShuffle4[seed % len(clientHintShuffle4)][:]
+	}
+	return []int{}
+}
+func getUngreasedChUa(majorVersion int, forkName string) []string {
+	// Set the capacity to 4, the maximum allowed brand size, so Go will never allocate memory twice
+	baseChUa := make([]string, 0, 4)
+	baseChUa = append(baseChUa, getGreasedChInvalidBrand(majorVersion),
+	"\"Chromium\";v=\"" + strconv.Itoa(majorVersion) + "\"")
+	switch forkName {
+	case "chrome":
+		baseChUa = append(baseChUa, "\"Google Chrome\";v=\"" + strconv.Itoa(majorVersion) + "\"")
+	case "edge":
+		baseChUa = append(baseChUa, "\"Microsoft Edge\";v=\"" + strconv.Itoa(majorVersion) + "\"")
+	}
+	return baseChUa
+}
+func getGreasedChUa(majorVersion int, forkName string) string {
+	ungreasedCh := getUngreasedChUa(majorVersion, forkName)
+	shuffleMap := getGreasedChOrder(len(ungreasedCh), majorVersion)
+	shuffledCh := make([]string, len(ungreasedCh))
+	for i, e := range shuffleMap {
+		shuffledCh[e] = ungreasedCh[i]
+	}
+	return strings.Join(shuffledCh, ", ")
+}
+
+// It's better to pin on Firefox ESR releases, and there could be a Firefox ESR version generator later.
+// However, if the Firefox fingerprint in uTLS doesn't have its update cadence match that of Firefox ESR, then it's better to update the Firefox version manually instead every time a new major ESR release is available.
+var FirefoxUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0"
+
+// The code below provides a coherent default browser user agent string based on a CPU-seeded PRNG.
+var AnchoredChromeVersion = ChromeVersion()
+var ChromeUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/" + strconv.Itoa(AnchoredChromeVersion) + ".0.0.0 Safari/537.36"
+var ChromeUACH = getGreasedChUa(AnchoredChromeVersion, "chrome")
+var MSEdgeUA = ChromeUA + "Edg/" + strconv.Itoa(AnchoredChromeVersion) + ".0.0.0"
+var MSEdgeUACH = getGreasedChUa(AnchoredChromeVersion, "edge")
+
+func applyMasqueradedHeaders(header http.Header, browser string, variant string) {
+	// Browser-specific.
+	switch browser {
+	case "chrome":
+		header["Sec-CH-UA"] = []string{ChromeUACH}
+		header["Sec-CH-UA-Mobile"] = []string{"?0"}
+		header["Sec-CH-UA-Platform"] = []string{"\"Windows\""}
+		header["DNT"] = []string{"1"}
+		header.Set("User-Agent", ChromeUA)
+		header.Set("Accept-Language", "en-US,en;q=0.9")
+	case "edge":
+		header["Sec-CH-UA"] = []string{MSEdgeUACH}
+		header["Sec-CH-UA-Mobile"] = []string{"?0"}
+		header["Sec-CH-UA-Platform"] = []string{"\"Windows\""}
+		header["DNT"] = []string{"1"}
+		header.Set("User-Agent", MSEdgeUA)
+		header.Set("Accept-Language", "en-US,en;q=0.9")
+	case "firefox":
+		header.Set("User-Agent", FirefoxUA)
+		header["DNT"] = []string{"1"}
+		header.Set("Accept-Language", "en-US,en;q=0.5")
+	case "golang":
+		// Expose the default net/http header.
+		header.Del("User-Agent")
+		return
+	}
+	// Context-specific.
+	switch variant {
+	case "nav":
+		if header.Get("Cache-Control") == "" {
+			switch browser {
+			case "chrome", "edge":
+				header.Set("Cache-Control", "max-age=0")
+			}
+		}
+		header.Set("Upgrade-Insecure-Requests", "1")
+		if header.Get("Accept") == "" {
+			switch browser {
+			case "chrome", "edge":
+				header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/jxl,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
+			case "firefox":
+				header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+			}
+		}
+		header.Set("Sec-Fetch-Site", "none")
+		header.Set("Sec-Fetch-Mode", "navigate")
+		header.Set("Sec-Fetch-User", "?1")
+		header.Set("Sec-Fetch-Dest", "document")
+		header.Set("Priority", "u=0, i")
+	case "ws":
+		header.Set("Sec-Fetch-Mode", "websocket")
+		header.Set("Sec-Fetch-Dest", "empty")
+		header.Set("Sec-Fetch-Site", "same-origin")
+		if header.Get("Cache-Control") == "" {
+			header.Set("Cache-Control", "no-cache")
+		}
+		if header.Get("Pragma") == "" {
+			header.Set("Pragma", "no-cache")
+		}
+		if header.Get("Accept") == "" {
+			header.Set("Accept", "*/*")
+		}
+	case "fetch":
+		header.Set("Sec-Fetch-Mode", "cors")
+		header.Set("Sec-Fetch-Dest", "empty")
+		header.Set("Sec-Fetch-Site", "same-origin")
+		if header.Get("Priority") == "" {
+			switch browser {
+			case "chrome", "edge":
+				header.Set("Priority", "u=1, i")
+			case "firefox":
+				header.Set("Priority", "u=4")
+			}
+		}
+		if header.Get("Cache-Control") == "" {
+			header.Set("Cache-Control", "no-cache")
+		}
+		if header.Get("Pragma") == "" {
+			header.Set("Pragma", "no-cache")
+		}
+		if header.Get("Accept") == "" {
+			header.Set("Accept", "*/*")
+		}
+	}
+}
+
+func TryDefaultHeadersWith(header http.Header, variant string) {
+	// The global UA special value handler for transports. Used to be called HandleTransportUASettings.
+	// Just a FYI to whoever needing to fix this piece of code after some spontaneous event, I tried to make the two methods separate to let the code be cleaner and more organized.
+	if len(header.Values("User-Agent")) < 1 {
+		applyMasqueradedHeaders(header, "chrome", variant)
+	} else {
+		switch header.Get("User-Agent") {
+		case "chrome":
+			applyMasqueradedHeaders(header, "chrome", variant)
+		case "firefox":
+			applyMasqueradedHeaders(header, "firefox", variant)
+		case "edge":
+			applyMasqueradedHeaders(header, "edge", variant)
+		case "golang":
+			applyMasqueradedHeaders(header, "golang", variant)
+		}
+	}
+}

infra/conf/transport_authenticators.go

@@ -44,6 +44,34 @@ func (v *AuthenticatorRequest) Build() (*http.RequestConfig, error) {
 				Name:  "User-Agent",
 				Value: []string{utils.ChromeUA},
 			},
+			{
+				Name: "Sec-CH-UA",
+				Value: []string{utils.ChromeUACH},
+			},
+			{
+				Name: "Sec-CH-UA-Mobile",
+				Value: []string{"?0"},
+			},
+			{
+				Name: "Sec-CH-UA-Platform",
+				Value: []string{"Windows"},
+			},
+			{
+				Name: "Sec-Fetch-Mode",
+				Value: []string{"no-cors", "cors", "same-origin"},
+			},
+			{
+				Name: "Sec-Fetch-Dest",
+				Value: []string{"empty"},
+			},
+			{
+				Name: "Sec-Fetch-Site",
+				Value: []string{"none"},
+			},
+			{
+				Name: "Sec-Fetch-User",
+				Value: []string{"?1"},
+			},
 			{
 				Name:  "Accept-Encoding",
 				Value: []string{"gzip, deflate"},

proxy/http/client.go

@@ -220,9 +220,7 @@ func setUpHTTPTunnel(ctx context.Context, dest net.Destination, target string, u
 	for _, h := range header {
 		req.Header.Set(h.Key, h.Value)
 	}
-	if req.Header.Get("User-Agent") == "" {
-		req.Header.Set("User-Agent", utils.ChromeUA)
-	}
+	utils.TryDefaultHeadersWith(req.Header, "nav")
 
 	connectHTTP1 := func(rawConn net.Conn) (net.Conn, error) {
 		req.Header.Set("Proxy-Connection", "Keep-Alive")

transport/internet/grpc/dial.go

@@ -10,8 +10,8 @@ import (
 	c "github.com/xtls/xray-core/common/ctx"
 	"github.com/xtls/xray-core/common/errors"
 	"github.com/xtls/xray-core/common/net"
-	"github.com/xtls/xray-core/common/session"
 	"github.com/xtls/xray-core/common/utils"
+	"github.com/xtls/xray-core/common/session"
 	"github.com/xtls/xray-core/transport/internet"
 	"github.com/xtls/xray-core/transport/internet/grpc/encoding"
 	"github.com/xtls/xray-core/transport/internet/reality"
@@ -191,8 +191,16 @@ func getGrpcClient(ctx context.Context, dest net.Destination, streamSettings *in
 	)
 	if err == nil {
 		userAgent := grpcSettings.UserAgent
-		if userAgent == "" {
+		// It's NOT recommended to set the UA of gRPC connections to that of real browsers, as they are fundamentally incapable of initiating real gRPC connections.
+		switch userAgent {
+		case "chrome", "":
 			userAgent = utils.ChromeUA
+		case "firefox":
+			userAgent = utils.FirefoxUA
+		case "edge":
+			userAgent = utils.MSEdgeUA
+		case "golang":
+			userAgent = ""
 		}
 		setUserAgent(conn, userAgent)
 		conn.Connect()

transport/internet/httpupgrade/dialer.go

@@ -96,9 +96,7 @@ func dialhttpUpgrade(ctx context.Context, dest net.Destination, streamSettings *
 	for key, value := range transportConfiguration.Header {
 		AddHeader(req.Header, key, value)
 	}
-	if req.Header.Get("User-Agent") == "" {
-		req.Header.Set("User-Agent", utils.ChromeUA)
-	}
+	utils.TryDefaultHeadersWith(req.Header, "ws")
 	req.Header.Set("Connection", "Upgrade")
 	req.Header.Set("Upgrade", "websocket")
 

transport/internet/reality/reality.go

@@ -223,7 +223,7 @@ func UClient(c net.Conn, config *Config, ctx context.Context, dest net.Destinati
 				if req == nil {
 					return
 				}
-				req.Header.Set("User-Agent", utils.ChromeUA)
+				utils.TryDefaultHeadersWith(req.Header, "nav")
 				if first && config.Show {
 					fmt.Printf("REALITY localAddr: %v\treq.UserAgent(): %v\n", localAddr, req.UserAgent())
 				}

transport/internet/splithttp/config.go

@@ -51,9 +51,7 @@ func (c *Config) GetRequestHeader() http.Header {
 	for k, v := range c.Headers {
 		header.Add(k, v)
 	}
-	if header.Get("User-Agent") == "" {
-		header.Set("User-Agent", utils.ChromeUA)
-	}
+	utils.TryDefaultHeadersWith(header, "fetch")
 	return header
 }
 

transport/internet/tls/ech.go

@@ -253,7 +253,7 @@ func dnsQuery(server string, domain string, sockopt *internet.SocketConfig) ([]b
 		}
 		req.Header.Set("Accept", "application/dns-message")
 		req.Header.Set("Content-Type", "application/dns-message")
-		req.Header.Set("User-Agent", utils.ChromeUA)
+		utils.TryDefaultHeadersWith(req.Header, "fetch")
 		req.Header.Set("X-Padding", utils.H2Base62Pad(crypto.RandBetween(100, 1000)))
 
 		resp, err := client.Do(req)

transport/internet/websocket/config.go

@@ -24,9 +24,7 @@ func (c *Config) GetRequestHeader() http.Header {
 	for k, v := range c.Header {
 		header.Add(k, v)
 	}
-	if header.Get("User-Agent") == "" {
-		header.Set("User-Agent", utils.ChromeUA)
-	}
+	utils.TryDefaultHeadersWith(header, "ws")
 	return header
 }