Geofiles: Revert related changes for now, waiting for better changes (#5557)

Reverts 5d94a62a83 c715154309 961c352127 36425d2a6e

Fixes https://github.com/XTLS/Xray-core/issues/5538 https://github.com/XTLS/Xray-core/issues/5536

app/dns/dns.go

@@ -12,15 +12,12 @@ import (
 	"sync"
 	"time"
 
-	router "github.com/xtls/xray-core/app/router"
 	"github.com/xtls/xray-core/common"
 	"github.com/xtls/xray-core/common/errors"
 	"github.com/xtls/xray-core/common/net"
-	"github.com/xtls/xray-core/common/platform/filesystem"
 	"github.com/xtls/xray-core/common/session"
 	"github.com/xtls/xray-core/common/strmatcher"
 	"github.com/xtls/xray-core/features/dns"
-	"google.golang.org/protobuf/proto"
 )
 
 // DNS is a DNS rely server.
@@ -100,25 +97,6 @@ func New(ctx context.Context, config *Config) (*DNS, error) {
 	}
 
 	for _, ns := range config.NameServer {
-		if runtime.GOOS != "windows" && runtime.GOOS != "wasm" {
-			err := parseDomains(ns)
-			if err != nil {
-				return nil, errors.New("failed to parse dns domain rules: ").Base(err)
-			}
-
-			expectedGeoip, err := router.GetGeoIPList(ns.ExpectedGeoip)
-			if err != nil {
-				return nil, errors.New("failed to parse dns expectIPs rules: ").Base(err)
-			}
-			ns.ExpectedGeoip = expectedGeoip
-
-			unexpectedGeoip, err := router.GetGeoIPList(ns.UnexpectedGeoip)
-			if err != nil {
-				return nil, errors.New("failed to parse dns unexpectedGeoip rules: ").Base(err)
-			}
-			ns.UnexpectedGeoip = unexpectedGeoip
-
-		}
 		domainRuleCount += len(ns.PrioritizedDomain)
 	}
 
@@ -602,76 +580,3 @@ func detectGUIPlatform() bool {
 	}
 	return false
 }
-
-func parseDomains(ns *NameServer) error {
-	pureDomains := []*router.Domain{}
-
-	// convert to pure domain
-	for _, pd := range ns.PrioritizedDomain {
-		pureDomains = append(pureDomains, &router.Domain{
-			Type:  router.Domain_Type(pd.Type),
-			Value: pd.Domain,
-		})
-	}
-
-	domainList := []*router.Domain{}
-	for _, domain := range pureDomains {
-		val := strings.Split(domain.Value, "_")
-		if len(val) >= 2 {
-
-			fileName := val[0]
-			code := val[1]
-
-			bs, err := filesystem.ReadAsset(fileName)
-			if err != nil {
-				return errors.New("failed to load file: ", fileName).Base(err)
-			}
-			bs = filesystem.Find(bs, []byte(code))
-			var geosite router.GeoSite
-
-			if err := proto.Unmarshal(bs, &geosite); err != nil {
-				return errors.New("failed Unmarshal :").Base(err)
-			}
-
-			// parse attr
-			if len(val) == 3 {
-				siteWithAttr := strings.Split(val[2], ",")
-				attrs := router.ParseAttrs(siteWithAttr)
-				if !attrs.IsEmpty() {
-					filteredDomains := make([]*router.Domain, 0, len(pureDomains))
-					for _, domain := range geosite.Domain {
-						if attrs.Match(domain) {
-							filteredDomains = append(filteredDomains, domain)
-						}
-					}
-					geosite.Domain = filteredDomains
-				}
-
-			}
-
-			domainList = append(domainList, geosite.Domain...)
-
-			// update ns.OriginalRules Size
-			ruleTag := strings.Join(val, ":")
-			for i, oRule := range ns.OriginalRules {
-				if oRule.Rule == strings.ToLower(ruleTag) {
-					ns.OriginalRules[i].Size = uint32(len(geosite.Domain))
-				}
-			}
-
-		} else {
-			domainList = append(domainList, domain)
-		}
-	}
-
-	// convert back to NameServer_PriorityDomain
-	ns.PrioritizedDomain = []*NameServer_PriorityDomain{}
-	for _, pd := range domainList {
-		ns.PrioritizedDomain = append(ns.PrioritizedDomain, &NameServer_PriorityDomain{
-			Type:   ToDomainMatchingType(pd.Type),
-			Domain: pd.Value,
-		})
-	}
-
-	return nil
-}

app/dns/dns_test.go

@@ -541,7 +541,7 @@ func TestIPMatch(t *testing.T) {
 						},
 						ExpectedGeoip: []*router.GeoIP{
 							{
-								// local
+								CountryCode: "local",
 								Cidr: []*router.CIDR{
 									{
 										// inner ip, will not match
@@ -565,7 +565,7 @@ func TestIPMatch(t *testing.T) {
 						},
 						ExpectedGeoip: []*router.GeoIP{
 							{
-								// test
+								CountryCode: "test",
 								Cidr: []*router.CIDR{
 									{
 										Ip:     []byte{8, 8, 8, 8},
@@ -574,7 +574,7 @@ func TestIPMatch(t *testing.T) {
 								},
 							},
 							{
-								// test
+								CountryCode: "test",
 								Cidr: []*router.CIDR{
 									{
 										Ip:     []byte{8, 8, 8, 4},
@@ -669,7 +669,7 @@ func TestLocalDomain(t *testing.T) {
 						},
 						ExpectedGeoip: []*router.GeoIP{
 							{ // Will match localhost, localhost-a and localhost-b,
-								// local
+								CountryCode: "local",
 								Cidr: []*router.CIDR{
 									{Ip: []byte{127, 0, 0, 2}, Prefix: 32},
 									{Ip: []byte{127, 0, 0, 3}, Prefix: 32},

app/dns/nameserver.go

@@ -297,18 +297,3 @@ func ResolveIpOptionOverride(queryStrategy QueryStrategy, ipOption dns.IPOption)
 		return ipOption
 	}
 }
-
-func ToDomainMatchingType(t router.Domain_Type) DomainMatchingType {
-	switch t {
-	case router.Domain_Domain:
-		return DomainMatchingType_Subdomain
-	case router.Domain_Full:
-		return DomainMatchingType_Full
-	case router.Domain_Plain:
-		return DomainMatchingType_Keyword
-	case router.Domain_Regex:
-		return DomainMatchingType_Regex
-	default:
-		panic("unknown domain type")
-	}
-}

app/router/condition.go

@@ -309,48 +309,6 @@ func (m *AttributeMatcher) Apply(ctx routing.Context) bool {
 	return m.Match(attributes)
 }
 
-// Geo attribute
-type GeoAttributeMatcher interface {
-	Match(*Domain) bool
-}
-
-type GeoBooleanMatcher string
-
-func (m GeoBooleanMatcher) Match(domain *Domain) bool {
-	for _, attr := range domain.Attribute {
-		if attr.Key == string(m) {
-			return true
-		}
-	}
-	return false
-}
-
-type GeoAttributeList struct {
-	Matcher []GeoAttributeMatcher
-}
-
-func (al *GeoAttributeList) Match(domain *Domain) bool {
-	for _, matcher := range al.Matcher {
-		if !matcher.Match(domain) {
-			return false
-		}
-	}
-	return true
-}
-
-func (al *GeoAttributeList) IsEmpty() bool {
-	return len(al.Matcher) == 0
-}
-
-func ParseAttrs(attrs []string) *GeoAttributeList {
-	al := new(GeoAttributeList)
-	for _, attr := range attrs {
-		lc := strings.ToLower(attr)
-		al.Matcher = append(al.Matcher, GeoBooleanMatcher(lc))
-	}
-	return al
-}
-
 type ProcessNameMatcher struct {
 	ProcessNames  []string
 	AbsPaths      []string
@@ -439,4 +397,4 @@ func (m *ProcessNameMatcher) Apply(ctx routing.Context) bool {
 		}
 	}
 	return false
-}
+}

app/router/condition_geoip_test.go

@@ -1,17 +1,40 @@
 package router_test
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
-	"runtime"
 	"testing"
 
 	"github.com/xtls/xray-core/app/router"
 	"github.com/xtls/xray-core/common"
 	"github.com/xtls/xray-core/common/net"
-	"github.com/xtls/xray-core/infra/conf"
+	"github.com/xtls/xray-core/common/platform"
+	"github.com/xtls/xray-core/common/platform/filesystem"
+	"google.golang.org/protobuf/proto"
 )
 
+func getAssetPath(file string) (string, error) {
+	path := platform.GetAssetLocation(file)
+	_, err := os.Stat(path)
+	if os.IsNotExist(err) {
+		path := filepath.Join("..", "..", "resources", file)
+		_, err := os.Stat(path)
+		if os.IsNotExist(err) {
+			return "", fmt.Errorf("can't find %s in standard asset locations or {project_root}/resources", file)
+		}
+		if err != nil {
+			return "", fmt.Errorf("can't stat %s: %v", path, err)
+		}
+		return path, nil
+	}
+	if err != nil {
+		return "", fmt.Errorf("can't stat %s: %v", path, err)
+	}
+
+	return path, nil
+}
+
 func TestGeoIPMatcher(t *testing.T) {
 	cidrList := []*router.CIDR{
 		{Ip: []byte{0, 0, 0, 0}, Prefix: 8},
@@ -159,11 +182,12 @@ func TestGeoIPReverseMatcher(t *testing.T) {
 }
 
 func TestGeoIPMatcher4CN(t *testing.T) {
-	geo := "geoip:cn"
-	geoip, err := loadGeoIP(geo)
+	ips, err := loadGeoIP("CN")
 	common.Must(err)
 
-	matcher, err := router.BuildOptimizedGeoIPMatcher(geoip)
+	matcher, err := router.BuildOptimizedGeoIPMatcher(&router.GeoIP{
+		Cidr: ips,
+	})
 	common.Must(err)
 
 	if matcher.Match([]byte{8, 8, 8, 8}) {
@@ -172,11 +196,12 @@ func TestGeoIPMatcher4CN(t *testing.T) {
 }
 
 func TestGeoIPMatcher6US(t *testing.T) {
-	geo := "geoip:us"
-	geoip, err := loadGeoIP(geo)
+	ips, err := loadGeoIP("US")
 	common.Must(err)
 
-	matcher, err := router.BuildOptimizedGeoIPMatcher(geoip)
+	matcher, err := router.BuildOptimizedGeoIPMatcher(&router.GeoIP{
+		Cidr: ips,
+	})
 	common.Must(err)
 
 	if !matcher.Match(net.ParseAddress("2001:4860:4860::8888").IP()) {
@@ -184,34 +209,37 @@ func TestGeoIPMatcher6US(t *testing.T) {
 	}
 }
 
-func loadGeoIP(geo string) (*router.GeoIP, error) {
-	os.Setenv("XRAY_LOCATION_ASSET", filepath.Join("..", "..", "resources"))
-
-	geoip, err := conf.ToCidrList([]string{geo})
+func loadGeoIP(country string) ([]*router.CIDR, error) {
+	path, err := getAssetPath("geoip.dat")
+	if err != nil {
+		return nil, err
+	}
+	geoipBytes, err := filesystem.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
 
-	if runtime.GOOS != "windows" && runtime.GOOS != "wasm" {
-		geoip, err = router.GetGeoIPList(geoip)
-		if err != nil {
-			return nil, err
+	var geoipList router.GeoIPList
+	if err := proto.Unmarshal(geoipBytes, &geoipList); err != nil {
+		return nil, err
+	}
+
+	for _, geoip := range geoipList.Entry {
+		if geoip.CountryCode == country {
+			return geoip.Cidr, nil
 		}
 	}
 
-	if len(geoip) == 0 {
-		panic("country not found: " + geo)
-	}
-
-	return geoip[0], nil
+	panic("country not found: " + country)
 }
 
 func BenchmarkGeoIPMatcher4CN(b *testing.B) {
-	geo := "geoip:cn"
-	geoip, err := loadGeoIP(geo)
+	ips, err := loadGeoIP("CN")
 	common.Must(err)
 
-	matcher, err := router.BuildOptimizedGeoIPMatcher(geoip)
+	matcher, err := router.BuildOptimizedGeoIPMatcher(&router.GeoIP{
+		Cidr: ips,
+	})
 	common.Must(err)
 
 	b.ResetTimer()
@@ -222,11 +250,12 @@ func BenchmarkGeoIPMatcher4CN(b *testing.B) {
 }
 
 func BenchmarkGeoIPMatcher6US(b *testing.B) {
-	geo := "geoip:us"
-	geoip, err := loadGeoIP(geo)
+	ips, err := loadGeoIP("US")
 	common.Must(err)
 
-	matcher, err := router.BuildOptimizedGeoIPMatcher(geoip)
+	matcher, err := router.BuildOptimizedGeoIPMatcher(&router.GeoIP{
+		Cidr: ips,
+	})
 	common.Must(err)
 
 	b.ResetTimer()

app/router/condition_test.go

@@ -1,22 +1,20 @@
 package router_test
 
 import (
-	"os"
-	"path/filepath"
-	"runtime"
 	"strconv"
 	"testing"
 
-	"github.com/xtls/xray-core/app/router"
 	. "github.com/xtls/xray-core/app/router"
 	"github.com/xtls/xray-core/common"
+	"github.com/xtls/xray-core/common/errors"
 	"github.com/xtls/xray-core/common/net"
+	"github.com/xtls/xray-core/common/platform/filesystem"
 	"github.com/xtls/xray-core/common/protocol"
 	"github.com/xtls/xray-core/common/protocol/http"
 	"github.com/xtls/xray-core/common/session"
 	"github.com/xtls/xray-core/features/routing"
 	routing_session "github.com/xtls/xray-core/features/routing/session"
-	"github.com/xtls/xray-core/infra/conf"
+	"google.golang.org/protobuf/proto"
 )
 
 func withBackground() routing.Context {
@@ -302,25 +300,32 @@ func TestRoutingRule(t *testing.T) {
 	}
 }
 
-func loadGeoSiteDomains(geo string) ([]*Domain, error) {
-	os.Setenv("XRAY_LOCATION_ASSET", filepath.Join("..", "..", "resources"))
-
-	domains, err := conf.ParseDomainRule(geo)
+func loadGeoSite(country string) ([]*Domain, error) {
+	path, err := getAssetPath("geosite.dat")
+	if err != nil {
+		return nil, err
+	}
+	geositeBytes, err := filesystem.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
 
-	if runtime.GOOS != "windows" && runtime.GOOS != "wasm" {
-		domains, err = router.GetDomainList(domains)
-		if err != nil {
-			return nil, err
+	var geositeList GeoSiteList
+	if err := proto.Unmarshal(geositeBytes, &geositeList); err != nil {
+		return nil, err
+	}
+
+	for _, site := range geositeList.Entry {
+		if site.CountryCode == country {
+			return site.Domain, nil
 		}
 	}
-	return domains, nil
+
+	return nil, errors.New("country not found: " + country)
 }
 
 func TestChinaSites(t *testing.T) {
-	domains, err := loadGeoSiteDomains("geosite:cn")
+	domains, err := loadGeoSite("CN")
 	common.Must(err)
 
 	acMatcher, err := NewMphMatcherGroup(domains)
@@ -361,50 +366,8 @@ func TestChinaSites(t *testing.T) {
 	}
 }
 
-func TestChinaSitesWithAttrs(t *testing.T) {
-	domains, err := loadGeoSiteDomains("geosite:google@cn")
-	common.Must(err)
-
-	acMatcher, err := NewMphMatcherGroup(domains)
-	common.Must(err)
-
-	type TestCase struct {
-		Domain string
-		Output bool
-	}
-	testCases := []TestCase{
-		{
-			Domain: "google.cn",
-			Output: true,
-		},
-		{
-			Domain: "recaptcha.net",
-			Output: true,
-		},
-		{
-			Domain: "164.com",
-			Output: false,
-		},
-		{
-			Domain: "164.com",
-			Output: false,
-		},
-	}
-
-	for i := 0; i < 1024; i++ {
-		testCases = append(testCases, TestCase{Domain: strconv.Itoa(i) + ".not-exists.com", Output: false})
-	}
-
-	for _, testCase := range testCases {
-		r := acMatcher.ApplyDomain(testCase.Domain)
-		if r != testCase.Output {
-			t.Error("ACDomainMatcher expected output ", testCase.Output, " for domain ", testCase.Domain, " but got ", r)
-		}
-	}
-}
-
 func BenchmarkMphDomainMatcher(b *testing.B) {
-	domains, err := loadGeoSiteDomains("geosite:cn")
+	domains, err := loadGeoSite("CN")
 	common.Must(err)
 
 	matcher, err := NewMphMatcherGroup(domains)
@@ -449,11 +412,11 @@ func BenchmarkMultiGeoIPMatcher(b *testing.B) {
 	var geoips []*GeoIP
 
 	{
-		ips, err := loadGeoIP("geoip:cn")
+		ips, err := loadGeoIP("CN")
 		common.Must(err)
 		geoips = append(geoips, &GeoIP{
 			CountryCode: "CN",
-			Cidr:        ips.Cidr,
+			Cidr:        ips,
 		})
 	}
 
@@ -462,25 +425,25 @@ func BenchmarkMultiGeoIPMatcher(b *testing.B) {
 		common.Must(err)
 		geoips = append(geoips, &GeoIP{
 			CountryCode: "JP",
-			Cidr:        ips.Cidr,
+			Cidr:        ips,
 		})
 	}
 
 	{
-		ips, err := loadGeoIP("geoip:ca")
+		ips, err := loadGeoIP("CA")
 		common.Must(err)
 		geoips = append(geoips, &GeoIP{
 			CountryCode: "CA",
-			Cidr:        ips.Cidr,
+			Cidr:        ips,
 		})
 	}
 
 	{
-		ips, err := loadGeoIP("geoip:us")
+		ips, err := loadGeoIP("US")
 		common.Must(err)
 		geoips = append(geoips, &GeoIP{
 			CountryCode: "US",
-			Cidr:        ips.Cidr,
+			Cidr:        ips,
 		})
 	}
 

app/router/config.go

@@ -3,14 +3,11 @@ package router
 import (
 	"context"
 	"regexp"
-	"runtime"
 	"strings"
 
 	"github.com/xtls/xray-core/common/errors"
-	"github.com/xtls/xray-core/common/platform/filesystem"
 	"github.com/xtls/xray-core/features/outbound"
 	"github.com/xtls/xray-core/features/routing"
-	"google.golang.org/protobuf/proto"
 )
 
 type Rule struct {
@@ -76,15 +73,7 @@ func (rr *RoutingRule) BuildCondition() (Condition, error) {
 	}
 
 	if len(rr.Geoip) > 0 {
-		geoip := rr.Geoip
-		if runtime.GOOS != "windows" && runtime.GOOS != "wasm" {
-			var err error
-			geoip, err = GetGeoIPList(rr.Geoip)
-			if err != nil {
-				return nil, errors.New("failed to build geoip from mmap").Base(err)
-			}
-		}
-		cond, err := NewIPMatcher(geoip, MatcherAsType_Target)
+		cond, err := NewIPMatcher(rr.Geoip, MatcherAsType_Target)
 		if err != nil {
 			return nil, err
 		}
@@ -109,20 +98,11 @@ func (rr *RoutingRule) BuildCondition() (Condition, error) {
 	}
 
 	if len(rr.Domain) > 0 {
-		domains := rr.Domain
-		if runtime.GOOS != "windows" && runtime.GOOS != "wasm" {
-			var err error
-			domains, err = GetDomainList(rr.Domain)
-			if err != nil {
-				return nil, errors.New("failed to build domains from mmap").Base(err)
-			}
-		}
-
-		matcher, err := NewMphMatcherGroup(domains)
+		matcher, err := NewMphMatcherGroup(rr.Domain)
 		if err != nil {
 			return nil, errors.New("failed to build domain condition with MphDomainMatcher").Base(err)
 		}
-		errors.LogDebug(context.Background(), "MphDomainMatcher is enabled for ", len(domains), " domain rule(s)")
+		errors.LogDebug(context.Background(), "MphDomainMatcher is enabled for ", len(rr.Domain), " domain rule(s)")
 		conds.Add(matcher)
 	}
 
@@ -183,80 +163,3 @@ func (br *BalancingRule) Build(ohm outbound.Manager, dispatcher routing.Dispatch
 		return nil, errors.New("unrecognized balancer type")
 	}
 }
-
-func GetGeoIPList(ips []*GeoIP) ([]*GeoIP, error) {
-	geoipList := []*GeoIP{}
-	for _, ip := range ips {
-		if ip.CountryCode != "" {
-			val := strings.Split(ip.CountryCode, "_")
-			fileName := "geoip.dat"
-			if len(val) == 2 {
-				fileName = strings.ToLower(val[0])
-			}
-			bs, err := filesystem.ReadAsset(fileName)
-			if err != nil {
-				return nil, errors.New("failed to load file: ", fileName).Base(err)
-			}
-			bs = filesystem.Find(bs, []byte(ip.CountryCode))
-
-			var geoip GeoIP
-
-			if err := proto.Unmarshal(bs, &geoip); err != nil {
-				return nil, errors.New("failed Unmarshal :").Base(err)
-			}
-			geoipList = append(geoipList, &geoip)
-
-		} else {
-			geoipList = append(geoipList, ip)
-		}
-	}
-	return geoipList, nil
-
-}
-
-func GetDomainList(domains []*Domain) ([]*Domain, error) {
-	domainList := []*Domain{}
-	for _, domain := range domains {
-		val := strings.Split(domain.Value, "_")
-
-		if len(val) >= 2 {
-
-			fileName := val[0]
-			code := val[1]
-
-			bs, err := filesystem.ReadAsset(fileName)
-			if err != nil {
-				return nil, errors.New("failed to load file: ", fileName).Base(err)
-			}
-			bs = filesystem.Find(bs, []byte(code))
-			var geosite GeoSite
-
-			if err := proto.Unmarshal(bs, &geosite); err != nil {
-				return nil, errors.New("failed Unmarshal :").Base(err)
-			}
-
-			// parse attr
-			if len(val) == 3 {
-				siteWithAttr := strings.Split(val[2], ",")
-				attrs := ParseAttrs(siteWithAttr)
-
-				if !attrs.IsEmpty() {
-					filteredDomains := make([]*Domain, 0, len(domains))
-					for _, domain := range geosite.Domain {
-						if attrs.Match(domain) {
-							filteredDomains = append(filteredDomains, domain)
-						}
-					}
-					geosite.Domain = filteredDomains
-				}
-
-			}
-
-			domainList = append(domainList, geosite.Domain...)
-
-		} else {
-			domainList = append(domainList, domain)
-		}
-	}
-	return domainList, nil
-}