Geodata: Support automatically updating .dat files and hot reloading (#5992)

https://github.com/XTLS/Xray-core/pull/5992#issuecomment-4320551920

Usage: https://github.com/XTLS/Xray-core/pull/5992#issuecomment-4291168039

infra/conf/geodata.go

@@ -0,0 +1,71 @@
+package conf
+
+import (
+	"net/url"
+
+	"github.com/robfig/cron/v3"
+	"github.com/xtls/xray-core/app/geodata"
+	"github.com/xtls/xray-core/common/errors"
+	"github.com/xtls/xray-core/common/platform/filesystem"
+	"google.golang.org/protobuf/proto"
+)
+
+type GeodataAssetConfig struct {
+	URL  string `json:"url"`
+	File string `json:"file"`
+}
+
+func (c *GeodataAssetConfig) Build() (*geodata.Asset, error) {
+	if err := validateHTTPS(c.URL); err != nil {
+		return nil, errors.New("invalid geodata asset url: ", c.URL).Base(err)
+	}
+	if _, err := filesystem.StatAsset(c.File); err != nil {
+		return nil, errors.New("invalid geodata asset file: ", c.File).Base(err)
+	}
+	return &geodata.Asset{
+		Url:  c.URL,
+		File: c.File,
+	}, nil
+}
+
+func validateHTTPS(s string) error {
+	u, err := url.ParseRequestURI(s)
+	if err != nil {
+		return err
+	}
+	if u.Scheme != "https" || u.Host == "" {
+		return errors.New("scheme must be https")
+	}
+	return nil
+}
+
+type GeodataConfig struct {
+	Cron     *string               `json:"cron"`
+	Outbound string                `json:"outbound"`
+	Assets   []*GeodataAssetConfig `json:"assets"`
+}
+
+func (c *GeodataConfig) Build() (proto.Message, error) {
+	config := &geodata.Config{}
+
+	if c.Cron != nil {
+		if _, err := cron.ParseStandard(*c.Cron); err != nil {
+			return nil, errors.New("invalid geodata cron").Base(err)
+		}
+		config.Cron = *c.Cron
+	}
+
+	config.Outbound = c.Outbound
+
+	assets := make([]*geodata.Asset, 0, len(c.Assets))
+	for _, asset := range c.Assets {
+		built, err := asset.Build()
+		if err != nil {
+			return nil, err
+		}
+		assets = append(assets, built)
+	}
+	config.Assets = assets
+
+	return config, nil
+}

infra/conf/geodata_test.go

@@ -0,0 +1,75 @@
+package conf_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/xtls/xray-core/app/geodata"
+	. "github.com/xtls/xray-core/infra/conf"
+)
+
+func TestGeodataConfig(t *testing.T) {
+	t.Setenv("xray.location.asset", filepath.Join("..", "..", "resources"))
+
+	creator := func() Buildable {
+		return new(GeodataConfig)
+	}
+
+	runMultiTestCase(t, []TestCase{
+		{
+			Input: `{
+				"cron": "0 4 * * *",
+				"outbound": "proxy",
+				"assets": [
+					{"url": "https://example.com/geoip.dat", "file": "geoip.dat"},
+					{"url": "https://example.com/geosite.dat", "file": "geosite.dat"}
+				]
+			}`,
+			Parser: loadJSON(creator),
+			Output: &geodata.Config{
+				Cron:     "0 4 * * *",
+				Outbound: "proxy",
+				Assets: []*geodata.Asset{
+					{Url: "https://example.com/geoip.dat", File: "geoip.dat"},
+					{Url: "https://example.com/geosite.dat", File: "geosite.dat"},
+				},
+			},
+		},
+	})
+}
+
+func TestGeodataAssetConfig(t *testing.T) {
+	t.Setenv("xray.location.asset", filepath.Join("..", "..", "resources"))
+
+	if _, err := (&GeodataAssetConfig{
+		URL:  "https://example.com/geoip.dat",
+		File: "geoip.dat",
+	}).Build(); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := (&GeodataAssetConfig{
+		URL:  "https://example.com/geoip.dat",
+		File: "missing.dat",
+	}).Build(); err == nil {
+		t.Fatal("expected error")
+	}
+}
+
+func TestGeodataAssetConfigInvalidURL(t *testing.T) {
+	t.Setenv("xray.location.asset", filepath.Join("..", "..", "resources"))
+
+	for _, rawURL := range []string{
+		"",
+		"http://example.com/geoip.dat",
+		"ftp://example.com/geoip.dat",
+		"https:///geoip.dat",
+	} {
+		if _, err := (&GeodataAssetConfig{
+			URL:  rawURL,
+			File: "geoip.dat",
+		}).Build(); err == nil {
+			t.Fatalf("expected error for %q", rawURL)
+		}
+	}
+}

infra/conf/xray.go

@@ -361,6 +361,7 @@ type Config struct {
 	Observatory      *ObservatoryConfig      `json:"observatory"`
 	BurstObservatory *BurstObservatoryConfig `json:"burstObservatory"`
 	Version          *VersionConfig          `json:"version"`
+	Geodata          *GeodataConfig          `json:"geodata"`
 }
 
 func (c *Config) findInboundTag(tag string) int {
@@ -433,6 +434,10 @@ func (c *Config) Override(o *Config, fn string) {
 		c.Version = o.Version
 	}
 
+	if o.Geodata != nil {
+		c.Geodata = o.Geodata
+	}
+
 	// update the Inbound in slice if the only one in override config has same tag
 	if len(o.InboundConfigs) > 0 {
 		for i := range o.InboundConfigs {
@@ -581,6 +586,14 @@ func (c *Config) Build() (*core.Config, error) {
 		config.App = append(config.App, serial.ToTypedMessage(r))
 	}
 
+	if c.Geodata != nil {
+		r, err := c.Geodata.Build()
+		if err != nil {
+			return nil, errors.New("failed to build geodata configuration").Base(err)
+		}
+		config.App = append(config.App, serial.ToTypedMessage(r))
+	}
+
 	var inbounds []InboundDetourConfig
 
 	if len(c.InboundConfigs) > 0 {