Optimize dynamic detect logic

conf/conf.go

@@ -3,7 +3,6 @@ package conf
 import (
 	"flag"
 	"os"
-	"sync"
 	"time"
 
 	toml "github.com/pelletier/go-toml"
@@ -29,9 +28,9 @@ type client struct {
 		DetectLevel   int               `toml:"detect_level"`
 		DetectTimeout string            `toml:"detect_timeout"`
 
-		ProxyList    []string `toml:"proxy_list"`
-		DirectList   []string `toml:"direct_list"`
-		DynamicList  []string `toml:"dynamic_list"`
+		ProxyList    []string       `toml:"proxy_list"`
+		DirectList   []string       `toml:"direct_list"`
+		DynamicList  map[string]int `toml:"dynamic_list"` // toml has a bug with dot
 		directRules  *util.Node
 		proxyRules   *util.Node
 		dynamicRules *util.Node
@@ -47,10 +46,6 @@ type server struct {
 var (
 	version, date string
 
-	flushOnce = sync.Once{}
-	flushMu   = sync.Mutex{}
-	flushCh   = make(chan struct{})
-
 	Server = server{}
 	Client = client{}
 	conf   = struct {
@@ -58,6 +53,7 @@ var (
 		Server *server `toml:"server"`
 		Client *client `toml:"client"`
 	}{"", &Server, &Client}
+	flushCh       = make(chan struct{})
 	Password      string
 	installCmd    string
 	uninstallFlag bool
@@ -108,6 +104,8 @@ func init() {
 			log.Fatalw("load config", "config", conf.file, "step", loadConfigFns[i].step, "err", err)
 		}
 	}
+
+	go flushConfDaemon()
 }
 
 // refreshFns will be executed while init and write new config
@@ -121,12 +119,12 @@ var loadConfigFns = []struct {
 	}
 	defer f.Close()
 
+	Client.Router.DynamicList = map[string]int{}
 	return toml.NewDecoder(f).Decode(&conf)
 
 }}, {"load_rules", func() error {
 	Client.Router.directRules = util.NewNodeFromRules(Client.Router.DirectList...)
 	Client.Router.proxyRules = util.NewNodeFromRules(Client.Router.ProxyList...)
-	Client.Router.dynamicRules = util.NewNodeFromRules(Client.Router.DynamicList...)
 	return nil
 
 }}, {"flush_dns", func() error {
@@ -136,9 +134,9 @@ var loadConfigFns = []struct {
 	return nil
 }}}
 
-func flushConf() {
+func flushConfDaemon() {
 	for range flushCh {
-		// safe write
+		// safe write file
 		if conf.file != "" {
 			f, err := os.OpenFile(conf.file+"~", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
 			if err != nil {
@@ -146,14 +144,11 @@ func flushConf() {
 				continue
 			}
 
-			flushMu.Lock()
 			if err := toml.NewEncoder(f).ArraysWithOneElementPerLine(true).Encode(conf); err != nil {
 				log.Errorw("flush config", "step", "flush", "err", err)
-				flushMu.Unlock()
 				f.Close()
 				continue
 			}
-			flushMu.Unlock()
 			f.Close()
 
 			if err = os.Rename(conf.file+"~", conf.file); err != nil {

conf/dynamic_rule.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/wweir/sower/internal/http"
 	"github.com/wweir/sower/internal/socks5"
-	"github.com/wweir/sower/util"
 	"github.com/wweir/utils/log"
 	"github.com/wweir/utils/mem"
 )
@@ -19,10 +18,12 @@ type dynamic struct {
 	port http.Port
 }
 
-var cache = mem.New(2 * time.Hour)
+var cache = mem.New(4 * time.Hour)
 var detect = &dynamic{}
 var passwordData []byte
 var timeout time.Duration
+var dynamicCache sync.Map
+var dynamicMu = sync.Mutex{}
 
 // ShouldProxy check if the domain shoule request though proxy
 func ShouldProxy(domain string) bool {
@@ -35,12 +36,10 @@ func ShouldProxy(domain string) bool {
 	if Client.Router.proxyRules.Match(domain) {
 		return true
 	}
-	if Client.Router.dynamicRules.Match(domain) {
-		return true
-	}
 
 	cache.Remember(detect, domain)
-	return Client.Router.dynamicRules.Match(domain)
+	val, _ := dynamicCache.Load(domain)
+	return val.(int) >= Client.Router.DetectLevel
 }
 
 func (d *dynamic) Get(key interface{}) (err error) {
@@ -49,7 +48,53 @@ func (d *dynamic) Get(key interface{}) (err error) {
 	if strings.Count(domain, ".") > 10 {
 		return nil
 	}
+	domainUnderscore := strings.ReplaceAll(domain, ".", "_")
+	var score int
 
+	defer func() {
+		dynamicCache.Store(domain, score)
+
+		if score < conf.Client.Router.DetectLevel {
+			delete(Client.Router.DynamicList, domainUnderscore)
+		} else {
+			Client.Router.DynamicList[domainUnderscore] = score
+
+			// persist when add new domain
+			select {
+			case flushCh <- struct{}{}:
+			default:
+			}
+			log.Infow("persist rule", "domain", domain, "score", score)
+		}
+	}()
+
+	if val, ok := dynamicCache.Load(domain); ok {
+		score = val.(int)
+	} else {
+		dynamicMu.Lock()
+		score = Client.Router.DynamicList[domainUnderscore]
+		dynamicMu.Unlock()
+	}
+
+	// detect range: [0,conf.Client.Router.DetectLevel)
+	switch {
+	case score < -1:
+		score++
+	case score == -1:
+		score++
+		score += d.detect(domain)
+	case score > conf.Client.Router.DetectLevel:
+		score--
+	case score == conf.Client.Router.DetectLevel:
+		score--
+		score += d.detect(domain)
+	}
+
+	return nil
+}
+
+// detect and caculate direct connection and proxy connection score
+func (d *dynamic) detect(domain string) int {
 	wg := sync.WaitGroup{}
 	httpScore, httpsScore := new(int32), new(int32)
 	for _, ping := range [...]dynamic{{port: http.HTTP}, {port: http.HTTPS}} {
@@ -63,12 +108,12 @@ func (d *dynamic) Get(key interface{}) (err error) {
 
 			switch ping.port {
 			case http.HTTP:
-				if !atomic.CompareAndSwapInt32(httpScore, 0, 2) {
-					atomic.AddInt32(httpScore, 1)
+				if !atomic.CompareAndSwapInt32(httpScore, 0, -2) {
+					atomic.AddInt32(httpScore, -1)
 				}
 			case http.HTTPS:
-				if !atomic.CompareAndSwapInt32(httpsScore, 0, 2) {
-					atomic.AddInt32(httpScore, 1)
+				if !atomic.CompareAndSwapInt32(httpsScore, 0, -2) {
+					atomic.AddInt32(httpScore, -1)
 				}
 			}
 		}(ping)
@@ -79,6 +124,7 @@ func (d *dynamic) Get(key interface{}) (err error) {
 			defer wg.Done()
 
 			var conn net.Conn
+			var err error
 			if addr, ok := socks5.IsSocks5Schema(Client.Address); ok {
 				conn, err = net.Dial("tcp", addr)
 				conn = socks5.ToSocks5(conn, domain, uint16(ping.port))
@@ -102,41 +148,17 @@ func (d *dynamic) Get(key interface{}) (err error) {
 
 			switch ping.port {
 			case http.HTTP:
-				if !atomic.CompareAndSwapInt32(httpScore, 0, -2) {
-					atomic.AddInt32(httpScore, -1)
+				if !atomic.CompareAndSwapInt32(httpScore, 0, 2) {
+					atomic.AddInt32(httpScore, 1)
 				}
 			case http.HTTPS:
-				if !atomic.CompareAndSwapInt32(httpsScore, 0, -2) {
-					atomic.AddInt32(httpScore, -1)
+				if !atomic.CompareAndSwapInt32(httpsScore, 0, 2) {
+					atomic.AddInt32(httpScore, 1)
 				}
 			}
 		}(ping)
 	}
 
 	wg.Wait()
-	if int(*httpScore+*httpsScore)+conf.Client.Router.DetectLevel < 0 {
-		addDynamic(domain)
-		log.Infow("add rule", "domain", domain, "http_score", *httpScore, "https_score", *httpsScore)
-	}
-	return nil
-}
-
-// addDynamic add new domain into dynamic list
-func addDynamic(domain string) {
-	flushMu.Lock()
-	Client.Router.DynamicList = util.NewReverseSecSlice(
-		append(Client.Router.DynamicList, domain)).Sort().Uniq()
-	Client.Router.dynamicRules = util.NewNodeFromRules(Client.Router.DynamicList...)
-	flushMu.Unlock()
-
-	flushOnce.Do(func() {
-		if conf.file != "" {
-			go flushConf()
-		}
-	})
-
-	select {
-	case flushCh <- struct{}{}:
-	default:
-	}
+	return int(*httpScore + *httpsScore)
 }

conf/sower.toml

@@ -22,7 +22,6 @@
       "pop.*.*.*",
       "**.cn",
     ]
-    dynamic_list = []
     proxy_list = [
       "**.google.*",
       "**.goo.gl",
@@ -44,6 +43,7 @@
       "**.amazon.com",
       "**.amazonaws.com",
       "*.githubusercontent.com",
+      "*.githubassets.com",
       "*.github.*",
     ]
 

go.sum

@@ -5,6 +5,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
@@ -88,6 +89,7 @@ golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnf
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=

proxy/proxy.go

@@ -147,7 +147,7 @@ func StartServer(relayTarget, password, certFile, keyFile, email string) {
 
 			rc, err := net.Dial("tcp", addr)
 			if err != nil {
-				log.Errorw("tcp dial", "host", domain, "addr", addr, "err", err)
+				log.Errorw("tcp dial", "addr", addr, "err", err)
 				return
 			}
 			defer rc.Close()

util/rev_sec.go

@@ -1,52 +0,0 @@
-package util
-
-import (
-	"sort"
-	"strings"
-)
-
-type ReverseSecSlice struct {
-	sort.StringSlice
-}
-
-func NewReverseSecSlice(a []string) *ReverseSecSlice {
-	return &ReverseSecSlice{sort.StringSlice(a)}
-}
-
-func (p *ReverseSecSlice) Sort() *ReverseSecSlice {
-	sort.Sort(p)
-	return p
-}
-func (p *ReverseSecSlice) Uniq() []string {
-	olds := []string(p.StringSlice)
-
-	last := ""
-	strs := make([]string, 0, len(olds))
-	for _, str := range olds {
-		if str != last {
-			strs = append(strs, str)
-		}
-		last = str
-	}
-	return strs
-}
-
-func (p *ReverseSecSlice) Less(i, j int) bool {
-	secsI := strings.Split(p.StringSlice[i], ".")
-	secsJ := strings.Split(p.StringSlice[j], ".")
-
-	lenI := len(secsI) - 1
-	lenJ := len(secsJ) - 1
-	length := lenI
-	if lenI > lenJ {
-		length = lenJ
-	}
-
-	for idx := 0; idx <= length; idx++ {
-		if secsI[lenI-idx] == secsJ[lenJ-idx] {
-			continue
-		}
-		return secsI[lenI-idx] < secsJ[lenJ-idx]
-	}
-	return lenI < lenJ
-}

util/rev_sec_test.go

@@ -1,65 +0,0 @@
-package util
-
-import (
-	"sort"
-	"testing"
-)
-
-func TestReverseSecSlice_Less(t *testing.T) {
-	type fields struct {
-		StringSlice sort.StringSlice
-	}
-	type args struct {
-		i int
-		j int
-	}
-	slice := fields{sort.StringSlice([]string{
-		"a.b.c",
-		"a.b.c",
-		"d.b.c",
-		"d.a.b.c",
-		"",
-		"",
-	})}
-	tests := []struct {
-		name   string
-		fields fields
-		args   args
-		want   bool
-	}{{
-		name:   "equal",
-		fields: slice,
-		args:   args{0, 1},
-		want:   false,
-	}, {
-		name:   "less",
-		fields: slice,
-		args:   args{0, 2},
-		want:   true,
-	}, {
-		name:   "length",
-		fields: slice,
-		args:   args{0, 3},
-		want:   true,
-	}, {
-		name:   "single_empty",
-		fields: slice,
-		args:   args{0, 4},
-		want:   false,
-	}, {
-		name:   "all_empty",
-		fields: slice,
-		args:   args{4, 5},
-		want:   false,
-	}}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			p := &ReverseSecSlice{
-				StringSlice: tt.fields.StringSlice,
-			}
-			if got := p.Less(tt.args.i, tt.args.j); got != tt.want {
-				t.Errorf("ReverseSecSlice.Less() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}