From 5d1827b2da12809880bd10f08efb121bd09d3c86 Mon Sep 17 00:00:00 2001 From: wweir Date: Tue, 18 Feb 2020 16:02:23 +0800 Subject: [PATCH] Optimize dynamic detect logic --- conf/conf.go | 23 ++++------ conf/dynamic_rule.go | 100 ++++++++++++++++++++++++++----------------- conf/sower.toml | 2 +- go.sum | 2 + proxy/proxy.go | 2 +- util/rev_sec.go | 52 ---------------------- util/rev_sec_test.go | 65 ---------------------------- 7 files changed, 74 insertions(+), 172 deletions(-) delete mode 100644 util/rev_sec.go delete mode 100644 util/rev_sec_test.go diff --git a/conf/conf.go b/conf/conf.go index fb5221b..9ac2d77 100644 --- a/conf/conf.go +++ b/conf/conf.go @@ -3,7 +3,6 @@ package conf import ( "flag" "os" - "sync" "time" toml "github.com/pelletier/go-toml" @@ -29,9 +28,9 @@ type client struct { DetectLevel int `toml:"detect_level"` DetectTimeout string `toml:"detect_timeout"` - ProxyList []string `toml:"proxy_list"` - DirectList []string `toml:"direct_list"` - DynamicList []string `toml:"dynamic_list"` + ProxyList []string `toml:"proxy_list"` + DirectList []string `toml:"direct_list"` + DynamicList map[string]int `toml:"dynamic_list"` // toml has a bug with dot directRules *util.Node proxyRules *util.Node dynamicRules *util.Node @@ -47,10 +46,6 @@ type server struct { var ( version, date string - flushOnce = sync.Once{} - flushMu = sync.Mutex{} - flushCh = make(chan struct{}) - Server = server{} Client = client{} conf = struct { @@ -58,6 +53,7 @@ var ( Server *server `toml:"server"` Client *client `toml:"client"` }{"", &Server, &Client} + flushCh = make(chan struct{}) Password string installCmd string uninstallFlag bool @@ -108,6 +104,8 @@ func init() { log.Fatalw("load config", "config", conf.file, "step", loadConfigFns[i].step, "err", err) } } + + go flushConfDaemon() } // refreshFns will be executed while init and write new config @@ -121,12 +119,12 @@ var loadConfigFns = []struct { } defer f.Close() + Client.Router.DynamicList = map[string]int{} return toml.NewDecoder(f).Decode(&conf) }}, {"load_rules", func() error { Client.Router.directRules = util.NewNodeFromRules(Client.Router.DirectList...) Client.Router.proxyRules = util.NewNodeFromRules(Client.Router.ProxyList...) - Client.Router.dynamicRules = util.NewNodeFromRules(Client.Router.DynamicList...) return nil }}, {"flush_dns", func() error { @@ -136,9 +134,9 @@ var loadConfigFns = []struct { return nil }}} -func flushConf() { +func flushConfDaemon() { for range flushCh { - // safe write + // safe write file if conf.file != "" { f, err := os.OpenFile(conf.file+"~", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { @@ -146,14 +144,11 @@ func flushConf() { continue } - flushMu.Lock() if err := toml.NewEncoder(f).ArraysWithOneElementPerLine(true).Encode(conf); err != nil { log.Errorw("flush config", "step", "flush", "err", err) - flushMu.Unlock() f.Close() continue } - flushMu.Unlock() f.Close() if err = os.Rename(conf.file+"~", conf.file); err != nil { diff --git a/conf/dynamic_rule.go b/conf/dynamic_rule.go index b563d62..7c1759c 100644 --- a/conf/dynamic_rule.go +++ b/conf/dynamic_rule.go @@ -10,7 +10,6 @@ import ( "github.com/wweir/sower/internal/http" "github.com/wweir/sower/internal/socks5" - "github.com/wweir/sower/util" "github.com/wweir/utils/log" "github.com/wweir/utils/mem" ) @@ -19,10 +18,12 @@ type dynamic struct { port http.Port } -var cache = mem.New(2 * time.Hour) +var cache = mem.New(4 * time.Hour) var detect = &dynamic{} var passwordData []byte var timeout time.Duration +var dynamicCache sync.Map +var dynamicMu = sync.Mutex{} // ShouldProxy check if the domain shoule request though proxy func ShouldProxy(domain string) bool { @@ -35,12 +36,10 @@ func ShouldProxy(domain string) bool { if Client.Router.proxyRules.Match(domain) { return true } - if Client.Router.dynamicRules.Match(domain) { - return true - } cache.Remember(detect, domain) - return Client.Router.dynamicRules.Match(domain) + val, _ := dynamicCache.Load(domain) + return val.(int) >= Client.Router.DetectLevel } func (d *dynamic) Get(key interface{}) (err error) { @@ -49,7 +48,53 @@ func (d *dynamic) Get(key interface{}) (err error) { if strings.Count(domain, ".") > 10 { return nil } + domainUnderscore := strings.ReplaceAll(domain, ".", "_") + var score int + defer func() { + dynamicCache.Store(domain, score) + + if score < conf.Client.Router.DetectLevel { + delete(Client.Router.DynamicList, domainUnderscore) + } else { + Client.Router.DynamicList[domainUnderscore] = score + + // persist when add new domain + select { + case flushCh <- struct{}{}: + default: + } + log.Infow("persist rule", "domain", domain, "score", score) + } + }() + + if val, ok := dynamicCache.Load(domain); ok { + score = val.(int) + } else { + dynamicMu.Lock() + score = Client.Router.DynamicList[domainUnderscore] + dynamicMu.Unlock() + } + + // detect range: [0,conf.Client.Router.DetectLevel) + switch { + case score < -1: + score++ + case score == -1: + score++ + score += d.detect(domain) + case score > conf.Client.Router.DetectLevel: + score-- + case score == conf.Client.Router.DetectLevel: + score-- + score += d.detect(domain) + } + + return nil +} + +// detect and caculate direct connection and proxy connection score +func (d *dynamic) detect(domain string) int { wg := sync.WaitGroup{} httpScore, httpsScore := new(int32), new(int32) for _, ping := range [...]dynamic{{port: http.HTTP}, {port: http.HTTPS}} { @@ -63,12 +108,12 @@ func (d *dynamic) Get(key interface{}) (err error) { switch ping.port { case http.HTTP: - if !atomic.CompareAndSwapInt32(httpScore, 0, 2) { - atomic.AddInt32(httpScore, 1) + if !atomic.CompareAndSwapInt32(httpScore, 0, -2) { + atomic.AddInt32(httpScore, -1) } case http.HTTPS: - if !atomic.CompareAndSwapInt32(httpsScore, 0, 2) { - atomic.AddInt32(httpScore, 1) + if !atomic.CompareAndSwapInt32(httpsScore, 0, -2) { + atomic.AddInt32(httpScore, -1) } } }(ping) @@ -79,6 +124,7 @@ func (d *dynamic) Get(key interface{}) (err error) { defer wg.Done() var conn net.Conn + var err error if addr, ok := socks5.IsSocks5Schema(Client.Address); ok { conn, err = net.Dial("tcp", addr) conn = socks5.ToSocks5(conn, domain, uint16(ping.port)) @@ -102,41 +148,17 @@ func (d *dynamic) Get(key interface{}) (err error) { switch ping.port { case http.HTTP: - if !atomic.CompareAndSwapInt32(httpScore, 0, -2) { - atomic.AddInt32(httpScore, -1) + if !atomic.CompareAndSwapInt32(httpScore, 0, 2) { + atomic.AddInt32(httpScore, 1) } case http.HTTPS: - if !atomic.CompareAndSwapInt32(httpsScore, 0, -2) { - atomic.AddInt32(httpScore, -1) + if !atomic.CompareAndSwapInt32(httpsScore, 0, 2) { + atomic.AddInt32(httpScore, 1) } } }(ping) } wg.Wait() - if int(*httpScore+*httpsScore)+conf.Client.Router.DetectLevel < 0 { - addDynamic(domain) - log.Infow("add rule", "domain", domain, "http_score", *httpScore, "https_score", *httpsScore) - } - return nil -} - -// addDynamic add new domain into dynamic list -func addDynamic(domain string) { - flushMu.Lock() - Client.Router.DynamicList = util.NewReverseSecSlice( - append(Client.Router.DynamicList, domain)).Sort().Uniq() - Client.Router.dynamicRules = util.NewNodeFromRules(Client.Router.DynamicList...) - flushMu.Unlock() - - flushOnce.Do(func() { - if conf.file != "" { - go flushConf() - } - }) - - select { - case flushCh <- struct{}{}: - default: - } + return int(*httpScore + *httpsScore) } diff --git a/conf/sower.toml b/conf/sower.toml index a4ae296..b74106a 100644 --- a/conf/sower.toml +++ b/conf/sower.toml @@ -22,7 +22,6 @@ "pop.*.*.*", "**.cn", ] - dynamic_list = [] proxy_list = [ "**.google.*", "**.goo.gl", @@ -44,6 +43,7 @@ "**.amazon.com", "**.amazonaws.com", "*.githubusercontent.com", + "*.githubassets.com", "*.github.*", ] diff --git a/go.sum b/go.sum index c273591..9bcf0a2 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= @@ -88,6 +89,7 @@ golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnf golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= diff --git a/proxy/proxy.go b/proxy/proxy.go index 5becb09..091ec47 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -147,7 +147,7 @@ func StartServer(relayTarget, password, certFile, keyFile, email string) { rc, err := net.Dial("tcp", addr) if err != nil { - log.Errorw("tcp dial", "host", domain, "addr", addr, "err", err) + log.Errorw("tcp dial", "addr", addr, "err", err) return } defer rc.Close() diff --git a/util/rev_sec.go b/util/rev_sec.go deleted file mode 100644 index 281159d..0000000 --- a/util/rev_sec.go +++ /dev/null @@ -1,52 +0,0 @@ -package util - -import ( - "sort" - "strings" -) - -type ReverseSecSlice struct { - sort.StringSlice -} - -func NewReverseSecSlice(a []string) *ReverseSecSlice { - return &ReverseSecSlice{sort.StringSlice(a)} -} - -func (p *ReverseSecSlice) Sort() *ReverseSecSlice { - sort.Sort(p) - return p -} -func (p *ReverseSecSlice) Uniq() []string { - olds := []string(p.StringSlice) - - last := "" - strs := make([]string, 0, len(olds)) - for _, str := range olds { - if str != last { - strs = append(strs, str) - } - last = str - } - return strs -} - -func (p *ReverseSecSlice) Less(i, j int) bool { - secsI := strings.Split(p.StringSlice[i], ".") - secsJ := strings.Split(p.StringSlice[j], ".") - - lenI := len(secsI) - 1 - lenJ := len(secsJ) - 1 - length := lenI - if lenI > lenJ { - length = lenJ - } - - for idx := 0; idx <= length; idx++ { - if secsI[lenI-idx] == secsJ[lenJ-idx] { - continue - } - return secsI[lenI-idx] < secsJ[lenJ-idx] - } - return lenI < lenJ -} diff --git a/util/rev_sec_test.go b/util/rev_sec_test.go deleted file mode 100644 index 5946250..0000000 --- a/util/rev_sec_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package util - -import ( - "sort" - "testing" -) - -func TestReverseSecSlice_Less(t *testing.T) { - type fields struct { - StringSlice sort.StringSlice - } - type args struct { - i int - j int - } - slice := fields{sort.StringSlice([]string{ - "a.b.c", - "a.b.c", - "d.b.c", - "d.a.b.c", - "", - "", - })} - tests := []struct { - name string - fields fields - args args - want bool - }{{ - name: "equal", - fields: slice, - args: args{0, 1}, - want: false, - }, { - name: "less", - fields: slice, - args: args{0, 2}, - want: true, - }, { - name: "length", - fields: slice, - args: args{0, 3}, - want: true, - }, { - name: "single_empty", - fields: slice, - args: args{0, 4}, - want: false, - }, { - name: "all_empty", - fields: slice, - args: args{4, 5}, - want: false, - }} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - p := &ReverseSecSlice{ - StringSlice: tt.fields.StringSlice, - } - if got := p.Less(tt.args.i, tt.args.j); got != tt.want { - t.Errorf("ReverseSecSlice.Less() = %v, want %v", got, tt.want) - } - }) - } -}