Przeglądaj źródła

Do not use ristretto

Memory usage is too big
tags/v2.0.0-rc1
9seconds 5 lat temu
rodzic
commit
87a720dd58
5 zmienionych plików z 49 dodań i 50 usunięć
  1. 2
    3
      go.mod
  2. 0
    7
      go.sum
  3. 45
    38
      network/dns_resolver.go
  4. 1
    1
      network/dns_resolver_internal_test.go
  5. 1
    1
      network/network.go

+ 2
- 3
go.mod Wyświetl plik

9
 	github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
9
 	github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
10
 	github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6
10
 	github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6
11
 	github.com/d4l3k/messagediff v1.2.1 // indirect
11
 	github.com/d4l3k/messagediff v1.2.1 // indirect
12
-	github.com/dgraph-io/ristretto v0.0.3 // indirect
13
 	github.com/jarcoal/httpmock v1.0.8
12
 	github.com/jarcoal/httpmock v1.0.8
14
 	github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1
13
 	github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1
15
 	github.com/libp2p/go-reuseport v0.0.2
14
 	github.com/libp2p/go-reuseport v0.0.2
23
 	github.com/stretchr/testify v1.7.0
22
 	github.com/stretchr/testify v1.7.0
24
 	github.com/tylertreat/BoomFilters v0.0.0-20200520150052-42a7b4300c0c
23
 	github.com/tylertreat/BoomFilters v0.0.0-20200520150052-42a7b4300c0c
25
 	github.com/xeipuuv/gojsonschema v1.2.0
24
 	github.com/xeipuuv/gojsonschema v1.2.0
26
-	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect
25
+	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
27
 	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
26
 	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
28
-	golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 // indirect
27
+	golang.org/x/sys v0.0.0-20210309074719-68d13333faf2
29
 )
28
 )

+ 0
- 7
go.sum Wyświetl plik

2
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
2
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
3
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
3
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
4
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
4
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
5
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
6
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
5
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
7
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
6
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
8
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
7
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
39
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
38
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
40
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
39
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
41
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
40
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
42
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
43
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
44
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
41
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
45
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
42
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
46
 github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
43
 github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
58
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
55
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
59
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
56
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
60
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
57
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
61
-github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI=
62
-github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
63
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
58
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
64
-github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
65
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
59
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
66
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
60
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
67
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
61
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
283
 github.com/smira/go-statsd v1.3.2/go.mod h1:1srXJ9/pbnN04G8f4F1jUzsGOnwkPKXciyqpewGlkC4=
277
 github.com/smira/go-statsd v1.3.2/go.mod h1:1srXJ9/pbnN04G8f4F1jUzsGOnwkPKXciyqpewGlkC4=
284
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
278
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
285
 github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
279
 github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
286
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
287
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
280
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
288
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
281
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
289
 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
282
 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=

+ 45
- 38
network/dns_resolver.go Wyświetl plik

2
 
2
 
3
 import (
3
 import (
4
 	"net/http"
4
 	"net/http"
5
+	"sync"
5
 	"time"
6
 	"time"
6
 
7
 
7
 	doh "github.com/babolivier/go-doh-client"
8
 	doh "github.com/babolivier/go-doh-client"
8
-	"github.com/dgraph-io/ristretto"
9
 )
9
 )
10
 
10
 
11
-const (
12
-	dnsResolverSize     = 1024 * 1024 // 1mb
13
-	dnsResolverKeepTime = 10 * time.Minute
14
-)
11
+const dnsResolverKeepTime = 10 * time.Minute
12
+
13
+type dnsResolverCacheEntry struct {
14
+	ips       []string
15
+	createdAt time.Time
16
+}
17
+
18
+func (c dnsResolverCacheEntry) Ok() bool {
19
+	return time.Since(c.createdAt) < dnsResolverKeepTime
20
+}
15
 
21
 
16
 type dnsResolver struct {
22
 type dnsResolver struct {
17
-	resolver doh.Resolver
18
-	cache    *ristretto.Cache
23
+	resolver   doh.Resolver
24
+	cache      map[string]dnsResolverCacheEntry
25
+	cacheMutex sync.RWMutex
19
 }
26
 }
20
 
27
 
21
-func (d dnsResolver) LookupA(hostname string) []string {
22
-	key := "\x00." + hostname
28
+func (d *dnsResolver) LookupA(hostname string) []string {
29
+	key := "\x00" + hostname
30
+
31
+	d.cacheMutex.RLock()
32
+	entry, ok := d.cache[key]
33
+	d.cacheMutex.RUnlock()
23
 
34
 
24
-	if value, ok := d.cache.Get(key); ok {
25
-		return value.([]string)
35
+	if ok && entry.Ok() {
36
+		return entry.ips
26
 	}
37
 	}
27
 
38
 
28
 	var ips []string
39
 	var ips []string
32
 			ips = append(ips, v.IP4)
43
 			ips = append(ips, v.IP4)
33
 		}
44
 		}
34
 
45
 
35
-		d.cache.SetWithTTL(key, ips, 0, dnsResolverKeepTime)
46
+		d.cacheMutex.Lock()
47
+		d.cache[key] = dnsResolverCacheEntry{
48
+			ips:       ips,
49
+			createdAt: time.Now(),
50
+		}
51
+		d.cacheMutex.Unlock()
36
 	}
52
 	}
37
 
53
 
38
 	return ips
54
 	return ips
39
 }
55
 }
40
 
56
 
41
-func (d dnsResolver) LookupAAAA(hostname string) []string {
42
-	key := "\x01." + hostname
57
+func (d *dnsResolver) LookupAAAA(hostname string) []string {
58
+	key := "\x01" + hostname
43
 
59
 
44
-	if value, ok := d.cache.Get(key); ok {
45
-		return value.([]string)
60
+	d.cacheMutex.RLock()
61
+	entry, ok := d.cache[key]
62
+	d.cacheMutex.RUnlock()
63
+
64
+	if ok && entry.Ok() {
65
+		return entry.ips
46
 	}
66
 	}
47
 
67
 
48
 	var ips []string
68
 	var ips []string
52
 			ips = append(ips, v.IP6)
72
 			ips = append(ips, v.IP6)
53
 		}
73
 		}
54
 
74
 
55
-		d.cache.SetWithTTL(key, ips, 0, dnsResolverKeepTime)
75
+		d.cacheMutex.Lock()
76
+		d.cache[key] = dnsResolverCacheEntry{
77
+			ips:       ips,
78
+			createdAt: time.Now(),
79
+		}
80
+		d.cacheMutex.Unlock()
56
 	}
81
 	}
57
 
82
 
58
 	return ips
83
 	return ips
59
 }
84
 }
60
 
85
 
61
-func newDNSResolver(hostname string, httpClient *http.Client) dnsResolver {
62
-	cache, err := ristretto.NewCache(&ristretto.Config{
63
-		NumCounters: 10 * dnsResolverSize, // nolint: gomnd // taken from official doc as a best practice value
64
-		MaxCost:     dnsResolverSize,
65
-		BufferItems: 64, // nolint: gomnd // taken from official doc as a best practice value
66
-		Cost: func(value interface{}) int64 {
67
-			var cost int64
68
-
69
-			for _, v := range value.([]string) {
70
-				cost += int64(len([]byte(v)))
71
-			}
72
-
73
-			return cost
74
-		},
75
-	})
76
-	if err != nil {
77
-		panic(err)
78
-	}
79
-
80
-	return dnsResolver{
86
+func newDNSResolver(hostname string, httpClient *http.Client) *dnsResolver {
87
+	return &dnsResolver{
81
 		resolver: doh.Resolver{
88
 		resolver: doh.Resolver{
82
 			Host:       hostname,
89
 			Host:       hostname,
83
 			Class:      doh.IN,
90
 			Class:      doh.IN,
84
 			HTTPClient: httpClient,
91
 			HTTPClient: httpClient,
85
 		},
92
 		},
86
-		cache: cache,
93
+		cache: map[string]dnsResolverCacheEntry{},
87
 	}
94
 	}
88
 }
95
 }

+ 1
- 1
network/dns_resolver_internal_test.go Wyświetl plik

12
 type DNSResolverTestSuite struct {
12
 type DNSResolverTestSuite struct {
13
 	suite.Suite
13
 	suite.Suite
14
 
14
 
15
-	d dnsResolver
15
+	d *dnsResolver
16
 }
16
 }
17
 
17
 
18
 func (suite *DNSResolverTestSuite) TestLookupA() {
18
 func (suite *DNSResolverTestSuite) TestLookupA() {

+ 1
- 1
network/network.go Wyświetl plik

25
 
25
 
26
 type network struct {
26
 type network struct {
27
 	dialer      Dialer
27
 	dialer      Dialer
28
-	dns         dnsResolver
29
 	httpTimeout time.Duration
28
 	httpTimeout time.Duration
30
 	userAgent   string
29
 	userAgent   string
30
+	dns         *dnsResolver
31
 }
31
 }
32
 
32
 
33
 func (n *network) Dial(protocol, address string) (net.Conn, error) {
33
 func (n *network) Dial(protocol, address string) (net.Conn, error) {

Ładowanie…
Anuluj
Zapisz