Merge upstream/master: idiomatic Go, IP priority, flaky test fixes

Upstream changes:
- More idiomatic Golang (b6427ee)
- Change IP address set priority (1fcec38)
- Flaky CI test fixes (eedee63, 73c6a3a, e54d9d6)

Conflicts resolved:
- README.md: keep fork version
- default.pgo: removed (causes SEGV in fork builds)
- proxy.go: keep fork's counting conn, use upstream's tracker
- proxy_opts.go: keep fork's APIBindTo field

antireplay/stable_bloom_filter_test.go

@@ -12,7 +12,7 @@ type StableBloomFilterTestSuite struct {
 }
 
 func (suite *StableBloomFilterTestSuite) TestOp() {
-	filter := antireplay.NewStableBloomFilter(500, 0.001)
+	filter := antireplay.NewStableBloomFilter(100000, 0.001)
 
 	suite.False(filter.SeenBefore([]byte{1, 2, 3}))
 	suite.False(filter.SeenBefore([]byte{4, 5, 6}))

internal/config/config.go

@@ -53,10 +53,10 @@ type Config struct {
 		Blocklist    ListConfig `json:"blocklist"`
 		Allowlist    ListConfig `json:"allowlist"`
 		Doppelganger struct {
-			URLs            []TypeHttpsURL  `json:"urls"`
-			Repeats         TypeConcurrency `json:"repeats_per_raid"`
-			UpdateEach      TypeDuration    `json:"raid_each"`
-			DRS             TypeBool        `json:"drs"`
+			URLs       []TypeHttpsURL  `json:"urls"`
+			Repeats    TypeConcurrency `json:"repeats_per_raid"`
+			UpdateEach TypeDuration    `json:"raid_each"`
+			DRS        TypeBool        `json:"drs"`
 		} `json:"doppelganger"`
 	} `json:"defense"`
 	Network struct {

internal/config/parse.go

@@ -48,10 +48,10 @@ type tomlConfig struct {
 			UpdateEach          string   `toml:"update-each" json:"updateEach,omitempty"`
 		} `toml:"allowlist" json:"allowlist,omitempty"`
 		Doppelganger struct {
-			URLs            []string `toml:"urls" json:"urls,omitempty"`
-			Repeats         uint     `toml:"repeats-per-raid" json:"repeats_per_raid,omitempty"`
-			UpdateEach      string   `toml:"raid-each" json:"raid_each,omitempty"`
-			DRS             bool     `toml:"drs" json:"drs,omitempty"`
+			URLs       []string `toml:"urls" json:"urls,omitempty"`
+			Repeats    uint     `toml:"repeats-per-raid" json:"repeats_per_raid,omitempty"`
+			UpdateEach string   `toml:"raid-each" json:"raid_each,omitempty"`
+			DRS        bool     `toml:"drs" json:"drs,omitempty"`
 		} `toml:"doppelganger" json:"doppelganger,omitempty"`
 	} `toml:"defense" json:"defense,omitempty"`
 	Network struct {

mtglib/conns.go

@@ -3,9 +3,11 @@ package mtglib
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net"
+	"sync/atomic"
 	"time"
 
 	"github.com/9seconds/mtg/v2/essentials"
@@ -97,20 +99,63 @@ func newConnProxyProtocol(source, target essentials.Conn) *connProxyProtocol {
 	}
 }
 
+// idleTracker is a shared idle tracker for a pair of relay connections.
+// Both directions update the same timestamp so that activity in one direction
+// prevents the other (idle) direction from timing out.
+type idleTracker struct {
+	lastActive atomic.Pointer[time.Time]
+	timeout    time.Duration
+}
+
+func newIdleTracker(timeout time.Duration) *idleTracker {
+	t := &idleTracker{timeout: timeout}
+	t.touch()
+
+	return t
+}
+
+func (t *idleTracker) touch() {
+	stamp := time.Now()
+	t.lastActive.Store(&stamp)
+}
+
+func (t *idleTracker) isIdle() bool {
+	return time.Since(*t.lastActive.Load()) >= t.timeout
+}
+
 type connIdleTimeout struct {
 	essentials.Conn
 
-	timeout time.Duration
+	tracker *idleTracker
 }
 
 func (c connIdleTimeout) Read(b []byte) (int, error) {
-	c.SetReadDeadline(time.Now().Add(c.timeout)) //nolint: errcheck
+	var netErr net.Error
+
+	for {
+		c.SetReadDeadline(time.Now().Add(c.tracker.timeout)) //nolint: errcheck
 
-	return c.Conn.Read(b) //nolint: wrapcheck
+		n, err := c.Conn.Read(b)
+
+		switch {
+		case err == nil:
+			c.tracker.touch()
+			return n, nil
+		case errors.As(err, &netErr) && netErr.Timeout() && !c.tracker.isIdle():
+			continue
+		}
+
+		return n, err
+	}
 }
 
 func (c connIdleTimeout) Write(b []byte) (int, error) {
-	c.SetWriteDeadline(time.Now().Add(c.timeout)) //nolint: errcheck
+	c.SetWriteDeadline(time.Now().Add(c.tracker.timeout)) //nolint: errcheck
 
-	return c.Conn.Write(b) //nolint: wrapcheck
+	n, err := c.Conn.Write(b)
+	if n > 0 {
+		c.tracker.touch()
+	}
+
+	return n, err //nolint: wrapcheck
 }

mtglib/conns_internal_test.go

@@ -16,6 +16,12 @@ import (
 	"github.com/stretchr/testify/suite"
 )
 
+type netTimeoutError struct{}
+
+func (e netTimeoutError) Error() string   { return "i/o timeout" }
+func (e netTimeoutError) Timeout() bool   { return true }
+func (e netTimeoutError) Temporary() bool { return true }
+
 type ConnRewindBaseConn struct {
 	testlib.EssentialsConnMock
 
@@ -291,6 +297,141 @@ func (suite *ConnProxyProtocolTestSuite) TearDownTest() {
 	suite.targetConnMock.AssertExpectations(suite.T())
 }
 
+type IdleTrackerTestSuite struct {
+	suite.Suite
+}
+
+func (suite *IdleTrackerTestSuite) TestNewNotIdle() {
+	tracker := newIdleTracker(time.Second)
+	suite.False(tracker.isIdle())
+}
+
+func (suite *IdleTrackerTestSuite) TestIdleAfterTimeout() {
+	tracker := newIdleTracker(10 * time.Millisecond)
+	time.Sleep(20 * time.Millisecond)
+
+	suite.True(tracker.isIdle())
+}
+
+func (suite *IdleTrackerTestSuite) TestTouchResetsIdle() {
+	tracker := newIdleTracker(50 * time.Millisecond)
+	time.Sleep(30 * time.Millisecond)
+
+	tracker.touch()
+
+	suite.False(tracker.isIdle())
+}
+
+type ConnIdleTimeoutTestSuite struct {
+	suite.Suite
+
+	connMock *testlib.EssentialsConnMock
+	tracker  *idleTracker
+	conn     connIdleTimeout
+}
+
+func (suite *ConnIdleTimeoutTestSuite) SetupTest() {
+	suite.connMock = &testlib.EssentialsConnMock{}
+	suite.tracker = newIdleTracker(time.Second)
+	suite.conn = connIdleTimeout{
+		Conn:    suite.connMock,
+		tracker: suite.tracker,
+	}
+}
+
+func (suite *ConnIdleTimeoutTestSuite) TearDownTest() {
+	suite.connMock.AssertExpectations(suite.T())
+}
+
+func (suite *ConnIdleTimeoutTestSuite) TestReadOk() {
+	suite.connMock.On("SetReadDeadline", mock.Anything).Return(nil)
+	suite.connMock.On("Read", mock.Anything).Once().Return(5, nil)
+
+	n, err := suite.conn.Read(make([]byte, 10))
+	suite.NoError(err)
+	suite.Equal(5, n)
+}
+
+func (suite *ConnIdleTimeoutTestSuite) TestReadNonTimeoutErr() {
+	suite.connMock.On("SetReadDeadline", mock.Anything).Return(nil)
+	suite.connMock.On("Read", mock.Anything).Once().Return(0, io.EOF)
+
+	n, err := suite.conn.Read(make([]byte, 10))
+	suite.True(errors.Is(err, io.EOF))
+	suite.Equal(0, n)
+}
+
+func (suite *ConnIdleTimeoutTestSuite) TestReadTimeoutRetriesWhenNotIdle() {
+	suite.connMock.On("SetReadDeadline", mock.Anything).Return(nil)
+	suite.connMock.On("Read", mock.Anything).Once().Return(0, netTimeoutError{})
+	suite.connMock.On("Read", mock.Anything).Once().Return(5, nil)
+
+	n, err := suite.conn.Read(make([]byte, 10))
+	suite.NoError(err)
+	suite.Equal(5, n)
+}
+
+func (suite *ConnIdleTimeoutTestSuite) TestReadTimeoutClosesWhenIdle() {
+	suite.tracker = newIdleTracker(time.Millisecond)
+	suite.conn = connIdleTimeout{
+		Conn:    suite.connMock,
+		tracker: suite.tracker,
+	}
+
+	time.Sleep(5 * time.Millisecond)
+
+	suite.connMock.On("SetReadDeadline", mock.Anything).Return(nil)
+	suite.connMock.On("Read", mock.Anything).Once().Return(0, netTimeoutError{})
+
+	n, err := suite.conn.Read(make([]byte, 10))
+	suite.Equal(0, n)
+
+	netErr, ok := err.(net.Error) //nolint: errorlint
+	suite.True(ok)
+	suite.True(netErr.Timeout())
+}
+
+func (suite *ConnIdleTimeoutTestSuite) TestSharedTrackerPreventsFalseTimeout() {
+	connMock2 := &testlib.EssentialsConnMock{}
+	conn2 := connIdleTimeout{
+		Conn:    connMock2,
+		tracker: suite.tracker,
+	}
+
+	connMock2.On("SetWriteDeadline", mock.Anything).Return(nil)
+	connMock2.On("Write", mock.Anything).Once().Return(5, nil)
+
+	_, _ = conn2.Write(make([]byte, 5))
+
+	suite.connMock.On("SetReadDeadline", mock.Anything).Return(nil)
+	suite.connMock.On("Read", mock.Anything).Once().Return(0, netTimeoutError{})
+	suite.connMock.On("Read", mock.Anything).Once().Return(3, nil)
+
+	n, err := suite.conn.Read(make([]byte, 10))
+	suite.NoError(err)
+	suite.Equal(3, n)
+
+	connMock2.AssertExpectations(suite.T())
+}
+
+func (suite *ConnIdleTimeoutTestSuite) TestWriteOk() {
+	suite.connMock.On("SetWriteDeadline", mock.Anything).Return(nil)
+	suite.connMock.On("Write", mock.Anything).Once().Return(5, nil)
+
+	n, err := suite.conn.Write(make([]byte, 5))
+	suite.NoError(err)
+	suite.Equal(5, n)
+}
+
+func (suite *ConnIdleTimeoutTestSuite) TestWriteErr() {
+	suite.connMock.On("SetWriteDeadline", mock.Anything).Return(nil)
+	suite.connMock.On("Write", mock.Anything).Once().Return(0, io.EOF)
+
+	n, err := suite.conn.Write(make([]byte, 5))
+	suite.True(errors.Is(err, io.EOF))
+	suite.Equal(0, n)
+}
+
 func TestConnTraffic(t *testing.T) {
 	t.Parallel()
 	suite.Run(t, &ConnTrafficTestSuite{})
@@ -305,3 +446,13 @@ func TestConnProxyProtocol(t *testing.T) {
 	t.Parallel()
 	suite.Run(t, &ConnProxyProtocolTestSuite{})
 }
+
+func TestIdleTracker(t *testing.T) {
+	t.Parallel()
+	suite.Run(t, &IdleTrackerTestSuite{})
+}
+
+func TestConnIdleTimeout(t *testing.T) {
+	t.Parallel()
+	suite.Run(t, &ConnIdleTimeoutTestSuite{})
+}

mtglib/internal/dc/view.go

@@ -5,15 +5,19 @@ type dcView struct {
 }
 
 func (d dcView) getV4(dc int) []Addr {
-	addrs := d.publicConfigs.getV4(dc)
+	var addrs []Addr
+
 	addrs = append(addrs, defaultDCAddrSet.getV4(dc)...)
+	addrs = append(addrs, d.publicConfigs.getV4(dc)...)
 
 	return addrs
 }
 
 func (d dcView) getV6(dc int) []Addr {
-	addrs := d.publicConfigs.getV6(dc)
+	var addrs []Addr
+
 	addrs = append(addrs, defaultDCAddrSet.getV6(dc)...)
+	addrs = append(addrs, d.publicConfigs.getV6(dc)...)
 
 	return addrs
 }

mtglib/internal/doppel/scout.go

@@ -61,23 +61,29 @@ func (s Scout) learn(ctx context.Context, url string) (ScoutResult, error) {
 		client.CloseIdleConnections()
 	}
 
-	if err != nil || len(results.data) == 0 {
+	if err != nil {
 		return ScoutResult{}, err
 	}
 
+	data, writeIndex := results.Snapshot()
+
+	if len(data) == 0 {
+		return ScoutResult{}, nil
+	}
+
 	var result ScoutResult
 
 	// Compute inter-record durations (existing logic).
 	lastTimestamp := time.Time{}
 
-	for i, v := range results.data {
+	for i, v := range data {
 		if v.recordType != tls.TypeApplicationData {
 			continue
 		}
 
 		if lastTimestamp.IsZero() {
 			if i > 0 {
-				lastTimestamp = results.data[i-1].timestamp
+				lastTimestamp = data[i-1].timestamp
 			} else {
 				lastTimestamp = v.timestamp
 			}
@@ -90,12 +96,12 @@ func (s Scout) learn(ctx context.Context, url string) (ScoutResult, error) {
 	// Compute cert size: sum of ApplicationData payload between CCS and
 	// the first client Write (which marks the end of server handshake).
 	seenCCS := false
-	boundary := results.writeIndex
+	boundary := writeIndex
 	if boundary < 0 {
-		boundary = len(results.data)
+		boundary = len(data)
 	}
 
-	for i, v := range results.data {
+	for i, v := range data {
 		if i >= boundary {
 			break
 		}

mtglib/internal/doppel/scout_conn_collected.go

@@ -1,6 +1,10 @@
 package doppel
 
-import "time"
+import (
+	"slices"
+	"sync"
+	"time"
+)
 
 const (
 	ScoutConnCollectedPreallocSize = 100
@@ -13,23 +17,38 @@ type ScoutConnResult struct {
 }
 
 type ScoutConnCollected struct {
+	mu         sync.Mutex
 	data       []ScoutConnResult
 	writeIndex int // index at which client first wrote post-handshake data; -1 if not set
 }
 
 func (s *ScoutConnCollected) Add(record byte, payloadLen int) {
+	s.mu.Lock()
 	s.data = append(s.data, ScoutConnResult{
 		timestamp:  time.Now(),
 		recordType: record,
 		payloadLen: payloadLen,
 	})
+	s.mu.Unlock()
 }
 
 // MarkWrite records the current data length as the handshake boundary.
 func (s *ScoutConnCollected) MarkWrite() {
+	s.mu.Lock()
 	if s.writeIndex < 0 {
 		s.writeIndex = len(s.data)
 	}
+	s.mu.Unlock()
+}
+
+// Snapshot returns a copy of the collected data and the write index.
+func (s *ScoutConnCollected) Snapshot() ([]ScoutConnResult, int) {
+	s.mu.Lock()
+	snapshot := slices.Clone(s.data)
+	writeIndex := s.writeIndex
+	s.mu.Unlock()
+
+	return snapshot, writeIndex
 }
 
 func NewScoutConnCollected() *ScoutConnCollected {

mtglib/internal/doppel/scout_conn_collected_test.go

@@ -1,6 +1,7 @@
 package doppel
 
 import (
+	"sync"
 	"testing"
 	"time"
 
@@ -16,8 +17,10 @@ func (suite *ScoutConnCollectedTestSuite) TestAddSingle() {
 	collected := NewScoutConnCollected()
 	collected.Add(tls.TypeApplicationData, 100)
 
-	suite.Len(collected.data, 1)
-	suite.Equal(byte(tls.TypeApplicationData), collected.data[0].recordType)
+	data, _ := collected.Snapshot()
+
+	suite.Len(data, 1)
+	suite.Equal(byte(tls.TypeApplicationData), data[0].recordType)
 }
 
 func (suite *ScoutConnCollectedTestSuite) TestAddTimestampsAreMonotonic() {
@@ -31,11 +34,52 @@ func (suite *ScoutConnCollectedTestSuite) TestAddTimestampsAreMonotonic() {
 	time.Sleep(time.Microsecond)
 	collected.Add(tls.TypeApplicationData, 100)
 
-	for i := 1; i < len(collected.data); i++ {
-		suite.True(collected.data[i].timestamp.After(collected.data[i-1].timestamp))
+	data, _ := collected.Snapshot()
+
+	for i := 1; i < len(data); i++ {
+		suite.True(data[i].timestamp.After(data[i-1].timestamp))
 	}
 }
 
+func (suite *ScoutConnCollectedTestSuite) TestConcurrentAddSnapshot() {
+	collected := NewScoutConnCollected()
+
+	var wg sync.WaitGroup
+
+	wg.Add(3)
+
+	go func() {
+		defer wg.Done()
+
+		for i := 0; i < 1000; i++ {
+			collected.Add(tls.TypeApplicationData, i)
+		}
+	}()
+
+	go func() {
+		defer wg.Done()
+
+		for i := 0; i < 100; i++ {
+			collected.MarkWrite()
+		}
+	}()
+
+	go func() {
+		defer wg.Done()
+
+		for i := 0; i < 1000; i++ {
+			// call Snapshot concurrently to exercise the lock under -race
+			collected.Snapshot() //nolint:errcheck
+		}
+	}()
+
+	wg.Wait()
+
+	data, writeIndex := collected.Snapshot()
+	suite.Len(data, 1000)
+	suite.GreaterOrEqual(writeIndex, 0)
+}
+
 func TestScoutConnCollected(t *testing.T) {
 	t.Parallel()
 	suite.Run(t, &ScoutConnCollectedTestSuite{})

mtglib/proxy.go

@@ -111,11 +111,13 @@ func (p *Proxy) ServeConn(conn essentials.Conn) {
 		return
 	}
 
+	tracker := newIdleTracker(p.idleTimeout)
+
 	relay.Relay(
 		ctx,
 		ctx.logger.Named("relay"),
-		connIdleTimeout{Conn: ctx.telegramConn, timeout: p.idleTimeout},
-		newCountingConn(connIdleTimeout{Conn: ctx.clientConn, timeout: p.idleTimeout}, p.stats, ctx.secretName),
+		connIdleTimeout{Conn: ctx.telegramConn, tracker: tracker},
+		newCountingConn(connIdleTimeout{Conn: ctx.clientConn, tracker: tracker}, p.stats, ctx.secretName),
 	)
 }
 
@@ -330,11 +332,13 @@ func (p *Proxy) doDomainFronting(ctx *streamContext, conn *connRewind) {
 		stream:   p.eventStream,
 	}
 
+	tracker := newIdleTracker(p.idleTimeout)
+
 	relay.Relay(
 		ctx,
 		ctx.logger.Named("domain-fronting"),
-		connIdleTimeout{Conn: frontConn, timeout: p.idleTimeout},
-		connIdleTimeout{Conn: conn, timeout: p.idleTimeout},
+		connIdleTimeout{Conn: frontConn, tracker: tracker},
+		connIdleTimeout{Conn: conn, tracker: tracker},
 	)
 }
 

mtglib/proxy_test.go

@@ -175,7 +175,7 @@ func (suite *ProxyTestSuite) TestHTTPSRequest() {
 	addr := fmt.Sprintf("https://%s/headers", suite.ProxyAddress())
 
 	resp, err := client.Get(addr) //nolint: noctx
-	suite.NoError(err)
+	suite.Require().NoError(err)
 
 	defer resp.Body.Close() //nolint: errcheck