Add firehol blocklist

go.mod

 	github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
 	github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6
 	github.com/jarcoal/httpmock v1.0.8
+	github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1 // indirect
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/libp2p/go-reuseport v0.0.2
 	github.com/mccutchen/go-httpbin v1.1.1
+	github.com/panjf2000/ants v1.3.0 // indirect
 	github.com/pelletier/go-toml v1.8.1
 	github.com/rs/zerolog v1.20.0 // indirect
 	github.com/stretchr/objx v0.3.0 // indirect

go.sum

 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k=
 github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
+github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1 h1:D7qhJP3R49ZjUzpzKQ6B2H3lgejPs6DTO5gRomhhOpE=
+github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1/go.mod h1:2OfLA+0esiUJpwMjrH39pEk79cb8MvGTBS9YlZpejJ4=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ=
 github.com/mccutchen/go-httpbin v1.1.1 h1:aEws49HEJEyXHLDnshQVswfUlCVoS8g6h9YaDyaW7RE=
 github.com/mccutchen/go-httpbin v1.1.1/go.mod h1:fhpOYavp5g2K74XDl/ao2y4KvhqVtKlkg1e+0UaQv7I=
+github.com/panjf2000/ants v1.3.0 h1:8pQ+8leaLc9lys2viEEr8md0U4RN6uOSUCE9bOYjQ9M=
+github.com/panjf2000/ants v1.3.0/go.mod h1:AaACblRPzq35m1g3enqYcxspbbiOJJYaxU2wMpm1cXY=
 github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=
 github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
 github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.1.5-0.20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=

ipblocklist/firehol.go

+package ipblocklist
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/9seconds/mtg/v2/mtglib"
+	"github.com/kentik/patricia"
+	"github.com/kentik/patricia/bool_tree"
+	"github.com/panjf2000/ants"
+)
+
+const (
+	fireholIPv4DefaultCIDR = 32
+	fireholIPv6DefaultCIDR = 128
+)
+
+var fireholRegexpComment = regexp.MustCompile(`\s*#.*?$`)
+
+type Firehol struct {
+	logger     mtglib.Logger
+	rwMutex    sync.RWMutex
+	remoteURLs []string
+	localFiles []string
+	httpClient *http.Client
+	workerPool *ants.Pool
+	treeV4     *bool_tree.TreeV4
+	treeV6     *bool_tree.TreeV6
+}
+
+func (f *Firehol) Contains(ip net.IP) bool {
+	if ip == nil {
+		return true
+	}
+
+	ip4 := ip.To4()
+
+	f.rwMutex.RLock()
+	defer f.rwMutex.RUnlock()
+
+	if ip4 != nil {
+		return f.containsIPv4(ip4)
+	}
+
+	return f.containsIPv6(ip.To16())
+}
+
+func (f *Firehol) containsIPv4(addr net.IP) bool {
+	ip := patricia.NewIPv4AddressFromBytes(addr, 32)
+
+	if ok, _, err := f.treeV4.FindDeepestTag(ip); ok && err == nil {
+		return true
+	}
+
+	return false
+}
+
+func (f *Firehol) containsIPv6(addr net.IP) bool {
+	ip := patricia.NewIPv6Address(addr, 128)
+
+	if ok, _, err := f.treeV6.FindDeepestTag(ip); ok && err == nil {
+		return true
+	}
+
+	return false
+}
+
+func (f *Firehol) Run(ctx context.Context, updateEach time.Duration) {
+	ticker := time.NewTicker(updateEach)
+
+	defer func() {
+		ticker.Stop()
+
+		select {
+		case <-ticker.C:
+		default:
+		}
+	}()
+
+	if err := f.update(ctx); err != nil {
+		f.logger.WarningError("cannot update blocklist", err)
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			if err := f.update(ctx); err != nil {
+				f.logger.WarningError("cannot update blocklist", err)
+			}
+		}
+	}
+}
+
+func (f *Firehol) update(ctx context.Context) error { // nolint: funlen, cyclop
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	wg := &sync.WaitGroup{}
+	wg.Add(len(f.remoteURLs) + len(f.localFiles))
+
+	treeMutex := &sync.Mutex{}
+	v4tree := bool_tree.NewTreeV4()
+	v6tree := bool_tree.NewTreeV6()
+
+	errorChan := make(chan error, 1)
+	defer close(errorChan)
+
+	for _, v := range f.localFiles {
+		go func(filename string) {
+			defer wg.Done()
+
+			if err := f.updateLocalFile(ctx, filename, treeMutex, v4tree, v6tree); err != nil {
+				cancel()
+				f.logger.BindStr("filename", filename).WarningError("cannot update", err)
+
+				select {
+				case errorChan <- err:
+				default:
+				}
+			}
+		}(v)
+	}
+
+	for _, v := range f.remoteURLs {
+		value := v
+
+		f.workerPool.Submit(func() { // nolint: errcheck
+			defer wg.Done()
+
+			if err := f.updateRemoteURL(ctx, value, treeMutex, v4tree, v6tree); err != nil {
+				cancel()
+				f.logger.BindStr("url", value).WarningError("cannot update", err)
+
+				select {
+				case errorChan <- err:
+				default:
+				}
+			}
+		})
+	}
+
+	wg.Wait()
+
+	select {
+	case err := <-errorChan:
+		return fmt.Errorf("cannot update trees: %w", err)
+	default:
+	}
+
+	f.rwMutex.Lock()
+	defer f.rwMutex.Unlock()
+
+	f.treeV4 = v4tree
+	f.treeV6 = v6tree
+
+	return nil
+}
+
+func (f *Firehol) updateLocalFile(ctx context.Context, filename string,
+	mutex sync.Locker,
+	v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) error {
+	filefp, err := os.Open(filename)
+	if err != nil {
+		return fmt.Errorf("cannot open file: %w", err)
+	}
+
+	defer filefp.Close()
+
+	return f.updateTrees(ctx, mutex, filefp, v4tree, v6tree)
+}
+
+func (f *Firehol) updateRemoteURL(ctx context.Context, url string,
+	mutex sync.Locker,
+	v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return fmt.Errorf("cannot build a request: %w", err)
+	}
+
+	resp, err := f.httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("cannot request a remote URL %s: %w", url, err)
+	}
+
+	defer func() {
+		io.Copy(ioutil.Discard, resp.Body) // nolint: errcheck
+		resp.Body.Close()
+	}()
+
+	return f.updateTrees(ctx, mutex, resp.Body, v4tree, v6tree)
+}
+
+func (f *Firehol) updateTrees(ctx context.Context,
+	mutex sync.Locker,
+	reader io.Reader,
+	v4tree *bool_tree.TreeV4,
+	v6tree *bool_tree.TreeV6) error {
+	scanner := bufio.NewScanner(reader)
+
+	for scanner.Scan() {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+
+		text := scanner.Text()
+		text = fireholRegexpComment.ReplaceAllLiteralString(text, "")
+		text = strings.TrimSpace(text)
+
+		if text == "" {
+			continue
+		}
+
+		ip, cidr, err := f.updateParseLine(text)
+		if err != nil {
+			return fmt.Errorf("cannot parse a line: %w", err)
+		}
+
+		if err := f.updateAddToTrees(ip, cidr, mutex, v4tree, v6tree); err != nil {
+			return fmt.Errorf("cannot add a node to the tree: %w", err)
+		}
+	}
+
+	if scanner.Err() != nil {
+		return fmt.Errorf("cannot parse a response: %w", scanner.Err())
+	}
+
+	return nil
+}
+
+func (f *Firehol) updateParseLine(text string) (net.IP, uint, error) {
+	_, ipnet, err := net.ParseCIDR(text)
+	if err != nil {
+		ipaddr := net.ParseIP(text)
+		if ipaddr == nil {
+			return nil, 0, fmt.Errorf("incorrect ip address %s", text)
+		}
+
+		ip4 := ipaddr.To4()
+		if ip4 != nil {
+			return ip4, fireholIPv4DefaultCIDR, nil
+		}
+
+		return ipaddr.To16(), fireholIPv6DefaultCIDR, nil
+	}
+
+	ones, _ := ipnet.Mask.Size()
+
+	return ipnet.IP, uint(ones), nil
+}
+
+func (f *Firehol) updateAddToTrees(ip net.IP, cidr uint,
+	mutex sync.Locker,
+	v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) error {
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	if ip.To4() != nil {
+		addr := patricia.NewIPv4AddressFromBytes(ip, cidr)
+
+		if _, _, err := v4tree.Set(addr, true); err != nil {
+			return err // nolint: wrapcheck
+		}
+	} else {
+		addr := patricia.NewIPv6Address(ip, cidr)
+
+		if _, _, err := v6tree.Set(addr, true); err != nil {
+			return err // nolint: wrapcheck
+		}
+	}
+
+	return nil
+}
+
+func NewFirehol(logger mtglib.Logger, network mtglib.Network,
+	downloadConcurrency uint,
+	remoteURLs []string,
+	localFiles []string) (*Firehol, error) {
+	for _, v := range remoteURLs {
+		parsed, err := url.Parse(v)
+		if err != nil {
+			return nil, fmt.Errorf("incorrect url %s: %w", v, err)
+		}
+
+		switch parsed.Scheme {
+		case "http", "https":
+		default:
+			return nil, fmt.Errorf("unsupported url %s", v)
+		}
+	}
+
+	for _, v := range localFiles {
+		if stat, err := os.Stat(v); os.IsNotExist(err) || stat.IsDir() || stat.Mode().Perm()&0o400 == 0 {
+			return nil, fmt.Errorf("%s is not a readable file", v)
+		}
+	}
+
+	if downloadConcurrency == 0 {
+		downloadConcurrency = 1
+	}
+
+	workerPool, _ := ants.NewPool(int(downloadConcurrency))
+
+	return &Firehol{
+		logger:     logger.Named("firehol"),
+		httpClient: network.MakeHTTPClient(nil),
+		treeV4:     bool_tree.NewTreeV4(),
+		treeV6:     bool_tree.NewTreeV6(),
+		workerPool: workerPool,
+		remoteURLs: remoteURLs,
+		localFiles: localFiles,
+	}, nil
+}

ipblocklist/firehol_test.go

+package ipblocklist_test
+
+import (
+	"context"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/9seconds/mtg/v2/ipblocklist"
+	"github.com/9seconds/mtg/v2/logger"
+	"github.com/9seconds/mtg/v2/network"
+	"github.com/9seconds/mtg/v2/testlib"
+	"github.com/jarcoal/httpmock"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/suite"
+)
+
+type FireholTestSuite struct {
+	suite.Suite
+
+	networkMock *testlib.MtglibNetworkMock
+	httpServer  *httptest.Server
+}
+
+func (suite *FireholTestSuite) SetupSuite() {
+	mux := http.NewServeMux()
+
+	mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
+		filefp, err := os.Open(filepath.Join("testdata", "remote_ipset.ipset"))
+		if err != nil {
+			panic(err)
+		}
+
+		defer filefp.Close()
+
+		io.Copy(w, filefp) // nolint: errcheck
+	})
+
+	suite.httpServer = httptest.NewServer(mux)
+}
+
+func (suite *FireholTestSuite) SetupTest() {
+	httpClient := &http.Client{}
+	suite.networkMock = &testlib.MtglibNetworkMock{}
+
+	httpmock.ActivateNonDefault(httpClient)
+
+	suite.networkMock.
+		On("MakeHTTPClient", mock.Anything).
+		Maybe().
+		Return(httpClient)
+}
+
+func (suite *FireholTestSuite) TearDownTest() {
+	suite.networkMock.AssertExpectations(suite.T())
+	httpmock.DeactivateAndReset()
+}
+
+func (suite *FireholTestSuite) TearDownSuite() {
+	suite.httpServer.Close()
+}
+
+func (suite *FireholTestSuite) TestLocalFail() {
+	blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(),
+		suite.networkMock, 2,
+		nil, []string{filepath.Join("testdata", "broken_ipset.ipset")})
+
+	suite.NoError(err)
+
+	go blocklist.Run(context.Background(), time.Hour)
+
+	time.Sleep(500 * time.Millisecond)
+
+	suite.False(blocklist.Contains(net.ParseIP("10.0.0.10")))
+	suite.False(blocklist.Contains(net.ParseIP("127.0.0.1")))
+}
+
+func (suite *FireholTestSuite) TestLocalOk() {
+	blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(),
+		suite.networkMock, 2,
+		nil, []string{filepath.Join("testdata", "good_ipset.ipset")})
+
+	suite.NoError(err)
+
+	go blocklist.Run(context.Background(), time.Hour)
+
+	time.Sleep(500 * time.Millisecond)
+
+	suite.True(blocklist.Contains(net.ParseIP("10.0.0.10")))
+	suite.False(blocklist.Contains(net.ParseIP("127.0.0.1")))
+}
+
+func (suite *FireholTestSuite) TestRemoteFail() {
+	blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(),
+		suite.networkMock, 2,
+		[]string{"https://google.com"}, nil)
+
+	suite.NoError(err)
+
+	go blocklist.Run(context.Background(), time.Hour)
+
+	time.Sleep(500 * time.Millisecond)
+
+	suite.False(blocklist.Contains(net.ParseIP("10.2.2.2")))
+}
+
+func (suite *FireholTestSuite) TestMixed() {
+	dialer, _ := network.NewDefaultDialer(0, 0)
+	ntw, _ := network.NewNetwork(dialer, "mtg", "1.1.1.1", 0, 0)
+
+	blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(),
+		ntw, 2,
+		[]string{
+			suite.httpServer.URL,
+		}, []string{
+			filepath.Join("testdata", "good_ipset.ipset"),
+		})
+
+	suite.NoError(err)
+
+	go blocklist.Run(context.Background(), time.Hour)
+
+	time.Sleep(500 * time.Millisecond)
+
+	suite.True(blocklist.Contains(net.ParseIP("10.2.2.2")))
+	suite.True(blocklist.Contains(net.ParseIP("10.1.0.100")))
+}
+
+func TestFirehol(t *testing.T) {
+	t.Parallel()
+	suite.Run(t, &FireholTestSuite{})
+}

ipblocklist/noop.go

+package ipblocklist
+
+import (
+	"net"
+
+	"github.com/9seconds/mtg/v2/mtglib"
+)
+
+type noop struct{}
+
+func (n noop) Contains(ip net.IP) bool { return false }
+
+func NewNoop() mtglib.IPBlocklist {
+	return noop{}
+}

ipblocklist/noop_test.go

+package ipblocklist_test
+
+import (
+	"net"
+	"testing"
+
+	"github.com/9seconds/mtg/v2/ipblocklist"
+	"github.com/stretchr/testify/suite"
+)
+
+type NoopTestSuite struct {
+	suite.Suite
+}
+
+func (suite *NoopTestSuite) TestOp() {
+	suite.False(ipblocklist.NewNoop().Contains(net.ParseIP("10.0.0.10")))
+	suite.False(ipblocklist.NewNoop().Contains(net.ParseIP("10.0.0.10")))
+}
+
+func TestNoop(t *testing.T) {
+	t.Parallel()
+	suite.Run(t, &NoopTestSuite{})
+}

ipblocklist/testdata/broken_ipset.ipset

+#
+# This is an intentionally broken ipset.
+#
+
+ajsdkfbd

ipblocklist/testdata/good_ipset.ipset

+#
+# This is very good ipset
+#
+
+10.0.0.10  # just an example
+10.1.0.0/24
+2001:0db8:85a3:0000:0000:8a2e:0370:7334

ipblocklist/testdata/remote_ipset.ipset

+10.2.2.2

mtglib/init.go

 	SeenBefore(data []byte) bool
 }
 
+type IPBlocklist interface {
+	Contains(net.IP) bool
+}
+
 type Logger interface {
 	Named(name string) Logger
 

network/network.go

 func NewNetwork(dialer Dialer,
 	userAgent, dohHostname string,
 	httpTimeout, idleTimeout time.Duration) (mtglib.Network, error) {
+	switch {
+	case httpTimeout < 0:
+		return nil, fmt.Errorf("timeout should be positive number %s", httpTimeout)
+	case httpTimeout == 0:
+		httpTimeout = DefaultHTTPTimeout
+	}
+
 	switch {
 	case idleTimeout < 0:
 		return nil, fmt.Errorf("timeout should be positive number %s", idleTimeout)