Add blocklist to config

config/config.go

@@ -27,6 +27,12 @@ type Config struct {
 			MaxSize   TypeBytes     `json:"max-size"`
 			ErrorRate TypeErrorRate `json:"error-rate"`
 		} `json:"anti-replay"`
+		Blocklist struct {
+			Enabled             bool               `json:"enabled"`
+			DownloadConcurrency uint               `json:"download-concurrency"`
+			URLs                []TypeBlocklistURI `json:"urls"`
+			UpdateEach          TypeDuration       `json:"update-each"`
+		} `json:"blocklist"`
 	} `json:"defense"`
 	Network struct {
 		PublicIP struct {
@@ -60,6 +66,7 @@ func (c *Config) Validate() error {
 	if !c.Secret.Valid() {
 		return fmt.Errorf("invalid secret %s", c.Secret.String())
 	}
+
 	if len(c.BindTo.HostValue(nil)) == 0 || c.BindTo.PortValue(0) == 0 {
 		return fmt.Errorf("incorrect bind-to parameter %s", c.BindTo.String())
 	}
@@ -98,6 +105,12 @@ type configRaw struct {
 			MaxSize   string  `toml:"max-size" json:"max-size,omitempty"`
 			ErrorRate float64 `toml:"error-rate" json:"error-rate,omitempty"`
 		} `toml:"anti-replay" json:"anti-replay,omitempty"`
+		Blocklist struct {
+			Enabled             bool     `toml:"enabled" json:"enabled,omitempty"`
+			DownloadConcurrency uint     `toml:"download-concurrency" json:"download-concurrency,omitempty"`
+			URLs                []string `toml:"urls" json:"urls,omitempty"`
+			UpdateEach          string   `toml:"update-each" json:"update-each,omitempty"`
+		} `toml:"blocklist" json:"blocklist,omitempty"`
 	} `toml:"defense" json:"defense,omitempty"`
 	Network struct {
 		PublicIP struct {

config/config_test.go

@@ -39,7 +39,7 @@ func (suite *ConfigTestSuite) TestParseMinimalConfig() {
 	conf, err := config.Parse(suite.ReadConfig("minimal.toml"))
 	suite.NoError(err)
 	suite.Equal("7oe1GqLy6TBc38CV3jx7q09nb29nbGUuY29t", conf.Secret.Base64())
-    suite.Equal("0.0.0.0:3128", conf.BindTo.String())
+	suite.Equal("0.0.0.0:3128", conf.BindTo.String())
 }
 
 func TestConfig(t *testing.T) {

config/type_blocklist_uri.go

@@ -0,0 +1,68 @@
+package config
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+)
+
+type TypeBlocklistURI struct {
+	value string
+}
+
+func (c *TypeBlocklistURI) UnmarshalText(data []byte) error {
+	if len(data) == 0 {
+		return nil
+	}
+
+	text := string(data)
+	if filepath.IsAbs(text) {
+		if _, err := os.Stat(text); os.IsNotExist(err) {
+			return fmt.Errorf("filepath %s does not exist", text)
+		}
+
+		c.value = text
+
+		return nil
+	}
+
+	parsedURL, err := url.Parse(text)
+	if err != nil {
+		return fmt.Errorf("incorrect url: %w", err)
+	}
+
+	switch parsedURL.Scheme {
+	case "http", "https": // nolint: goconst
+	default:
+		return fmt.Errorf("unknown schema %s", parsedURL.Scheme)
+	}
+
+	if parsedURL.Host == "" {
+		return fmt.Errorf("incorrect url %s", text)
+	}
+
+	c.value = parsedURL.String()
+
+	return nil
+}
+
+func (c TypeBlocklistURI) MarshalText() ([]byte, error) {
+	return []byte(c.value), nil
+}
+
+func (c TypeBlocklistURI) String() string {
+	return c.value
+}
+
+func (c TypeBlocklistURI) IsRemote() bool {
+	return !filepath.IsAbs(c.value)
+}
+
+func (c TypeBlocklistURI) Value(defaultValue string) string {
+	if c.value == "" {
+		return defaultValue
+	}
+
+	return c.value
+}

config/type_blocklist_uri_test.go

@@ -0,0 +1,156 @@
+package config_test
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strconv"
+	"testing"
+
+	"github.com/9seconds/mtg/v2/config"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+)
+
+type typeBlocklistURITestStruct struct {
+	Value config.TypeBlocklistURI `json:"value"`
+}
+
+type TypeBlocklistURITestSuite struct {
+	suite.Suite
+}
+
+func (suite *TypeBlocklistURITestSuite) TestUnmarshalFail() {
+	rnd := make([]byte, 48)
+
+	rand.Read(rnd) // nolint: errcheck
+
+	unknownPath := base64.StdEncoding.EncodeToString(rnd)
+
+	testData := []string{
+		"1",
+		unknownPath,
+		"/" + unknownPath,
+		"http:/",
+		"gopher://lalalal",
+	}
+
+	for _, v := range testData {
+		data, err := json.Marshal(map[string]string{
+			"value": v,
+		})
+		suite.NoError(err)
+
+		suite.T().Run(v, func(t *testing.T) {
+			assert.Error(t, json.Unmarshal(data, &typeBlocklistURITestStruct{}))
+		})
+	}
+}
+
+func (suite *TypeBlocklistURITestSuite) TestUnmarshalOk() {
+	dir, _ := os.Getwd()
+	dir, _ = filepath.Abs(dir)
+
+	testData := []string{
+		"http://lalala",
+		filepath.Join(dir, "config.go"),
+		"https://lalala",
+	}
+
+	for _, v := range testData {
+		value := v
+
+		data, err := json.Marshal(map[string]string{
+			"value": v,
+		})
+		suite.NoError(err)
+
+		suite.T().Run(v, func(t *testing.T) {
+			testStruct := &typeBlocklistURITestStruct{}
+
+			assert.NoError(t, json.Unmarshal(data, testStruct))
+			assert.EqualValues(t, value, testStruct.Value.Value(""))
+		})
+	}
+}
+
+func (suite *TypeBlocklistURITestSuite) TestMarshalOk() {
+	dir, _ := os.Getwd()
+	dir, _ = filepath.Abs(dir)
+
+	testData := []string{
+		"http://lalalal",
+		filepath.Join(dir, "config.go"),
+	}
+
+	for _, v := range testData {
+		name := v
+
+		data, err := json.Marshal(map[string]string{
+			"value": name,
+		})
+		suite.NoError(err)
+
+		suite.T().Run(name, func(t *testing.T) {
+			testStruct := &typeBlocklistURITestStruct{}
+
+			assert.NoError(t, json.Unmarshal(data, testStruct))
+			assert.Equal(t, name, testStruct.Value.String())
+
+			marshalled, err := testStruct.Value.MarshalText()
+			assert.NoError(t, err)
+			assert.Equal(t, name, string(marshalled))
+		})
+	}
+}
+
+func (suite *TypeBlocklistURITestSuite) TestValue() {
+	testStruct := &typeBlocklistURITestStruct{}
+
+	suite.Equal("http://lalala", testStruct.Value.Value("http://lalala"))
+
+	data, err := json.Marshal(map[string]string{
+		"value": "http://blablabla",
+	})
+	suite.NoError(err)
+	suite.NoError(json.Unmarshal(data, testStruct))
+
+	suite.Equal("http://blablabla", testStruct.Value.Value(""))
+}
+
+func (suite *TypeBlocklistURITestSuite) TestIsRemote() {
+	dir, _ := os.Getwd()
+	dir, _ = filepath.Abs(dir)
+
+	testData := map[bool]string{
+		true:  "http://lalalal",
+		false: filepath.Join(dir, "config.go"),
+	}
+
+	for k, v := range testData {
+		ok := k
+
+		data, err := json.Marshal(map[string]string{
+			"value": v,
+		})
+		suite.NoError(err)
+
+		suite.T().Run(strconv.FormatBool(ok), func(t *testing.T) {
+			testStruct := &typeBlocklistURITestStruct{}
+			assert.NoError(t, json.Unmarshal(data, testStruct))
+
+			if ok {
+				assert.True(t, testStruct.Value.IsRemote())
+			} else {
+				assert.False(t, testStruct.Value.IsRemote())
+			}
+		})
+	}
+}
+
+func TestTypeBlocklistURI(t *testing.T) {
+	t.Parallel()
+	suite.Run(t, &TypeBlocklistURITestSuite{})
+}

example.config.toml

@@ -149,6 +149,30 @@ max-size = "16mb"
 # to maintain a desired error ratio.
 error-rate = 0.0001
 
+# You can protect proxies by using different blocklists. If client has
+# ip from the given range, we do not try to do a proper handshake. We
+# actually route it to fronting domain. So, this client will never ever
+# have a chance to use mtg to access Telegram.
+#
+# Please remember that blocklists are initialized in async way. So,
+# when you start a proxy, blocklists are empty, they are populated and
+# processed in backgrounds. An error in any URL is ignored.
+[defense.blocklist]
+# You can enable/disable this feature.
+enabled = true
+# This is a limiter for concurrency. In order to protect website
+# from overloading, we download files in this number of threads.
+download-concurrency = 2
+# A list of URLs in FireHOL format (https://iplists.firehol.org/)
+# You can provider links here (starts with https:// or http://) or
+# path to a local file, but in this case it should be absolute.
+urls = [
+    # "https://iplists.firehol.org/files/firehol_level1.netset",
+    # "/local.file"
+]
+# How often do we need to update a blocklist set.
+update-each = "1d"
+
 # statsd statistics integration.
 [stats.statsd]
 # enabled/disabled