Merge remote-tracking branch 'origin/master' into stable

Makefile

@@ -3,26 +3,28 @@ IMAGE_NAME   := mtg
 APP_NAME     := $(IMAGE_NAME)
 
 CC_BINARIES  := $(shell bash -c "echo -n $(APP_NAME)-{linux,freebsd,openbsd}-{386,amd64} $(APP_NAME)-linux-{arm,arm64}")
-APP_DEPS     := version.go
 
-GOLANGCI_LINT_VERSION := v1.10.2
+GOLANGCI_LINT_VERSION := v1.11.2
 
-COMMON_BUILD_FLAGS := -ldflags="-s -w"
+VERSION_GO         := $(shell go version)
+VERSION_DATE       := $(shell date -Ru)
+VERSION_TAG        := $(shell git describe --tags --always)
+COMMON_BUILD_FLAGS := -ldflags="-s -w -X 'main.version=$(VERSION_TAG) ($(VERSION_GO)) [$(VERSION_DATE)]'"
 
 MOD_ON  := env GO111MODULE=on
 MOD_OFF := env GO111MODULE=auto
 
 # -----------------------------------------------------------------------------
 
-$(APP_NAME): $(APP_DEPS)
+$(APP_NAME):
 	@$(MOD_ON) go build $(COMMON_BUILD_FLAGS) -o "$(APP_NAME)"
 
-static-$(APP_NAME): $(APP_DEPS)
+static-$(APP_NAME):
 	@$(MOD_ON) env CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo $(COMMON_BUILD_FLAGS) -o "$(APP_NAME)"
 
 $(APP_NAME)-%: GOOS=$(shell echo -n "$@" | sed 's?$(APP_NAME)-??' | cut -f1 -d-)
 $(APP_NAME)-%: GOARCH=$(shell echo -n "$@" | sed 's?$(APP_NAME)-??' | cut -f2 -d-)
-$(APP_NAME)-%: $(APP_DEPS) ccbuilds
+$(APP_NAME)-%: ccbuilds
 	@$(MOD_ON) env "GOOS=$(GOOS)" "GOARCH=$(GOARCH)" \
 		go build \
 		$(COMMON_BUILD_FLAGS) \
@@ -31,9 +33,6 @@ $(APP_NAME)-%: $(APP_DEPS) ccbuilds
 ccbuilds:
 	@rm -rf ./ccbuilds && mkdir -p ./ccbuilds
 
-version.go:
-	@$(MOD_ON) go generate main.go
-
 vendor: go.mod go.sum
 	@$(MOD_ON) go mod vendor
 
@@ -53,15 +52,15 @@ crosscompile-dir:
 	@rm -rf "$(CC_DIR)" && mkdir -p "$(CC_DIR)"
 
 .PHONY: test
-test: vendor $(APP_DEPS)
+test: vendor
 	@$(MOD_ON) go test -v ./...
 
 .PHONY: lint
-lint: vendor $(APP_DEPS)
+lint: vendor
 	@$(MOD_OFF) golangci-lint run
 
 .PHONY: critic
-critic: vendor $(APP_DEPS)
+critic: vendor
 	@$(MOD_OFF) gocritic check-project "$(ROOT_DIR)"
 
 .PHONY: clean

README.md

@@ -113,11 +113,49 @@ head -c 512 /dev/urandom | md5sum | cut -f 1 -d ' '
 
 ## Secure mode
 
-If you want to support new secure mode, please prepend `dd` to the
-secret. For example, secret `cf18fa8ea0267057e2c61a5f7322a8e7` should
-be `ddcf18fa8ea0267057e2c61a5f7322a8e7`. But pay attention that some
-old clients won't support this mode. If this is not your case, I would
-suggest to go with this mode.
+Secure mode is not the best name and of course, it creates a lot of
+confusion. To explain what it means, we need to tell you some bits on
+dd-secrets.
+
+MTPROTO proxy protocol requires 16-byte secret. You usually
+propagate it as a 32 characters hexadecimal string like
+`282831900f371ca182feb0e4e1e1aeef` (if you decode this string
+to bytes, you will get a real secret which is used in the
+protocol). Everything went quite good until the moment when
+developers found an evidence that [protocol is quite weak to
+DPI](https://github.com/TelegramMessenger/MTProxy/issues/35) and some
+enthusiasts even created simple proofs of concepts on [detecting MTPROTO
+traffic](https://github.com/darkk/poormansmtproto).
+
+Telegram team has introduced a patch called dd-secrets. If you have
+a secret `282831900f371ca182feb0e4e1e1aeef` then your dd-secret is
+`dd282831900f371ca182feb0e4e1e1aeef`. That is, you just add dd prefix
+to the secret, prepend it with dd. In that case, original secret
+`282831900f371ca182feb0e4e1e1aeef` is used but client and server start
+to act a little bit different: they start to add random noise to the
+packets so they can't be detected by their length. In order to keep
+backward compatibility, all proxies a quite liberal to the secrets to
+use: if the client uses plain secret, without dd prefix, they fall back
+to the normal behavior. If dd-secret is used (proxy can extract this
+information on the handshake), then more secured, the hardened behavior
+is used.
+
+Yes, it can look like a hack but it is as it is.
+
+Now going back to the secure mode: if you do not pass `-s` flag to the
+mtg, then it checks what mode is requested by the client. If the client
+uses plain secret, without dd prefix, then proxy falls back to the
+original behavior and do not play with paddings. If dd-secret is used
+and client demands this mode, then proxy start to add that random noise
+to the packets. But if you pass `-s`, then only clients with dd-secrets
+can connect. How to migrate existing clients then? If a client is new
+enough, you can just prepend the secret with dd string in the settings.
+If it is an old guy, then nothing to do, sorry.
+
+Why this mode matters? We do not have evidence but there is quite a big
+suspicion that some ISPs start to filter MTPROTO traffic. If they detect
+the IP address which acts as a proxy, they block it and no clients can
+use this proxy. This is an attempt to prevent such a situation.
 
 Oneliners to generate such secrets:
 
@@ -131,10 +169,6 @@ or
 echo dd$(head -c 512 /dev/urandom | md5sum | cut -f 1 -d ' ')
 ```
 
-If you want to enforce the usage of secure mode, please pass `-s` or
-`--secure-only` flags. In that case, clients which do not use dd-secrets
-are going to be disconnected from the proxy.
-
 
 ## Environment variables
 
@@ -192,13 +226,13 @@ This tool will listen on port 3128 by default with the given secret.
 # One-line runner
 
 ```console
-docker run --name mtg --restart=unless-stopped -p 3128:3128 -p 3129:3129 -d nineseconds/mtg:stable $(openssl rand -hex 16)
+docker run --name mtg --restart=unless-stopped -p 3128:3128 -q 3129:3129 -d nineseconds/mtg:stable $(openssl rand -hex 16)
 ```
 
 or in secret mode:
 
 ```console
-docker run --name mtg --restart=unless-stopped -p 3128:3128 -p 3129:3129 -d nineseconds/mtg:stable dd$(openssl rand -hex 16)
+docker run --name mtg --restart=unless-stopped -p 3128:3128 -q 3129:3129 -d nineseconds/mtg:stable dd$(openssl rand -hex 16)
 ```
 
 You will have this tool up and running on port 3128. Now curl

go.mod

@@ -19,13 +19,13 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v0.9.0
 	github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect
-	github.com/prometheus/common v0.0.0-20181015124227-bcb74de08d37 // indirect
+	github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39 // indirect
 	github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d // indirect
 	github.com/stretchr/testify v1.2.2
 	go.uber.org/atomic v1.3.2 // indirect
 	go.uber.org/multierr v1.1.0 // indirect
 	go.uber.org/zap v1.9.1
-	golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1 // indirect
+	golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc // indirect
 	golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/alexcesaro/statsd.v2 v2.0.0

go.sum

@@ -37,8 +37,8 @@ github.com/prometheus/client_golang v0.9.0 h1:tXuTFVHC03mW0D+Ua1Q2d1EAVqLTuggX50
 github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/common v0.0.0-20181015124227-bcb74de08d37 h1:Y7YdJ9Xb3MoQOzAWXnDunAJYpvhVwZdTirNfGUgPKaA=
-github.com/prometheus/common v0.0.0-20181015124227-bcb74de08d37/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39 h1:Cto4X6SVMWRPBkJ/3YHn1iDGDGc/Z+sW+AEMKHMVvN4=
+github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d h1:GoAlyOgbOEIFdaDqxJVlbOQ1DtGmZWs/Qau0hIlk+WQ=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
@@ -49,8 +49,8 @@ go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o=
 go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1 h1:Y/KGZSOdz/2r0WJ9Mkmz6NJBusp0kiNx1Cn82lzJQ6w=
-golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc h1:ZMCWScCvS2fUVFw8LOpxyUUW5qiviqr4Dg5NdjLeiLU=
+golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=

main.go

@@ -1,7 +1,5 @@
 package main
 
-//go:generate scripts/generate_version.sh
-
 import (
 	"encoding/json"
 	"fmt"
@@ -22,6 +20,8 @@ import (
 	"github.com/9seconds/mtg/stats"
 )
 
+var version = "dev" // this has to be set by build ld flags
+
 var (
 	app = kingpin.New("mtg", "Simple MTPROTO proxy.")
 
@@ -183,6 +183,7 @@ func main() { // nolint: gocyclo
 	defer logger.Sync() // nolint: errcheck
 
 	printURLs(conf.GetURLs())
+	zap.S().Debugw("Configuration", "config", conf)
 
 	if conf.UseMiddleProxy() {
 		zap.S().Infow("Use middle proxy connection to Telegram")

scripts/generate_version.sh

@@ -1,12 +0,0 @@
-#!/bin/sh
-set -eu
-
-PROJECT_DIR="$(git rev-parse --show-toplevel)"
-OUTPUT_FILE="${PROJECT_DIR}/version.go"
-
-cat > "$OUTPUT_FILE" <<EOF
-package main
-// autogenerated by $(basename "$0") on $(date -Ru)
-
-const version = "$(git describe --long --always) ($(go version)) [$(date -Ru)]"
-EOF

stats/init.go

@@ -22,7 +22,7 @@ func Init(conf *config.Config) error {
 	go prometheus.run()
 
 	go NewStats(conf).start()
-	go startServer(conf)
+	go startServer(conf, prometheus.getHTTPHandler())
 
 	return nil
 }

stats/prometheus.go

@@ -1,10 +1,12 @@
 package stats
 
 import (
+	"net/http"
 	"time"
 
 	"github.com/juju/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	"github.com/9seconds/mtg/config"
 )
@@ -12,6 +14,8 @@ import (
 const prometheusPollTime = time.Second
 
 type prometheusExporter struct {
+	registry prometheus.Gatherer
+
 	connections *prometheus.GaugeVec
 	traffic     *prometheus.GaugeVec
 	speed       *prometheus.GaugeVec
@@ -36,7 +40,13 @@ func (p *prometheusExporter) run() {
 	}
 }
 
+func (p *prometheusExporter) getHTTPHandler() http.Handler {
+	return promhttp.HandlerFor(p.registry, promhttp.HandlerOpts{})
+}
+
 func newPrometheus(conf *config.Config) (*prometheusExporter, error) {
+	registry := prometheus.NewRegistry()
+
 	connections := prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: conf.Prometheus.Prefix,
 		Name:      "connections",
@@ -58,20 +68,21 @@ func newPrometheus(conf *config.Config) (*prometheusExporter, error) {
 		Help:      "How many crashes happened.",
 	})
 
-	if err := prometheus.Register(connections); err != nil {
+	if err := registry.Register(connections); err != nil {
 		return nil, errors.Annotate(err, "Cannot register connections collector")
 	}
-	if err := prometheus.Register(traffic); err != nil {
+	if err := registry.Register(traffic); err != nil {
 		return nil, errors.Annotate(err, "cannot register traffic collector")
 	}
-	if err := prometheus.Register(speed); err != nil {
+	if err := registry.Register(speed); err != nil {
 		return nil, errors.Annotate(err, "cannot register speed collector")
 	}
-	if err := prometheus.Register(crashes); err != nil {
+	if err := registry.Register(crashes); err != nil {
 		return nil, errors.Annotate(err, "cannot register crashes collector")
 	}
 
 	return &prometheusExporter{
+		registry:    registry,
 		connections: connections,
 		traffic:     traffic,
 		speed:       speed,

stats/server.go

@@ -4,13 +4,12 @@ import (
 	"encoding/json"
 	"net/http"
 
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"go.uber.org/zap"
 
 	"github.com/9seconds/mtg/config"
 )
 
-func startServer(conf *config.Config) {
+func startServer(conf *config.Config, prometheusHandler http.Handler) {
 	log := zap.S().Named("stats")
 
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
@@ -33,7 +32,7 @@ func startServer(conf *config.Config) {
 			log.Errorw("Cannot encode json", "error", err)
 		}
 	})
-	http.Handle("/prometheus/", promhttp.Handler())
+	http.Handle("/prometheus/", prometheusHandler)
 
 	if err := http.ListenAndServe(conf.StatAddr(), nil); err != nil {
 		log.Fatalw("Stats server has been stopped", "error", err)