Pull request: all: add idna handling, imp domain validation

Updates #2915. Squashed commit of the following: commit b907324426c87ee7334edbd61e43c44444ad27a9 Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Wed Apr 7 16:26:41 2021 +0300 all: imp docs, upd commit c022f75cac006e077095cad283fea0a91d3a0eea Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Wed Apr 7 15:51:30 2021 +0300 all: add idna handling, imp domain validation
2021-04-07 16:36:38 +03:00
parent 00a61fdea0
commit c133b01ef7
13 changed files with 375 additions and 215 deletions
--- a/internal/dnsforward/clientid.go
+++ b/internal/dnsforward/clientid.go
@@ -6,33 +6,14 @@ import (
 	"path"
 	"strings"

+	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
 	"github.com/AdguardTeam/dnsproxy/proxy"
 	"github.com/lucas-clemente/quic-go"
 )

-// maxDomainLabelLen is the maximum allowed length of a domain name label
-// according to RFC 1035.
-const maxDomainLabelLen = 63
-
-// validateDomainNameLabel returns an error if label is not a valid label of
-// a domain name.
-func validateDomainNameLabel(label string) (err error) {
-	if len(label) > maxDomainLabelLen {
-		return fmt.Errorf("%q is too long, max: %d", label, maxDomainLabelLen)
-	}
-
-	for i, r := range label {
-		if (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '-' {
-			return fmt.Errorf("invalid char %q at index %d in %q", r, i, label)
-		}
-	}
-
-	return nil
-}
-
 // ValidateClientID returns an error if clientID is not a valid client ID.
 func ValidateClientID(clientID string) (err error) {
-	err = validateDomainNameLabel(clientID)
+	err = aghnet.ValidateDomainNameLabel(clientID)
 	if err != nil {
 		return fmt.Errorf("invalid client id: %w", err)
 	}
--- a/internal/dnsforward/clientid_test.go
+++ b/internal/dnsforward/clientid_test.go
@@ -238,8 +238,8 @@ func TestProcessClientID_https(t *testing.T) {
 		name:         "invalid_client_id",
 		path:         "/dns-query/!!!",
 		wantClientID: "",
-		wantErrMsg: `client id check: invalid client id: invalid char '!'` +
-			` at index 0 in "!!!"`,
+		wantErrMsg: `client id check: invalid client id: invalid char '!' ` +
+			`at index 0 in "!!!"`,
 		wantRes: resultCodeError,
 	}}

--- a/internal/dnsforward/dnsforward.go
+++ b/internal/dnsforward/dnsforward.go
@@ -114,7 +114,7 @@ func NewServer(p DNSCreateParams) (s *Server, err error) {
 	if p.AutohostTLD == "" {
 		autohostSuffix = defaultAutohostSuffix
 	} else {
-		err = validateDomainNameLabel(p.AutohostTLD)
+		err = aghnet.ValidateDomainNameLabel(p.AutohostTLD)
 		if err != nil {
 			return nil, fmt.Errorf("autohost tld: %w", err)
 		}
--- a/internal/dnsforward/dnsforward_test.go
+++ b/internal/dnsforward/dnsforward_test.go
@@ -947,145 +947,6 @@ func publicKey(priv interface{}) interface{} {
 	}
 }

-func TestValidateUpstream(t *testing.T) {
-	testCases := []struct {
-		name     string
-		upstream string
-		valid    bool
-		wantDef  bool
-	}{{
-		name:     "invalid",
-		upstream: "1.2.3.4.5",
-		valid:    false,
-	}, {
-		name:     "invalid",
-		upstream: "123.3.7m",
-		valid:    false,
-	}, {
-		name:     "invalid",
-		upstream: "htttps://google.com/dns-query",
-		valid:    false,
-	}, {
-		name:     "invalid",
-		upstream: "[/host.com]tls://dns.adguard.com",
-		valid:    false,
-	}, {
-		name:     "invalid",
-		upstream: "[host.ru]#",
-		valid:    false,
-	}, {
-		name:     "valid_default",
-		upstream: "1.1.1.1",
-		valid:    true,
-		wantDef:  true,
-	}, {
-		name:     "valid_default",
-		upstream: "tls://1.1.1.1",
-		valid:    true,
-		wantDef:  true,
-	}, {
-		name:     "valid_default",
-		upstream: "https://dns.adguard.com/dns-query",
-		valid:    true,
-		wantDef:  true,
-	}, {
-		name:     "valid_default",
-		upstream: "sdns://AQMAAAAAAAAAFDE3Ni4xMDMuMTMwLjEzMDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20",
-		valid:    true,
-		wantDef:  true,
-	}, {
-		name:     "valid",
-		upstream: "[/host.com/]1.1.1.1",
-		valid:    true,
-		wantDef:  false,
-	}, {
-		name:     "valid",
-		upstream: "[//]tls://1.1.1.1",
-		valid:    true,
-		wantDef:  false,
-	}, {
-		name:     "valid",
-		upstream: "[/www.host.com/]#",
-		valid:    true,
-		wantDef:  false,
-	}, {
-		name:     "valid",
-		upstream: "[/host.com/google.com/]8.8.8.8",
-		valid:    true,
-		wantDef:  false,
-	}, {
-		name:     "valid",
-		upstream: "[/host/]sdns://AQMAAAAAAAAAFDE3Ni4xMDMuMTMwLjEzMDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20",
-		valid:    true,
-		wantDef:  false,
-	}}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			defaultUpstream, err := validateUpstream(tc.upstream)
-			require.Equal(t, tc.valid, err == nil)
-			if tc.valid {
-				assert.Equal(t, tc.wantDef, defaultUpstream)
-			}
-		})
-	}
-}
-
-func TestValidateUpstreamsSet(t *testing.T) {
-	testCases := []struct {
-		name    string
-		msg     string
-		set     []string
-		wantNil bool
-	}{{
-		name:    "empty",
-		msg:     "empty upstreams array should be valid",
-		set:     nil,
-		wantNil: true,
-	}, {
-		name:    "comment",
-		msg:     "comments should not be validated",
-		set:     []string{"# comment"},
-		wantNil: true,
-	}, {
-		name: "valid_no_default",
-		msg:  "there is no default upstream",
-		set: []string{
-			"[/host.com/]1.1.1.1",
-			"[//]tls://1.1.1.1",
-			"[/www.host.com/]#",
-			"[/host.com/google.com/]8.8.8.8",
-			"[/host/]sdns://AQMAAAAAAAAAFDE3Ni4xMDMuMTMwLjEzMDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20",
-		},
-		wantNil: false,
-	}, {
-		name: "valid_with_default",
-		msg:  "upstreams set is valid, but doesn't pass through validation cause: %s",
-		set: []string{
-			"[/host.com/]1.1.1.1",
-			"[//]tls://1.1.1.1",
-			"[/www.host.com/]#",
-			"[/host.com/google.com/]8.8.8.8",
-			"[/host/]sdns://AQMAAAAAAAAAFDE3Ni4xMDMuMTMwLjEzMDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20",
-			"8.8.8.8",
-		},
-		wantNil: true,
-	}, {
-		name:    "invalid",
-		msg:     "there is an invalid upstream in set, but it pass through validation",
-		set:     []string{"dhcp://fake.dns"},
-		wantNil: false,
-	}}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			err := ValidateUpstreams(tc.set)
-
-			assert.Equalf(t, tc.wantNil, err == nil, tc.msg, err)
-		})
-	}
-}
-
 func TestIPStringFromAddr(t *testing.T) {
 	t.Run("not_nil", func(t *testing.T) {
 		addr := net.UDPAddr{
--- a/internal/dnsforward/http.go
+++ b/internal/dnsforward/http.go
@@ -8,10 +8,11 @@ import (
 	"strconv"
 	"strings"

+	"github.com/AdguardTeam/AdGuardHome/internal/agherr"
+	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
 	"github.com/AdguardTeam/dnsproxy/proxy"
 	"github.com/AdguardTeam/dnsproxy/upstream"
 	"github.com/AdguardTeam/golibs/log"
-	"github.com/AdguardTeam/golibs/utils"
 	"github.com/miekg/dns"
 )

@@ -302,7 +303,7 @@ type upstreamJSON struct {
 }

 // ValidateUpstreams validates each upstream and returns an error if any upstream is invalid or if there are no default upstreams specified
-func ValidateUpstreams(upstreams []string) error {
+func ValidateUpstreams(upstreams []string) (err error) {
 	// No need to validate comments
 	upstreams = filterOutComments(upstreams)

@@ -311,7 +312,7 @@ func ValidateUpstreams(upstreams []string) error {
 		return nil
 	}

-	_, err := proxy.ParseUpstreamsConfig(
+	_, err = proxy.ParseUpstreamsConfig(
 		upstreams,
 		upstream.Options{
 			Bootstrap: []string{},
@@ -345,56 +346,61 @@ func ValidateUpstreams(upstreams []string) error {
 var protocols = []string{"tls://", "https://", "tcp://", "sdns://", "quic://"}

 func validateUpstream(u string) (bool, error) {
-	// Check if user tries to specify upstream for domain
-	u, defaultUpstream, err := separateUpstream(u)
+	// Check if the user tries to specify upstream for domain.
+	u, useDefault, err := separateUpstream(u)
 	if err != nil {
-		return defaultUpstream, err
+		return useDefault, err
 	}

 	// The special server address '#' means "use the default servers"
-	if u == "#" && !defaultUpstream {
-		return defaultUpstream, nil
+	if u == "#" && !useDefault {
+		return useDefault, nil
 	}

 	// Check if the upstream has a valid protocol prefix
 	for _, proto := range protocols {
 		if strings.HasPrefix(u, proto) {
-			return defaultUpstream, nil
+			return useDefault, nil
 		}
 	}

 	// Return error if the upstream contains '://' without any valid protocol
 	if strings.Contains(u, "://") {
-		return defaultUpstream, fmt.Errorf("wrong protocol")
+		return useDefault, fmt.Errorf("wrong protocol")
 	}

 	// Check if upstream is valid plain DNS
-	return defaultUpstream, checkPlainDNS(u)
+	return useDefault, checkPlainDNS(u)
 }

-// separateUpstream returns upstream without specified domains and a bool flag that indicates if no domains were specified
-// error will be returned if upstream per domain specification is invalid
-func separateUpstream(upstream string) (string, bool, error) {
-	defaultUpstream := true
-	if strings.HasPrefix(upstream, "[/") {
-		defaultUpstream = false
-		// split domains and upstream string
-		domainsAndUpstream := strings.Split(strings.TrimPrefix(upstream, "[/"), "/]")
-		if len(domainsAndUpstream) != 2 {
-			return "", defaultUpstream, fmt.Errorf("wrong dns upstream per domain specification: %s", upstream)
+// separateUpstream returns the upstream without the specified domains.
+// useDefault is true when a default upstream must be used.
+func separateUpstream(upstreamStr string) (upstream string, useDefault bool, err error) {
+	defer agherr.Annotate("bad upstream for domain spec %q: %w", &err, upstreamStr)
+
+	if !strings.HasPrefix(upstreamStr, "[/") {
+		return upstreamStr, true, nil
+	}
+
+	parts := strings.Split(upstreamStr[2:], "/]")
+	if len(parts) != 2 {
+		return "", false, agherr.Error("duplicated separator")
+	}
+
+	domains := parts[0]
+	upstream = parts[1]
+	for i, host := range strings.Split(domains, "/") {
+		if host == "" {
+			continue
 		}

-		// split domains list and validate each one
-		for _, host := range strings.Split(domainsAndUpstream[0], "/") {
-			if host != "" {
-				if err := utils.IsValidHostname(host); err != nil {
-					return "", defaultUpstream, err
-				}
-			}
+		err = aghnet.ValidateDomainName(host)
+		if err != nil {
+			return "", false, fmt.Errorf("domain at index %d: %w", i, err)
 		}
-		upstream = domainsAndUpstream[1]
 	}
-	return upstream, defaultUpstream, nil
+
+	return upstream, false, nil
 }

 // checkPlainDNS checks if host is plain DNS
@@ -462,13 +468,13 @@ func checkDNS(input string, bootstrap []string) error {
 	}

 	// separate upstream from domains list
-	input, defaultUpstream, err := separateUpstream(input)
+	input, useDefault, err := separateUpstream(input)
 	if err != nil {
 		return fmt.Errorf("wrong upstream format: %w", err)
 	}

 	// No need to check this DNS server
-	if !defaultUpstream {
+	if !useDefault {
 		return nil
 	}

--- a/internal/dnsforward/http_test.go
+++ b/internal/dnsforward/http_test.go
@@ -213,3 +213,158 @@ func TestDNSForwardHTTTP_handleSetConfig(t *testing.T) {
 		})
 	}
 }
+
+// TODO(a.garipov): Rewrite to check the actual error messages.
+func TestValidateUpstream(t *testing.T) {
+	testCases := []struct {
+		name     string
+		upstream string
+		valid    bool
+		wantDef  bool
+	}{{
+		name:     "invalid",
+		upstream: "1.2.3.4.5",
+		valid:    false,
+		wantDef:  false,
+	}, {
+		name:     "invalid",
+		upstream: "123.3.7m",
+		valid:    false,
+		wantDef:  false,
+	}, {
+		name:     "invalid",
+		upstream: "htttps://google.com/dns-query",
+		valid:    false,
+		wantDef:  false,
+	}, {
+		name:     "invalid",
+		upstream: "[/host.com]tls://dns.adguard.com",
+		valid:    false,
+		wantDef:  false,
+	}, {
+		name:     "invalid",
+		upstream: "[host.ru]#",
+		valid:    false,
+		wantDef:  false,
+	}, {
+		name:     "valid_default",
+		upstream: "1.1.1.1",
+		valid:    true,
+		wantDef:  true,
+	}, {
+		name:     "valid_default",
+		upstream: "tls://1.1.1.1",
+		valid:    true,
+		wantDef:  true,
+	}, {
+		name:     "valid_default",
+		upstream: "https://dns.adguard.com/dns-query",
+		valid:    true,
+		wantDef:  true,
+	}, {
+		name:     "valid_default",
+		upstream: "sdns://AQMAAAAAAAAAFDE3Ni4xMDMuMTMwLjEzMDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20",
+		valid:    true,
+		wantDef:  true,
+	}, {
+		name:     "valid",
+		upstream: "[/host.com/]1.1.1.1",
+		valid:    true,
+		wantDef:  false,
+	}, {
+		name:     "valid",
+		upstream: "[//]tls://1.1.1.1",
+		valid:    true,
+		wantDef:  false,
+	}, {
+		name:     "valid",
+		upstream: "[/www.host.com/]#",
+		valid:    true,
+		wantDef:  false,
+	}, {
+		name:     "valid",
+		upstream: "[/host.com/google.com/]8.8.8.8",
+		valid:    true,
+		wantDef:  false,
+	}, {
+		name:     "valid",
+		upstream: "[/host/]sdns://AQMAAAAAAAAAFDE3Ni4xMDMuMTMwLjEzMDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20",
+		valid:    true,
+		wantDef:  false,
+	}, {
+		name:     "idna",
+		upstream: "[/пример.рф/]8.8.8.8",
+		valid:    true,
+		wantDef:  false,
+	}, {
+		name:     "bad_domain",
+		upstream: "[/!/]8.8.8.8",
+		valid:    false,
+		wantDef:  false,
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			defaultUpstream, err := validateUpstream(tc.upstream)
+			require.Equal(t, tc.valid, err == nil)
+			if tc.valid {
+				assert.Equal(t, tc.wantDef, defaultUpstream)
+			}
+		})
+	}
+}
+
+func TestValidateUpstreamsSet(t *testing.T) {
+	testCases := []struct {
+		name    string
+		msg     string
+		set     []string
+		wantNil bool
+	}{{
+		name:    "empty",
+		msg:     "empty upstreams array should be valid",
+		set:     nil,
+		wantNil: true,
+	}, {
+		name:    "comment",
+		msg:     "comments should not be validated",
+		set:     []string{"# comment"},
+		wantNil: true,
+	}, {
+		name: "valid_no_default",
+		msg:  "there is no default upstream",
+		set: []string{
+			"[/host.com/]1.1.1.1",
+			"[//]tls://1.1.1.1",
+			"[/www.host.com/]#",
+			"[/host.com/google.com/]8.8.8.8",
+			"[/host/]sdns://AQMAAAAAAAAAFDE3Ni4xMDMuMTMwLjEzMDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20",
+		},
+		wantNil: false,
+	}, {
+		name: "valid_with_default",
+		msg:  "upstreams set is valid, but doesn't pass through validation cause: %s",
+		set: []string{
+			"[/host.com/]1.1.1.1",
+			"[//]tls://1.1.1.1",
+			"[/www.host.com/]#",
+			"[/host.com/google.com/]8.8.8.8",
+			"[/host/]sdns://AQMAAAAAAAAAFDE3Ni4xMDMuMTMwLjEzMDo1NDQzINErR_JS3PLCu_iZEIbq95zkSV2LFsigxDIuUso_OQhzIjIuZG5zY3J5cHQuZGVmYXVsdC5uczEuYWRndWFyZC5jb20",
+			"8.8.8.8",
+		},
+		wantNil: true,
+	}, {
+		name:    "invalid",
+		msg:     "there is an invalid upstream in set, but it pass through validation",
+		set:     []string{"dhcp://fake.dns"},
+		wantNil: false,
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := ValidateUpstreams(tc.set)
+
+			assert.Equalf(t, tc.wantNil, err == nil, tc.msg, err)
+		})
+	}
+}
--- a/internal/dnsforward/util.go
+++ b/internal/dnsforward/util.go
@@ -5,7 +5,7 @@ import (
 	"sort"
 	"strings"

-	"github.com/AdguardTeam/golibs/utils"
+	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
 )

 // IPFromAddr gets IP address from addr.
@@ -58,9 +58,10 @@ func matchDomainWildcard(host, wildcard string) bool {

 // Return TRUE if client's SNI value matches DNS names from certificate
 func matchDNSName(dnsNames []string, sni string) bool {
-	if utils.IsValidHostname(sni) != nil {
+	if aghnet.ValidateDomainName(sni) != nil {
 		return false
 	}
+
 	if findSorted(dnsNames, sni) != -1 {
 		return true
 	}