Pull request 2100: v0.107.42-rc

Squashed commit of the following:

commit 284190f748345c7556e60b67f051ec5f6f080948
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Dec 6 19:36:00 2023 +0300

    all: sync with master; upd chlog

internal/dnsforward/upstreams.go

@@ -1,16 +1,17 @@
 package dnsforward
 
 import (
-	"bytes"
 	"fmt"
 	"net"
-	"net/url"
+	"net/netip"
 	"os"
 	"strings"
 	"time"
 
+	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
 	"github.com/AdguardTeam/dnsproxy/proxy"
 	"github.com/AdguardTeam/dnsproxy/upstream"
+	"github.com/AdguardTeam/golibs/errors"
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/AdguardTeam/golibs/netutil"
 	"github.com/AdguardTeam/golibs/stringutil"
@@ -18,6 +19,28 @@ import (
 	"golang.org/x/exp/slices"
 )
 
+const (
+	// errNotDomainSpecific is returned when the upstream should be
+	// domain-specific, but isn't.
+	errNotDomainSpecific errors.Error = "not a domain-specific upstream"
+
+	// errMissingSeparator is returned when the domain-specific part of the
+	// upstream configuration line isn't closed.
+	errMissingSeparator errors.Error = "missing separator"
+
+	// errDupSeparator is returned when the domain-specific part of the upstream
+	// configuration line contains more than one ending separator.
+	errDupSeparator errors.Error = "duplicated separator"
+
+	// errNoDefaultUpstreams is returned when there are no default upstreams
+	// specified in the upstream configuration.
+	errNoDefaultUpstreams errors.Error = "no default upstreams specified"
+
+	// errWrongResponse is returned when the checked upstream replies in an
+	// unexpected way.
+	errWrongResponse errors.Error = "wrong response"
+)
+
 // loadUpstreams parses upstream DNS servers from the configured file or from
 // the configuration itself.
 func (s *Server) loadUpstreams() (upstreams []string, err error) {
@@ -39,7 +62,7 @@ func (s *Server) loadUpstreams() (upstreams []string, err error) {
 }
 
 // prepareUpstreamSettings sets upstream DNS server settings.
-func (s *Server) prepareUpstreamSettings() (err error) {
+func (s *Server) prepareUpstreamSettings(boot upstream.Resolver) (err error) {
 	// Load upstreams either from the file, or from the settings
 	var upstreams []string
 	upstreams, err = s.loadUpstreams()
@@ -48,7 +71,7 @@ func (s *Server) prepareUpstreamSettings() (err error) {
 	}
 
 	s.conf.UpstreamConfig, err = s.prepareUpstreamConfig(upstreams, defaultDNS, &upstream.Options{
-		Bootstrap:    s.conf.BootstrapDNS,
+		Bootstrap:    boot,
 		Timeout:      s.conf.UpstreamTimeout,
 		HTTPVersions: UpstreamHTTPVersions(s.conf.UseHTTP3Upstreams),
 		PreferIPv6:   s.conf.BootstrapPreferIPv6,
@@ -92,178 +115,9 @@ func (s *Server) prepareUpstreamConfig(
 		uc.Upstreams = defaultUpstreamConfig.Upstreams
 	}
 
-	// dnsFilter can be nil during application update.
-	if s.dnsFilter != nil {
-		err = s.replaceUpstreamsWithHosts(uc, opts)
-		if err != nil {
-			return nil, fmt.Errorf("resolving upstreams with hosts: %w", err)
-		}
-	}
-
 	return uc, nil
 }
 
-// replaceUpstreamsWithHosts replaces unique upstreams with their resolved
-// versions based on the system hosts file.
-//
-// TODO(e.burkov):  This should be performed inside dnsproxy, which should
-// actually consider /etc/hosts.  See TODO on [aghnet.HostsContainer].
-func (s *Server) replaceUpstreamsWithHosts(
-	upsConf *proxy.UpstreamConfig,
-	opts *upstream.Options,
-) (err error) {
-	resolved := map[string]*upstream.Options{}
-
-	err = s.resolveUpstreamsWithHosts(resolved, upsConf.Upstreams, opts)
-	if err != nil {
-		return fmt.Errorf("resolving upstreams: %w", err)
-	}
-
-	hosts := maps.Keys(upsConf.DomainReservedUpstreams)
-	// TODO(e.burkov):  Think of extracting sorted range into an util function.
-	slices.Sort(hosts)
-	for _, host := range hosts {
-		err = s.resolveUpstreamsWithHosts(resolved, upsConf.DomainReservedUpstreams[host], opts)
-		if err != nil {
-			return fmt.Errorf("resolving upstreams reserved for %s: %w", host, err)
-		}
-	}
-
-	hosts = maps.Keys(upsConf.SpecifiedDomainUpstreams)
-	slices.Sort(hosts)
-	for _, host := range hosts {
-		err = s.resolveUpstreamsWithHosts(resolved, upsConf.SpecifiedDomainUpstreams[host], opts)
-		if err != nil {
-			return fmt.Errorf("resolving upstreams specific for %s: %w", host, err)
-		}
-	}
-
-	return nil
-}
-
-// resolveUpstreamsWithHosts resolves the IP addresses of each of the upstreams
-// and replaces those both in upstreams and resolved.  Upstreams that failed to
-// resolve are placed to resolved as-is.  This function only returns error of
-// upstreams closing.
-func (s *Server) resolveUpstreamsWithHosts(
-	resolved map[string]*upstream.Options,
-	upstreams []upstream.Upstream,
-	opts *upstream.Options,
-) (err error) {
-	for i := range upstreams {
-		u := upstreams[i]
-		addr := u.Address()
-		host := extractUpstreamHost(addr)
-
-		withIPs, ok := resolved[host]
-		if !ok {
-			recs := s.dnsFilter.EtcHostsRecords(host)
-			if len(recs) == 0 {
-				resolved[host] = nil
-
-				return nil
-			}
-
-			withIPs = opts.Clone()
-			withIPs.ServerIPAddrs = make([]net.IP, 0, len(recs))
-			for _, rec := range recs {
-				withIPs.ServerIPAddrs = append(withIPs.ServerIPAddrs, rec.Addr.AsSlice())
-			}
-
-			sortNetIPAddrs(withIPs.ServerIPAddrs, opts.PreferIPv6)
-			resolved[host] = withIPs
-		} else if withIPs == nil {
-			continue
-		}
-
-		if err = u.Close(); err != nil {
-			return fmt.Errorf("closing upstream %s: %w", addr, err)
-		}
-
-		upstreams[i], err = upstream.AddressToUpstream(addr, withIPs)
-		if err != nil {
-			return fmt.Errorf("replacing upstream %s with resolved %s: %w", addr, host, err)
-		}
-
-		log.Debug("dnsforward: using %s for %s", withIPs.ServerIPAddrs, upstreams[i].Address())
-	}
-
-	return nil
-}
-
-// extractUpstreamHost returns the hostname of addr without port with an
-// assumption that any address passed here has already been successfully parsed
-// by [upstream.AddressToUpstream].  This function essentially mirrors the logic
-// of [upstream.AddressToUpstream], see TODO on [replaceUpstreamsWithHosts].
-func extractUpstreamHost(addr string) (host string) {
-	var err error
-	if strings.Contains(addr, "://") {
-		var u *url.URL
-		u, err = url.Parse(addr)
-		if err != nil {
-			log.Debug("dnsforward: parsing upstream %s: %s", addr, err)
-
-			return addr
-		}
-
-		return u.Hostname()
-	}
-
-	// Probably, plain UDP upstream defined by address or address:port.
-	host, err = netutil.SplitHost(addr)
-	if err != nil {
-		return addr
-	}
-
-	return host
-}
-
-// sortNetIPAddrs sorts addrs in accordance with the protocol preferences.
-// Invalid addresses are sorted near the end.
-//
-// TODO(e.burkov):  This function taken from dnsproxy, which also already
-// contains a few similar functions.  Think of moving to golibs.
-func sortNetIPAddrs(addrs []net.IP, preferIPv6 bool) {
-	l := len(addrs)
-	if l <= 1 {
-		return
-	}
-
-	slices.SortStableFunc(addrs, func(addrA, addrB net.IP) (res int) {
-		switch len(addrA) {
-		case net.IPv4len, net.IPv6len:
-			switch len(addrB) {
-			case net.IPv4len, net.IPv6len:
-				// Go on.
-			default:
-				return -1
-			}
-		default:
-			return 1
-		}
-
-		// Treat IPv6-mapped IPv4 addresses as IPv6 addresses.
-		aIs4, bIs4 := addrA.To4() != nil, addrB.To4() != nil
-		if aIs4 == bIs4 {
-			return bytes.Compare(addrA, addrB)
-		}
-
-		if aIs4 {
-			if preferIPv6 {
-				return 1
-			}
-
-			return -1
-		}
-
-		if preferIPv6 {
-			return -1
-		}
-
-		return 1
-	})
-}
-
 // UpstreamHTTPVersions returns the HTTP versions for upstream configuration
 // depending on configuration.
 func UpstreamHTTPVersions(http3 bool) (v []upstream.HTTPVersion) {
@@ -295,3 +149,221 @@ func setProxyUpstreamMode(
 		conf.UpstreamMode = proxy.UModeLoadBalance
 	}
 }
+
+// createBootstrap returns a bootstrap resolver based on the configuration of s.
+// boots are the upstream resolvers that should be closed after use.  r is the
+// actual bootstrap resolver, which may include the system hosts.
+//
+// TODO(e.burkov):  This function currently returns a resolver and a slice of
+// the upstream resolvers, which are essentially the same.  boots are returned
+// for being able to close them afterwards, but it introduces an implicit
+// contract that r could only be used before that.  Anyway, this code should
+// improve when the [proxy.UpstreamConfig] will become an [upstream.Resolver]
+// and be used here.
+func (s *Server) createBootstrap(
+	addrs []string,
+	opts *upstream.Options,
+) (r upstream.Resolver, boots []*upstream.UpstreamResolver, err error) {
+	if len(addrs) == 0 {
+		addrs = defaultBootstrap
+	}
+
+	boots, err = aghnet.ParseBootstraps(addrs, opts)
+	if err != nil {
+		// Don't wrap the error, since it's informative enough as is.
+		return nil, nil, err
+	}
+
+	var parallel upstream.ParallelResolver
+	for _, b := range boots {
+		parallel = append(parallel, b)
+	}
+
+	if s.etcHosts != nil {
+		r = upstream.ConsequentResolver{s.etcHosts, parallel}
+	} else {
+		r = parallel
+	}
+
+	return r, boots, nil
+}
+
+// IsCommentOrEmpty returns true if s starts with a "#" character or is empty.
+// This function is useful for filtering out non-upstream lines from upstream
+// configs.
+func IsCommentOrEmpty(s string) (ok bool) {
+	return len(s) == 0 || s[0] == '#'
+}
+
+// newUpstreamConfig validates upstreams and returns an appropriate upstream
+// configuration or nil if it can't be built.
+//
+// TODO(e.burkov):  Perhaps proxy.ParseUpstreamsConfig should validate upstreams
+// slice already so that this function may be considered useless.
+func newUpstreamConfig(upstreams []string) (conf *proxy.UpstreamConfig, err error) {
+	// No need to validate comments and empty lines.
+	upstreams = stringutil.FilterOut(upstreams, IsCommentOrEmpty)
+	if len(upstreams) == 0 {
+		// Consider this case valid since it means the default server should be
+		// used.
+		return nil, nil
+	}
+
+	err = validateUpstreamConfig(upstreams)
+	if err != nil {
+		// Don't wrap the error since it's informative enough as is.
+		return nil, err
+	}
+
+	conf, err = proxy.ParseUpstreamsConfig(
+		upstreams,
+		&upstream.Options{
+			Bootstrap: net.DefaultResolver,
+			Timeout:   DefaultTimeout,
+		},
+	)
+	if err != nil {
+		// Don't wrap the error since it's informative enough as is.
+		return nil, err
+	} else if len(conf.Upstreams) == 0 {
+		return nil, errNoDefaultUpstreams
+	}
+
+	return conf, nil
+}
+
+// validateUpstreamConfig validates each upstream from the upstream
+// configuration and returns an error if any upstream is invalid.
+//
+// TODO(e.burkov):  Merge with [upstreamConfigValidator] somehow.
+func validateUpstreamConfig(conf []string) (err error) {
+	for _, u := range conf {
+		var ups []string
+		var isSpecific bool
+		ups, isSpecific, err = splitUpstreamLine(u)
+		if err != nil {
+			// Don't wrap the error since it's informative enough as is.
+			return err
+		}
+
+		for _, addr := range ups {
+			_, err = validateUpstream(addr, isSpecific)
+			if err != nil {
+				return fmt.Errorf("validating upstream %q: %w", addr, err)
+			}
+		}
+	}
+
+	return nil
+}
+
+// ValidateUpstreams validates each upstream and returns an error if any
+// upstream is invalid or if there are no default upstreams specified.
+//
+// TODO(e.burkov):  Merge with [upstreamConfigValidator] somehow.
+func ValidateUpstreams(upstreams []string) (err error) {
+	_, err = newUpstreamConfig(upstreams)
+
+	return err
+}
+
+// ValidateUpstreamsPrivate validates each upstream and returns an error if any
+// upstream is invalid or if there are no default upstreams specified.  It also
+// checks each domain of domain-specific upstreams for being ARPA pointing to
+// a locally-served network.  privateNets must not be nil.
+func ValidateUpstreamsPrivate(upstreams []string, privateNets netutil.SubnetSet) (err error) {
+	conf, err := newUpstreamConfig(upstreams)
+	if err != nil {
+		return fmt.Errorf("creating config: %w", err)
+	}
+
+	if conf == nil {
+		return nil
+	}
+
+	keys := maps.Keys(conf.DomainReservedUpstreams)
+	slices.Sort(keys)
+
+	var errs []error
+	for _, domain := range keys {
+		var subnet netip.Prefix
+		subnet, err = extractARPASubnet(domain)
+		if err != nil {
+			errs = append(errs, err)
+
+			continue
+		}
+
+		if !privateNets.Contains(subnet.Addr().AsSlice()) {
+			errs = append(
+				errs,
+				fmt.Errorf("arpa domain %q should point to a locally-served network", domain),
+			)
+		}
+	}
+
+	return errors.Annotate(errors.Join(errs...), "checking domain-specific upstreams: %w")
+}
+
+// protocols are the supported URL schemes for upstreams.
+var protocols = []string{"h3", "https", "quic", "sdns", "tcp", "tls", "udp"}
+
+// validateUpstream returns an error if u alongside with domains is not a valid
+// upstream configuration.  useDefault is true if the upstream is
+// domain-specific and is configured to point at the default upstream server
+// which is validated separately.  The upstream is considered domain-specific
+// only if domains is at least not nil.
+func validateUpstream(u string, isSpecific bool) (useDefault bool, err error) {
+	// The special server address '#' means that default server must be used.
+	if useDefault = u == "#" && isSpecific; useDefault {
+		return useDefault, nil
+	}
+
+	// Check if the upstream has a valid protocol prefix.
+	//
+	// TODO(e.burkov):  Validate the domain name.
+	if proto, _, ok := strings.Cut(u, "://"); ok {
+		if !slices.Contains(protocols, proto) {
+			return false, fmt.Errorf("bad protocol %q", proto)
+		}
+	} else if _, err = netip.ParseAddr(u); err == nil {
+		return false, nil
+	} else if _, err = netip.ParseAddrPort(u); err == nil {
+		return false, nil
+	}
+
+	return false, err
+}
+
+// splitUpstreamLine returns the upstreams and the specified domains.  domains
+// is nil when the upstream is not domains-specific.  Otherwise it may also be
+// empty.
+func splitUpstreamLine(upstreamStr string) (upstreams []string, isSpecific bool, err error) {
+	if !strings.HasPrefix(upstreamStr, "[/") {
+		return []string{upstreamStr}, false, nil
+	}
+
+	defer func() { err = errors.Annotate(err, "splitting upstream line %q: %w", upstreamStr) }()
+
+	doms, ups, found := strings.Cut(upstreamStr[2:], "/]")
+	if !found {
+		return nil, false, errMissingSeparator
+	} else if strings.Contains(ups, "/]") {
+		return nil, false, errDupSeparator
+	}
+
+	for i, host := range strings.Split(doms, "/") {
+		if host == "" {
+			continue
+		}
+
+		err = netutil.ValidateDomainName(strings.TrimPrefix(host, "*."))
+		if err != nil {
+			return nil, false, fmt.Errorf("domain at index %d: %w", i, err)
+		}
+
+		isSpecific = true
+	}
+
+	return strings.Fields(ups), isSpecific, nil
+}