all: add tests; imp addrproc, docs

Updates #6003.

Squashed commit of the following:

commit 1cc42303c29edc621802fc182ccb5701e412f099
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Jul 13 13:47:41 2023 +0300

    all: fix chlog

commit e835084c7aac6384ea7b0886e6b3b1d614438baa
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Jul 13 13:40:45 2023 +0300

    rulelist: imp longer line handling

.github/workflows/build.yml

@@ -1,7 +1,7 @@
 'name': 'build'
 
 'env':
-  'GO_VERSION': '1.19.10'
+  'GO_VERSION': '1.19.11'
   'NODE_VERSION': '14'
 
 'on':

.github/workflows/lint.yml

@@ -1,7 +1,7 @@
 'name': 'lint'
 
 'env':
-  'GO_VERSION': '1.19.10'
+  'GO_VERSION': '1.19.11'
 
 'on':
   'push':

CHANGELOG.md

@@ -14,15 +14,50 @@ and this project adheres to
 <!--
 ## [v0.108.0] - TBA
 
-## [v0.107.34] - 2023-07-26 (APPROX.)
+## [v0.107.35] - 2023-08-02 (APPROX.)
 
-See also the [v0.107.34 GitHub milestone][ms-v0.107.34].
+See also the [v0.107.35 GitHub milestone][ms-v0.107.35].
 
-[ms-v0.107.34]: https://github.com/AdguardTeam/AdGuardHome/milestone/69?closed=1
+[ms-v0.107.35]: https://github.com/AdguardTeam/AdGuardHome/milestone/70?closed=1
 
 NOTE: Add new changes BELOW THIS COMMENT.
 -->
 
+### Fixed
+
+- Occasional client information lookup failures that could lead to the DNS
+  server getting stuck ([#6006]).
+- `bufio.Scanner: token too long` errors when trying to add filtering-rule lists
+  with lines over 1024 bytes long ([#6003]).
+
+### Removed
+
+- Default exposure of the non-standard ports 784 and 8853 for DNS-over-QUIC in
+  the `Dockerfile`.
+
+[#6003]: https://github.com/AdguardTeam/AdGuardHome/issues/6003
+[#6006]: https://github.com/AdguardTeam/AdGuardHome/issues/6006
+
+<!--
+NOTE: Add new changes ABOVE THIS COMMENT.
+-->
+
+
+
+## [v0.107.34] - 2023-07-12
+
+See also the [v0.107.34 GitHub milestone][ms-v0.107.34].
+
+### Security
+
+- Go version has been updated to prevent the possibility of exploiting the
+  CVE-2023-29406 Go vulnerability fixed in [Go 1.19.11][go-1.19.11].
+
+### Added
+
+- Ability to ignore queries for the root domain, such as `NS .` queries
+  ([#5990]).
+
 ### Changed
 
 - Improved CPU and RAM consumption during updates of filtering-rule lists.
@@ -32,8 +67,9 @@ NOTE: Add new changes BELOW THIS COMMENT.
 In this release, the schema version has changed from 23 to 24.
 
 - Properties starting with `log_`, and `verbose` property, which used to set up
-  logging are now moved to the new object `log` containing new properties `file`,
-  `max_backups`, `max_size`, `max_age`, `compress`, `local_time`, and `verbose`:
+  logging are now moved to the new object `log` containing new properties
+  `file`, `max_backups`, `max_size`, `max_age`, `compress`, `local_time`, and
+  `verbose`:
 
   ```yaml
   # BEFORE:
@@ -47,13 +83,13 @@ In this release, the schema version has changed from 23 to 24.
 
   # AFTER:
   'log':
-    'file': ""
-    'max_backups': 0
-    'max_size': 100
-    'max_age': 3
-    'compress': false
-    'local_time': false
-    'verbose': false
+      'file': ""
+      'max_backups': 0
+      'max_size': 100
+      'max_age': 3
+      'compress': false
+      'local_time': false
+      'verbose': false
   ```
 
   To rollback this change, remove the new object `log`, set back `log_` and
@@ -66,6 +102,8 @@ In this release, the schema version has changed from 23 to 24.
 
 ### Fixed
 
+- Two unspecified IPs when a host is blocked in two filter lists ([#5972]).
+- Incorrect setting of Parental Control cache size.
 - Excessive RAM and CPU consumption by Safe Browsing and Parental Control
   filters ([#5896]).
 
@@ -80,10 +118,11 @@ In this release, the schema version has changed from 23 to 24.
   image, and reload it from scratch.
 
 [#5896]: https://github.com/AdguardTeam/AdGuardHome/issues/5896
+[#5972]: https://github.com/AdguardTeam/AdGuardHome/issues/5972
+[#5990]: https://github.com/AdguardTeam/AdGuardHome/issues/5990
 
-<!--
-NOTE: Add new changes ABOVE THIS COMMENT.
--->
+[go-1.19.11]:   https://groups.google.com/g/golang-announce/c/2q13H6LEEx0/m/sduSepLLBwAJ
+[ms-v0.107.34]: https://github.com/AdguardTeam/AdGuardHome/milestone/69?closed=1
 
 
 
@@ -2218,11 +2257,12 @@ See also the [v0.104.2 GitHub milestone][ms-v0.104.2].
 
 
 <!--
-[Unreleased]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.34...HEAD
-[v0.107.34]:  https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.33...v0.107.34
+[Unreleased]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.35...HEAD
+[v0.107.35]:  https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.34...v0.107.35
 -->
 
-[Unreleased]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.33...HEAD
+[Unreleased]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.34...HEAD
+[v0.107.34]:  https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.33...v0.107.34
 [v0.107.33]:  https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.32...v0.107.33
 [v0.107.32]:  https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.31...v0.107.32
 [v0.107.31]:  https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.30...v0.107.31

bamboo-specs/release.yaml

@@ -7,7 +7,7 @@
 # Make sure to sync any changes with the branch overrides below.
 'variables':
     'channel': 'edge'
-    'dockerGo': 'adguard/golang-ubuntu:6.7'
+    'dockerGo': 'adguard/golang-ubuntu:6.8'
 
 'stages':
   - 'Build frontend':
@@ -272,7 +272,7 @@
         # need to build a few of these.
         'variables':
             'channel': 'beta'
-            'dockerGo': 'adguard/golang-ubuntu:6.7'
+            'dockerGo': 'adguard/golang-ubuntu:6.8'
     # release-vX.Y.Z branches are the branches from which the actual final
     # release is built.
   - '^release-v[0-9]+\.[0-9]+\.[0-9]+':
@@ -287,4 +287,4 @@
         # are the ones that actually get released.
         'variables':
             'channel': 'release'
-            'dockerGo': 'adguard/golang-ubuntu:6.7'
+            'dockerGo': 'adguard/golang-ubuntu:6.8'

bamboo-specs/snapcraft.yaml

@@ -10,7 +10,7 @@
 # Make sure to sync any changes with the branch overrides below.
 'variables':
     'channel': 'edge'
-    'dockerGo': 'adguard/golang-ubuntu:6.7'
+    'dockerGo': 'adguard/golang-ubuntu:6.8'
     'snapcraftChannel': 'edge'
 
 'stages':
@@ -191,7 +191,7 @@
         # need to build a few of these.
         'variables':
             'channel': 'beta'
-            'dockerGo': 'adguard/golang-ubuntu:6.7'
+            'dockerGo': 'adguard/golang-ubuntu:6.8'
             'snapcraftChannel': 'beta'
     # release-vX.Y.Z branches are the branches from which the actual final
     # release is built.
@@ -207,5 +207,5 @@
         # are the ones that actually get released.
         'variables':
             'channel': 'release'
-            'dockerGo': 'adguard/golang-ubuntu:6.7'
+            'dockerGo': 'adguard/golang-ubuntu:6.8'
             'snapcraftChannel': 'candidate'

bamboo-specs/test.yaml

@@ -5,7 +5,7 @@
     'key': 'AHBRTSPECS'
     'name': 'AdGuard Home - Build and run tests'
 'variables':
-    'dockerGo': 'adguard/golang-ubuntu:6.7'
+    'dockerGo': 'adguard/golang-ubuntu:6.8'
 
 'stages':
   - 'Tests':

client/src/__locales/en.json

@@ -444,7 +444,7 @@
     "client_confirm_delete": "Are you sure you want to delete client \"{{key}}\"?",
     "list_confirm_delete": "Are you sure you want to delete this list?",
     "auto_clients_title": "Runtime clients",
-    "auto_clients_desc": "Devices not on the list of Persistent clients that may still use AdGuard Home",
+    "auto_clients_desc": "Information about IP addresses of devices that are using or may use AdGuard Home. This information is gathered from several sources, including hosts files, reverse DNS, etc.",
     "access_title": "Access settings",
     "access_desc": "Here you can configure access rules for the AdGuard Home DNS server",
     "access_allowed_title": "Allowed clients",

client/src/components/ui/Card.css

@@ -1,7 +1,7 @@
 .card-header {
     align-items: center;
     justify-content: space-between;
-    padding: 10px 24px;
+    padding: 0.6rem 1.5rem;
 }
 
 .card-subtitle {
@@ -16,11 +16,11 @@
 
 .card-table-overflow--limited {
     overflow-y: auto;
-    max-height: 280px;
+    max-height: 17.5rem;
 }
 
 .dashboard .card-table-overflow--limited {
-    max-height: 288px;
+    max-height: 18rem;
 }
 
 .card-actions {
@@ -38,7 +38,7 @@
 }
 
 .card-body--status {
-    padding: 40px 24px;
+    padding: 2.5rem 1.5rem;
     text-align: center;
 }
 
@@ -81,14 +81,14 @@
     position: relative;
     flex: 1 1 auto;
     margin: 0;
-    padding: 16px 24px;
+    padding: 1rem 1.5rem;
 }
 
 .card-value-stats {
     display: block;
     font-size: 2.1rem;
     line-height: 2.7rem;
-    height: 44px;
+    height: 2.7rem;
     text-overflow: ellipsis;
     white-space: nowrap;
     overflow: hidden;
@@ -108,7 +108,7 @@
 }
 
 .card--full {
-    height: calc(100% - 22px);
+    height: calc(100% - 1.5rem);
 }
 
 .card-wrap {
@@ -125,7 +125,7 @@
 
 @media (min-width: 992px) {
     .dashboard .card:not(.card--full) {
-        height: 350px;
+        height: 22rem;
     }
 }
 

client/src/components/ui/Tabler.css

@@ -13363,7 +13363,7 @@ a.icon:hover {
     padding: 0.5rem 1.5rem;
     display: -ms-flexbox;
     display: flex;
-    min-height: 60px;
+    min-height: 3.5rem;
     -ms-flex-align: center;
     align-items: center;
 }

docker/Dockerfile

@@ -41,18 +41,12 @@ RUN setcap 'cap_net_bind_service=+eip' /opt/adguardhome/AdGuardHome
 # 68     :      UDP : DHCP (client)
 # 80     : TCP      : HTTP (main)
 # 443    : TCP, UDP : HTTPS, DNS-over-HTTPS (incl. HTTP/3), DNSCrypt (main)
-# 784    :      UDP : DNS-over-QUIC (deprecated; use 853)
 # 853    : TCP, UDP : DNS-over-TLS, DNS-over-QUIC
 # 3000   : TCP, UDP : HTTP(S) (alt, incl. HTTP/3)
 # 5443   : TCP, UDP : DNSCrypt (alt)
 # 6060   : TCP      : HTTP (pprof)
-# 8853   :      UDP : DNS-over-QUIC (deprecated; use 853)
-#
-# TODO(a.garipov): Remove the old, non-standard 784 and 8853 ports for
-# DNS-over-QUIC in a future release.
-EXPOSE 53/tcp 53/udp 67/udp 68/udp 80/tcp 443/tcp 443/udp 784/udp\
-	853/tcp 853/udp 3000/tcp 3000/udp 5443/tcp 5443/udp 6060/tcp\
-	8853/udp
+EXPOSE 53/tcp 53/udp 67/udp 68/udp 80/tcp 443/tcp 443/udp 853/tcp\
+	853/udp 3000/tcp 3000/udp 5443/tcp 5443/udp 6060/tcp
 
 WORKDIR /opt/adguardhome/work
 

go.mod

@@ -5,7 +5,7 @@ go 1.19
 require (
 	// TODO(a.garipov): Update to a tagged version when it's released.
 	github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768
-	github.com/AdguardTeam/golibs v0.13.3
+	github.com/AdguardTeam/golibs v0.13.4
 	github.com/AdguardTeam/urlfilter v0.16.1
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/ameshkov/dnscrypt/v2 v2.2.7

go.sum

@@ -2,8 +2,8 @@ github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768 h1:5Ia6wA+
 github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768/go.mod h1:CQhZTkqC8X0ID6glrtyaxgqRRdiYfn1gJulC1cZ5Dn8=
 github.com/AdguardTeam/golibs v0.4.0/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4=
 github.com/AdguardTeam/golibs v0.10.4/go.mod h1:rSfQRGHIdgfxriDDNgNJ7HmE5zRoURq8R+VdR81Zuzw=
-github.com/AdguardTeam/golibs v0.13.3 h1:RT3QbzThtaLiFLkIUDS6/hlGEXrh0zYvdf4bd7UWpGo=
-github.com/AdguardTeam/golibs v0.13.3/go.mod h1:wkJ6EUsN4np/9Gp7+9QeooY9E2U2WCLJYAioLCzkHsI=
+github.com/AdguardTeam/golibs v0.13.4 h1:ACTwIR1pEENBijHcEWtiMbSh4wWQOlIHRxmUB8oBHf8=
+github.com/AdguardTeam/golibs v0.13.4/go.mod h1:wkJ6EUsN4np/9Gp7+9QeooY9E2U2WCLJYAioLCzkHsI=
 github.com/AdguardTeam/gomitmproxy v0.2.0/go.mod h1:Qdv0Mktnzer5zpdpi5rAwixNJzW2FN91LjKJCkVbYGU=
 github.com/AdguardTeam/urlfilter v0.16.1 h1:ZPi0rjqo8cQf2FVdzo6cqumNoHZx2KPXj2yZa1A5BBw=
 github.com/AdguardTeam/urlfilter v0.16.1/go.mod h1:46YZDOV1+qtdRDuhZKVPSSp7JWWes0KayqHrKAFBdEI=

internal/aghnet/addr.go

@@ -0,0 +1,43 @@
+package aghnet
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/AdguardTeam/golibs/stringutil"
+)
+
+// NormalizeDomain returns a lowercased version of host without the final dot,
+// unless host is ".", in which case it returns it unchanged.  That is a special
+// case that to allow matching queries like:
+//
+//	dig IN NS '.'
+func NormalizeDomain(host string) (norm string) {
+	if host == "." {
+		return host
+	}
+
+	return strings.ToLower(strings.TrimSuffix(host, "."))
+}
+
+// NewDomainNameSet returns nil and error, if list has duplicate or empty domain
+// name.  Otherwise returns a set, which contains domain names normalized using
+// [NormalizeDomain].
+func NewDomainNameSet(list []string) (set *stringutil.Set, err error) {
+	set = stringutil.NewSet()
+
+	for i, host := range list {
+		if host == "" {
+			return nil, fmt.Errorf("at index %d: hostname is empty", i)
+		}
+
+		host = NormalizeDomain(host)
+		if set.Has(host) {
+			return nil, fmt.Errorf("duplicate hostname %q at index %d", host, i)
+		}
+
+		set.Add(host)
+	}
+
+	return set, nil
+}

internal/aghnet/addr_test.go

@@ -0,0 +1,59 @@
+package aghnet_test
+
+import (
+	"testing"
+
+	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
+	"github.com/AdguardTeam/golibs/testutil"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewDomainNameSet(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		name       string
+		wantErrMsg string
+		in         []string
+	}{{
+		name:       "nil",
+		wantErrMsg: "",
+		in:         nil,
+	}, {
+		name:       "success",
+		wantErrMsg: "",
+		in: []string{
+			"Domain.Example",
+			".",
+		},
+	}, {
+		name:       "dups",
+		wantErrMsg: `duplicate hostname "domain.example" at index 1`,
+		in: []string{
+			"Domain.Example",
+			"domain.example",
+		},
+	}, {
+		name:       "bad_domain",
+		wantErrMsg: "at index 0: hostname is empty",
+		in: []string{
+			"",
+		},
+	}}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			set, err := aghnet.NewDomainNameSet(tc.in)
+			testutil.AssertErrorMsg(t, tc.wantErrMsg, err)
+			if err != nil {
+				return
+			}
+
+			for _, host := range tc.in {
+				assert.Truef(t, set.Has(aghnet.NormalizeDomain(host)), "%q not matched", host)
+			}
+		})
+	}
+}

internal/aghnet/hostgen.go

@@ -1,12 +1,8 @@
 package aghnet
 
 import (
-	"fmt"
 	"net/netip"
 	"strings"
-
-	"github.com/AdguardTeam/golibs/errors"
-	"github.com/AdguardTeam/golibs/stringutil"
 )
 
 // GenerateHostname generates the hostname from ip.  In case of using IPv4 the
@@ -29,32 +25,8 @@ func GenerateHostname(ip netip.Addr) (hostname string) {
 	hostname = ip.StringExpanded()
 
 	if ip.Is4() {
-		return strings.Replace(hostname, ".", "-", -1)
+		return strings.ReplaceAll(hostname, ".", "-")
 	}
 
-	return strings.Replace(hostname, ":", "-", -1)
-}
-
-// NewDomainNameSet returns nil and error, if list has duplicate or empty
-// domain name.  Otherwise returns a set, which contains non-FQDN domain names,
-// and nil error.
-func NewDomainNameSet(list []string) (set *stringutil.Set, err error) {
-	set = stringutil.NewSet()
-
-	for i, v := range list {
-		host := strings.ToLower(strings.TrimSuffix(v, "."))
-		// TODO(a.garipov): Think about ignoring empty (".") names in the
-		// future.
-		if host == "" {
-			return nil, errors.Error("host name is empty")
-		}
-
-		if set.Has(host) {
-			return nil, fmt.Errorf("duplicate host name %q at index %d", host, i)
-		}
-
-		set.Add(host)
-	}
-
-	return set, nil
+	return strings.ReplaceAll(hostname, ":", "-")
 }

internal/dnsforward/config.go

@@ -270,7 +270,10 @@ type ServerConfig struct {
 	UDPListenAddrs []*net.UDPAddr        // UDP listen address
 	TCPListenAddrs []*net.TCPAddr        // TCP listen address
 	UpstreamConfig *proxy.UpstreamConfig // Upstream DNS servers config
-	OnDNSRequest   func(d *proxy.DNSContext)
+
+	// ClientIPs, if not nil, is used to send clients' IP addresses to other
+	// parts of AdGuard Home that may use it for resolving rDNS, WHOIS, etc.
+	ClientIPs chan netip.Addr
 
 	FilteringConfig
 	TLSConfig

internal/dnsforward/dnsforward.go

@@ -99,6 +99,10 @@ type Server struct {
 	// must be a valid domain name plus dots on each side.
 	localDomainSuffix string
 
+	// ClientIPs, if not nil, is used to send clients' IP addresses to other
+	// parts of AdGuard Home that may use it for resolving rDNS, WHOIS, etc.
+	clientIPs chan<- netip.Addr
+
 	ipset          ipsetCtx
 	privateNets    netutil.SubnetSet
 	localResolvers *proxy.Proxy
@@ -318,7 +322,8 @@ func (s *Server) Exchange(ip netip.Addr) (host string, err error) {
 			Qclass: dns.ClassINET,
 		}},
 	}
-	ctx := &proxy.DNSContext{
+
+	dctx := &proxy.DNSContext{
 		Proto:     "udp",
 		Req:       req,
 		StartTime: time.Now(),
@@ -336,11 +341,11 @@ func (s *Server) Exchange(ip netip.Addr) (host string, err error) {
 		resolver = s.internalProxy
 	}
 
-	if err = resolver.Resolve(ctx); err != nil {
+	if err = resolver.Resolve(dctx); err != nil {
 		return "", err
 	}
 
-	return hostFromPTR(ctx.Res)
+	return hostFromPTR(dctx.Res)
 }
 
 // hostFromPTR returns domain name from the PTR response or error.
@@ -555,6 +560,8 @@ func (s *Server) Prepare(conf *ServerConfig) (err error) {
 
 	s.recDetector.clear()
 
+	s.clientIPs = s.conf.ClientIPs
+
 	return nil
 }
 
@@ -696,6 +703,9 @@ func (s *Server) Reconfigure(conf *ServerConfig) error {
 	// TODO(a.garipov): This whole piece of API is weird and needs to be remade.
 	if conf == nil {
 		conf = &s.conf
+	} else if s.clientIPs != nil {
+		close(s.clientIPs)
+		s.clientIPs = nil
 	}
 
 	err = s.Prepare(conf)

internal/dnsforward/dnsforward_test.go

@@ -39,11 +39,29 @@ func TestMain(m *testing.M) {
 	testutil.DiscardLogOutput(m)
 }
 
+// testTimeout is the common timeout for tests.
+//
+// TODO(a.garipov): Use more.
+const testTimeout = 1 * time.Second
+
+// testQuestionTarget is the common question target for tests.
+//
+// TODO(a.garipov): Use more.
+const testQuestionTarget = "target.example"
+
 const (
 	tlsServerName     = "testdns.adguard.com"
 	testMessagesCount = 10
 )
 
+// testClientAddr is the common net.Addr for tests.
+//
+// TODO(a.garipov): Use more.
+var testClientAddr net.Addr = &net.TCPAddr{
+	IP:   net.IP{1, 2, 3, 4},
+	Port: 12345,
+}
+
 func startDeferStop(t *testing.T, s *Server) {
 	t.Helper()
 
@@ -53,6 +71,13 @@ func startDeferStop(t *testing.T, s *Server) {
 	testutil.CleanupAndRequireSuccess(t, s.Stop)
 }
 
+// packageUpstreamVariableMu is used to serialize access to the package-level
+// variables of package upstream.
+//
+// TODO(s.chzhen): Move these parameters to upstream options and remove this
+// crutch.
+var packageUpstreamVariableMu = &sync.Mutex{}
+
 func createTestServer(
 	t *testing.T,
 	filterConf *filtering.Config,
@@ -61,6 +86,9 @@ func createTestServer(
 ) (s *Server) {
 	t.Helper()
 
+	packageUpstreamVariableMu.Lock()
+	defer packageUpstreamVariableMu.Unlock()
+
 	rules := `||nxdomain.example.org
 ||NULL.example.org^
 127.0.0.1	host.example.org
@@ -307,11 +335,9 @@ func TestServer(t *testing.T) {
 }
 
 func TestServer_timeout(t *testing.T) {
-	const timeout time.Duration = time.Second
-
 	t.Run("custom", func(t *testing.T) {
 		srvConf := &ServerConfig{
-			UpstreamTimeout: timeout,
+			UpstreamTimeout: testTimeout,
 			FilteringConfig: FilteringConfig{
 				BlockingMode:     BlockingModeDefault,
 				EDNSClientSubnet: &EDNSClientSubnet{Enabled: false},
@@ -324,7 +350,7 @@ func TestServer_timeout(t *testing.T) {
 		err = s.Prepare(srvConf)
 		require.NoError(t, err)
 
-		assert.Equal(t, timeout, s.conf.UpstreamTimeout)
+		assert.Equal(t, testTimeout, s.conf.UpstreamTimeout)
 	})
 
 	t.Run("default", func(t *testing.T) {
@@ -545,7 +571,7 @@ func TestInvalidRequest(t *testing.T) {
 
 	// Send a DNS request without question.
 	_, _, err := (&dns.Client{
-		Timeout: 500 * time.Millisecond,
+		Timeout: testTimeout,
 	}).Exchange(&req, addr)
 
 	assert.NoErrorf(t, err, "got a response to an invalid query")

internal/dnsforward/filter.go

@@ -50,10 +50,10 @@ func (s *Server) beforeRequestHandler(
 	return true, nil
 }
 
-// getClientRequestFilteringSettings looks up client filtering settings using
-// the client's IP address and ID, if any, from dctx.
-func (s *Server) getClientRequestFilteringSettings(dctx *dnsContext) *filtering.Settings {
-	setts := s.dnsFilter.Settings()
+// clientRequestFilteringSettings looks up client filtering settings using the
+// client's IP address and ID, if any, from dctx.
+func (s *Server) clientRequestFilteringSettings(dctx *dnsContext) (setts *filtering.Settings) {
+	setts = s.dnsFilter.Settings()
 	setts.ProtectionEnabled = dctx.protectionEnabled
 	if s.conf.FilterHandler != nil {
 		ip, _ := netutil.IPAndPortFromAddr(dctx.proxyCtx.Addr)

internal/dnsforward/filter_test.go

@@ -21,6 +21,8 @@ func TestHandleDNSRequest_filterDNSResponse(t *testing.T) {
 ||cname.specific^$dnstype=~CNAME
 ||0.0.0.1^$dnstype=~A
 ||::1^$dnstype=~AAAA
+0.0.0.0 duplicate.domain
+0.0.0.0 duplicate.domain
 `
 
 	forwardConf := ServerConfig{
@@ -137,6 +139,17 @@ func TestHandleDNSRequest_filterDNSResponse(t *testing.T) {
 			},
 			A: netutil.IPv4Zero(),
 		}},
+	}, {
+		req:  createTestMessage("duplicate.domain."),
+		name: "duplicate_domain",
+		wantAns: []dns.RR{&dns.A{
+			Hdr: dns.RR_Header{
+				Name:   "duplicate.domain.",
+				Rrtype: dns.TypeA,
+				Class:  dns.ClassINET,
+			},
+			A: netutil.IPv4Zero(),
+		}},
 	}}
 
 	for _, tc := range testCases {

internal/dnsforward/msg.go

@@ -26,11 +26,25 @@ func (s *Server) makeResponse(req *dns.Msg) (resp *dns.Msg) {
 	return resp
 }
 
-// ipsFromRules extracts non-IP addresses from the filtering result rules.
+// containsIP returns true if the IP is already in the list.
+func containsIP(ips []net.IP, ip net.IP) bool {
+	for _, a := range ips {
+		if a.Equal(ip) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// ipsFromRules extracts unique non-IP addresses from the filtering result
+// rules.
 func ipsFromRules(resRules []*filtering.ResultRule) (ips []net.IP) {
 	for _, r := range resRules {
-		if r.IP != nil {
-			ips = append(ips, r.IP)
+		// len(resRules) and len(ips) are actually small enough for O(n^2) to do
+		// not raise performance questions.
+		if ip := r.IP; ip != nil && !containsIP(ips, ip) {
+			ips = append(ips, ip)
 		}
 	}
 

internal/dnsforward/process.go

@@ -30,6 +30,7 @@ type dnsContext struct {
 	setts *filtering.Settings
 
 	result *filtering.Result
+
 	// origResp is the response received from upstream.  It is set when the
 	// response is modified by filters.
 	origResp *dns.Msg
@@ -48,13 +49,13 @@ type dnsContext struct {
 	// clientID is the ClientID from DoH, DoQ, or DoT, if provided.
 	clientID string
 
+	// startTime is the time at which the processing of the request has started.
+	startTime time.Time
+
 	// origQuestion is the question received from the client.  It is set
 	// when the request is modified by rewrites.
 	origQuestion dns.Question
 
-	// startTime is the time at which the processing of the request has started.
-	startTime time.Time
-
 	// protectionEnabled shows if the filtering is enabled, and if the
 	// server's DNS filter is ready.
 	protectionEnabled bool
@@ -160,6 +161,22 @@ func (s *Server) processRecursion(dctx *dnsContext) (rc resultCode) {
 	return resultCodeSuccess
 }
 
+// mozillaFQDN is the domain used to signal the Firefox browser to not use its
+// own DoH server.
+//
+// See https://support.mozilla.org/en-US/kb/canary-domain-use-application-dnsnet.
+const mozillaFQDN = "use-application-dns.net."
+
+// healthcheckFQDN is a reserved domain-name used for healthchecking.
+//
+// [Section 6.2 of RFC 6761] states that DNS Registries/Registrars must not
+// grant requests to register test names in the normal way to any person or
+// entity, making domain names under the test. TLD free to use in internal
+// purposes.
+//
+// [Section 6.2 of RFC 6761]: https://www.rfc-editor.org/rfc/rfc6761.html#section-6.2
+const healthcheckFQDN = "healthcheck.adguardhome.test."
+
 // processInitial terminates the following processing for some requests if
 // needed and enriches dctx with some client-specific information.
 //
@@ -169,6 +186,8 @@ func (s *Server) processInitial(dctx *dnsContext) (rc resultCode) {
 	defer log.Debug("dnsforward: finished processing initial")
 
 	pctx := dctx.proxyCtx
+	s.processClientIP(pctx.Addr)
+
 	q := pctx.Req.Question[0]
 	qt := q.Qtype
 	if s.conf.AAAADisabled && qt == dns.TypeAAAA {
@@ -177,28 +196,13 @@ func (s *Server) processInitial(dctx *dnsContext) (rc resultCode) {
 		return resultCodeFinish
 	}
 
-	if s.conf.OnDNSRequest != nil {
-		s.conf.OnDNSRequest(pctx)
-	}
-
-	// Disable Mozilla DoH.
-	//
-	// See https://support.mozilla.org/en-US/kb/canary-domain-use-application-dnsnet.
-	if (qt == dns.TypeA || qt == dns.TypeAAAA) && q.Name == "use-application-dns.net." {
+	if (qt == dns.TypeA || qt == dns.TypeAAAA) && q.Name == mozillaFQDN {
 		pctx.Res = s.genNXDomain(pctx.Req)
 
 		return resultCodeFinish
 	}
 
-	// Handle a reserved domain healthcheck.adguardhome.test.
-	//
-	// [Section 6.2 of RFC 6761] states that DNS Registries/Registrars must not
-	// grant requests to register test names in the normal way to any person or
-	// entity, making domain names under test. TLD free to use in internal
-	// purposes.
-	//
-	// [Section 6.2 of RFC 6761]: https://www.rfc-editor.org/rfc/rfc6761.html#section-6.2
-	if q.Name == "healthcheck.adguardhome.test." {
+	if q.Name == healthcheckFQDN {
 		// Generate a NODATA negative response to make nslookup exit with 0.
 		pctx.Res = s.makeResponse(pctx.Req)
 
@@ -213,11 +217,33 @@ func (s *Server) processInitial(dctx *dnsContext) (rc resultCode) {
 
 	// Get the client-specific filtering settings.
 	dctx.protectionEnabled, _ = s.UpdatedProtectionStatus()
-	dctx.setts = s.getClientRequestFilteringSettings(dctx)
+	dctx.setts = s.clientRequestFilteringSettings(dctx)
 
 	return resultCodeSuccess
 }
 
+// processClientIP sends the client IP address to s.clientIPs, if needed.
+func (s *Server) processClientIP(addr net.Addr) {
+	clientIP := netutil.NetAddrToAddrPort(addr).Addr()
+	if clientIP == (netip.Addr{}) {
+		log.Info("dnsforward: warning: bad client addr %q", addr)
+
+		return
+	}
+
+	// Do not assign s.clientIPs to a local variable to then use, since this
+	// lock also serializes the closure of s.clientIPs.
+	s.serverLock.RLock()
+	defer s.serverLock.RUnlock()
+
+	select {
+	case s.clientIPs <- clientIP:
+		// Go on.
+	default:
+		log.Debug("dnsforward: client ip channel is nil or full; len: %d", len(s.clientIPs))
+	}
+}
+
 func (s *Server) setTableHostToIP(t hostToIPTable) {
 	s.tableHostToIPLock.Lock()
 	defer s.tableHostToIPLock.Unlock()

internal/dnsforward/process_internal_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/AdguardTeam/dnsproxy/upstream"
 	"github.com/AdguardTeam/golibs/netutil"
 	"github.com/AdguardTeam/golibs/testutil"
+	"github.com/AdguardTeam/urlfilter/rules"
 	"github.com/miekg/dns"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -22,6 +23,95 @@ const (
 	ddrTestFQDN       = ddrTestDomainName + "."
 )
 
+func TestServer_ProcessInitial(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		name         string
+		target       string
+		wantRCode    rules.RCode
+		qType        rules.RRType
+		aaaaDisabled bool
+		wantRC       resultCode
+	}{{
+		name:         "success",
+		target:       testQuestionTarget,
+		wantRCode:    -1,
+		qType:        dns.TypeA,
+		aaaaDisabled: false,
+		wantRC:       resultCodeSuccess,
+	}, {
+		name:         "aaaa_disabled",
+		target:       testQuestionTarget,
+		wantRCode:    dns.RcodeSuccess,
+		qType:        dns.TypeAAAA,
+		aaaaDisabled: true,
+		wantRC:       resultCodeFinish,
+	}, {
+		name:         "aaaa_disabled_a",
+		target:       testQuestionTarget,
+		wantRCode:    -1,
+		qType:        dns.TypeA,
+		aaaaDisabled: true,
+		wantRC:       resultCodeSuccess,
+	}, {
+		name:         "mozilla_canary",
+		target:       mozillaFQDN,
+		wantRCode:    dns.RcodeNameError,
+		qType:        dns.TypeA,
+		aaaaDisabled: false,
+		wantRC:       resultCodeFinish,
+	}, {
+		name:         "adguardhome_healthcheck",
+		target:       healthcheckFQDN,
+		wantRCode:    dns.RcodeSuccess,
+		qType:        dns.TypeA,
+		aaaaDisabled: false,
+		wantRC:       resultCodeFinish,
+	}}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			clientIPs := make(chan netip.Addr, 1)
+
+			c := ServerConfig{
+				FilteringConfig: FilteringConfig{
+					AAAADisabled:     tc.aaaaDisabled,
+					EDNSClientSubnet: &EDNSClientSubnet{Enabled: false},
+				},
+				ClientIPs: clientIPs,
+			}
+
+			s := createTestServer(t, &filtering.Config{}, c, nil)
+
+			dctx := &dnsContext{
+				proxyCtx: &proxy.DNSContext{
+					Req:       createTestMessageWithType(tc.target, tc.qType),
+					Addr:      testClientAddr,
+					RequestID: 1234,
+				},
+			}
+
+			gotRC := s.processInitial(dctx)
+			assert.Equal(t, tc.wantRC, gotRC)
+
+			gotAddr, _ := testutil.RequireReceive(t, clientIPs, testTimeout)
+			assert.Equal(t, netutil.NetAddrToAddrPort(testClientAddr).Addr(), gotAddr)
+
+			if tc.wantRCode > 0 {
+				gotResp := dctx.proxyCtx.Res
+				require.NotNil(t, gotResp)
+
+				assert.Equal(t, tc.wantRCode, gotResp.Rcode)
+			}
+		})
+	}
+}
+
 func TestServer_ProcessDDRQuery(t *testing.T) {
 	dohSVCB := &dns.SVCB{
 		Priority: 1,
@@ -64,7 +154,7 @@ func TestServer_ProcessDDRQuery(t *testing.T) {
 	}{{
 		name:       "pass_host",
 		wantRes:    resultCodeSuccess,
-		host:       "example.net.",
+		host:       testQuestionTarget,
 		qtype:      dns.TypeSVCB,
 		ddrEnabled: true,
 		portDoH:    8043,
@@ -234,33 +324,33 @@ func TestServer_ProcessDetermineLocal(t *testing.T) {
 func TestServer_ProcessDHCPHosts_localRestriction(t *testing.T) {
 	knownIP := netip.MustParseAddr("1.2.3.4")
 	testCases := []struct {
+		wantIP     netip.Addr
 		name       string
 		host       string
-		wantIP     netip.Addr
 		wantRes    resultCode
 		isLocalCli bool
 	}{{
+		wantIP:     knownIP,
 		name:       "local_client_success",
 		host:       "example.lan",
-		wantIP:     knownIP,
 		wantRes:    resultCodeSuccess,
 		isLocalCli: true,
 	}, {
+		wantIP:     netip.Addr{},
 		name:       "local_client_unknown_host",
 		host:       "wronghost.lan",
-		wantIP:     netip.Addr{},
 		wantRes:    resultCodeSuccess,
 		isLocalCli: true,
 	}, {
+		wantIP:     netip.Addr{},
 		name:       "external_client_known_host",
 		host:       "example.lan",
-		wantIP:     netip.Addr{},
 		wantRes:    resultCodeFinish,
 		isLocalCli: false,
 	}, {
+		wantIP:     netip.Addr{},
 		name:       "external_client_unknown_host",
 		host:       "wronghost.lan",
-		wantIP:     netip.Addr{},
 		wantRes:    resultCodeFinish,
 		isLocalCli: false,
 	}}
@@ -332,52 +422,52 @@ func TestServer_ProcessDHCPHosts(t *testing.T) {
 
 	knownIP := netip.MustParseAddr("1.2.3.4")
 	testCases := []struct {
+		wantIP  netip.Addr
 		name    string
 		host    string
 		suffix  string
-		wantIP  netip.Addr
 		wantRes resultCode
 		qtyp    uint16
 	}{{
+		wantIP:  netip.Addr{},
 		name:    "success_external",
 		host:    examplecom,
 		suffix:  defaultLocalDomainSuffix,
-		wantIP:  netip.Addr{},
 		wantRes: resultCodeSuccess,
 		qtyp:    dns.TypeA,
 	}, {
+		wantIP:  netip.Addr{},
 		name:    "success_external_non_a",
 		host:    examplecom,
 		suffix:  defaultLocalDomainSuffix,
-		wantIP:  netip.Addr{},
 		wantRes: resultCodeSuccess,
 		qtyp:    dns.TypeCNAME,
 	}, {
+		wantIP:  knownIP,
 		name:    "success_internal",
 		host:    examplelan,
 		suffix:  defaultLocalDomainSuffix,
-		wantIP:  knownIP,
 		wantRes: resultCodeSuccess,
 		qtyp:    dns.TypeA,
 	}, {
+		wantIP:  netip.Addr{},
 		name:    "success_internal_unknown",
 		host:    "example-new.lan",
 		suffix:  defaultLocalDomainSuffix,
-		wantIP:  netip.Addr{},
 		wantRes: resultCodeSuccess,
 		qtyp:    dns.TypeA,
 	}, {
+		wantIP:  netip.Addr{},
 		name:    "success_internal_aaaa",
 		host:    examplelan,
 		suffix:  defaultLocalDomainSuffix,
-		wantIP:  netip.Addr{},
 		wantRes: resultCodeSuccess,
 		qtyp:    dns.TypeAAAA,
 	}, {
+		wantIP:  knownIP,
 		name:    "success_custom_suffix",
 		host:    "example.custom",
 		suffix:  "custom",
-		wantIP:  knownIP,
 		wantRes: resultCodeSuccess,
 		qtyp:    dns.TypeA,
 	}}
@@ -560,10 +650,8 @@ func TestServer_ProcessLocalPTR_usingResolvers(t *testing.T) {
 	var dnsCtx *dnsContext
 	setup := func(use bool) {
 		proxyCtx = &proxy.DNSContext{
-			Addr: &net.TCPAddr{
-				IP: net.IP{127, 0, 0, 1},
-			},
-			Req: createTestMessageWithType(reqAddr, dns.TypePTR),
+			Addr: testClientAddr,
+			Req:  createTestMessageWithType(reqAddr, dns.TypePTR),
 		}
 		dnsCtx = &dnsContext{
 			proxyCtx:        proxyCtx,

internal/dnsforward/stats.go

@@ -2,9 +2,9 @@ package dnsforward
 
 import (
 	"net"
-	"strings"
 	"time"
 
+	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
 	"github.com/AdguardTeam/AdGuardHome/internal/filtering"
 	"github.com/AdguardTeam/AdGuardHome/internal/querylog"
 	"github.com/AdguardTeam/AdGuardHome/internal/stats"
@@ -24,7 +24,7 @@ func (s *Server) processQueryLogsAndStats(dctx *dnsContext) (rc resultCode) {
 	pctx := dctx.proxyCtx
 
 	q := pctx.Req.Question[0]
-	host := strings.ToLower(strings.TrimSuffix(q.Name, "."))
+	host := aghnet.NormalizeDomain(q.Name)
 
 	ip, _ := netutil.IPAndPortFromAddr(pctx.Addr)
 	ip = slices.Clone(ip)
@@ -139,11 +139,10 @@ func (s *Server) updateStats(
 	clientIP string,
 ) {
 	pctx := ctx.proxyCtx
-	e := stats.Entry{}
-	e.Domain = strings.ToLower(pctx.Req.Question[0].Name)
-	if e.Domain != "." {
-		// Remove last ".", but save the domain as is for "." queries.
-		e.Domain = e.Domain[:len(e.Domain)-1]
+	e := stats.Entry{
+		Domain: aghnet.NormalizeDomain(pctx.Req.Question[0].Name),
+		Result: stats.RNotFiltered,
+		Time:   uint32(elapsed / 1000),
 	}
 
 	if clientID := ctx.clientID; clientID != "" {
@@ -152,9 +151,6 @@ func (s *Server) updateStats(
 		e.Client = clientIP
 	}
 
-	e.Time = uint32(elapsed / 1000)
-	e.Result = stats.RNotFiltered
-
 	switch res.Reason {
 	case filtering.FilteredSafeBrowsing:
 		e.Result = stats.RSafeBrowsing
@@ -162,7 +158,8 @@ func (s *Server) updateStats(
 		e.Result = stats.RParental
 	case filtering.FilteredSafeSearch:
 		e.Result = stats.RSafeSearch
-	case filtering.FilteredBlockList,
+	case
+		filtering.FilteredBlockList,
 		filtering.FilteredInvalid,
 		filtering.FilteredBlockedService:
 		e.Result = stats.RFiltered

internal/dnsforward/upstreams.go

@@ -42,11 +42,13 @@ func (s *Server) loadUpstreams() (upstreams []string, err error) {
 
 // prepareUpstreamSettings sets upstream DNS server settings.
 func (s *Server) prepareUpstreamSettings() (err error) {
-	// We're setting a customized set of RootCAs.  The reason is that Go default
-	// mechanism of loading TLS roots does not always work properly on some
-	// routers so we're loading roots manually and pass it here.
+	// Use a customized set of RootCAs, because Go's default mechanism of
+	// loading TLS roots does not always work properly on some routers so we're
+	// loading roots manually and pass it here.
 	//
 	// See [aghtls.SystemRootCAs].
+	//
+	// TODO(a.garipov): Investigate if that's true.
 	upstream.RootCAs = s.conf.TLSv12Roots
 	upstream.CipherSuites = s.conf.TLSCiphers
 
@@ -190,7 +192,7 @@ func (s *Server) resolveUpstreamsWithHosts(
 
 // extractUpstreamHost returns the hostname of addr without port with an
 // assumption that any address passed here has already been successfully parsed
-// by [upstream.AddressToUpstream].  This function eesentially mirrors the logic
+// by [upstream.AddressToUpstream].  This function essentially mirrors the logic
 // of [upstream.AddressToUpstream], see TODO on [replaceUpstreamsWithHosts].
 func extractUpstreamHost(addr string) (host string) {
 	var err error

internal/filtering/filtering.go

@@ -943,7 +943,7 @@ func New(c *Config, blockFilters []Filter) (d *DNSFilter, err error) {
 	d = &DNSFilter{
 		bufPool: &sync.Pool{
 			New: func() (buf any) {
-				bufVal := make([]byte, rulelist.MaxRuleLen)
+				bufVal := make([]byte, rulelist.DefaultRuleBufSize)
 
 				return &bufVal
 			},

internal/filtering/hashprefix/cache.go

@@ -47,7 +47,7 @@ func fromCacheItem(item *cacheItem) (data []byte) {
 	data = binary.BigEndian.AppendUint64(data, uint64(expiry))
 
 	for _, v := range item.hashes {
-		// nolint:looppointer // The subsilce is used for a copy.
+		// nolint:looppointer // The subslice of v is used for a copy.
 		data = append(data, v[:]...)
 	}
 
@@ -63,7 +63,7 @@ func (c *Checker) findInCache(
 
 	i := 0
 	for _, hash := range hashes {
-		// nolint:looppointer // The subsilce is used for a safe cache lookup.
+		// nolint:looppointer // The has subslice is used for a cache lookup.
 		data := c.cache.Get(hash[:prefixLen])
 		if data == nil {
 			hashes[i] = hash
@@ -98,34 +98,36 @@ func (c *Checker) storeInCache(hashesToRequest, respHashes []hostnameHash) {
 
 	for _, hash := range respHashes {
 		var pref prefix
-		// nolint:looppointer // The subsilce is used for a copy.
+		// nolint:looppointer // The hash subslice is used for a copy.
 		copy(pref[:], hash[:])
 
 		hashToStore[pref] = append(hashToStore[pref], hash)
 	}
 
 	for pref, hash := range hashToStore {
-		// nolint:looppointer // The subsilce is used for a safe cache lookup.
-		c.setCache(pref[:], hash)
+		c.setCache(pref, hash)
 	}
 
 	for _, hash := range hashesToRequest {
-		// nolint:looppointer // The subsilce is used for a safe cache lookup.
-		pref := hash[:prefixLen]
-		val := c.cache.Get(pref)
+		// nolint:looppointer // The hash subslice is used for a cache lookup.
+		val := c.cache.Get(hash[:prefixLen])
 		if val == nil {
+			var pref prefix
+			// nolint:looppointer // The hash subslice is used for a copy.
+			copy(pref[:], hash[:])
+
 			c.setCache(pref, nil)
 		}
 	}
 }
 
 // setCache stores hash in cache.
-func (c *Checker) setCache(pref []byte, hashes []hostnameHash) {
+func (c *Checker) setCache(pref prefix, hashes []hostnameHash) {
 	item := &cacheItem{
 		expiry: time.Now().Add(c.cacheTime),
 		hashes: hashes,
 	}
 
-	c.cache.Set(pref, fromCacheItem(item))
+	c.cache.Set(pref[:], fromCacheItem(item))
 	log.Debug("%s: stored in cache: %v", c.svc, pref)
 }

internal/filtering/hashprefix/hashprefix.go

@@ -173,7 +173,7 @@ func (c *Checker) getQuestion(hashes []hostnameHash) (q string) {
 	b := &strings.Builder{}
 
 	for _, hash := range hashes {
-		// nolint:looppointer // The subsilce is used for safe hex encoding.
+		// nolint:looppointer // The hash subslice is used for hex encoding.
 		stringutil.WriteToBuilder(b, hex.EncodeToString(hash[:prefixLen]), ".")
 	}
 

internal/filtering/rulelist/parser.go

@@ -7,6 +7,7 @@ import (
 	"hash/crc32"
 	"io"
 	"unicode"
+	"unicode/utf8"
 
 	"github.com/AdguardTeam/golibs/errors"
 )
@@ -48,19 +49,29 @@ type ParseResult struct {
 // nil.
 func (p *Parser) Parse(dst io.Writer, src io.Reader, buf []byte) (r *ParseResult, err error) {
 	s := bufio.NewScanner(src)
-	s.Buffer(buf, MaxRuleLen)
 
-	lineIdx := 0
+	// Don't use [DefaultRuleBufSize] as the maximum size, since some
+	// filtering-rule lists compressed by e.g. HostlistsCompiler can have very
+	// large lines.  The buffer optimization still works for the more common
+	// case of reasonably-sized lines.
+	//
+	// See https://github.com/AdguardTeam/AdGuardHome/issues/6003.
+	s.Buffer(buf, bufio.MaxScanTokenSize)
+
+	// Use a one-based index for lines and columns, since these errors end up in
+	// the frontend, and users are more familiar with one-based line and column
+	// indexes.
+	lineNum := 1
 	for s.Scan() {
 		var n int
-		n, err = p.processLine(dst, s.Bytes(), lineIdx)
+		n, err = p.processLine(dst, s.Bytes(), lineNum)
 		p.written += n
 		if err != nil {
 			// Don't wrap the error, because it's informative enough as is.
 			return p.result(), err
 		}
 
-		lineIdx++
+		lineNum++
 	}
 
 	r = p.result()
@@ -81,7 +92,7 @@ func (p *Parser) result() (r *ParseResult) {
 
 // processLine processes a single line.  It may write to dst, and if it does, n
 // is the number of bytes written.
-func (p *Parser) processLine(dst io.Writer, line []byte, lineIdx int) (n int, err error) {
+func (p *Parser) processLine(dst io.Writer, line []byte, lineNum int) (n int, err error) {
 	trimmed := bytes.TrimSpace(line)
 	if p.written == 0 && isHTMLLine(trimmed) {
 		return 0, ErrHTML
@@ -94,10 +105,13 @@ func (p *Parser) processLine(dst io.Writer, line []byte, lineIdx int) (n int, er
 		badIdx, isRule = p.parseLineTitle(trimmed)
 	}
 	if badIdx != -1 {
+		badRune, _ := utf8.DecodeRune(trimmed[badIdx:])
+
 		return 0, fmt.Errorf(
-			"line at index %d: character at index %d: non-printable character",
-			lineIdx,
-			badIdx+bytes.Index(line, trimmed),
+			"line %d: character %d: non-printable character %q",
+			lineNum,
+			badIdx+bytes.Index(line, trimmed)+1,
+			badRune,
 		)
 	}
 

internal/filtering/rulelist/parser_test.go

@@ -17,6 +17,9 @@ import (
 func TestParser_Parse(t *testing.T) {
 	t.Parallel()
 
+	longRule := strings.Repeat("a", rulelist.DefaultRuleBufSize+1) + "\n"
+	tooLongRule := strings.Repeat("a", bufio.MaxScanTokenSize+1) + "\n"
+
 	testCases := []struct {
 		name         string
 		in           string
@@ -80,20 +83,28 @@ func TestParser_Parse(t *testing.T) {
 			testRuleTextBlocked +
 			">>>\x7F<<<",
 		wantDst: testRuleTextBlocked,
-		wantErrMsg: "line at index 2: " +
-			"character at index 3: " +
-			"non-printable character",
+		wantErrMsg: "line 3: " +
+			"character 4: " +
+			"non-printable character '\\x7f'",
 		wantTitle:    "Test Title",
 		wantRulesNum: 1,
 		wantWritten:  len(testRuleTextBlocked),
 	}, {
 		name:         "too_long",
-		in:           strings.Repeat("a", rulelist.MaxRuleLen+1),
+		in:           tooLongRule,
 		wantDst:      "",
-		wantErrMsg:   "scanning filter contents: " + bufio.ErrTooLong.Error(),
+		wantErrMsg:   "scanning filter contents: bufio.Scanner: token too long",
 		wantTitle:    "",
 		wantRulesNum: 0,
 		wantWritten:  0,
+	}, {
+		name:         "longer_than_default",
+		in:           longRule,
+		wantDst:      longRule,
+		wantErrMsg:   "",
+		wantTitle:    "",
+		wantRulesNum: 1,
+		wantWritten:  len(longRule),
 	}, {
 		name:         "bad_tab_and_comment",
 		in:           testRuleTextBadTab,
@@ -118,7 +129,7 @@ func TestParser_Parse(t *testing.T) {
 			t.Parallel()
 
 			dst := &bytes.Buffer{}
-			buf := make([]byte, rulelist.MaxRuleLen)
+			buf := make([]byte, rulelist.DefaultRuleBufSize)
 
 			p := rulelist.NewParser()
 			r, err := p.Parse(dst, strings.NewReader(tc.in), buf)
@@ -145,7 +156,7 @@ func TestParser_Parse_writeError(t *testing.T) {
 			return 1, errors.Error("test error")
 		},
 	}
-	buf := make([]byte, rulelist.MaxRuleLen)
+	buf := make([]byte, rulelist.DefaultRuleBufSize)
 
 	p := rulelist.NewParser()
 	r, err := p.Parse(dst, strings.NewReader(testRuleTextBlocked), buf)
@@ -165,7 +176,7 @@ func TestParser_Parse_checksums(t *testing.T) {
 			"# Another comment.\n"
 	)
 
-	buf := make([]byte, rulelist.MaxRuleLen)
+	buf := make([]byte, rulelist.DefaultRuleBufSize)
 
 	p := rulelist.NewParser()
 	r, err := p.Parse(&bytes.Buffer{}, strings.NewReader(withoutComments), buf)
@@ -192,7 +203,7 @@ var (
 func BenchmarkParser_Parse(b *testing.B) {
 	dst := &bytes.Buffer{}
 	src := strings.NewReader(strings.Repeat(testRuleTextBlocked, 1000))
-	buf := make([]byte, rulelist.MaxRuleLen)
+	buf := make([]byte, rulelist.DefaultRuleBufSize)
 	p := rulelist.NewParser()
 
 	b.ReportAllocs()

internal/filtering/rulelist/rulelist.go

@@ -4,8 +4,6 @@
 // TODO(a.garipov): Expand.
 package rulelist
 
-// MaxRuleLen is the maximum length of a line with a filtering rule, in bytes.
-//
-// TODO(a.garipov): Consider changing this to a rune length, like AdGuardDNS
-// does.
-const MaxRuleLen = 1024
+// DefaultRuleBufSize is the default length of a buffer used to read a line with
+// a filtering rule, in bytes.
+const DefaultRuleBufSize = 1024

internal/filtering/servicelist.go

@@ -1505,6 +1505,7 @@ var blockedServices = []blockedService{{
 		"||aus.social^",
 		"||awscommunity.social^",
 		"||climatejustice.social^",
+		"||cupoftea.social^",
 		"||cyberplace.social^",
 		"||defcon.social^",
 		"||det.social^",
@@ -1530,6 +1531,7 @@ var blockedServices = []blockedService{{
 		"||masto.pt^",
 		"||mastodon.au^",
 		"||mastodon.bida.im^",
+		"||mastodon.com.tr^",
 		"||mastodon.eus^",
 		"||mastodon.green^",
 		"||mastodon.ie^",
@@ -1551,11 +1553,11 @@ var blockedServices = []blockedService{{
 		"||mastodont.cat^",
 		"||mastodontech.de^",
 		"||mastodontti.fi^",
-		"||mastouille.fr^",
 		"||mathstodon.xyz^",
 		"||metalhead.club^",
 		"||mindly.social^",
 		"||mstdn.ca^",
+		"||mstdn.jp^",
 		"||mstdn.party^",
 		"||mstdn.plus^",
 		"||mstdn.social^",
@@ -1567,7 +1569,6 @@ var blockedServices = []blockedService{{
 		"||nrw.social^",
 		"||o3o.ca^",
 		"||ohai.social^",
-		"||pewtix.com^",
 		"||piaille.fr^",
 		"||pol.social^",
 		"||ravenation.club^",
@@ -1582,20 +1583,19 @@ var blockedServices = []blockedService{{
 		"||social.linux.pizza^",
 		"||social.politicaconciencia.org^",
 		"||social.vivaldi.net^",
-		"||sself.co^",
 		"||stranger.social^",
 		"||sueden.social^",
 		"||tech.lgbt^",
 		"||techhub.social^",
 		"||theblower.au^",
 		"||tkz.one^",
-		"||todon.eu^",
 		"||toot.aquilenet.fr^",
 		"||toot.community^",
 		"||toot.funami.tech^",
 		"||toot.io^",
 		"||toot.wales^",
 		"||troet.cafe^",
+		"||twingyeo.kr^",
 		"||union.place^",
 		"||universeodon.com^",
 		"||urbanists.social^",

internal/home/clientaddr.go

@@ -0,0 +1,145 @@
+package home
+
+import (
+	"context"
+	"net/netip"
+	"time"
+
+	"github.com/AdguardTeam/AdGuardHome/internal/rdns"
+	"github.com/AdguardTeam/AdGuardHome/internal/whois"
+	"github.com/AdguardTeam/golibs/log"
+)
+
+// TODO(a.garipov): It is currently hard to add tests for this structure due to
+// strong coupling between it and Context.dnsServer with Context.clients.
+// Resolve this coupling and add proper testing.
+
+// clientAddrProcessor processes incoming client addresses with rDNS and WHOIS,
+// if configured.
+type clientAddrProcessor struct {
+	rdns  rdns.Interface
+	whois whois.Interface
+}
+
+const (
+	// defaultQueueSize is the size of queue of IPs for rDNS and WHOIS
+	// processing.
+	defaultQueueSize = 255
+
+	// defaultCacheSize is the maximum size of the cache for rDNS and WHOIS
+	// processing.  It must be greater than zero.
+	defaultCacheSize = 10_000
+
+	// defaultIPTTL is the Time to Live duration for IP addresses cached by
+	// rDNS and WHOIS.
+	defaultIPTTL = 1 * time.Hour
+)
+
+// newClientAddrProcessor returns a new client address processor.  c must not be
+// nil.
+func newClientAddrProcessor(c *clientSourcesConfig) (p *clientAddrProcessor) {
+	p = &clientAddrProcessor{
+		rdns:  &rdns.Empty{},
+		whois: &whois.Empty{},
+	}
+
+	if c.RDNS {
+		p.rdns = rdns.New(&rdns.Config{
+			Exchanger: Context.dnsServer,
+			CacheSize: defaultCacheSize,
+			CacheTTL:  defaultIPTTL,
+		})
+	}
+
+	if c.WHOIS {
+		// TODO(s.chzhen):  Consider making configurable.
+		const (
+			// defaultTimeout is the timeout for WHOIS requests.
+			defaultTimeout = 5 * time.Second
+
+			// defaultMaxConnReadSize is an upper limit in bytes for reading from a
+			// net.Conn.
+			defaultMaxConnReadSize = 64 * 1024
+
+			// defaultMaxRedirects is the maximum redirects count.
+			defaultMaxRedirects = 5
+
+			// defaultMaxInfoLen is the maximum length of whois.Info fields.
+			defaultMaxInfoLen = 250
+		)
+
+		p.whois = whois.New(&whois.Config{
+			DialContext:     customDialContext,
+			ServerAddr:      whois.DefaultServer,
+			Port:            whois.DefaultPort,
+			Timeout:         defaultTimeout,
+			CacheSize:       defaultCacheSize,
+			MaxConnReadSize: defaultMaxConnReadSize,
+			MaxRedirects:    defaultMaxRedirects,
+			MaxInfoLen:      defaultMaxInfoLen,
+			CacheTTL:        defaultIPTTL,
+		})
+	}
+
+	return p
+}
+
+// process processes the incoming client IP-address information.  It is intended
+// to be used as a goroutine.  Once clientIPs is closed, process exits.
+func (p *clientAddrProcessor) process(clientIPs <-chan netip.Addr) {
+	defer log.OnPanic("clientAddrProcessor.process")
+
+	log.Info("home: processing client addresses")
+
+	for ip := range clientIPs {
+		p.processRDNS(ip)
+		p.processWHOIS(ip)
+	}
+
+	log.Info("home: finished processing client addresses")
+}
+
+// processRDNS resolves the clients' IP addresses using reverse DNS.
+func (p *clientAddrProcessor) processRDNS(ip netip.Addr) {
+	start := time.Now()
+	log.Debug("home: processing client %s with rdns", ip)
+	defer func() {
+		log.Debug("home: finished processing client %s with rdns in %s", ip, time.Since(start))
+	}()
+
+	ok := Context.dnsServer.ShouldResolveClient(ip)
+	if !ok {
+		return
+	}
+
+	host, changed := p.rdns.Process(ip)
+	if host == "" || !changed {
+		return
+	}
+
+	ok = Context.clients.AddHost(ip, host, ClientSourceRDNS)
+	if ok {
+		return
+	}
+
+	log.Debug("dns: setting rdns info for client %q: already set with higher priority source", ip)
+}
+
+// processWHOIS looks up the information aobut clients' IP addresses in the
+// WHOIS databases.
+func (p *clientAddrProcessor) processWHOIS(ip netip.Addr) {
+	start := time.Now()
+	log.Debug("home: processing client %s with whois", ip)
+	defer func() {
+		log.Debug("home: finished processing client %s with whois in %s", ip, time.Since(start))
+	}()
+
+	// TODO(s.chzhen):  Move the timeout logic from WHOIS configuration to the
+	// context.
+	info, changed := p.whois.Process(context.Background(), ip)
+	if info == nil || !changed {
+		return
+	}
+
+	Context.clients.setWHOISInfo(ip, info)
+}

internal/home/clients.go

@@ -141,7 +141,7 @@ func (clients *clientsContainer) handleHostsUpdates() {
 	}
 }
 
-// webHandlersRegistered prevents a [clientsContainer] from regisering its web
+// webHandlersRegistered prevents a [clientsContainer] from registering its web
 // handlers more than once.
 //
 // TODO(a.garipov): Refactor HTTP handler registration logic.

internal/home/config.go

@@ -240,6 +240,7 @@ type tlsConfigSettings struct {
 
 type queryLogConfig struct {
 	// Ignored is the list of host names, which should not be written to log.
+	// "." is considered to be the root domain.
 	Ignored []string `yaml:"ignored"`
 
 	// Interval is the interval for query log's files rotation.

internal/home/dns.go

@@ -17,10 +17,7 @@ import (
 	"github.com/AdguardTeam/AdGuardHome/internal/dnsforward"
 	"github.com/AdguardTeam/AdGuardHome/internal/filtering"
 	"github.com/AdguardTeam/AdGuardHome/internal/querylog"
-	"github.com/AdguardTeam/AdGuardHome/internal/rdns"
 	"github.com/AdguardTeam/AdGuardHome/internal/stats"
-	"github.com/AdguardTeam/AdGuardHome/internal/whois"
-	"github.com/AdguardTeam/dnsproxy/proxy"
 	"github.com/AdguardTeam/golibs/errors"
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/AdguardTeam/golibs/netutil"
@@ -154,134 +151,32 @@ func initDNSServer(
 
 	Context.clients.dnsServer = Context.dnsServer
 
-	dnsConf, err := generateServerConfig(tlsConf, httpReg)
+	dnsConf, err := newServerConfig(tlsConf, httpReg)
 	if err != nil {
 		closeDNSServer()
 
-		return fmt.Errorf("generateServerConfig: %w", err)
+		return fmt.Errorf("newServerConfig: %w", err)
 	}
 
-	err = Context.dnsServer.Prepare(&dnsConf)
+	err = Context.dnsServer.Prepare(dnsConf)
 	if err != nil {
 		closeDNSServer()
 
 		return fmt.Errorf("dnsServer.Prepare: %w", err)
 	}
 
-	initRDNS()
-	initWHOIS()
+	clientIPs := dnsConf.ClientIPs
+	addrProc := newClientAddrProcessor(config.Clients.Sources)
+	go addrProc.process(clientIPs)
+
+	const topClientsNumber = 100
+	for _, ip := range Context.stats.TopClientsIP(topClientsNumber) {
+		clientIPs <- ip
+	}
 
 	return nil
 }
 
-const (
-	// defaultQueueSize is the size of queue of IPs for rDNS and WHOIS
-	// processing.
-	defaultQueueSize = 255
-
-	// defaultCacheSize is the maximum size of the cache for rDNS and WHOIS
-	// processing.  It must be greater than zero.
-	defaultCacheSize = 10_000
-
-	// defaultIPTTL is the Time to Live duration for IP addresses cached by
-	// rDNS and WHOIS.
-	defaultIPTTL = 1 * time.Hour
-)
-
-// initRDNS initializes the rDNS.
-func initRDNS() {
-	Context.rdnsCh = make(chan netip.Addr, defaultQueueSize)
-
-	// TODO(s.chzhen):  Add ability to disable it on dns server configuration
-	// update in [dnsforward] package.
-	r := rdns.New(&rdns.Config{
-		Exchanger: Context.dnsServer,
-		CacheSize: defaultCacheSize,
-		CacheTTL:  defaultIPTTL,
-	})
-
-	go processRDNS(r)
-}
-
-// processRDNS processes reverse DNS lookup queries.  It is intended to be used
-// as a goroutine.
-func processRDNS(r rdns.Interface) {
-	defer log.OnPanic("rdns")
-
-	for ip := range Context.rdnsCh {
-		ok := Context.dnsServer.ShouldResolveClient(ip)
-		if !ok {
-			continue
-		}
-
-		host, changed := r.Process(ip)
-		if host == "" || !changed {
-			continue
-		}
-
-		ok = Context.clients.AddHost(ip, host, ClientSourceRDNS)
-		if ok {
-			continue
-		}
-
-		log.Debug(
-			"dns: can't set rdns info for client %q: already set with higher priority source",
-			ip,
-		)
-	}
-}
-
-// initWHOIS initializes the WHOIS.
-//
-// TODO(s.chzhen):  Consider making configurable.
-func initWHOIS() {
-	const (
-		// defaultTimeout is the timeout for WHOIS requests.
-		defaultTimeout = 5 * time.Second
-
-		// defaultMaxConnReadSize is an upper limit in bytes for reading from
-		// net.Conn.
-		defaultMaxConnReadSize = 64 * 1024
-
-		// defaultMaxRedirects is the maximum redirects count.
-		defaultMaxRedirects = 5
-
-		// defaultMaxInfoLen is the maximum length of whois.Info fields.
-		defaultMaxInfoLen = 250
-	)
-
-	Context.whoisCh = make(chan netip.Addr, defaultQueueSize)
-
-	var w whois.Interface
-
-	if config.Clients.Sources.WHOIS {
-		w = whois.New(&whois.Config{
-			DialContext:     customDialContext,
-			ServerAddr:      whois.DefaultServer,
-			Port:            whois.DefaultPort,
-			Timeout:         defaultTimeout,
-			CacheSize:       defaultCacheSize,
-			MaxConnReadSize: defaultMaxConnReadSize,
-			MaxRedirects:    defaultMaxRedirects,
-			MaxInfoLen:      defaultMaxInfoLen,
-			CacheTTL:        defaultIPTTL,
-		})
-	} else {
-		w = whois.Empty{}
-	}
-
-	go func() {
-		defer log.OnPanic("whois")
-
-		for ip := range Context.whoisCh {
-			info, changed := w.Process(context.Background(), ip)
-			if info != nil && changed {
-				Context.clients.setWHOISInfo(ip, info)
-			}
-		}
-	}()
-}
-
 // parseSubnetSet parses a slice of subnets.  If the slice is empty, it returns
 // a subnet set that matches all locally served networks, see
 // [netutil.IsLocallyServed].
@@ -312,17 +207,6 @@ func isRunning() bool {
 	return Context.dnsServer != nil && Context.dnsServer.IsRunning()
 }
 
-func onDNSRequest(pctx *proxy.DNSContext) {
-	ip := netutil.NetAddrToAddrPort(pctx.Addr).Addr()
-	if ip == (netip.Addr{}) {
-		// This would be quite weird if we get here.
-		return
-	}
-
-	Context.rdnsCh <- ip
-	Context.whoisCh <- ip
-}
-
 func ipsToTCPAddrs(ips []netip.Addr, port int) (tcpAddrs []*net.TCPAddr) {
 	if ips == nil {
 		return nil
@@ -349,19 +233,20 @@ func ipsToUDPAddrs(ips []netip.Addr, port int) (udpAddrs []*net.UDPAddr) {
 	return udpAddrs
 }
 
-func generateServerConfig(
+func newServerConfig(
 	tlsConf *tlsConfigSettings,
 	httpReg aghhttp.RegisterFunc,
-) (newConf dnsforward.ServerConfig, err error) {
+) (newConf *dnsforward.ServerConfig, err error) {
 	dnsConf := config.DNS
 	hosts := aghalg.CoalesceSlice(dnsConf.BindHosts, []netip.Addr{netutil.IPv4Localhost()})
-	newConf = dnsforward.ServerConfig{
+	clientIPs := make(chan netip.Addr, defaultQueueSize)
+	newConf = &dnsforward.ServerConfig{
 		UDPListenAddrs:  ipsToUDPAddrs(hosts, dnsConf.Port),
 		TCPListenAddrs:  ipsToTCPAddrs(hosts, dnsConf.Port),
 		FilteringConfig: dnsConf.FilteringConfig,
 		ConfigModified:  onConfigModified,
 		HTTPRegister:    httpReg,
-		OnDNSRequest:    onDNSRequest,
+		ClientIPs:       clientIPs,
 		UseDNS64:        config.DNS.UseDNS64,
 		DNS64Prefixes:   config.DNS.DNS64Prefixes,
 	}
@@ -385,9 +270,9 @@ func generateServerConfig(
 		if tlsConf.PortDNSCrypt != 0 {
 			newConf.DNSCryptConfig, err = newDNSCrypt(hosts, *tlsConf)
 			if err != nil {
-				// Don't wrap the error, because it's already
-				// wrapped by newDNSCrypt.
-				return dnsforward.ServerConfig{}, err
+				// Don't wrap the error, because it's already wrapped by
+				// newDNSCrypt.
+				return nil, err
 			}
 		}
 	}
@@ -556,31 +441,26 @@ func startDNSServer() error {
 	Context.stats.Start()
 	Context.queryLog.Start()
 
-	const topClientsNumber = 100 // the number of clients to get
-	for _, ip := range Context.stats.TopClientsIP(topClientsNumber) {
-		Context.rdnsCh <- ip
-		Context.whoisCh <- ip
-	}
-
 	return nil
 }
 
 func reconfigureDNSServer() (err error) {
-	var newConf dnsforward.ServerConfig
-
 	tlsConf := &tlsConfigSettings{}
 	Context.tls.WriteDiskConfig(tlsConf)
 
-	newConf, err = generateServerConfig(tlsConf, httpRegister)
+	newConf, err := newServerConfig(tlsConf, httpRegister)
 	if err != nil {
 		return fmt.Errorf("generating forwarding dns server config: %w", err)
 	}
 
-	err = Context.dnsServer.Reconfigure(&newConf)
+	err = Context.dnsServer.Reconfigure(newConf)
 	if err != nil {
 		return fmt.Errorf("starting forwarding dns server: %w", err)
 	}
 
+	addrProc := newClientAddrProcessor(config.Clients.Sources)
+	go addrProc.process(newConf.ClientIPs)
+
 	return nil
 }
 

internal/home/home.go

@@ -82,12 +82,6 @@ type homeContext struct {
 	client           *http.Client
 	appSignalChannel chan os.Signal // Channel for receiving OS signals by the console app
 
-	// rdnsCh is the channel for receiving IPs for rDNS processing.
-	rdnsCh chan netip.Addr
-
-	// whoisCh is the channel for receiving IPs for WHOIS processing.
-	whoisCh chan netip.Addr
-
 	// tlsCipherIDs are the ID of the cipher suites that AdGuard Home must use.
 	tlsCipherIDs []uint16
 
@@ -470,7 +464,7 @@ func setupDNSFilteringConf(conf *filtering.Config) (err error) {
 		ServiceName: pcService,
 		TXTSuffix:   pcTXTSuffix,
 		CacheTime:   cacheTime,
-		CacheSize:   conf.SafeBrowsingCacheSize,
+		CacheSize:   conf.ParentalCacheSize,
 	})
 
 	conf.SafeSearchConf.CustomResolver = safeSearchResolver{}
@@ -634,10 +628,10 @@ func run(opts options, clientBuildFS fs.FS) {
 		Context.tls.start()
 
 		go func() {
-			sErr := startDNSServer()
-			if sErr != nil {
+			startErr := startDNSServer()
+			if startErr != nil {
 				closeDNSServer()
-				fatalOnError(sErr)
+				fatalOnError(startErr)
 			}
 		}()
 

internal/querylog/qlog.go

@@ -4,7 +4,6 @@ package querylog
 import (
 	"fmt"
 	"os"
-	"strings"
 	"sync"
 	"time"
 
@@ -161,10 +160,7 @@ func (l *queryLog) clear() {
 // newLogEntry creates an instance of logEntry from parameters.
 func newLogEntry(params *AddParams) (entry *logEntry) {
 	q := params.Question.Question[0]
-	qHost := q.Name
-	if qHost != "." {
-		qHost = strings.ToLower(q.Name[:len(q.Name)-1])
-	}
+	qHost := aghnet.NormalizeDomain(q.Name)
 
 	entry = &logEntry{
 		// TODO(d.kolyshev): Export this timestamp to func params.

internal/stats/http_test.go

@@ -86,7 +86,7 @@ func TestHandleStatsConfig(t *testing.T) {
 			},
 		},
 		wantCode: http.StatusUnprocessableEntity,
-		wantErr:  "ignored: duplicate host name \"ignor.ed\" at index 1\n",
+		wantErr:  "ignored: duplicate hostname \"ignor.ed\" at index 1\n",
 	}, {
 		name: "ignored_empty",
 		body: getConfigResp{
@@ -97,7 +97,7 @@ func TestHandleStatsConfig(t *testing.T) {
 			},
 		},
 		wantCode: http.StatusUnprocessableEntity,
-		wantErr:  "ignored: host name is empty\n",
+		wantErr:  "ignored: at index 0: hostname is empty\n",
 	}, {
 		name: "enabled_is_null",
 		body: getConfigResp{

internal/tools/go.mod

@@ -13,7 +13,7 @@ require (
 	golang.org/x/tools v0.11.0
 	golang.org/x/vuln v0.2.0
 	// TODO(a.garipov): Return to tagged releases once a new one appears.
-	honnef.co/go/tools v0.5.0-0.dev.0.20230706211743-ddee6bbaa341
+	honnef.co/go/tools v0.5.0-0.dev.0.20230709092525-bc759185c5ee
 	mvdan.cc/gofumpt v0.5.0
 	mvdan.cc/unparam v0.0.0-20230610194454-9ea02bef9868
 )
@@ -27,7 +27,7 @@ require (
 	github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
-	golang.org/x/exp/typeparams v0.0.0-20230626212559-97b1e661b5df // indirect
+	golang.org/x/exp/typeparams v0.0.0-20230711023510-fffb14384f22 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/sys v0.10.0 // indirect

internal/tools/go.sum

@@ -52,8 +52,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug=
 golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
-golang.org/x/exp/typeparams v0.0.0-20230626212559-97b1e661b5df h1:jfUqBujZx2dktJVEmZpCkyngz7MWrVv1y9kLOqFNsqw=
-golang.org/x/exp/typeparams v0.0.0-20230626212559-97b1e661b5df/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
+golang.org/x/exp/typeparams v0.0.0-20230711023510-fffb14384f22 h1:e8iSCQYXZ4EB6q3kIfy2fgPFTvDbozqzRe4OuIOyrL4=
+golang.org/x/exp/typeparams v0.0.0-20230711023510-fffb14384f22/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
@@ -107,8 +107,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.5.0-0.dev.0.20230706211743-ddee6bbaa341 h1:jNlTAPEjbDiN9qda/1wple0GSpewFnWhvc1GO7bZX1U=
-honnef.co/go/tools v0.5.0-0.dev.0.20230706211743-ddee6bbaa341/go.mod h1:GUV+uIBCLpdf0/v6UhHHG/yzI/z6qPskBeQCjcNB96k=
+honnef.co/go/tools v0.5.0-0.dev.0.20230709092525-bc759185c5ee h1:mpyvMqtlVZTwEv78QL3S2ZDTMHMO1fgNwr2kC7+K7oU=
+honnef.co/go/tools v0.5.0-0.dev.0.20230709092525-bc759185c5ee/go.mod h1:GUV+uIBCLpdf0/v6UhHHG/yzI/z6qPskBeQCjcNB96k=
 mvdan.cc/gofumpt v0.5.0 h1:0EQ+Z56k8tXjj/6TQD25BFNKQXpCvT0rnansIc7Ug5E=
 mvdan.cc/gofumpt v0.5.0/go.mod h1:HBeVDtMKRZpXyxFciAirzdKklDlGu8aAy1wEbH5Y9js=
 mvdan.cc/unparam v0.0.0-20230610194454-9ea02bef9868 h1:F4Q7pXcrU9UiU1fq0ZWqSOxKjNAteRuDr7JDk7uVLRQ=

scripts/README.md

@@ -269,25 +269,29 @@ Optional environment:
 
    ###  Usage
 
- *  `go run main.go help`: print usage.
+ *  `go run ./scripts/translations help`: print usage.
 
- *  `go run main.go download [-n <count>]`: download and save all translations.
-    `n` is optional flag where count is a number of concurrent downloads.
+ *  `go run ./scripts/translations download [-n <count>]`: download and save
+    all translations.  `n` is optional flag where count is a number of
+    concurrent downloads.
 
- *  `go run main.go upload`: upload the base `en` locale.
+ *  `go run ./scripts/translations upload`: upload the base `en` locale.
 
- *  `go run main.go summary`: show the current locales summary.
+ *  `go run ./scripts/translations summary`: show the current locales summary.
 
- *  `go run main.go unused`: show the list of unused strings.
+ *  `go run ./scripts/translations unused`: show the list of unused strings.
 
- *  `go run main.go auto-add`: add locales with additions to the git and
-    restore locales with deletions.
+ *  `go run ./scripts/translations auto-add`: add locales with additions to the
+    git and restore locales with deletions.
 
 After the download you'll find the output locales in the `client/src/__locales/`
 directory.
 
 Optional environment:
 
+ *  `DOWNLOAD_LANGUAGES`: set a list of specific languages to `download`.  For
+    example `ar be bg`.
+
  *  `UPLOAD_LANGUAGE`: set an alternative language for `upload`.
 
  *  `TWOSKY_URI`: set an alternative URL for `download` or `upload`.

scripts/make/go-lint.sh

@@ -35,7 +35,7 @@ set -f -u
 go_version="$( "${GO:-go}" version )"
 readonly go_version
 
-go_min_version='go1.19.10'
+go_min_version='go1.19.11'
 go_version_msg="
 warning: your go version (${go_version}) is different from the recommended minimal one (${go_min_version}).
 if you have the version installed, please set the GO environment variable.
@@ -183,6 +183,7 @@ run_linter gocognit --over 10\
 	./internal/tools/\
 	./internal/version/\
 	./internal/whois/\
+	./scripts/\
 	;
 
 run_linter ineffassign ./...

scripts/translations/download.go

@@ -0,0 +1,177 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/AdguardTeam/AdGuardHome/internal/aghio"
+	"github.com/AdguardTeam/golibs/errors"
+	"github.com/AdguardTeam/golibs/log"
+	"golang.org/x/exp/slices"
+)
+
+// download and save all translations.
+func (c *twoskyClient) download() (err error) {
+	var numWorker int
+
+	flagSet := flag.NewFlagSet("download", flag.ExitOnError)
+	flagSet.Usage = func() {
+		usage("download command error")
+	}
+	flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")
+
+	err = flagSet.Parse(os.Args[2:])
+	if err != nil {
+		// Don't wrap the error since it's informative enough as is.
+		return err
+	}
+
+	if numWorker < 1 {
+		usage("count must be positive")
+	}
+
+	downloadURI := c.uri.JoinPath("download")
+
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+
+	wg := &sync.WaitGroup{}
+	failed := &sync.Map{}
+	uriCh := make(chan *url.URL, len(c.langs))
+
+	for i := 0; i < numWorker; i++ {
+		wg.Add(1)
+		go downloadWorker(wg, failed, client, uriCh)
+	}
+
+	for lang := range c.langs {
+		uri := translationURL(downloadURI, defaultBaseFile, c.projectID, lang)
+
+		uriCh <- uri
+	}
+
+	close(uriCh)
+	wg.Wait()
+
+	printFailedLocales(failed)
+
+	return nil
+}
+
+// printFailedLocales prints sorted list of failed downloads, if any.
+func printFailedLocales(failed *sync.Map) {
+	keys := []string{}
+	failed.Range(func(k, _ any) bool {
+		s, ok := k.(string)
+		if !ok {
+			panic("unexpected type")
+		}
+
+		keys = append(keys, s)
+
+		return true
+	})
+
+	if len(keys) == 0 {
+		return
+	}
+
+	slices.Sort(keys)
+	log.Info("failed locales: %s", strings.Join(keys, " "))
+}
+
+// downloadWorker downloads translations by received urls and saves them.
+// Where failed is a map for storing failed downloads.
+func downloadWorker(
+	wg *sync.WaitGroup,
+	failed *sync.Map,
+	client *http.Client,
+	uriCh <-chan *url.URL,
+) {
+	defer wg.Done()
+
+	for uri := range uriCh {
+		q := uri.Query()
+		code := q.Get("language")
+
+		err := saveToFile(client, uri, code)
+		if err != nil {
+			log.Error("download: worker: %s", err)
+			failed.Store(code, struct{}{})
+		}
+	}
+}
+
+// saveToFile downloads translation by url and saves it to a file, or returns
+// error.
+func saveToFile(client *http.Client, uri *url.URL, code string) (err error) {
+	data, err := getTranslation(client, uri.String())
+	if err != nil {
+		log.Info("%s", data)
+
+		return fmt.Errorf("getting translation: %s", err)
+	}
+
+	name := filepath.Join(localesDir, code+".json")
+	err = os.WriteFile(name, data, 0o664)
+	if err != nil {
+		return fmt.Errorf("writing file: %s", err)
+	}
+
+	fmt.Println(name)
+
+	return nil
+}
+
+// getTranslation returns received translation data and error.  If err is not
+// nil, data may contain a response from server for inspection.
+func getTranslation(client *http.Client, url string) (data []byte, err error) {
+	resp, err := client.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("requesting: %w", err)
+	}
+
+	defer log.OnCloserError(resp.Body, log.ERROR)
+
+	if resp.StatusCode != http.StatusOK {
+		err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
+
+		// Go on and download the body for inspection.
+	}
+
+	limitReader, lrErr := aghio.LimitReader(resp.Body, readLimit)
+	if lrErr != nil {
+		// Generally shouldn't happen, since the only error returned by
+		// [aghio.LimitReader] is an argument error.
+		panic(fmt.Errorf("limit reading: %w", lrErr))
+	}
+
+	data, readErr := io.ReadAll(limitReader)
+
+	return data, errors.WithDeferred(err, readErr)
+}
+
+// translationURL returns a new url.URL with provided query parameters.
+func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
+	uri = &url.URL{}
+	*uri = *oldURL
+
+	q := uri.Query()
+	q.Set("format", "json")
+	q.Set("filename", baseFile)
+	q.Set("project", projectID)
+	q.Set("language", string(lang))
+
+	uri.RawQuery = q.Encode()
+
+	return uri
+}

scripts/translations/main.go

@@ -6,25 +6,16 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/json"
-	"flag"
 	"fmt"
-	"io"
-	"mime/multipart"
-	"net/http"
-	"net/textproto"
 	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
-	"sync"
 	"time"
 
-	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
-	"github.com/AdguardTeam/AdGuardHome/internal/aghio"
 	"github.com/AdguardTeam/AdGuardHome/internal/aghos"
 	"github.com/AdguardTeam/golibs/errors"
-	"github.com/AdguardTeam/golibs/httphdr"
 	"github.com/AdguardTeam/golibs/log"
 	"golang.org/x/exp/maps"
 	"golang.org/x/exp/slices"
@@ -38,7 +29,8 @@ const (
 	srcDir           = "./client/src"
 	twoskyURI        = "https://twosky.int.agrd.dev/api/v1"
 
-	readLimit = 1 * 1024 * 1024
+	readLimit     = 1 * 1024 * 1024
+	uploadTimeout = 10 * time.Second
 )
 
 // langCode is a language code.
@@ -62,31 +54,26 @@ func main() {
 		usage("")
 	}
 
-	uriStr := os.Getenv("TWOSKY_URI")
-	if uriStr == "" {
-		uriStr = twoskyURI
-	}
-
-	uri, err := url.Parse(uriStr)
+	conf, err := readTwoskyConfig()
 	check(err)
 
-	projectID := os.Getenv("TWOSKY_PROJECT_ID")
-	if projectID == "" {
-		projectID = defaultProjectID
-	}
-
-	conf, err := readTwoskyConf()
-	check(err)
+	var cli *twoskyClient
 
 	switch os.Args[1] {
 	case "summary":
 		err = summary(conf.Languages)
 	case "download":
-		err = download(uri, projectID, conf.Languages)
+		cli, err = conf.toClient()
+		check(err)
+
+		err = cli.download()
 	case "unused":
 		err = unused(conf.LocalizableFiles[0])
 	case "upload":
-		err = upload(uri, projectID, conf.BaseLangcode)
+		cli, err = conf.toClient()
+		check(err)
+
+		err = cli.upload()
 	case "auto-add":
 		err = autoAdd(conf.LocalizableFiles[0])
 	default:
@@ -133,51 +120,131 @@ Commands:
 	os.Exit(0)
 }
 
-// twoskyConf is the configuration structure for localization.
-type twoskyConf struct {
+// twoskyConfig is the configuration structure for localization.
+type twoskyConfig struct {
 	Languages        languages `json:"languages"`
 	ProjectID        string    `json:"project_id"`
 	BaseLangcode     langCode  `json:"base_locale"`
 	LocalizableFiles []string  `json:"localizable_files"`
 }
 
-// readTwoskyConf returns configuration.
-func readTwoskyConf() (t twoskyConf, err error) {
-	defer func() { err = errors.Annotate(err, "parsing twosky conf: %w") }()
+// readTwoskyConfig returns twosky configuration.
+func readTwoskyConfig() (t *twoskyConfig, err error) {
+	defer func() { err = errors.Annotate(err, "parsing twosky config: %w") }()
 
 	b, err := os.ReadFile(twoskyConfFile)
 	if err != nil {
 		// Don't wrap the error since it's informative enough as is.
-		return twoskyConf{}, err
+		return nil, err
 	}
 
-	var tsc []twoskyConf
+	var tsc []twoskyConfig
 	err = json.Unmarshal(b, &tsc)
 	if err != nil {
 		err = fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err)
 
-		return twoskyConf{}, err
+		return nil, err
 	}
 
 	if len(tsc) == 0 {
 		err = fmt.Errorf("%q is empty", twoskyConfFile)
 
-		return twoskyConf{}, err
+		return nil, err
 	}
 
 	conf := tsc[0]
 
 	for _, lang := range conf.Languages {
 		if lang == "" {
-			return twoskyConf{}, errors.Error("language is empty")
+			return nil, errors.Error("language is empty")
 		}
 	}
 
 	if len(conf.LocalizableFiles) == 0 {
-		return twoskyConf{}, errors.Error("no localizable files specified")
+		return nil, errors.Error("no localizable files specified")
 	}
 
-	return conf, nil
+	return &conf, nil
+}
+
+// twoskyClient is the twosky client with methods for download and upload
+// translations.
+type twoskyClient struct {
+	// uri is the base URL.
+	uri *url.URL
+
+	// langs is the map of languages to download.
+	langs languages
+
+	// projectID is the name of the project.
+	projectID string
+
+	// baseLang is the base language code.
+	baseLang langCode
+}
+
+// toClient reads values from environment variables or defaults, validates
+// them, and returns the twosky client.
+func (t *twoskyConfig) toClient() (cli *twoskyClient, err error) {
+	defer func() { err = errors.Annotate(err, "filling config: %w") }()
+
+	uriStr := os.Getenv("TWOSKY_URI")
+	if uriStr == "" {
+		uriStr = twoskyURI
+	}
+	uri, err := url.Parse(uriStr)
+	if err != nil {
+		return nil, err
+	}
+
+	projectID := os.Getenv("TWOSKY_PROJECT_ID")
+	if projectID == "" {
+		projectID = defaultProjectID
+	}
+
+	baseLang := t.BaseLangcode
+	uLangStr := os.Getenv("UPLOAD_LANGUAGE")
+	if uLangStr != "" {
+		baseLang = langCode(uLangStr)
+	}
+
+	langs := t.Languages
+	dlLangStr := os.Getenv("DOWNLOAD_LANGUAGES")
+	if dlLangStr != "" {
+		var dlLangs languages
+		dlLangs, err = validateLanguageStr(dlLangStr, langs)
+		if err != nil {
+			return nil, err
+		}
+
+		langs = dlLangs
+	}
+
+	return &twoskyClient{
+		uri:       uri,
+		projectID: projectID,
+		baseLang:  baseLang,
+		langs:     langs,
+	}, nil
+}
+
+// validateLanguageStr validates languages codes that contain in the str and
+// returns language map, where key is language code and value is display name.
+func validateLanguageStr(str string, all languages) (langs languages, err error) {
+	langs = make(languages)
+	codes := strings.Fields(str)
+
+	for _, k := range codes {
+		lc := langCode(k)
+		name, ok := all[lc]
+		if !ok {
+			return nil, fmt.Errorf("validating languages: unexpected language code %q", k)
+		}
+
+		langs[lc] = name
+	}
+
+	return langs, nil
 }
 
 // readLocales reads file with name fn and returns a map, where key is text
@@ -233,163 +300,33 @@ func summary(langs languages) (err error) {
 	return nil
 }
 
-// download and save all translations.  uri is the base URL.  projectID is the
-// name of the project.
-func download(uri *url.URL, projectID string, langs languages) (err error) {
-	var numWorker int
+// unused prints unused text labels.
+func unused(basePath string) (err error) {
+	defer func() { err = errors.Annotate(err, "unused: %w") }()
 
-	flagSet := flag.NewFlagSet("download", flag.ExitOnError)
-	flagSet.Usage = func() {
-		usage("download command error")
-	}
-	flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")
-
-	err = flagSet.Parse(os.Args[2:])
+	baseLoc, err := readLocales(basePath)
 	if err != nil {
-		// Don't wrap the error since it's informative enough as is.
 		return err
 	}
 
-	if numWorker < 1 {
-		usage("count must be positive")
-	}
-
-	downloadURI := uri.JoinPath("download")
-
-	client := &http.Client{
-		Timeout: 10 * time.Second,
-	}
-
-	wg := &sync.WaitGroup{}
-	uriCh := make(chan *url.URL, len(langs))
-
-	for i := 0; i < numWorker; i++ {
-		wg.Add(1)
-		go downloadWorker(wg, client, uriCh)
-	}
-
-	for lang := range langs {
-		uri = translationURL(downloadURI, defaultBaseFile, projectID, lang)
-
-		uriCh <- uri
-	}
-
-	close(uriCh)
-	wg.Wait()
-
-	return nil
-}
-
-// downloadWorker downloads translations by received urls and saves them.
-func downloadWorker(wg *sync.WaitGroup, client *http.Client, uriCh <-chan *url.URL) {
-	defer wg.Done()
-
-	for uri := range uriCh {
-		data, err := getTranslation(client, uri.String())
-		if err != nil {
-			log.Error("download worker: getting translation: %s", err)
-			log.Info("download worker: error response:\n%s", data)
-
-			continue
-		}
-
-		q := uri.Query()
-		code := q.Get("language")
-
-		// Fix some TwoSky weirdnesses.
-		//
-		// TODO(a.garipov): Remove when those are fixed.
-		code = strings.ToLower(code)
-
-		name := filepath.Join(localesDir, code+".json")
-		err = os.WriteFile(name, data, 0o664)
-		if err != nil {
-			log.Error("download worker: writing file: %s", err)
-
-			continue
-		}
-
-		fmt.Println(name)
-	}
-}
-
-// getTranslation returns received translation data and error.  If err is not
-// nil, data may contain a response from server for inspection.
-func getTranslation(client *http.Client, url string) (data []byte, err error) {
-	resp, err := client.Get(url)
-	if err != nil {
-		return nil, fmt.Errorf("requesting: %w", err)
-	}
-
-	defer log.OnCloserError(resp.Body, log.ERROR)
-
-	if resp.StatusCode != http.StatusOK {
-		err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
-
-		// Go on and download the body for inspection.
-	}
-
-	limitReader, lrErr := aghio.LimitReader(resp.Body, readLimit)
-	if lrErr != nil {
-		// Generally shouldn't happen, since the only error returned by
-		// [aghio.LimitReader] is an argument error.
-		panic(fmt.Errorf("limit reading: %w", lrErr))
-	}
-
-	data, readErr := io.ReadAll(limitReader)
-
-	return data, errors.WithDeferred(err, readErr)
-}
-
-// translationURL returns a new url.URL with provided query parameters.
-func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
-	uri = &url.URL{}
-	*uri = *oldURL
-
-	// Fix some TwoSky weirdnesses.
-	//
-	// TODO(a.garipov): Remove when those are fixed.
-	switch lang {
-	case "si-lk":
-		lang = "si-LK"
-	case "zh-hk":
-		lang = "zh-HK"
-	default:
-		// Go on.
-	}
-
-	q := uri.Query()
-	q.Set("format", "json")
-	q.Set("filename", baseFile)
-	q.Set("project", projectID)
-	q.Set("language", string(lang))
-
-	uri.RawQuery = q.Encode()
-
-	return uri
-}
-
-// unused prints unused text labels.
-func unused(basePath string) (err error) {
-	baseLoc, err := readLocales(basePath)
-	if err != nil {
-		return fmt.Errorf("unused: %w", err)
-	}
-
 	locDir := filepath.Clean(localesDir)
+	js, err := findJS(locDir)
+	if err != nil {
+		return err
+	}
 
-	fileNames := []string{}
-	err = filepath.Walk(srcDir, func(name string, info os.FileInfo, err error) error {
+	return findUnused(js, baseLoc)
+}
+
+// findJS returns list of JavaScript and JSON files or error.
+func findJS(locDir string) (fileNames []string, err error) {
+	walkFn := func(name string, _ os.FileInfo, err error) error {
 		if err != nil {
 			log.Info("warning: accessing a path %q: %s", name, err)
 
 			return nil
 		}
 
-		if info.IsDir() {
-			return nil
-		}
-
 		if strings.HasPrefix(name, locDir) {
 			return nil
 		}
@@ -400,13 +337,14 @@ func unused(basePath string) (err error) {
 		}
 
 		return nil
-	})
-
-	if err != nil {
-		return fmt.Errorf("filepath walking %q: %w", srcDir, err)
 	}
 
-	return findUnused(fileNames, baseLoc)
+	err = filepath.Walk(srcDir, walkFn)
+	if err != nil {
+		return nil, fmt.Errorf("filepath walking %q: %w", srcDir, err)
+	}
+
+	return fileNames, nil
 }
 
 // findUnused prints unused text labels from fileNames.
@@ -445,118 +383,6 @@ func findUnused(fileNames []string, loc locales) (err error) {
 	return nil
 }
 
-// upload base translation.  uri is the base URL.  projectID is the name of the
-// project.  baseLang is the base language code.
-func upload(uri *url.URL, projectID string, baseLang langCode) (err error) {
-	defer func() { err = errors.Annotate(err, "upload: %w") }()
-
-	uploadURI := uri.JoinPath("upload")
-
-	lang := baseLang
-
-	langStr := os.Getenv("UPLOAD_LANGUAGE")
-	if langStr != "" {
-		lang = langCode(langStr)
-	}
-
-	basePath := filepath.Join(localesDir, defaultBaseFile)
-
-	formData := map[string]string{
-		"format":   "json",
-		"language": string(lang),
-		"filename": defaultBaseFile,
-		"project":  projectID,
-	}
-
-	buf, cType, err := prepareMultipartMsg(formData, basePath)
-	if err != nil {
-		return fmt.Errorf("preparing multipart msg: %w", err)
-	}
-
-	err = send(uploadURI.String(), cType, buf)
-	if err != nil {
-		return fmt.Errorf("sending multipart msg: %w", err)
-	}
-
-	return nil
-}
-
-// prepareMultipartMsg prepares translation data for upload.
-func prepareMultipartMsg(
-	formData map[string]string,
-	basePath string,
-) (buf *bytes.Buffer, cType string, err error) {
-	buf = &bytes.Buffer{}
-	w := multipart.NewWriter(buf)
-	var fw io.Writer
-
-	for k, v := range formData {
-		err = w.WriteField(k, v)
-		if err != nil {
-			return nil, "", fmt.Errorf("writing field: %w", err)
-		}
-	}
-
-	file, err := os.Open(basePath)
-	if err != nil {
-		return nil, "", fmt.Errorf("opening file: %w", err)
-	}
-
-	defer func() {
-		err = errors.WithDeferred(err, file.Close())
-	}()
-
-	h := make(textproto.MIMEHeader)
-	h.Set(httphdr.ContentType, aghhttp.HdrValApplicationJSON)
-
-	d := fmt.Sprintf("form-data; name=%q; filename=%q", "file", defaultBaseFile)
-	h.Set(httphdr.ContentDisposition, d)
-
-	fw, err = w.CreatePart(h)
-	if err != nil {
-		return nil, "", fmt.Errorf("creating part: %w", err)
-	}
-
-	_, err = io.Copy(fw, file)
-	if err != nil {
-		return nil, "", fmt.Errorf("copying: %w", err)
-	}
-
-	err = w.Close()
-	if err != nil {
-		return nil, "", fmt.Errorf("closing writer: %w", err)
-	}
-
-	return buf, w.FormDataContentType(), nil
-}
-
-// send POST request to uriStr.
-func send(uriStr, cType string, buf *bytes.Buffer) (err error) {
-	var client http.Client
-
-	req, err := http.NewRequest(http.MethodPost, uriStr, buf)
-	if err != nil {
-		return fmt.Errorf("bad request: %w", err)
-	}
-
-	req.Header.Set(httphdr.ContentType, cType)
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("client post form: %w", err)
-	}
-
-	defer func() {
-		err = errors.WithDeferred(err, resp.Body.Close())
-	}()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode))
-	}
-
-	return nil
-}
-
 // autoAdd adds locales with additions to the git and restores locales with
 // deletions.
 func autoAdd(basePath string) (err error) {
@@ -572,28 +398,48 @@ func autoAdd(basePath string) (err error) {
 		return errors.Error("base locale contains deletions")
 	}
 
-	var (
-		args []string
-		code int
-		out  []byte
-	)
-
-	if len(adds) > 0 {
-		args = append([]string{"add"}, adds...)
-		code, out, err = aghos.RunCommand("git", args...)
-
-		if err != nil || code != 0 {
-			return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err)
-		}
+	err = handleAdds(adds)
+	if err != nil {
+		// Don't wrap the error since it's informative enough as is.
+		return nil
 	}
 
-	if len(dels) > 0 {
-		args = append([]string{"restore"}, dels...)
-		code, out, err = aghos.RunCommand("git", args...)
+	err = handleDels(dels)
+	if err != nil {
+		// Don't wrap the error since it's informative enough as is.
+		return nil
+	}
 
-		if err != nil || code != 0 {
-			return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err)
-		}
+	return nil
+}
+
+// handleAdds adds locales with additions to the git.
+func handleAdds(locales []string) (err error) {
+	if len(locales) == 0 {
+		return nil
+	}
+
+	args := append([]string{"add"}, locales...)
+	code, out, err := aghos.RunCommand("git", args...)
+
+	if err != nil || code != 0 {
+		return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err)
+	}
+
+	return nil
+}
+
+// handleDels restores locales with deletions.
+func handleDels(locales []string) (err error) {
+	if len(locales) == 0 {
+		return nil
+	}
+
+	args := append([]string{"restore"}, locales...)
+	code, out, err := aghos.RunCommand("git", args...)
+
+	if err != nil || code != 0 {
+		return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err)
 	}
 
 	return nil

scripts/translations/upload.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/textproto"
+	"os"
+	"path/filepath"
+
+	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
+	"github.com/AdguardTeam/golibs/errors"
+	"github.com/AdguardTeam/golibs/httphdr"
+	"github.com/AdguardTeam/golibs/mapsutil"
+)
+
+// upload base translation.
+func (c *twoskyClient) upload() (err error) {
+	defer func() { err = errors.Annotate(err, "upload: %w") }()
+
+	uploadURI := c.uri.JoinPath("upload")
+	basePath := filepath.Join(localesDir, defaultBaseFile)
+
+	formData := map[string]string{
+		"format":   "json",
+		"language": string(c.baseLang),
+		"filename": defaultBaseFile,
+		"project":  c.projectID,
+	}
+
+	buf, cType, err := prepareMultipartMsg(formData, basePath)
+	if err != nil {
+		return fmt.Errorf("preparing multipart msg: %w", err)
+	}
+
+	err = send(uploadURI.String(), cType, buf)
+	if err != nil {
+		return fmt.Errorf("sending multipart msg: %w", err)
+	}
+
+	return nil
+}
+
+// prepareMultipartMsg prepares translation data for upload.
+func prepareMultipartMsg(
+	formData map[string]string,
+	basePath string,
+) (buf *bytes.Buffer, cType string, err error) {
+	buf = &bytes.Buffer{}
+	w := multipart.NewWriter(buf)
+	var fw io.Writer
+
+	err = mapsutil.OrderedRangeError(formData, w.WriteField)
+	if err != nil {
+		return nil, "", fmt.Errorf("writing field: %w", err)
+	}
+
+	file, err := os.Open(basePath)
+	if err != nil {
+		return nil, "", fmt.Errorf("opening file: %w", err)
+	}
+
+	defer func() {
+		err = errors.WithDeferred(err, file.Close())
+	}()
+
+	h := make(textproto.MIMEHeader)
+	h.Set(httphdr.ContentType, aghhttp.HdrValApplicationJSON)
+
+	d := fmt.Sprintf("form-data; name=%q; filename=%q", "file", defaultBaseFile)
+	h.Set(httphdr.ContentDisposition, d)
+
+	fw, err = w.CreatePart(h)
+	if err != nil {
+		return nil, "", fmt.Errorf("creating part: %w", err)
+	}
+
+	_, err = io.Copy(fw, file)
+	if err != nil {
+		return nil, "", fmt.Errorf("copying: %w", err)
+	}
+
+	err = w.Close()
+	if err != nil {
+		return nil, "", fmt.Errorf("closing writer: %w", err)
+	}
+
+	return buf, w.FormDataContentType(), nil
+}
+
+// send POST request to uriStr.
+func send(uriStr, cType string, buf *bytes.Buffer) (err error) {
+	client := http.Client{
+		Timeout: uploadTimeout,
+	}
+
+	req, err := http.NewRequest(http.MethodPost, uriStr, buf)
+	if err != nil {
+		return fmt.Errorf("bad request: %w", err)
+	}
+
+	req.Header.Set(httphdr.ContentType, cType)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("client post form: %w", err)
+	}
+
+	defer func() {
+		err = errors.WithDeferred(err, resp.Body.Close())
+	}()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode))
+	}
+
+	return nil
+}