Pull request 1918: upd-chlog

Squashed commit of the following:

commit d5d21a8dd3ca892b8c9ba3d6c2154a99933d6dc3
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Jul 12 16:07:53 2023 +0300

    all: upd chlog

.github/workflows/build.yml

@@ -1,7 +1,7 @@
 'name': 'build'
 
 'env':
-  'GO_VERSION': '1.19.10'
+  'GO_VERSION': '1.19.11'
   'NODE_VERSION': '14'
 
 'on':

.github/workflows/lint.yml

@@ -1,7 +1,7 @@
 'name': 'lint'
 
 'env':
-  'GO_VERSION': '1.19.10'
+  'GO_VERSION': '1.19.11'
 
 'on':
   'push':

CHANGELOG.md

@@ -14,15 +14,35 @@ and this project adheres to
 <!--
 ## [v0.108.0] - TBA
 
-## [v0.107.34] - 2023-07-26 (APPROX.)
+## [v0.107.35] - 2023-08-02 (APPROX.)
 
-See also the [v0.107.34 GitHub milestone][ms-v0.107.34].
+See also the [v0.107.35 GitHub milestone][ms-v0.107.35].
 
-[ms-v0.107.34]: https://github.com/AdguardTeam/AdGuardHome/milestone/69?closed=1
+[ms-v0.107.35]: https://github.com/AdguardTeam/AdGuardHome/milestone/70?closed=1
 
 NOTE: Add new changes BELOW THIS COMMENT.
 -->
 
+<!--
+NOTE: Add new changes ABOVE THIS COMMENT.
+-->
+
+
+
+## [v0.107.34] - 2023-07-12
+
+See also the [v0.107.34 GitHub milestone][ms-v0.107.34].
+
+### Security
+
+- Go version has been updated to prevent the possibility of exploiting the
+  CVE-2023-29406 Go vulnerability fixed in [Go 1.19.11][go-1.19.11].
+
+### Added
+
+- Ability to ignore queries for the root domain, such as `NS .` queries
+  ([#5990]).
+
 ### Changed
 
 - Improved CPU and RAM consumption during updates of filtering-rule lists.
@@ -32,8 +52,9 @@ NOTE: Add new changes BELOW THIS COMMENT.
 In this release, the schema version has changed from 23 to 24.
 
 - Properties starting with `log_`, and `verbose` property, which used to set up
-  logging are now moved to the new object `log` containing new properties `file`,
-  `max_backups`, `max_size`, `max_age`, `compress`, `local_time`, and `verbose`:
+  logging are now moved to the new object `log` containing new properties
+  `file`, `max_backups`, `max_size`, `max_age`, `compress`, `local_time`, and
+  `verbose`:
 
   ```yaml
   # BEFORE:
@@ -47,13 +68,13 @@ In this release, the schema version has changed from 23 to 24.
 
   # AFTER:
   'log':
-    'file': ""
-    'max_backups': 0
-    'max_size': 100
-    'max_age': 3
-    'compress': false
-    'local_time': false
-    'verbose': false
+      'file': ""
+      'max_backups': 0
+      'max_size': 100
+      'max_age': 3
+      'compress': false
+      'local_time': false
+      'verbose': false
   ```
 
   To rollback this change, remove the new object `log`, set back `log_` and
@@ -83,10 +104,10 @@ In this release, the schema version has changed from 23 to 24.
 
 [#5896]: https://github.com/AdguardTeam/AdGuardHome/issues/5896
 [#5972]: https://github.com/AdguardTeam/AdGuardHome/issues/5972
+[#5990]: https://github.com/AdguardTeam/AdGuardHome/issues/5990
 
-<!--
-NOTE: Add new changes ABOVE THIS COMMENT.
--->
+[go-1.19.11]:   https://groups.google.com/g/golang-announce/c/2q13H6LEEx0/m/sduSepLLBwAJ
+[ms-v0.107.34]: https://github.com/AdguardTeam/AdGuardHome/milestone/69?closed=1
 
 
 
@@ -2221,11 +2242,12 @@ See also the [v0.104.2 GitHub milestone][ms-v0.104.2].
 
 
 <!--
-[Unreleased]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.34...HEAD
-[v0.107.34]:  https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.33...v0.107.34
+[Unreleased]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.35...HEAD
+[v0.107.35]:  https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.34...v0.107.35
 -->
 
-[Unreleased]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.33...HEAD
+[Unreleased]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.34...HEAD
+[v0.107.34]:  https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.33...v0.107.34
 [v0.107.33]:  https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.32...v0.107.33
 [v0.107.32]:  https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.31...v0.107.32
 [v0.107.31]:  https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.30...v0.107.31

bamboo-specs/release.yaml

@@ -7,7 +7,7 @@
 # Make sure to sync any changes with the branch overrides below.
 'variables':
     'channel': 'edge'
-    'dockerGo': 'adguard/golang-ubuntu:6.7'
+    'dockerGo': 'adguard/golang-ubuntu:6.8'
 
 'stages':
   - 'Build frontend':
@@ -272,7 +272,7 @@
         # need to build a few of these.
         'variables':
             'channel': 'beta'
-            'dockerGo': 'adguard/golang-ubuntu:6.7'
+            'dockerGo': 'adguard/golang-ubuntu:6.8'
     # release-vX.Y.Z branches are the branches from which the actual final
     # release is built.
   - '^release-v[0-9]+\.[0-9]+\.[0-9]+':
@@ -287,4 +287,4 @@
         # are the ones that actually get released.
         'variables':
             'channel': 'release'
-            'dockerGo': 'adguard/golang-ubuntu:6.7'
+            'dockerGo': 'adguard/golang-ubuntu:6.8'

bamboo-specs/snapcraft.yaml

@@ -10,7 +10,7 @@
 # Make sure to sync any changes with the branch overrides below.
 'variables':
     'channel': 'edge'
-    'dockerGo': 'adguard/golang-ubuntu:6.7'
+    'dockerGo': 'adguard/golang-ubuntu:6.8'
     'snapcraftChannel': 'edge'
 
 'stages':
@@ -191,7 +191,7 @@
         # need to build a few of these.
         'variables':
             'channel': 'beta'
-            'dockerGo': 'adguard/golang-ubuntu:6.7'
+            'dockerGo': 'adguard/golang-ubuntu:6.8'
             'snapcraftChannel': 'beta'
     # release-vX.Y.Z branches are the branches from which the actual final
     # release is built.
@@ -207,5 +207,5 @@
         # are the ones that actually get released.
         'variables':
             'channel': 'release'
-            'dockerGo': 'adguard/golang-ubuntu:6.7'
+            'dockerGo': 'adguard/golang-ubuntu:6.8'
             'snapcraftChannel': 'candidate'

bamboo-specs/test.yaml

@@ -5,7 +5,7 @@
     'key': 'AHBRTSPECS'
     'name': 'AdGuard Home - Build and run tests'
 'variables':
-    'dockerGo': 'adguard/golang-ubuntu:6.7'
+    'dockerGo': 'adguard/golang-ubuntu:6.8'
 
 'stages':
   - 'Tests':

go.mod

@@ -5,7 +5,7 @@ go 1.19
 require (
 	// TODO(a.garipov): Update to a tagged version when it's released.
 	github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768
-	github.com/AdguardTeam/golibs v0.13.3
+	github.com/AdguardTeam/golibs v0.13.4
 	github.com/AdguardTeam/urlfilter v0.16.1
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/ameshkov/dnscrypt/v2 v2.2.7

go.sum

@@ -2,8 +2,8 @@ github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768 h1:5Ia6wA+
 github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768/go.mod h1:CQhZTkqC8X0ID6glrtyaxgqRRdiYfn1gJulC1cZ5Dn8=
 github.com/AdguardTeam/golibs v0.4.0/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4=
 github.com/AdguardTeam/golibs v0.10.4/go.mod h1:rSfQRGHIdgfxriDDNgNJ7HmE5zRoURq8R+VdR81Zuzw=
-github.com/AdguardTeam/golibs v0.13.3 h1:RT3QbzThtaLiFLkIUDS6/hlGEXrh0zYvdf4bd7UWpGo=
-github.com/AdguardTeam/golibs v0.13.3/go.mod h1:wkJ6EUsN4np/9Gp7+9QeooY9E2U2WCLJYAioLCzkHsI=
+github.com/AdguardTeam/golibs v0.13.4 h1:ACTwIR1pEENBijHcEWtiMbSh4wWQOlIHRxmUB8oBHf8=
+github.com/AdguardTeam/golibs v0.13.4/go.mod h1:wkJ6EUsN4np/9Gp7+9QeooY9E2U2WCLJYAioLCzkHsI=
 github.com/AdguardTeam/gomitmproxy v0.2.0/go.mod h1:Qdv0Mktnzer5zpdpi5rAwixNJzW2FN91LjKJCkVbYGU=
 github.com/AdguardTeam/urlfilter v0.16.1 h1:ZPi0rjqo8cQf2FVdzo6cqumNoHZx2KPXj2yZa1A5BBw=
 github.com/AdguardTeam/urlfilter v0.16.1/go.mod h1:46YZDOV1+qtdRDuhZKVPSSp7JWWes0KayqHrKAFBdEI=

internal/aghnet/addr.go

@@ -0,0 +1,43 @@
+package aghnet
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/AdguardTeam/golibs/stringutil"
+)
+
+// NormalizeDomain returns a lowercased version of host without the final dot,
+// unless host is ".", in which case it returns it unchanged.  That is a special
+// case that to allow matching queries like:
+//
+//	dig IN NS '.'
+func NormalizeDomain(host string) (norm string) {
+	if host == "." {
+		return host
+	}
+
+	return strings.ToLower(strings.TrimSuffix(host, "."))
+}
+
+// NewDomainNameSet returns nil and error, if list has duplicate or empty domain
+// name.  Otherwise returns a set, which contains domain names normalized using
+// [NormalizeDomain].
+func NewDomainNameSet(list []string) (set *stringutil.Set, err error) {
+	set = stringutil.NewSet()
+
+	for i, host := range list {
+		if host == "" {
+			return nil, fmt.Errorf("at index %d: hostname is empty", i)
+		}
+
+		host = NormalizeDomain(host)
+		if set.Has(host) {
+			return nil, fmt.Errorf("duplicate hostname %q at index %d", host, i)
+		}
+
+		set.Add(host)
+	}
+
+	return set, nil
+}

internal/aghnet/addr_test.go

@@ -0,0 +1,59 @@
+package aghnet_test
+
+import (
+	"testing"
+
+	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
+	"github.com/AdguardTeam/golibs/testutil"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewDomainNameSet(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		name       string
+		wantErrMsg string
+		in         []string
+	}{{
+		name:       "nil",
+		wantErrMsg: "",
+		in:         nil,
+	}, {
+		name:       "success",
+		wantErrMsg: "",
+		in: []string{
+			"Domain.Example",
+			".",
+		},
+	}, {
+		name:       "dups",
+		wantErrMsg: `duplicate hostname "domain.example" at index 1`,
+		in: []string{
+			"Domain.Example",
+			"domain.example",
+		},
+	}, {
+		name:       "bad_domain",
+		wantErrMsg: "at index 0: hostname is empty",
+		in: []string{
+			"",
+		},
+	}}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			set, err := aghnet.NewDomainNameSet(tc.in)
+			testutil.AssertErrorMsg(t, tc.wantErrMsg, err)
+			if err != nil {
+				return
+			}
+
+			for _, host := range tc.in {
+				assert.Truef(t, set.Has(aghnet.NormalizeDomain(host)), "%q not matched", host)
+			}
+		})
+	}
+}

internal/aghnet/hostgen.go

@@ -1,12 +1,8 @@
 package aghnet
 
 import (
-	"fmt"
 	"net/netip"
 	"strings"
-
-	"github.com/AdguardTeam/golibs/errors"
-	"github.com/AdguardTeam/golibs/stringutil"
 )
 
 // GenerateHostname generates the hostname from ip.  In case of using IPv4 the
@@ -29,32 +25,8 @@ func GenerateHostname(ip netip.Addr) (hostname string) {
 	hostname = ip.StringExpanded()
 
 	if ip.Is4() {
-		return strings.Replace(hostname, ".", "-", -1)
+		return strings.ReplaceAll(hostname, ".", "-")
 	}
 
-	return strings.Replace(hostname, ":", "-", -1)
-}
-
-// NewDomainNameSet returns nil and error, if list has duplicate or empty
-// domain name.  Otherwise returns a set, which contains non-FQDN domain names,
-// and nil error.
-func NewDomainNameSet(list []string) (set *stringutil.Set, err error) {
-	set = stringutil.NewSet()
-
-	for i, v := range list {
-		host := strings.ToLower(strings.TrimSuffix(v, "."))
-		// TODO(a.garipov): Think about ignoring empty (".") names in the
-		// future.
-		if host == "" {
-			return nil, errors.Error("host name is empty")
-		}
-
-		if set.Has(host) {
-			return nil, fmt.Errorf("duplicate host name %q at index %d", host, i)
-		}
-
-		set.Add(host)
-	}
-
-	return set, nil
+	return strings.ReplaceAll(hostname, ":", "-")
 }

internal/dnsforward/stats.go

@@ -2,9 +2,9 @@ package dnsforward
 
 import (
 	"net"
-	"strings"
 	"time"
 
+	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
 	"github.com/AdguardTeam/AdGuardHome/internal/filtering"
 	"github.com/AdguardTeam/AdGuardHome/internal/querylog"
 	"github.com/AdguardTeam/AdGuardHome/internal/stats"
@@ -24,7 +24,7 @@ func (s *Server) processQueryLogsAndStats(dctx *dnsContext) (rc resultCode) {
 	pctx := dctx.proxyCtx
 
 	q := pctx.Req.Question[0]
-	host := strings.ToLower(strings.TrimSuffix(q.Name, "."))
+	host := aghnet.NormalizeDomain(q.Name)
 
 	ip, _ := netutil.IPAndPortFromAddr(pctx.Addr)
 	ip = slices.Clone(ip)
@@ -139,11 +139,10 @@ func (s *Server) updateStats(
 	clientIP string,
 ) {
 	pctx := ctx.proxyCtx
-	e := stats.Entry{}
-	e.Domain = strings.ToLower(pctx.Req.Question[0].Name)
-	if e.Domain != "." {
-		// Remove last ".", but save the domain as is for "." queries.
-		e.Domain = e.Domain[:len(e.Domain)-1]
+	e := stats.Entry{
+		Domain: aghnet.NormalizeDomain(pctx.Req.Question[0].Name),
+		Result: stats.RNotFiltered,
+		Time:   uint32(elapsed / 1000),
 	}
 
 	if clientID := ctx.clientID; clientID != "" {
@@ -152,9 +151,6 @@ func (s *Server) updateStats(
 		e.Client = clientIP
 	}
 
-	e.Time = uint32(elapsed / 1000)
-	e.Result = stats.RNotFiltered
-
 	switch res.Reason {
 	case filtering.FilteredSafeBrowsing:
 		e.Result = stats.RSafeBrowsing
@@ -162,7 +158,8 @@ func (s *Server) updateStats(
 		e.Result = stats.RParental
 	case filtering.FilteredSafeSearch:
 		e.Result = stats.RSafeSearch
-	case filtering.FilteredBlockList,
+	case
+		filtering.FilteredBlockList,
 		filtering.FilteredInvalid,
 		filtering.FilteredBlockedService:
 		e.Result = stats.RFiltered

internal/home/config.go

@@ -240,6 +240,7 @@ type tlsConfigSettings struct {
 
 type queryLogConfig struct {
 	// Ignored is the list of host names, which should not be written to log.
+	// "." is considered to be the root domain.
 	Ignored []string `yaml:"ignored"`
 
 	// Interval is the interval for query log's files rotation.

internal/querylog/qlog.go

@@ -4,7 +4,6 @@ package querylog
 import (
 	"fmt"
 	"os"
-	"strings"
 	"sync"
 	"time"
 
@@ -161,10 +160,7 @@ func (l *queryLog) clear() {
 // newLogEntry creates an instance of logEntry from parameters.
 func newLogEntry(params *AddParams) (entry *logEntry) {
 	q := params.Question.Question[0]
-	qHost := q.Name
-	if qHost != "." {
-		qHost = strings.ToLower(q.Name[:len(q.Name)-1])
-	}
+	qHost := aghnet.NormalizeDomain(q.Name)
 
 	entry = &logEntry{
 		// TODO(d.kolyshev): Export this timestamp to func params.

internal/stats/http_test.go

@@ -86,7 +86,7 @@ func TestHandleStatsConfig(t *testing.T) {
 			},
 		},
 		wantCode: http.StatusUnprocessableEntity,
-		wantErr:  "ignored: duplicate host name \"ignor.ed\" at index 1\n",
+		wantErr:  "ignored: duplicate hostname \"ignor.ed\" at index 1\n",
 	}, {
 		name: "ignored_empty",
 		body: getConfigResp{
@@ -97,7 +97,7 @@ func TestHandleStatsConfig(t *testing.T) {
 			},
 		},
 		wantCode: http.StatusUnprocessableEntity,
-		wantErr:  "ignored: host name is empty\n",
+		wantErr:  "ignored: at index 0: hostname is empty\n",
 	}, {
 		name: "enabled_is_null",
 		body: getConfigResp{

scripts/README.md

@@ -269,25 +269,29 @@ Optional environment:
 
    ###  Usage
 
- *  `go run main.go help`: print usage.
+ *  `go run ./scripts/translations help`: print usage.
 
- *  `go run main.go download [-n <count>]`: download and save all translations.
-    `n` is optional flag where count is a number of concurrent downloads.
+ *  `go run ./scripts/translations download [-n <count>]`: download and save
+    all translations.  `n` is optional flag where count is a number of
+    concurrent downloads.
 
- *  `go run main.go upload`: upload the base `en` locale.
+ *  `go run ./scripts/translations upload`: upload the base `en` locale.
 
- *  `go run main.go summary`: show the current locales summary.
+ *  `go run ./scripts/translations summary`: show the current locales summary.
 
- *  `go run main.go unused`: show the list of unused strings.
+ *  `go run ./scripts/translations unused`: show the list of unused strings.
 
- *  `go run main.go auto-add`: add locales with additions to the git and
-    restore locales with deletions.
+ *  `go run ./scripts/translations auto-add`: add locales with additions to the
+    git and restore locales with deletions.
 
 After the download you'll find the output locales in the `client/src/__locales/`
 directory.
 
 Optional environment:
 
+ *  `DOWNLOAD_LANGUAGES`: set a list of specific languages to `download`.  For
+    example `ar be bg`.
+
  *  `UPLOAD_LANGUAGE`: set an alternative language for `upload`.
 
  *  `TWOSKY_URI`: set an alternative URL for `download` or `upload`.

scripts/make/go-lint.sh

@@ -35,7 +35,7 @@ set -f -u
 go_version="$( "${GO:-go}" version )"
 readonly go_version
 
-go_min_version='go1.19.10'
+go_min_version='go1.19.11'
 go_version_msg="
 warning: your go version (${go_version}) is different from the recommended minimal one (${go_min_version}).
 if you have the version installed, please set the GO environment variable.
@@ -183,6 +183,7 @@ run_linter gocognit --over 10\
 	./internal/tools/\
 	./internal/version/\
 	./internal/whois/\
+	./scripts/\
 	;
 
 run_linter ineffassign ./...

scripts/translations/download.go

@@ -0,0 +1,177 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/AdguardTeam/AdGuardHome/internal/aghio"
+	"github.com/AdguardTeam/golibs/errors"
+	"github.com/AdguardTeam/golibs/log"
+	"golang.org/x/exp/slices"
+)
+
+// download and save all translations.
+func (c *twoskyClient) download() (err error) {
+	var numWorker int
+
+	flagSet := flag.NewFlagSet("download", flag.ExitOnError)
+	flagSet.Usage = func() {
+		usage("download command error")
+	}
+	flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")
+
+	err = flagSet.Parse(os.Args[2:])
+	if err != nil {
+		// Don't wrap the error since it's informative enough as is.
+		return err
+	}
+
+	if numWorker < 1 {
+		usage("count must be positive")
+	}
+
+	downloadURI := c.uri.JoinPath("download")
+
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+
+	wg := &sync.WaitGroup{}
+	failed := &sync.Map{}
+	uriCh := make(chan *url.URL, len(c.langs))
+
+	for i := 0; i < numWorker; i++ {
+		wg.Add(1)
+		go downloadWorker(wg, failed, client, uriCh)
+	}
+
+	for lang := range c.langs {
+		uri := translationURL(downloadURI, defaultBaseFile, c.projectID, lang)
+
+		uriCh <- uri
+	}
+
+	close(uriCh)
+	wg.Wait()
+
+	printFailedLocales(failed)
+
+	return nil
+}
+
+// printFailedLocales prints sorted list of failed downloads, if any.
+func printFailedLocales(failed *sync.Map) {
+	keys := []string{}
+	failed.Range(func(k, _ any) bool {
+		s, ok := k.(string)
+		if !ok {
+			panic("unexpected type")
+		}
+
+		keys = append(keys, s)
+
+		return true
+	})
+
+	if len(keys) == 0 {
+		return
+	}
+
+	slices.Sort(keys)
+	log.Info("failed locales: %s", strings.Join(keys, " "))
+}
+
+// downloadWorker downloads translations by received urls and saves them.
+// Where failed is a map for storing failed downloads.
+func downloadWorker(
+	wg *sync.WaitGroup,
+	failed *sync.Map,
+	client *http.Client,
+	uriCh <-chan *url.URL,
+) {
+	defer wg.Done()
+
+	for uri := range uriCh {
+		q := uri.Query()
+		code := q.Get("language")
+
+		err := saveToFile(client, uri, code)
+		if err != nil {
+			log.Error("download: worker: %s", err)
+			failed.Store(code, struct{}{})
+		}
+	}
+}
+
+// saveToFile downloads translation by url and saves it to a file, or returns
+// error.
+func saveToFile(client *http.Client, uri *url.URL, code string) (err error) {
+	data, err := getTranslation(client, uri.String())
+	if err != nil {
+		log.Info("%s", data)
+
+		return fmt.Errorf("getting translation: %s", err)
+	}
+
+	name := filepath.Join(localesDir, code+".json")
+	err = os.WriteFile(name, data, 0o664)
+	if err != nil {
+		return fmt.Errorf("writing file: %s", err)
+	}
+
+	fmt.Println(name)
+
+	return nil
+}
+
+// getTranslation returns received translation data and error.  If err is not
+// nil, data may contain a response from server for inspection.
+func getTranslation(client *http.Client, url string) (data []byte, err error) {
+	resp, err := client.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("requesting: %w", err)
+	}
+
+	defer log.OnCloserError(resp.Body, log.ERROR)
+
+	if resp.StatusCode != http.StatusOK {
+		err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
+
+		// Go on and download the body for inspection.
+	}
+
+	limitReader, lrErr := aghio.LimitReader(resp.Body, readLimit)
+	if lrErr != nil {
+		// Generally shouldn't happen, since the only error returned by
+		// [aghio.LimitReader] is an argument error.
+		panic(fmt.Errorf("limit reading: %w", lrErr))
+	}
+
+	data, readErr := io.ReadAll(limitReader)
+
+	return data, errors.WithDeferred(err, readErr)
+}
+
+// translationURL returns a new url.URL with provided query parameters.
+func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
+	uri = &url.URL{}
+	*uri = *oldURL
+
+	q := uri.Query()
+	q.Set("format", "json")
+	q.Set("filename", baseFile)
+	q.Set("project", projectID)
+	q.Set("language", string(lang))
+
+	uri.RawQuery = q.Encode()
+
+	return uri
+}

scripts/translations/main.go

@@ -6,25 +6,16 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/json"
-	"flag"
 	"fmt"
-	"io"
-	"mime/multipart"
-	"net/http"
-	"net/textproto"
 	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
-	"sync"
 	"time"
 
-	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
-	"github.com/AdguardTeam/AdGuardHome/internal/aghio"
 	"github.com/AdguardTeam/AdGuardHome/internal/aghos"
 	"github.com/AdguardTeam/golibs/errors"
-	"github.com/AdguardTeam/golibs/httphdr"
 	"github.com/AdguardTeam/golibs/log"
 	"golang.org/x/exp/maps"
 	"golang.org/x/exp/slices"
@@ -38,7 +29,8 @@ const (
 	srcDir           = "./client/src"
 	twoskyURI        = "https://twosky.int.agrd.dev/api/v1"
 
-	readLimit = 1 * 1024 * 1024
+	readLimit     = 1 * 1024 * 1024
+	uploadTimeout = 10 * time.Second
 )
 
 // langCode is a language code.
@@ -62,31 +54,26 @@ func main() {
 		usage("")
 	}
 
-	uriStr := os.Getenv("TWOSKY_URI")
-	if uriStr == "" {
-		uriStr = twoskyURI
-	}
-
-	uri, err := url.Parse(uriStr)
+	conf, err := readTwoskyConfig()
 	check(err)
 
-	projectID := os.Getenv("TWOSKY_PROJECT_ID")
-	if projectID == "" {
-		projectID = defaultProjectID
-	}
-
-	conf, err := readTwoskyConf()
-	check(err)
+	var cli *twoskyClient
 
 	switch os.Args[1] {
 	case "summary":
 		err = summary(conf.Languages)
 	case "download":
-		err = download(uri, projectID, conf.Languages)
+		cli, err = conf.toClient()
+		check(err)
+
+		err = cli.download()
 	case "unused":
 		err = unused(conf.LocalizableFiles[0])
 	case "upload":
-		err = upload(uri, projectID, conf.BaseLangcode)
+		cli, err = conf.toClient()
+		check(err)
+
+		err = cli.upload()
 	case "auto-add":
 		err = autoAdd(conf.LocalizableFiles[0])
 	default:
@@ -133,51 +120,131 @@ Commands:
 	os.Exit(0)
 }
 
-// twoskyConf is the configuration structure for localization.
-type twoskyConf struct {
+// twoskyConfig is the configuration structure for localization.
+type twoskyConfig struct {
 	Languages        languages `json:"languages"`
 	ProjectID        string    `json:"project_id"`
 	BaseLangcode     langCode  `json:"base_locale"`
 	LocalizableFiles []string  `json:"localizable_files"`
 }
 
-// readTwoskyConf returns configuration.
-func readTwoskyConf() (t twoskyConf, err error) {
-	defer func() { err = errors.Annotate(err, "parsing twosky conf: %w") }()
+// readTwoskyConfig returns twosky configuration.
+func readTwoskyConfig() (t *twoskyConfig, err error) {
+	defer func() { err = errors.Annotate(err, "parsing twosky config: %w") }()
 
 	b, err := os.ReadFile(twoskyConfFile)
 	if err != nil {
 		// Don't wrap the error since it's informative enough as is.
-		return twoskyConf{}, err
+		return nil, err
 	}
 
-	var tsc []twoskyConf
+	var tsc []twoskyConfig
 	err = json.Unmarshal(b, &tsc)
 	if err != nil {
 		err = fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err)
 
-		return twoskyConf{}, err
+		return nil, err
 	}
 
 	if len(tsc) == 0 {
 		err = fmt.Errorf("%q is empty", twoskyConfFile)
 
-		return twoskyConf{}, err
+		return nil, err
 	}
 
 	conf := tsc[0]
 
 	for _, lang := range conf.Languages {
 		if lang == "" {
-			return twoskyConf{}, errors.Error("language is empty")
+			return nil, errors.Error("language is empty")
 		}
 	}
 
 	if len(conf.LocalizableFiles) == 0 {
-		return twoskyConf{}, errors.Error("no localizable files specified")
+		return nil, errors.Error("no localizable files specified")
 	}
 
-	return conf, nil
+	return &conf, nil
+}
+
+// twoskyClient is the twosky client with methods for download and upload
+// translations.
+type twoskyClient struct {
+	// uri is the base URL.
+	uri *url.URL
+
+	// langs is the map of languages to download.
+	langs languages
+
+	// projectID is the name of the project.
+	projectID string
+
+	// baseLang is the base language code.
+	baseLang langCode
+}
+
+// toClient reads values from environment variables or defaults, validates
+// them, and returns the twosky client.
+func (t *twoskyConfig) toClient() (cli *twoskyClient, err error) {
+	defer func() { err = errors.Annotate(err, "filling config: %w") }()
+
+	uriStr := os.Getenv("TWOSKY_URI")
+	if uriStr == "" {
+		uriStr = twoskyURI
+	}
+	uri, err := url.Parse(uriStr)
+	if err != nil {
+		return nil, err
+	}
+
+	projectID := os.Getenv("TWOSKY_PROJECT_ID")
+	if projectID == "" {
+		projectID = defaultProjectID
+	}
+
+	baseLang := t.BaseLangcode
+	uLangStr := os.Getenv("UPLOAD_LANGUAGE")
+	if uLangStr != "" {
+		baseLang = langCode(uLangStr)
+	}
+
+	langs := t.Languages
+	dlLangStr := os.Getenv("DOWNLOAD_LANGUAGES")
+	if dlLangStr != "" {
+		var dlLangs languages
+		dlLangs, err = validateLanguageStr(dlLangStr, langs)
+		if err != nil {
+			return nil, err
+		}
+
+		langs = dlLangs
+	}
+
+	return &twoskyClient{
+		uri:       uri,
+		projectID: projectID,
+		baseLang:  baseLang,
+		langs:     langs,
+	}, nil
+}
+
+// validateLanguageStr validates languages codes that contain in the str and
+// returns language map, where key is language code and value is display name.
+func validateLanguageStr(str string, all languages) (langs languages, err error) {
+	langs = make(languages)
+	codes := strings.Fields(str)
+
+	for _, k := range codes {
+		lc := langCode(k)
+		name, ok := all[lc]
+		if !ok {
+			return nil, fmt.Errorf("validating languages: unexpected language code %q", k)
+		}
+
+		langs[lc] = name
+	}
+
+	return langs, nil
 }
 
 // readLocales reads file with name fn and returns a map, where key is text
@@ -233,163 +300,33 @@ func summary(langs languages) (err error) {
 	return nil
 }
 
-// download and save all translations.  uri is the base URL.  projectID is the
-// name of the project.
-func download(uri *url.URL, projectID string, langs languages) (err error) {
-	var numWorker int
+// unused prints unused text labels.
+func unused(basePath string) (err error) {
+	defer func() { err = errors.Annotate(err, "unused: %w") }()
 
-	flagSet := flag.NewFlagSet("download", flag.ExitOnError)
-	flagSet.Usage = func() {
-		usage("download command error")
-	}
-	flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")
-
-	err = flagSet.Parse(os.Args[2:])
+	baseLoc, err := readLocales(basePath)
 	if err != nil {
-		// Don't wrap the error since it's informative enough as is.
 		return err
 	}
 
-	if numWorker < 1 {
-		usage("count must be positive")
-	}
-
-	downloadURI := uri.JoinPath("download")
-
-	client := &http.Client{
-		Timeout: 10 * time.Second,
-	}
-
-	wg := &sync.WaitGroup{}
-	uriCh := make(chan *url.URL, len(langs))
-
-	for i := 0; i < numWorker; i++ {
-		wg.Add(1)
-		go downloadWorker(wg, client, uriCh)
-	}
-
-	for lang := range langs {
-		uri = translationURL(downloadURI, defaultBaseFile, projectID, lang)
-
-		uriCh <- uri
-	}
-
-	close(uriCh)
-	wg.Wait()
-
-	return nil
-}
-
-// downloadWorker downloads translations by received urls and saves them.
-func downloadWorker(wg *sync.WaitGroup, client *http.Client, uriCh <-chan *url.URL) {
-	defer wg.Done()
-
-	for uri := range uriCh {
-		data, err := getTranslation(client, uri.String())
-		if err != nil {
-			log.Error("download worker: getting translation: %s", err)
-			log.Info("download worker: error response:\n%s", data)
-
-			continue
-		}
-
-		q := uri.Query()
-		code := q.Get("language")
-
-		// Fix some TwoSky weirdnesses.
-		//
-		// TODO(a.garipov): Remove when those are fixed.
-		code = strings.ToLower(code)
-
-		name := filepath.Join(localesDir, code+".json")
-		err = os.WriteFile(name, data, 0o664)
-		if err != nil {
-			log.Error("download worker: writing file: %s", err)
-
-			continue
-		}
-
-		fmt.Println(name)
-	}
-}
-
-// getTranslation returns received translation data and error.  If err is not
-// nil, data may contain a response from server for inspection.
-func getTranslation(client *http.Client, url string) (data []byte, err error) {
-	resp, err := client.Get(url)
-	if err != nil {
-		return nil, fmt.Errorf("requesting: %w", err)
-	}
-
-	defer log.OnCloserError(resp.Body, log.ERROR)
-
-	if resp.StatusCode != http.StatusOK {
-		err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
-
-		// Go on and download the body for inspection.
-	}
-
-	limitReader, lrErr := aghio.LimitReader(resp.Body, readLimit)
-	if lrErr != nil {
-		// Generally shouldn't happen, since the only error returned by
-		// [aghio.LimitReader] is an argument error.
-		panic(fmt.Errorf("limit reading: %w", lrErr))
-	}
-
-	data, readErr := io.ReadAll(limitReader)
-
-	return data, errors.WithDeferred(err, readErr)
-}
-
-// translationURL returns a new url.URL with provided query parameters.
-func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
-	uri = &url.URL{}
-	*uri = *oldURL
-
-	// Fix some TwoSky weirdnesses.
-	//
-	// TODO(a.garipov): Remove when those are fixed.
-	switch lang {
-	case "si-lk":
-		lang = "si-LK"
-	case "zh-hk":
-		lang = "zh-HK"
-	default:
-		// Go on.
-	}
-
-	q := uri.Query()
-	q.Set("format", "json")
-	q.Set("filename", baseFile)
-	q.Set("project", projectID)
-	q.Set("language", string(lang))
-
-	uri.RawQuery = q.Encode()
-
-	return uri
-}
-
-// unused prints unused text labels.
-func unused(basePath string) (err error) {
-	baseLoc, err := readLocales(basePath)
-	if err != nil {
-		return fmt.Errorf("unused: %w", err)
-	}
-
 	locDir := filepath.Clean(localesDir)
+	js, err := findJS(locDir)
+	if err != nil {
+		return err
+	}
 
-	fileNames := []string{}
-	err = filepath.Walk(srcDir, func(name string, info os.FileInfo, err error) error {
+	return findUnused(js, baseLoc)
+}
+
+// findJS returns list of JavaScript and JSON files or error.
+func findJS(locDir string) (fileNames []string, err error) {
+	walkFn := func(name string, _ os.FileInfo, err error) error {
 		if err != nil {
 			log.Info("warning: accessing a path %q: %s", name, err)
 
 			return nil
 		}
 
-		if info.IsDir() {
-			return nil
-		}
-
 		if strings.HasPrefix(name, locDir) {
 			return nil
 		}
@@ -400,13 +337,14 @@ func unused(basePath string) (err error) {
 		}
 
 		return nil
-	})
-
-	if err != nil {
-		return fmt.Errorf("filepath walking %q: %w", srcDir, err)
 	}
 
-	return findUnused(fileNames, baseLoc)
+	err = filepath.Walk(srcDir, walkFn)
+	if err != nil {
+		return nil, fmt.Errorf("filepath walking %q: %w", srcDir, err)
+	}
+
+	return fileNames, nil
 }
 
 // findUnused prints unused text labels from fileNames.
@@ -445,118 +383,6 @@ func findUnused(fileNames []string, loc locales) (err error) {
 	return nil
 }
 
-// upload base translation.  uri is the base URL.  projectID is the name of the
-// project.  baseLang is the base language code.
-func upload(uri *url.URL, projectID string, baseLang langCode) (err error) {
-	defer func() { err = errors.Annotate(err, "upload: %w") }()
-
-	uploadURI := uri.JoinPath("upload")
-
-	lang := baseLang
-
-	langStr := os.Getenv("UPLOAD_LANGUAGE")
-	if langStr != "" {
-		lang = langCode(langStr)
-	}
-
-	basePath := filepath.Join(localesDir, defaultBaseFile)
-
-	formData := map[string]string{
-		"format":   "json",
-		"language": string(lang),
-		"filename": defaultBaseFile,
-		"project":  projectID,
-	}
-
-	buf, cType, err := prepareMultipartMsg(formData, basePath)
-	if err != nil {
-		return fmt.Errorf("preparing multipart msg: %w", err)
-	}
-
-	err = send(uploadURI.String(), cType, buf)
-	if err != nil {
-		return fmt.Errorf("sending multipart msg: %w", err)
-	}
-
-	return nil
-}
-
-// prepareMultipartMsg prepares translation data for upload.
-func prepareMultipartMsg(
-	formData map[string]string,
-	basePath string,
-) (buf *bytes.Buffer, cType string, err error) {
-	buf = &bytes.Buffer{}
-	w := multipart.NewWriter(buf)
-	var fw io.Writer
-
-	for k, v := range formData {
-		err = w.WriteField(k, v)
-		if err != nil {
-			return nil, "", fmt.Errorf("writing field: %w", err)
-		}
-	}
-
-	file, err := os.Open(basePath)
-	if err != nil {
-		return nil, "", fmt.Errorf("opening file: %w", err)
-	}
-
-	defer func() {
-		err = errors.WithDeferred(err, file.Close())
-	}()
-
-	h := make(textproto.MIMEHeader)
-	h.Set(httphdr.ContentType, aghhttp.HdrValApplicationJSON)
-
-	d := fmt.Sprintf("form-data; name=%q; filename=%q", "file", defaultBaseFile)
-	h.Set(httphdr.ContentDisposition, d)
-
-	fw, err = w.CreatePart(h)
-	if err != nil {
-		return nil, "", fmt.Errorf("creating part: %w", err)
-	}
-
-	_, err = io.Copy(fw, file)
-	if err != nil {
-		return nil, "", fmt.Errorf("copying: %w", err)
-	}
-
-	err = w.Close()
-	if err != nil {
-		return nil, "", fmt.Errorf("closing writer: %w", err)
-	}
-
-	return buf, w.FormDataContentType(), nil
-}
-
-// send POST request to uriStr.
-func send(uriStr, cType string, buf *bytes.Buffer) (err error) {
-	var client http.Client
-
-	req, err := http.NewRequest(http.MethodPost, uriStr, buf)
-	if err != nil {
-		return fmt.Errorf("bad request: %w", err)
-	}
-
-	req.Header.Set(httphdr.ContentType, cType)
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("client post form: %w", err)
-	}
-
-	defer func() {
-		err = errors.WithDeferred(err, resp.Body.Close())
-	}()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode))
-	}
-
-	return nil
-}
-
 // autoAdd adds locales with additions to the git and restores locales with
 // deletions.
 func autoAdd(basePath string) (err error) {
@@ -572,28 +398,48 @@ func autoAdd(basePath string) (err error) {
 		return errors.Error("base locale contains deletions")
 	}
 
-	var (
-		args []string
-		code int
-		out  []byte
-	)
-
-	if len(adds) > 0 {
-		args = append([]string{"add"}, adds...)
-		code, out, err = aghos.RunCommand("git", args...)
-
-		if err != nil || code != 0 {
-			return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err)
-		}
+	err = handleAdds(adds)
+	if err != nil {
+		// Don't wrap the error since it's informative enough as is.
+		return nil
 	}
 
-	if len(dels) > 0 {
-		args = append([]string{"restore"}, dels...)
-		code, out, err = aghos.RunCommand("git", args...)
+	err = handleDels(dels)
+	if err != nil {
+		// Don't wrap the error since it's informative enough as is.
+		return nil
+	}
 
-		if err != nil || code != 0 {
-			return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err)
-		}
+	return nil
+}
+
+// handleAdds adds locales with additions to the git.
+func handleAdds(locales []string) (err error) {
+	if len(locales) == 0 {
+		return nil
+	}
+
+	args := append([]string{"add"}, locales...)
+	code, out, err := aghos.RunCommand("git", args...)
+
+	if err != nil || code != 0 {
+		return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err)
+	}
+
+	return nil
+}
+
+// handleDels restores locales with deletions.
+func handleDels(locales []string) (err error) {
+	if len(locales) == 0 {
+		return nil
+	}
+
+	args := append([]string{"restore"}, locales...)
+	code, out, err := aghos.RunCommand("git", args...)
+
+	if err != nil || code != 0 {
+		return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err)
 	}
 
 	return nil

scripts/translations/upload.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/textproto"
+	"os"
+	"path/filepath"
+
+	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
+	"github.com/AdguardTeam/golibs/errors"
+	"github.com/AdguardTeam/golibs/httphdr"
+	"github.com/AdguardTeam/golibs/mapsutil"
+)
+
+// upload base translation.
+func (c *twoskyClient) upload() (err error) {
+	defer func() { err = errors.Annotate(err, "upload: %w") }()
+
+	uploadURI := c.uri.JoinPath("upload")
+	basePath := filepath.Join(localesDir, defaultBaseFile)
+
+	formData := map[string]string{
+		"format":   "json",
+		"language": string(c.baseLang),
+		"filename": defaultBaseFile,
+		"project":  c.projectID,
+	}
+
+	buf, cType, err := prepareMultipartMsg(formData, basePath)
+	if err != nil {
+		return fmt.Errorf("preparing multipart msg: %w", err)
+	}
+
+	err = send(uploadURI.String(), cType, buf)
+	if err != nil {
+		return fmt.Errorf("sending multipart msg: %w", err)
+	}
+
+	return nil
+}
+
+// prepareMultipartMsg prepares translation data for upload.
+func prepareMultipartMsg(
+	formData map[string]string,
+	basePath string,
+) (buf *bytes.Buffer, cType string, err error) {
+	buf = &bytes.Buffer{}
+	w := multipart.NewWriter(buf)
+	var fw io.Writer
+
+	err = mapsutil.OrderedRangeError(formData, w.WriteField)
+	if err != nil {
+		return nil, "", fmt.Errorf("writing field: %w", err)
+	}
+
+	file, err := os.Open(basePath)
+	if err != nil {
+		return nil, "", fmt.Errorf("opening file: %w", err)
+	}
+
+	defer func() {
+		err = errors.WithDeferred(err, file.Close())
+	}()
+
+	h := make(textproto.MIMEHeader)
+	h.Set(httphdr.ContentType, aghhttp.HdrValApplicationJSON)
+
+	d := fmt.Sprintf("form-data; name=%q; filename=%q", "file", defaultBaseFile)
+	h.Set(httphdr.ContentDisposition, d)
+
+	fw, err = w.CreatePart(h)
+	if err != nil {
+		return nil, "", fmt.Errorf("creating part: %w", err)
+	}
+
+	_, err = io.Copy(fw, file)
+	if err != nil {
+		return nil, "", fmt.Errorf("copying: %w", err)
+	}
+
+	err = w.Close()
+	if err != nil {
+		return nil, "", fmt.Errorf("closing writer: %w", err)
+	}
+
+	return buf, w.FormDataContentType(), nil
+}
+
+// send POST request to uriStr.
+func send(uriStr, cType string, buf *bytes.Buffer) (err error) {
+	client := http.Client{
+		Timeout: uploadTimeout,
+	}
+
+	req, err := http.NewRequest(http.MethodPost, uriStr, buf)
+	if err != nil {
+		return fmt.Errorf("bad request: %w", err)
+	}
+
+	req.Header.Set(httphdr.ContentType, cType)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("client post form: %w", err)
+	}
+
+	defer func() {
+		err = errors.WithDeferred(err, resp.Body.Close())
+	}()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode))
+	}
+
+	return nil
+}