client: fix dashboard table styles

.github/workflows/build.yml

@@ -1,7 +1,7 @@
 'name': 'build'
 
 'env':
-  'GO_VERSION': '1.19.11'
+  'GO_VERSION': '1.19.10'
   'NODE_VERSION': '14'
 
 'on':

.github/workflows/lint.yml

@@ -1,7 +1,7 @@
 'name': 'lint'
 
 'env':
-  'GO_VERSION': '1.19.11'
+  'GO_VERSION': '1.19.10'
 
 'on':
   'push':

CHANGELOG.md

@@ -14,34 +14,14 @@ and this project adheres to
 <!--
 ## [v0.108.0] - TBA
 
-## [v0.107.35] - 2023-08-02 (APPROX.)
+## [v0.107.34] - 2023-07-26 (APPROX.)
-
-See also the [v0.107.35 GitHub milestone][ms-v0.107.35].
-
-[ms-v0.107.35]: https://github.com/AdguardTeam/AdGuardHome/milestone/70?closed=1
-
-NOTE: Add new changes BELOW THIS COMMENT.
--->
-
-<!--
-NOTE: Add new changes ABOVE THIS COMMENT.
--->
-
-
-
-## [v0.107.34] - 2023-07-12
 
 See also the [v0.107.34 GitHub milestone][ms-v0.107.34].
 
-### Security
+[ms-v0.107.34]: https://github.com/AdguardTeam/AdGuardHome/milestone/69?closed=1
 
-- Go version has been updated to prevent the possibility of exploiting the
+NOTE: Add new changes BELOW THIS COMMENT.
-  CVE-2023-29406 Go vulnerability fixed in [Go 1.19.11][go-1.19.11].
+-->
-
-### Added
-
-- Ability to ignore queries for the root domain, such as `NS .` queries
-  ([#5990]).
 
 ### Changed
 
@@ -52,9 +32,8 @@ See also the [v0.107.34 GitHub milestone][ms-v0.107.34].
 In this release, the schema version has changed from 23 to 24.
 
 - Properties starting with `log_`, and `verbose` property, which used to set up
-  logging are now moved to the new object `log` containing new properties
+  logging are now moved to the new object `log` containing new properties `file`,
-  `file`, `max_backups`, `max_size`, `max_age`, `compress`, `local_time`, and
+  `max_backups`, `max_size`, `max_age`, `compress`, `local_time`, and `verbose`:
-  `verbose`:
 
   ```yaml
   # BEFORE:
@@ -68,13 +47,13 @@ In this release, the schema version has changed from 23 to 24.
 
   # AFTER:
   'log':
-      'file': ""
+    'file': ""
-      'max_backups': 0
+    'max_backups': 0
-      'max_size': 100
+    'max_size': 100
-      'max_age': 3
+    'max_age': 3
-      'compress': false
+    'compress': false
-      'local_time': false
+    'local_time': false
-      'verbose': false
+    'verbose': false
   ```
 
   To rollback this change, remove the new object `log`, set back `log_` and
@@ -87,8 +66,6 @@ In this release, the schema version has changed from 23 to 24.
 
 ### Fixed
 
-- Two unspecified IPs when a host is blocked in two filter lists ([#5972]).
-- Incorrect setting of Parental Control cache size.
 - Excessive RAM and CPU consumption by Safe Browsing and Parental Control
   filters ([#5896]).
 
@@ -103,11 +80,10 @@ In this release, the schema version has changed from 23 to 24.
   image, and reload it from scratch.
 
 [#5896]: https://github.com/AdguardTeam/AdGuardHome/issues/5896
-[#5972]: https://github.com/AdguardTeam/AdGuardHome/issues/5972
-[#5990]: https://github.com/AdguardTeam/AdGuardHome/issues/5990
 
-[go-1.19.11]:   https://groups.google.com/g/golang-announce/c/2q13H6LEEx0/m/sduSepLLBwAJ
+<!--
-[ms-v0.107.34]: https://github.com/AdguardTeam/AdGuardHome/milestone/69?closed=1
+NOTE: Add new changes ABOVE THIS COMMENT.
+-->
 
 
 
@@ -2242,12 +2218,11 @@ See also the [v0.104.2 GitHub milestone][ms-v0.104.2].
 
 
 <!--
-[Unreleased]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.35...HEAD
-[v0.107.35]:  https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.34...v0.107.35
--->
-
 [Unreleased]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.34...HEAD
 [v0.107.34]:  https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.33...v0.107.34
+-->
+
+[Unreleased]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.33...HEAD
 [v0.107.33]:  https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.32...v0.107.33
 [v0.107.32]:  https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.31...v0.107.32
 [v0.107.31]:  https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.30...v0.107.31

bamboo-specs/release.yaml

@@ -7,7 +7,7 @@
 # Make sure to sync any changes with the branch overrides below.
 'variables':
     'channel': 'edge'
-    'dockerGo': 'adguard/golang-ubuntu:6.8'
+    'dockerGo': 'adguard/golang-ubuntu:6.7'
 
 'stages':
   - 'Build frontend':
@@ -272,7 +272,7 @@
         # need to build a few of these.
         'variables':
             'channel': 'beta'
-            'dockerGo': 'adguard/golang-ubuntu:6.8'
+            'dockerGo': 'adguard/golang-ubuntu:6.7'
     # release-vX.Y.Z branches are the branches from which the actual final
     # release is built.
   - '^release-v[0-9]+\.[0-9]+\.[0-9]+':
@@ -287,4 +287,4 @@
         # are the ones that actually get released.
         'variables':
             'channel': 'release'
-            'dockerGo': 'adguard/golang-ubuntu:6.8'
+            'dockerGo': 'adguard/golang-ubuntu:6.7'

bamboo-specs/snapcraft.yaml

@@ -10,7 +10,7 @@
 # Make sure to sync any changes with the branch overrides below.
 'variables':
     'channel': 'edge'
-    'dockerGo': 'adguard/golang-ubuntu:6.8'
+    'dockerGo': 'adguard/golang-ubuntu:6.7'
     'snapcraftChannel': 'edge'
 
 'stages':
@@ -191,7 +191,7 @@
         # need to build a few of these.
         'variables':
             'channel': 'beta'
-            'dockerGo': 'adguard/golang-ubuntu:6.8'
+            'dockerGo': 'adguard/golang-ubuntu:6.7'
             'snapcraftChannel': 'beta'
     # release-vX.Y.Z branches are the branches from which the actual final
     # release is built.
@@ -207,5 +207,5 @@
         # are the ones that actually get released.
         'variables':
             'channel': 'release'
-            'dockerGo': 'adguard/golang-ubuntu:6.8'
+            'dockerGo': 'adguard/golang-ubuntu:6.7'
             'snapcraftChannel': 'candidate'

bamboo-specs/test.yaml

@@ -5,7 +5,7 @@
     'key': 'AHBRTSPECS'
     'name': 'AdGuard Home - Build and run tests'
 'variables':
-    'dockerGo': 'adguard/golang-ubuntu:6.8'
+    'dockerGo': 'adguard/golang-ubuntu:6.7'
 
 'stages':
   - 'Tests':

client/src/components/ui/Card.css

@@ -1,7 +1,7 @@
 .card-header {
     align-items: center;
     justify-content: space-between;
-    padding: 0.6rem 1.5rem;
+    padding: 10px 24px;
 }
 
 .card-subtitle {
@@ -16,11 +16,11 @@
 
 .card-table-overflow--limited {
     overflow-y: auto;
-    max-height: 17.5rem;
+    max-height: 280px;
 }
 
 .dashboard .card-table-overflow--limited {
-    max-height: 18rem;
+    max-height: 288px;
 }
 
 .card-actions {
@@ -38,7 +38,7 @@
 }
 
 .card-body--status {
-    padding: 2.5rem 1.5rem;
+    padding: 40px 24px;
     text-align: center;
 }
 
@@ -81,14 +81,14 @@
     position: relative;
     flex: 1 1 auto;
     margin: 0;
-    padding: 1rem 1.5rem;
+    padding: 16px 24px;
 }
 
 .card-value-stats {
     display: block;
     font-size: 2.1rem;
     line-height: 2.7rem;
-    height: 2.7rem;
+    height: 44px;
     text-overflow: ellipsis;
     white-space: nowrap;
     overflow: hidden;
@@ -108,7 +108,7 @@
 }
 
 .card--full {
-    height: calc(100% - 1.5rem);
+    height: calc(100% - 22px);
 }
 
 .card-wrap {
@@ -125,7 +125,7 @@
 
 @media (min-width: 992px) {
     .dashboard .card:not(.card--full) {
-        height: 22rem;
+        height: 350px;
     }
 }
 

client/src/components/ui/Tabler.css

@@ -13363,7 +13363,7 @@ a.icon:hover {
     padding: 0.5rem 1.5rem;
     display: -ms-flexbox;
     display: flex;
-    min-height: 3.5rem;
+    min-height: 60px;
     -ms-flex-align: center;
     align-items: center;
 }

go.mod

@@ -5,7 +5,7 @@ go 1.19
 require (
 	// TODO(a.garipov): Update to a tagged version when it's released.
 	github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768
-	github.com/AdguardTeam/golibs v0.13.4
+	github.com/AdguardTeam/golibs v0.13.3
 	github.com/AdguardTeam/urlfilter v0.16.1
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/ameshkov/dnscrypt/v2 v2.2.7

go.sum

@@ -2,8 +2,8 @@ github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768 h1:5Ia6wA+
 github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768/go.mod h1:CQhZTkqC8X0ID6glrtyaxgqRRdiYfn1gJulC1cZ5Dn8=
 github.com/AdguardTeam/golibs v0.4.0/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4=
 github.com/AdguardTeam/golibs v0.10.4/go.mod h1:rSfQRGHIdgfxriDDNgNJ7HmE5zRoURq8R+VdR81Zuzw=
-github.com/AdguardTeam/golibs v0.13.4 h1:ACTwIR1pEENBijHcEWtiMbSh4wWQOlIHRxmUB8oBHf8=
+github.com/AdguardTeam/golibs v0.13.3 h1:RT3QbzThtaLiFLkIUDS6/hlGEXrh0zYvdf4bd7UWpGo=
-github.com/AdguardTeam/golibs v0.13.4/go.mod h1:wkJ6EUsN4np/9Gp7+9QeooY9E2U2WCLJYAioLCzkHsI=
+github.com/AdguardTeam/golibs v0.13.3/go.mod h1:wkJ6EUsN4np/9Gp7+9QeooY9E2U2WCLJYAioLCzkHsI=
 github.com/AdguardTeam/gomitmproxy v0.2.0/go.mod h1:Qdv0Mktnzer5zpdpi5rAwixNJzW2FN91LjKJCkVbYGU=
 github.com/AdguardTeam/urlfilter v0.16.1 h1:ZPi0rjqo8cQf2FVdzo6cqumNoHZx2KPXj2yZa1A5BBw=
 github.com/AdguardTeam/urlfilter v0.16.1/go.mod h1:46YZDOV1+qtdRDuhZKVPSSp7JWWes0KayqHrKAFBdEI=

internal/aghnet/addr.go

@@ -1,43 +0,0 @@
-package aghnet
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/AdguardTeam/golibs/stringutil"
-)
-
-// NormalizeDomain returns a lowercased version of host without the final dot,
-// unless host is ".", in which case it returns it unchanged.  That is a special
-// case that to allow matching queries like:
-//
-//	dig IN NS '.'
-func NormalizeDomain(host string) (norm string) {
-	if host == "." {
-		return host
-	}
-
-	return strings.ToLower(strings.TrimSuffix(host, "."))
-}
-
-// NewDomainNameSet returns nil and error, if list has duplicate or empty domain
-// name.  Otherwise returns a set, which contains domain names normalized using
-// [NormalizeDomain].
-func NewDomainNameSet(list []string) (set *stringutil.Set, err error) {
-	set = stringutil.NewSet()
-
-	for i, host := range list {
-		if host == "" {
-			return nil, fmt.Errorf("at index %d: hostname is empty", i)
-		}
-
-		host = NormalizeDomain(host)
-		if set.Has(host) {
-			return nil, fmt.Errorf("duplicate hostname %q at index %d", host, i)
-		}
-
-		set.Add(host)
-	}
-
-	return set, nil
-}

internal/aghnet/addr_test.go

@@ -1,59 +0,0 @@
-package aghnet_test
-
-import (
-	"testing"
-
-	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
-	"github.com/AdguardTeam/golibs/testutil"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestNewDomainNameSet(t *testing.T) {
-	t.Parallel()
-
-	testCases := []struct {
-		name       string
-		wantErrMsg string
-		in         []string
-	}{{
-		name:       "nil",
-		wantErrMsg: "",
-		in:         nil,
-	}, {
-		name:       "success",
-		wantErrMsg: "",
-		in: []string{
-			"Domain.Example",
-			".",
-		},
-	}, {
-		name:       "dups",
-		wantErrMsg: `duplicate hostname "domain.example" at index 1`,
-		in: []string{
-			"Domain.Example",
-			"domain.example",
-		},
-	}, {
-		name:       "bad_domain",
-		wantErrMsg: "at index 0: hostname is empty",
-		in: []string{
-			"",
-		},
-	}}
-
-	for _, tc := range testCases {
-		tc := tc
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			set, err := aghnet.NewDomainNameSet(tc.in)
-			testutil.AssertErrorMsg(t, tc.wantErrMsg, err)
-			if err != nil {
-				return
-			}
-
-			for _, host := range tc.in {
-				assert.Truef(t, set.Has(aghnet.NormalizeDomain(host)), "%q not matched", host)
-			}
-		})
-	}
-}

internal/aghnet/hostgen.go

@@ -1,8 +1,12 @@
 package aghnet
 
 import (
+	"fmt"
 	"net/netip"
 	"strings"
+
+	"github.com/AdguardTeam/golibs/errors"
+	"github.com/AdguardTeam/golibs/stringutil"
 )
 
 // GenerateHostname generates the hostname from ip.  In case of using IPv4 the
@@ -25,8 +29,32 @@ func GenerateHostname(ip netip.Addr) (hostname string) {
 	hostname = ip.StringExpanded()
 
 	if ip.Is4() {
-		return strings.ReplaceAll(hostname, ".", "-")
+		return strings.Replace(hostname, ".", "-", -1)
 	}
 
-	return strings.ReplaceAll(hostname, ":", "-")
+	return strings.Replace(hostname, ":", "-", -1)
+}
+
+// NewDomainNameSet returns nil and error, if list has duplicate or empty
+// domain name.  Otherwise returns a set, which contains non-FQDN domain names,
+// and nil error.
+func NewDomainNameSet(list []string) (set *stringutil.Set, err error) {
+	set = stringutil.NewSet()
+
+	for i, v := range list {
+		host := strings.ToLower(strings.TrimSuffix(v, "."))
+		// TODO(a.garipov): Think about ignoring empty (".") names in the
+		// future.
+		if host == "" {
+			return nil, errors.Error("host name is empty")
+		}
+
+		if set.Has(host) {
+			return nil, fmt.Errorf("duplicate host name %q at index %d", host, i)
+		}
+
+		set.Add(host)
+	}
+
+	return set, nil
 }

internal/dnsforward/filter_test.go

@@ -21,8 +21,6 @@ func TestHandleDNSRequest_filterDNSResponse(t *testing.T) {
 ||cname.specific^$dnstype=~CNAME
 ||0.0.0.1^$dnstype=~A
 ||::1^$dnstype=~AAAA
-0.0.0.0 duplicate.domain
-0.0.0.0 duplicate.domain
 `
 
 	forwardConf := ServerConfig{
@@ -139,17 +137,6 @@ func TestHandleDNSRequest_filterDNSResponse(t *testing.T) {
 			},
 			A: netutil.IPv4Zero(),
 		}},
-	}, {
-		req:  createTestMessage("duplicate.domain."),
-		name: "duplicate_domain",
-		wantAns: []dns.RR{&dns.A{
-			Hdr: dns.RR_Header{
-				Name:   "duplicate.domain.",
-				Rrtype: dns.TypeA,
-				Class:  dns.ClassINET,
-			},
-			A: netutil.IPv4Zero(),
-		}},
 	}}
 
 	for _, tc := range testCases {

internal/dnsforward/msg.go

@@ -26,25 +26,11 @@ func (s *Server) makeResponse(req *dns.Msg) (resp *dns.Msg) {
 	return resp
 }
 
-// containsIP returns true if the IP is already in the list.
+// ipsFromRules extracts non-IP addresses from the filtering result rules.
-func containsIP(ips []net.IP, ip net.IP) bool {
-	for _, a := range ips {
-		if a.Equal(ip) {
-			return true
-		}
-	}
-
-	return false
-}
-
-// ipsFromRules extracts unique non-IP addresses from the filtering result
-// rules.
 func ipsFromRules(resRules []*filtering.ResultRule) (ips []net.IP) {
 	for _, r := range resRules {
-		// len(resRules) and len(ips) are actually small enough for O(n^2) to do
+		if r.IP != nil {
-		// not raise performance questions.
+			ips = append(ips, r.IP)
-		if ip := r.IP; ip != nil && !containsIP(ips, ip) {
-			ips = append(ips, ip)
 		}
 	}
 

internal/dnsforward/stats.go

@@ -2,9 +2,9 @@ package dnsforward
 
 import (
 	"net"
+	"strings"
 	"time"
 
-	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
 	"github.com/AdguardTeam/AdGuardHome/internal/filtering"
 	"github.com/AdguardTeam/AdGuardHome/internal/querylog"
 	"github.com/AdguardTeam/AdGuardHome/internal/stats"
@@ -24,7 +24,7 @@ func (s *Server) processQueryLogsAndStats(dctx *dnsContext) (rc resultCode) {
 	pctx := dctx.proxyCtx
 
 	q := pctx.Req.Question[0]
-	host := aghnet.NormalizeDomain(q.Name)
+	host := strings.ToLower(strings.TrimSuffix(q.Name, "."))
 
 	ip, _ := netutil.IPAndPortFromAddr(pctx.Addr)
 	ip = slices.Clone(ip)
@@ -139,10 +139,11 @@ func (s *Server) updateStats(
 	clientIP string,
 ) {
 	pctx := ctx.proxyCtx
-	e := stats.Entry{
+	e := stats.Entry{}
-		Domain: aghnet.NormalizeDomain(pctx.Req.Question[0].Name),
+	e.Domain = strings.ToLower(pctx.Req.Question[0].Name)
-		Result: stats.RNotFiltered,
+	if e.Domain != "." {
-		Time:   uint32(elapsed / 1000),
+		// Remove last ".", but save the domain as is for "." queries.
+		e.Domain = e.Domain[:len(e.Domain)-1]
 	}
 
 	if clientID := ctx.clientID; clientID != "" {
@@ -151,6 +152,9 @@ func (s *Server) updateStats(
 		e.Client = clientIP
 	}
 
+	e.Time = uint32(elapsed / 1000)
+	e.Result = stats.RNotFiltered
+
 	switch res.Reason {
 	case filtering.FilteredSafeBrowsing:
 		e.Result = stats.RSafeBrowsing
@@ -158,8 +162,7 @@ func (s *Server) updateStats(
 		e.Result = stats.RParental
 	case filtering.FilteredSafeSearch:
 		e.Result = stats.RSafeSearch
-	case
+	case filtering.FilteredBlockList,
-		filtering.FilteredBlockList,
 		filtering.FilteredInvalid,
 		filtering.FilteredBlockedService:
 		e.Result = stats.RFiltered

internal/filtering/hashprefix/cache.go

@@ -47,7 +47,7 @@ func fromCacheItem(item *cacheItem) (data []byte) {
 	data = binary.BigEndian.AppendUint64(data, uint64(expiry))
 
 	for _, v := range item.hashes {
-		// nolint:looppointer // The subslice of v is used for a copy.
+		// nolint:looppointer // The subsilce is used for a copy.
 		data = append(data, v[:]...)
 	}
 
@@ -63,7 +63,7 @@ func (c *Checker) findInCache(
 
 	i := 0
 	for _, hash := range hashes {
-		// nolint:looppointer // The has subslice is used for a cache lookup.
+		// nolint:looppointer // The subsilce is used for a safe cache lookup.
 		data := c.cache.Get(hash[:prefixLen])
 		if data == nil {
 			hashes[i] = hash
@@ -98,36 +98,34 @@ func (c *Checker) storeInCache(hashesToRequest, respHashes []hostnameHash) {
 
 	for _, hash := range respHashes {
 		var pref prefix
-		// nolint:looppointer // The hash subslice is used for a copy.
+		// nolint:looppointer // The subsilce is used for a copy.
 		copy(pref[:], hash[:])
 
 		hashToStore[pref] = append(hashToStore[pref], hash)
 	}
 
 	for pref, hash := range hashToStore {
-		c.setCache(pref, hash)
+		// nolint:looppointer // The subsilce is used for a safe cache lookup.
+		c.setCache(pref[:], hash)
 	}
 
 	for _, hash := range hashesToRequest {
-		// nolint:looppointer // The hash subslice is used for a cache lookup.
+		// nolint:looppointer // The subsilce is used for a safe cache lookup.
-		val := c.cache.Get(hash[:prefixLen])
+		pref := hash[:prefixLen]
+		val := c.cache.Get(pref)
 		if val == nil {
-			var pref prefix
-			// nolint:looppointer // The hash subslice is used for a copy.
-			copy(pref[:], hash[:])
-
 			c.setCache(pref, nil)
 		}
 	}
 }
 
 // setCache stores hash in cache.
-func (c *Checker) setCache(pref prefix, hashes []hostnameHash) {
+func (c *Checker) setCache(pref []byte, hashes []hostnameHash) {
 	item := &cacheItem{
 		expiry: time.Now().Add(c.cacheTime),
 		hashes: hashes,
 	}
 
-	c.cache.Set(pref[:], fromCacheItem(item))
+	c.cache.Set(pref, fromCacheItem(item))
 	log.Debug("%s: stored in cache: %v", c.svc, pref)
 }

internal/filtering/hashprefix/hashprefix.go

@@ -173,7 +173,7 @@ func (c *Checker) getQuestion(hashes []hostnameHash) (q string) {
 	b := &strings.Builder{}
 
 	for _, hash := range hashes {
-		// nolint:looppointer // The hash subslice is used for hex encoding.
+		// nolint:looppointer // The subsilce is used for safe hex encoding.
 		stringutil.WriteToBuilder(b, hex.EncodeToString(hash[:prefixLen]), ".")
 	}
 

internal/filtering/servicelist.go

@@ -1505,7 +1505,6 @@ var blockedServices = []blockedService{{
 		"||aus.social^",
 		"||awscommunity.social^",
 		"||climatejustice.social^",
-		"||cupoftea.social^",
 		"||cyberplace.social^",
 		"||defcon.social^",
 		"||det.social^",
@@ -1531,7 +1530,6 @@ var blockedServices = []blockedService{{
 		"||masto.pt^",
 		"||mastodon.au^",
 		"||mastodon.bida.im^",
-		"||mastodon.com.tr^",
 		"||mastodon.eus^",
 		"||mastodon.green^",
 		"||mastodon.ie^",
@@ -1553,11 +1551,11 @@ var blockedServices = []blockedService{{
 		"||mastodont.cat^",
 		"||mastodontech.de^",
 		"||mastodontti.fi^",
+		"||mastouille.fr^",
 		"||mathstodon.xyz^",
 		"||metalhead.club^",
 		"||mindly.social^",
 		"||mstdn.ca^",
-		"||mstdn.jp^",
 		"||mstdn.party^",
 		"||mstdn.plus^",
 		"||mstdn.social^",
@@ -1569,6 +1567,7 @@ var blockedServices = []blockedService{{
 		"||nrw.social^",
 		"||o3o.ca^",
 		"||ohai.social^",
+		"||pewtix.com^",
 		"||piaille.fr^",
 		"||pol.social^",
 		"||ravenation.club^",
@@ -1583,19 +1582,20 @@ var blockedServices = []blockedService{{
 		"||social.linux.pizza^",
 		"||social.politicaconciencia.org^",
 		"||social.vivaldi.net^",
+		"||sself.co^",
 		"||stranger.social^",
 		"||sueden.social^",
 		"||tech.lgbt^",
 		"||techhub.social^",
 		"||theblower.au^",
 		"||tkz.one^",
+		"||todon.eu^",
 		"||toot.aquilenet.fr^",
 		"||toot.community^",
 		"||toot.funami.tech^",
 		"||toot.io^",
 		"||toot.wales^",
 		"||troet.cafe^",
-		"||twingyeo.kr^",
 		"||union.place^",
 		"||universeodon.com^",
 		"||urbanists.social^",

internal/home/config.go

@@ -240,7 +240,6 @@ type tlsConfigSettings struct {
 
 type queryLogConfig struct {
 	// Ignored is the list of host names, which should not be written to log.
-	// "." is considered to be the root domain.
 	Ignored []string `yaml:"ignored"`
 
 	// Interval is the interval for query log's files rotation.

internal/home/home.go

@@ -470,7 +470,7 @@ func setupDNSFilteringConf(conf *filtering.Config) (err error) {
 		ServiceName: pcService,
 		TXTSuffix:   pcTXTSuffix,
 		CacheTime:   cacheTime,
-		CacheSize:   conf.ParentalCacheSize,
+		CacheSize:   conf.SafeBrowsingCacheSize,
 	})
 
 	conf.SafeSearchConf.CustomResolver = safeSearchResolver{}

internal/querylog/qlog.go

@@ -4,6 +4,7 @@ package querylog
 import (
 	"fmt"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
@@ -160,7 +161,10 @@ func (l *queryLog) clear() {
 // newLogEntry creates an instance of logEntry from parameters.
 func newLogEntry(params *AddParams) (entry *logEntry) {
 	q := params.Question.Question[0]
-	qHost := aghnet.NormalizeDomain(q.Name)
+	qHost := q.Name
+	if qHost != "." {
+		qHost = strings.ToLower(q.Name[:len(q.Name)-1])
+	}
 
 	entry = &logEntry{
 		// TODO(d.kolyshev): Export this timestamp to func params.

internal/stats/http_test.go

@@ -86,7 +86,7 @@ func TestHandleStatsConfig(t *testing.T) {
 			},
 		},
 		wantCode: http.StatusUnprocessableEntity,
-		wantErr:  "ignored: duplicate hostname \"ignor.ed\" at index 1\n",
+		wantErr:  "ignored: duplicate host name \"ignor.ed\" at index 1\n",
 	}, {
 		name: "ignored_empty",
 		body: getConfigResp{
@@ -97,7 +97,7 @@ func TestHandleStatsConfig(t *testing.T) {
 			},
 		},
 		wantCode: http.StatusUnprocessableEntity,
-		wantErr:  "ignored: at index 0: hostname is empty\n",
+		wantErr:  "ignored: host name is empty\n",
 	}, {
 		name: "enabled_is_null",
 		body: getConfigResp{

internal/tools/go.mod

@@ -13,7 +13,7 @@ require (
 	golang.org/x/tools v0.11.0
 	golang.org/x/vuln v0.2.0
 	// TODO(a.garipov): Return to tagged releases once a new one appears.
-	honnef.co/go/tools v0.5.0-0.dev.0.20230709092525-bc759185c5ee
+	honnef.co/go/tools v0.5.0-0.dev.0.20230706211743-ddee6bbaa341
 	mvdan.cc/gofumpt v0.5.0
 	mvdan.cc/unparam v0.0.0-20230610194454-9ea02bef9868
 )
@@ -27,7 +27,7 @@ require (
 	github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
-	golang.org/x/exp/typeparams v0.0.0-20230711023510-fffb14384f22 // indirect
+	golang.org/x/exp/typeparams v0.0.0-20230626212559-97b1e661b5df // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/sys v0.10.0 // indirect

internal/tools/go.sum

@@ -52,8 +52,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug=
 golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
-golang.org/x/exp/typeparams v0.0.0-20230711023510-fffb14384f22 h1:e8iSCQYXZ4EB6q3kIfy2fgPFTvDbozqzRe4OuIOyrL4=
+golang.org/x/exp/typeparams v0.0.0-20230626212559-97b1e661b5df h1:jfUqBujZx2dktJVEmZpCkyngz7MWrVv1y9kLOqFNsqw=
-golang.org/x/exp/typeparams v0.0.0-20230711023510-fffb14384f22/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
+golang.org/x/exp/typeparams v0.0.0-20230626212559-97b1e661b5df/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
@@ -107,8 +107,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.5.0-0.dev.0.20230709092525-bc759185c5ee h1:mpyvMqtlVZTwEv78QL3S2ZDTMHMO1fgNwr2kC7+K7oU=
+honnef.co/go/tools v0.5.0-0.dev.0.20230706211743-ddee6bbaa341 h1:jNlTAPEjbDiN9qda/1wple0GSpewFnWhvc1GO7bZX1U=
-honnef.co/go/tools v0.5.0-0.dev.0.20230709092525-bc759185c5ee/go.mod h1:GUV+uIBCLpdf0/v6UhHHG/yzI/z6qPskBeQCjcNB96k=
+honnef.co/go/tools v0.5.0-0.dev.0.20230706211743-ddee6bbaa341/go.mod h1:GUV+uIBCLpdf0/v6UhHHG/yzI/z6qPskBeQCjcNB96k=
 mvdan.cc/gofumpt v0.5.0 h1:0EQ+Z56k8tXjj/6TQD25BFNKQXpCvT0rnansIc7Ug5E=
 mvdan.cc/gofumpt v0.5.0/go.mod h1:HBeVDtMKRZpXyxFciAirzdKklDlGu8aAy1wEbH5Y9js=
 mvdan.cc/unparam v0.0.0-20230610194454-9ea02bef9868 h1:F4Q7pXcrU9UiU1fq0ZWqSOxKjNAteRuDr7JDk7uVLRQ=

scripts/README.md

@@ -269,29 +269,25 @@ Optional environment:
 
    ###  Usage
 
- *  `go run ./scripts/translations help`: print usage.
+ *  `go run main.go help`: print usage.
 
- *  `go run ./scripts/translations download [-n <count>]`: download and save
+ *  `go run main.go download [-n <count>]`: download and save all translations.
-    all translations.  `n` is optional flag where count is a number of
+    `n` is optional flag where count is a number of concurrent downloads.
-    concurrent downloads.
 
- *  `go run ./scripts/translations upload`: upload the base `en` locale.
+ *  `go run main.go upload`: upload the base `en` locale.
 
- *  `go run ./scripts/translations summary`: show the current locales summary.
+ *  `go run main.go summary`: show the current locales summary.
 
- *  `go run ./scripts/translations unused`: show the list of unused strings.
+ *  `go run main.go unused`: show the list of unused strings.
 
- *  `go run ./scripts/translations auto-add`: add locales with additions to the
+ *  `go run main.go auto-add`: add locales with additions to the git and
-    git and restore locales with deletions.
+    restore locales with deletions.
 
 After the download you'll find the output locales in the `client/src/__locales/`
 directory.
 
 Optional environment:
 
- *  `DOWNLOAD_LANGUAGES`: set a list of specific languages to `download`.  For
-    example `ar be bg`.
-
  *  `UPLOAD_LANGUAGE`: set an alternative language for `upload`.
 
  *  `TWOSKY_URI`: set an alternative URL for `download` or `upload`.

scripts/make/go-lint.sh

@@ -35,7 +35,7 @@ set -f -u
 go_version="$( "${GO:-go}" version )"
 readonly go_version
 
-go_min_version='go1.19.11'
+go_min_version='go1.19.10'
 go_version_msg="
 warning: your go version (${go_version}) is different from the recommended minimal one (${go_min_version}).
 if you have the version installed, please set the GO environment variable.
@@ -183,7 +183,6 @@ run_linter gocognit --over 10\
 	./internal/tools/\
 	./internal/version/\
 	./internal/whois/\
-	./scripts/\
 	;
 
 run_linter ineffassign ./...

scripts/translations/download.go

@@ -1,177 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"os"
-	"path/filepath"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/AdguardTeam/AdGuardHome/internal/aghio"
-	"github.com/AdguardTeam/golibs/errors"
-	"github.com/AdguardTeam/golibs/log"
-	"golang.org/x/exp/slices"
-)
-
-// download and save all translations.
-func (c *twoskyClient) download() (err error) {
-	var numWorker int
-
-	flagSet := flag.NewFlagSet("download", flag.ExitOnError)
-	flagSet.Usage = func() {
-		usage("download command error")
-	}
-	flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")
-
-	err = flagSet.Parse(os.Args[2:])
-	if err != nil {
-		// Don't wrap the error since it's informative enough as is.
-		return err
-	}
-
-	if numWorker < 1 {
-		usage("count must be positive")
-	}
-
-	downloadURI := c.uri.JoinPath("download")
-
-	client := &http.Client{
-		Timeout: 10 * time.Second,
-	}
-
-	wg := &sync.WaitGroup{}
-	failed := &sync.Map{}
-	uriCh := make(chan *url.URL, len(c.langs))
-
-	for i := 0; i < numWorker; i++ {
-		wg.Add(1)
-		go downloadWorker(wg, failed, client, uriCh)
-	}
-
-	for lang := range c.langs {
-		uri := translationURL(downloadURI, defaultBaseFile, c.projectID, lang)
-
-		uriCh <- uri
-	}
-
-	close(uriCh)
-	wg.Wait()
-
-	printFailedLocales(failed)
-
-	return nil
-}
-
-// printFailedLocales prints sorted list of failed downloads, if any.
-func printFailedLocales(failed *sync.Map) {
-	keys := []string{}
-	failed.Range(func(k, _ any) bool {
-		s, ok := k.(string)
-		if !ok {
-			panic("unexpected type")
-		}
-
-		keys = append(keys, s)
-
-		return true
-	})
-
-	if len(keys) == 0 {
-		return
-	}
-
-	slices.Sort(keys)
-	log.Info("failed locales: %s", strings.Join(keys, " "))
-}
-
-// downloadWorker downloads translations by received urls and saves them.
-// Where failed is a map for storing failed downloads.
-func downloadWorker(
-	wg *sync.WaitGroup,
-	failed *sync.Map,
-	client *http.Client,
-	uriCh <-chan *url.URL,
-) {
-	defer wg.Done()
-
-	for uri := range uriCh {
-		q := uri.Query()
-		code := q.Get("language")
-
-		err := saveToFile(client, uri, code)
-		if err != nil {
-			log.Error("download: worker: %s", err)
-			failed.Store(code, struct{}{})
-		}
-	}
-}
-
-// saveToFile downloads translation by url and saves it to a file, or returns
-// error.
-func saveToFile(client *http.Client, uri *url.URL, code string) (err error) {
-	data, err := getTranslation(client, uri.String())
-	if err != nil {
-		log.Info("%s", data)
-
-		return fmt.Errorf("getting translation: %s", err)
-	}
-
-	name := filepath.Join(localesDir, code+".json")
-	err = os.WriteFile(name, data, 0o664)
-	if err != nil {
-		return fmt.Errorf("writing file: %s", err)
-	}
-
-	fmt.Println(name)
-
-	return nil
-}
-
-// getTranslation returns received translation data and error.  If err is not
-// nil, data may contain a response from server for inspection.
-func getTranslation(client *http.Client, url string) (data []byte, err error) {
-	resp, err := client.Get(url)
-	if err != nil {
-		return nil, fmt.Errorf("requesting: %w", err)
-	}
-
-	defer log.OnCloserError(resp.Body, log.ERROR)
-
-	if resp.StatusCode != http.StatusOK {
-		err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
-
-		// Go on and download the body for inspection.
-	}
-
-	limitReader, lrErr := aghio.LimitReader(resp.Body, readLimit)
-	if lrErr != nil {
-		// Generally shouldn't happen, since the only error returned by
-		// [aghio.LimitReader] is an argument error.
-		panic(fmt.Errorf("limit reading: %w", lrErr))
-	}
-
-	data, readErr := io.ReadAll(limitReader)
-
-	return data, errors.WithDeferred(err, readErr)
-}
-
-// translationURL returns a new url.URL with provided query parameters.
-func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
-	uri = &url.URL{}
-	*uri = *oldURL
-
-	q := uri.Query()
-	q.Set("format", "json")
-	q.Set("filename", baseFile)
-	q.Set("project", projectID)
-	q.Set("language", string(lang))
-
-	uri.RawQuery = q.Encode()
-
-	return uri
-}

scripts/translations/main.go

@@ -6,16 +6,25 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/json"
+	"flag"
 	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/textproto"
 	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 
+	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
+	"github.com/AdguardTeam/AdGuardHome/internal/aghio"
 	"github.com/AdguardTeam/AdGuardHome/internal/aghos"
 	"github.com/AdguardTeam/golibs/errors"
+	"github.com/AdguardTeam/golibs/httphdr"
 	"github.com/AdguardTeam/golibs/log"
 	"golang.org/x/exp/maps"
 	"golang.org/x/exp/slices"
@@ -29,8 +38,7 @@ const (
 	srcDir           = "./client/src"
 	twoskyURI        = "https://twosky.int.agrd.dev/api/v1"
 
-	readLimit     = 1 * 1024 * 1024
+	readLimit = 1 * 1024 * 1024
-	uploadTimeout = 10 * time.Second
 )
 
 // langCode is a language code.
@@ -54,26 +62,31 @@ func main() {
 		usage("")
 	}
 
-	conf, err := readTwoskyConfig()
+	uriStr := os.Getenv("TWOSKY_URI")
+	if uriStr == "" {
+		uriStr = twoskyURI
+	}
+
+	uri, err := url.Parse(uriStr)
 	check(err)
 
-	var cli *twoskyClient
+	projectID := os.Getenv("TWOSKY_PROJECT_ID")
+	if projectID == "" {
+		projectID = defaultProjectID
+	}
+
+	conf, err := readTwoskyConf()
+	check(err)
 
 	switch os.Args[1] {
 	case "summary":
 		err = summary(conf.Languages)
 	case "download":
-		cli, err = conf.toClient()
+		err = download(uri, projectID, conf.Languages)
-		check(err)
-
-		err = cli.download()
 	case "unused":
 		err = unused(conf.LocalizableFiles[0])
 	case "upload":
-		cli, err = conf.toClient()
+		err = upload(uri, projectID, conf.BaseLangcode)
-		check(err)
-
-		err = cli.upload()
 	case "auto-add":
 		err = autoAdd(conf.LocalizableFiles[0])
 	default:
@@ -120,131 +133,51 @@ Commands:
 	os.Exit(0)
 }
 
-// twoskyConfig is the configuration structure for localization.
+// twoskyConf is the configuration structure for localization.
-type twoskyConfig struct {
+type twoskyConf struct {
 	Languages        languages `json:"languages"`
 	ProjectID        string    `json:"project_id"`
 	BaseLangcode     langCode  `json:"base_locale"`
 	LocalizableFiles []string  `json:"localizable_files"`
 }
 
-// readTwoskyConfig returns twosky configuration.
+// readTwoskyConf returns configuration.
-func readTwoskyConfig() (t *twoskyConfig, err error) {
+func readTwoskyConf() (t twoskyConf, err error) {
-	defer func() { err = errors.Annotate(err, "parsing twosky config: %w") }()
+	defer func() { err = errors.Annotate(err, "parsing twosky conf: %w") }()
 
 	b, err := os.ReadFile(twoskyConfFile)
 	if err != nil {
 		// Don't wrap the error since it's informative enough as is.
-		return nil, err
+		return twoskyConf{}, err
 	}
 
-	var tsc []twoskyConfig
+	var tsc []twoskyConf
 	err = json.Unmarshal(b, &tsc)
 	if err != nil {
 		err = fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err)
 
-		return nil, err
+		return twoskyConf{}, err
 	}
 
 	if len(tsc) == 0 {
 		err = fmt.Errorf("%q is empty", twoskyConfFile)
 
-		return nil, err
+		return twoskyConf{}, err
 	}
 
 	conf := tsc[0]
 
 	for _, lang := range conf.Languages {
 		if lang == "" {
-			return nil, errors.Error("language is empty")
+			return twoskyConf{}, errors.Error("language is empty")
 		}
 	}
 
 	if len(conf.LocalizableFiles) == 0 {
-		return nil, errors.Error("no localizable files specified")
+		return twoskyConf{}, errors.Error("no localizable files specified")
 	}
 
-	return &conf, nil
+	return conf, nil
-}
-
-// twoskyClient is the twosky client with methods for download and upload
-// translations.
-type twoskyClient struct {
-	// uri is the base URL.
-	uri *url.URL
-
-	// langs is the map of languages to download.
-	langs languages
-
-	// projectID is the name of the project.
-	projectID string
-
-	// baseLang is the base language code.
-	baseLang langCode
-}
-
-// toClient reads values from environment variables or defaults, validates
-// them, and returns the twosky client.
-func (t *twoskyConfig) toClient() (cli *twoskyClient, err error) {
-	defer func() { err = errors.Annotate(err, "filling config: %w") }()
-
-	uriStr := os.Getenv("TWOSKY_URI")
-	if uriStr == "" {
-		uriStr = twoskyURI
-	}
-	uri, err := url.Parse(uriStr)
-	if err != nil {
-		return nil, err
-	}
-
-	projectID := os.Getenv("TWOSKY_PROJECT_ID")
-	if projectID == "" {
-		projectID = defaultProjectID
-	}
-
-	baseLang := t.BaseLangcode
-	uLangStr := os.Getenv("UPLOAD_LANGUAGE")
-	if uLangStr != "" {
-		baseLang = langCode(uLangStr)
-	}
-
-	langs := t.Languages
-	dlLangStr := os.Getenv("DOWNLOAD_LANGUAGES")
-	if dlLangStr != "" {
-		var dlLangs languages
-		dlLangs, err = validateLanguageStr(dlLangStr, langs)
-		if err != nil {
-			return nil, err
-		}
-
-		langs = dlLangs
-	}
-
-	return &twoskyClient{
-		uri:       uri,
-		projectID: projectID,
-		baseLang:  baseLang,
-		langs:     langs,
-	}, nil
-}
-
-// validateLanguageStr validates languages codes that contain in the str and
-// returns language map, where key is language code and value is display name.
-func validateLanguageStr(str string, all languages) (langs languages, err error) {
-	langs = make(languages)
-	codes := strings.Fields(str)
-
-	for _, k := range codes {
-		lc := langCode(k)
-		name, ok := all[lc]
-		if !ok {
-			return nil, fmt.Errorf("validating languages: unexpected language code %q", k)
-		}
-
-		langs[lc] = name
-	}
-
-	return langs, nil
 }
 
 // readLocales reads file with name fn and returns a map, where key is text
@@ -300,33 +233,163 @@ func summary(langs languages) (err error) {
 	return nil
 }
 
+// download and save all translations.  uri is the base URL.  projectID is the
+// name of the project.
+func download(uri *url.URL, projectID string, langs languages) (err error) {
+	var numWorker int
+
+	flagSet := flag.NewFlagSet("download", flag.ExitOnError)
+	flagSet.Usage = func() {
+		usage("download command error")
+	}
+	flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")
+
+	err = flagSet.Parse(os.Args[2:])
+	if err != nil {
+		// Don't wrap the error since it's informative enough as is.
+		return err
+	}
+
+	if numWorker < 1 {
+		usage("count must be positive")
+	}
+
+	downloadURI := uri.JoinPath("download")
+
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+
+	wg := &sync.WaitGroup{}
+	uriCh := make(chan *url.URL, len(langs))
+
+	for i := 0; i < numWorker; i++ {
+		wg.Add(1)
+		go downloadWorker(wg, client, uriCh)
+	}
+
+	for lang := range langs {
+		uri = translationURL(downloadURI, defaultBaseFile, projectID, lang)
+
+		uriCh <- uri
+	}
+
+	close(uriCh)
+	wg.Wait()
+
+	return nil
+}
+
+// downloadWorker downloads translations by received urls and saves them.
+func downloadWorker(wg *sync.WaitGroup, client *http.Client, uriCh <-chan *url.URL) {
+	defer wg.Done()
+
+	for uri := range uriCh {
+		data, err := getTranslation(client, uri.String())
+		if err != nil {
+			log.Error("download worker: getting translation: %s", err)
+			log.Info("download worker: error response:\n%s", data)
+
+			continue
+		}
+
+		q := uri.Query()
+		code := q.Get("language")
+
+		// Fix some TwoSky weirdnesses.
+		//
+		// TODO(a.garipov): Remove when those are fixed.
+		code = strings.ToLower(code)
+
+		name := filepath.Join(localesDir, code+".json")
+		err = os.WriteFile(name, data, 0o664)
+		if err != nil {
+			log.Error("download worker: writing file: %s", err)
+
+			continue
+		}
+
+		fmt.Println(name)
+	}
+}
+
+// getTranslation returns received translation data and error.  If err is not
+// nil, data may contain a response from server for inspection.
+func getTranslation(client *http.Client, url string) (data []byte, err error) {
+	resp, err := client.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("requesting: %w", err)
+	}
+
+	defer log.OnCloserError(resp.Body, log.ERROR)
+
+	if resp.StatusCode != http.StatusOK {
+		err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
+
+		// Go on and download the body for inspection.
+	}
+
+	limitReader, lrErr := aghio.LimitReader(resp.Body, readLimit)
+	if lrErr != nil {
+		// Generally shouldn't happen, since the only error returned by
+		// [aghio.LimitReader] is an argument error.
+		panic(fmt.Errorf("limit reading: %w", lrErr))
+	}
+
+	data, readErr := io.ReadAll(limitReader)
+
+	return data, errors.WithDeferred(err, readErr)
+}
+
+// translationURL returns a new url.URL with provided query parameters.
+func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
+	uri = &url.URL{}
+	*uri = *oldURL
+
+	// Fix some TwoSky weirdnesses.
+	//
+	// TODO(a.garipov): Remove when those are fixed.
+	switch lang {
+	case "si-lk":
+		lang = "si-LK"
+	case "zh-hk":
+		lang = "zh-HK"
+	default:
+		// Go on.
+	}
+
+	q := uri.Query()
+	q.Set("format", "json")
+	q.Set("filename", baseFile)
+	q.Set("project", projectID)
+	q.Set("language", string(lang))
+
+	uri.RawQuery = q.Encode()
+
+	return uri
+}
+
 // unused prints unused text labels.
 func unused(basePath string) (err error) {
-	defer func() { err = errors.Annotate(err, "unused: %w") }()
-
 	baseLoc, err := readLocales(basePath)
 	if err != nil {
-		return err
+		return fmt.Errorf("unused: %w", err)
 	}
 
 	locDir := filepath.Clean(localesDir)
-	js, err := findJS(locDir)
-	if err != nil {
-		return err
-	}
 
-	return findUnused(js, baseLoc)
+	fileNames := []string{}
-}
+	err = filepath.Walk(srcDir, func(name string, info os.FileInfo, err error) error {
-
-// findJS returns list of JavaScript and JSON files or error.
-func findJS(locDir string) (fileNames []string, err error) {
-	walkFn := func(name string, _ os.FileInfo, err error) error {
 		if err != nil {
 			log.Info("warning: accessing a path %q: %s", name, err)
 
 			return nil
 		}
 
+		if info.IsDir() {
+			return nil
+		}
+
 		if strings.HasPrefix(name, locDir) {
 			return nil
 		}
@@ -337,14 +400,13 @@ func findJS(locDir string) (fileNames []string, err error) {
 		}
 
 		return nil
-	}
+	})
 
-	err = filepath.Walk(srcDir, walkFn)
 	if err != nil {
-		return nil, fmt.Errorf("filepath walking %q: %w", srcDir, err)
+		return fmt.Errorf("filepath walking %q: %w", srcDir, err)
 	}
 
-	return fileNames, nil
+	return findUnused(fileNames, baseLoc)
 }
 
 // findUnused prints unused text labels from fileNames.
@@ -383,6 +445,118 @@ func findUnused(fileNames []string, loc locales) (err error) {
 	return nil
 }
 
+// upload base translation.  uri is the base URL.  projectID is the name of the
+// project.  baseLang is the base language code.
+func upload(uri *url.URL, projectID string, baseLang langCode) (err error) {
+	defer func() { err = errors.Annotate(err, "upload: %w") }()
+
+	uploadURI := uri.JoinPath("upload")
+
+	lang := baseLang
+
+	langStr := os.Getenv("UPLOAD_LANGUAGE")
+	if langStr != "" {
+		lang = langCode(langStr)
+	}
+
+	basePath := filepath.Join(localesDir, defaultBaseFile)
+
+	formData := map[string]string{
+		"format":   "json",
+		"language": string(lang),
+		"filename": defaultBaseFile,
+		"project":  projectID,
+	}
+
+	buf, cType, err := prepareMultipartMsg(formData, basePath)
+	if err != nil {
+		return fmt.Errorf("preparing multipart msg: %w", err)
+	}
+
+	err = send(uploadURI.String(), cType, buf)
+	if err != nil {
+		return fmt.Errorf("sending multipart msg: %w", err)
+	}
+
+	return nil
+}
+
+// prepareMultipartMsg prepares translation data for upload.
+func prepareMultipartMsg(
+	formData map[string]string,
+	basePath string,
+) (buf *bytes.Buffer, cType string, err error) {
+	buf = &bytes.Buffer{}
+	w := multipart.NewWriter(buf)
+	var fw io.Writer
+
+	for k, v := range formData {
+		err = w.WriteField(k, v)
+		if err != nil {
+			return nil, "", fmt.Errorf("writing field: %w", err)
+		}
+	}
+
+	file, err := os.Open(basePath)
+	if err != nil {
+		return nil, "", fmt.Errorf("opening file: %w", err)
+	}
+
+	defer func() {
+		err = errors.WithDeferred(err, file.Close())
+	}()
+
+	h := make(textproto.MIMEHeader)
+	h.Set(httphdr.ContentType, aghhttp.HdrValApplicationJSON)
+
+	d := fmt.Sprintf("form-data; name=%q; filename=%q", "file", defaultBaseFile)
+	h.Set(httphdr.ContentDisposition, d)
+
+	fw, err = w.CreatePart(h)
+	if err != nil {
+		return nil, "", fmt.Errorf("creating part: %w", err)
+	}
+
+	_, err = io.Copy(fw, file)
+	if err != nil {
+		return nil, "", fmt.Errorf("copying: %w", err)
+	}
+
+	err = w.Close()
+	if err != nil {
+		return nil, "", fmt.Errorf("closing writer: %w", err)
+	}
+
+	return buf, w.FormDataContentType(), nil
+}
+
+// send POST request to uriStr.
+func send(uriStr, cType string, buf *bytes.Buffer) (err error) {
+	var client http.Client
+
+	req, err := http.NewRequest(http.MethodPost, uriStr, buf)
+	if err != nil {
+		return fmt.Errorf("bad request: %w", err)
+	}
+
+	req.Header.Set(httphdr.ContentType, cType)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("client post form: %w", err)
+	}
+
+	defer func() {
+		err = errors.WithDeferred(err, resp.Body.Close())
+	}()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode))
+	}
+
+	return nil
+}
+
 // autoAdd adds locales with additions to the git and restores locales with
 // deletions.
 func autoAdd(basePath string) (err error) {
@@ -398,48 +572,28 @@ func autoAdd(basePath string) (err error) {
 		return errors.Error("base locale contains deletions")
 	}
 
-	err = handleAdds(adds)
+	var (
-	if err != nil {
+		args []string
-		// Don't wrap the error since it's informative enough as is.
+		code int
-		return nil
+		out  []byte
+	)
+
+	if len(adds) > 0 {
+		args = append([]string{"add"}, adds...)
+		code, out, err = aghos.RunCommand("git", args...)
+
+		if err != nil || code != 0 {
+			return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err)
+		}
 	}
 
-	err = handleDels(dels)
+	if len(dels) > 0 {
-	if err != nil {
+		args = append([]string{"restore"}, dels...)
-		// Don't wrap the error since it's informative enough as is.
+		code, out, err = aghos.RunCommand("git", args...)
-		return nil
-	}
 
-	return nil
+		if err != nil || code != 0 {
-}
+			return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err)
-
+		}
-// handleAdds adds locales with additions to the git.
-func handleAdds(locales []string) (err error) {
-	if len(locales) == 0 {
-		return nil
-	}
-
-	args := append([]string{"add"}, locales...)
-	code, out, err := aghos.RunCommand("git", args...)
-
-	if err != nil || code != 0 {
-		return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err)
-	}
-
-	return nil
-}
-
-// handleDels restores locales with deletions.
-func handleDels(locales []string) (err error) {
-	if len(locales) == 0 {
-		return nil
-	}
-
-	args := append([]string{"restore"}, locales...)
-	code, out, err := aghos.RunCommand("git", args...)
-
-	if err != nil || code != 0 {
-		return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err)
 	}
 
 	return nil

scripts/translations/upload.go

@@ -1,120 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"mime/multipart"
-	"net/http"
-	"net/textproto"
-	"os"
-	"path/filepath"
-
-	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
-	"github.com/AdguardTeam/golibs/errors"
-	"github.com/AdguardTeam/golibs/httphdr"
-	"github.com/AdguardTeam/golibs/mapsutil"
-)
-
-// upload base translation.
-func (c *twoskyClient) upload() (err error) {
-	defer func() { err = errors.Annotate(err, "upload: %w") }()
-
-	uploadURI := c.uri.JoinPath("upload")
-	basePath := filepath.Join(localesDir, defaultBaseFile)
-
-	formData := map[string]string{
-		"format":   "json",
-		"language": string(c.baseLang),
-		"filename": defaultBaseFile,
-		"project":  c.projectID,
-	}
-
-	buf, cType, err := prepareMultipartMsg(formData, basePath)
-	if err != nil {
-		return fmt.Errorf("preparing multipart msg: %w", err)
-	}
-
-	err = send(uploadURI.String(), cType, buf)
-	if err != nil {
-		return fmt.Errorf("sending multipart msg: %w", err)
-	}
-
-	return nil
-}
-
-// prepareMultipartMsg prepares translation data for upload.
-func prepareMultipartMsg(
-	formData map[string]string,
-	basePath string,
-) (buf *bytes.Buffer, cType string, err error) {
-	buf = &bytes.Buffer{}
-	w := multipart.NewWriter(buf)
-	var fw io.Writer
-
-	err = mapsutil.OrderedRangeError(formData, w.WriteField)
-	if err != nil {
-		return nil, "", fmt.Errorf("writing field: %w", err)
-	}
-
-	file, err := os.Open(basePath)
-	if err != nil {
-		return nil, "", fmt.Errorf("opening file: %w", err)
-	}
-
-	defer func() {
-		err = errors.WithDeferred(err, file.Close())
-	}()
-
-	h := make(textproto.MIMEHeader)
-	h.Set(httphdr.ContentType, aghhttp.HdrValApplicationJSON)
-
-	d := fmt.Sprintf("form-data; name=%q; filename=%q", "file", defaultBaseFile)
-	h.Set(httphdr.ContentDisposition, d)
-
-	fw, err = w.CreatePart(h)
-	if err != nil {
-		return nil, "", fmt.Errorf("creating part: %w", err)
-	}
-
-	_, err = io.Copy(fw, file)
-	if err != nil {
-		return nil, "", fmt.Errorf("copying: %w", err)
-	}
-
-	err = w.Close()
-	if err != nil {
-		return nil, "", fmt.Errorf("closing writer: %w", err)
-	}
-
-	return buf, w.FormDataContentType(), nil
-}
-
-// send POST request to uriStr.
-func send(uriStr, cType string, buf *bytes.Buffer) (err error) {
-	client := http.Client{
-		Timeout: uploadTimeout,
-	}
-
-	req, err := http.NewRequest(http.MethodPost, uriStr, buf)
-	if err != nil {
-		return fmt.Errorf("bad request: %w", err)
-	}
-
-	req.Header.Set(httphdr.ContentType, cType)
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return fmt.Errorf("client post form: %w", err)
-	}
-
-	defer func() {
-		err = errors.WithDeferred(err, resp.Body.Close())
-	}()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode))
-	}
-
-	return nil
-}