all: sync with master; upd chlog

scripts/translations/download.go

@@ -0,0 +1,177 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/AdguardTeam/AdGuardHome/internal/aghio"
+	"github.com/AdguardTeam/golibs/errors"
+	"github.com/AdguardTeam/golibs/log"
+	"golang.org/x/exp/slices"
+)
+
+// download and save all translations.
+func (c *twoskyClient) download() (err error) {
+	var numWorker int
+
+	flagSet := flag.NewFlagSet("download", flag.ExitOnError)
+	flagSet.Usage = func() {
+		usage("download command error")
+	}
+	flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")
+
+	err = flagSet.Parse(os.Args[2:])
+	if err != nil {
+		// Don't wrap the error since it's informative enough as is.
+		return err
+	}
+
+	if numWorker < 1 {
+		usage("count must be positive")
+	}
+
+	downloadURI := c.uri.JoinPath("download")
+
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+
+	wg := &sync.WaitGroup{}
+	failed := &sync.Map{}
+	uriCh := make(chan *url.URL, len(c.langs))
+
+	for i := 0; i < numWorker; i++ {
+		wg.Add(1)
+		go downloadWorker(wg, failed, client, uriCh)
+	}
+
+	for _, lang := range c.langs {
+		uri := translationURL(downloadURI, defaultBaseFile, c.projectID, lang)
+
+		uriCh <- uri
+	}
+
+	close(uriCh)
+	wg.Wait()
+
+	printFailedLocales(failed)
+
+	return nil
+}
+
+// printFailedLocales prints sorted list of failed downloads, if any.
+func printFailedLocales(failed *sync.Map) {
+	keys := []string{}
+	failed.Range(func(k, _ any) bool {
+		s, ok := k.(string)
+		if !ok {
+			panic("unexpected type")
+		}
+
+		keys = append(keys, s)
+
+		return true
+	})
+
+	if len(keys) == 0 {
+		return
+	}
+
+	slices.Sort(keys)
+	log.Info("failed locales: %s", strings.Join(keys, " "))
+}
+
+// downloadWorker downloads translations by received urls and saves them.
+// Where failed is a map for storing failed downloads.
+func downloadWorker(
+	wg *sync.WaitGroup,
+	failed *sync.Map,
+	client *http.Client,
+	uriCh <-chan *url.URL,
+) {
+	defer wg.Done()
+
+	for uri := range uriCh {
+		q := uri.Query()
+		code := q.Get("language")
+
+		err := saveToFile(client, uri, code)
+		if err != nil {
+			log.Error("download: worker: %s", err)
+			failed.Store(code, struct{}{})
+		}
+	}
+}
+
+// saveToFile downloads translation by url and saves it to a file, or returns
+// error.
+func saveToFile(client *http.Client, uri *url.URL, code string) (err error) {
+	data, err := getTranslation(client, uri.String())
+	if err != nil {
+		log.Info("%s", data)
+
+		return fmt.Errorf("getting translation: %s", err)
+	}
+
+	name := filepath.Join(localesDir, code+".json")
+	err = os.WriteFile(name, data, 0o664)
+	if err != nil {
+		return fmt.Errorf("writing file: %s", err)
+	}
+
+	fmt.Println(name)
+
+	return nil
+}
+
+// getTranslation returns received translation data and error.  If err is not
+// nil, data may contain a response from server for inspection.
+func getTranslation(client *http.Client, url string) (data []byte, err error) {
+	resp, err := client.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("requesting: %w", err)
+	}
+
+	defer log.OnCloserError(resp.Body, log.ERROR)
+
+	if resp.StatusCode != http.StatusOK {
+		err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
+
+		// Go on and download the body for inspection.
+	}
+
+	limitReader, lrErr := aghio.LimitReader(resp.Body, readLimit)
+	if lrErr != nil {
+		// Generally shouldn't happen, since the only error returned by
+		// [aghio.LimitReader] is an argument error.
+		panic(fmt.Errorf("limit reading: %w", lrErr))
+	}
+
+	data, readErr := io.ReadAll(limitReader)
+
+	return data, errors.WithDeferred(err, readErr)
+}
+
+// translationURL returns a new url.URL with provided query parameters.
+func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
+	uri = &url.URL{}
+	*uri = *oldURL
+
+	q := uri.Query()
+	q.Set("format", "json")
+	q.Set("filename", baseFile)
+	q.Set("project", projectID)
+	q.Set("language", string(lang))
+
+	uri.RawQuery = q.Encode()
+
+	return uri
+}