all: sync with master; upd chlog
This commit is contained in:
Ainar Garipov
177
scripts/translations/download.go
Normal file
177
scripts/translations/download.go
Normal file
@@ -0,0 +1,177 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghio"
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
// download and save all translations.
|
||||
func (c *twoskyClient) download() (err error) {
|
||||
var numWorker int
|
||||
|
||||
flagSet := flag.NewFlagSet("download", flag.ExitOnError)
|
||||
flagSet.Usage = func() {
|
||||
usage("download command error")
|
||||
}
|
||||
flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")
|
||||
|
||||
err = flagSet.Parse(os.Args[2:])
|
||||
if err != nil {
|
||||
// Don't wrap the error since it's informative enough as is.
|
||||
return err
|
||||
}
|
||||
|
||||
if numWorker < 1 {
|
||||
usage("count must be positive")
|
||||
}
|
||||
|
||||
downloadURI := c.uri.JoinPath("download")
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
wg := &sync.WaitGroup{}
|
||||
failed := &sync.Map{}
|
||||
uriCh := make(chan *url.URL, len(c.langs))
|
||||
|
||||
for i := 0; i < numWorker; i++ {
|
||||
wg.Add(1)
|
||||
go downloadWorker(wg, failed, client, uriCh)
|
||||
}
|
||||
|
||||
for _, lang := range c.langs {
|
||||
uri := translationURL(downloadURI, defaultBaseFile, c.projectID, lang)
|
||||
|
||||
uriCh <- uri
|
||||
}
|
||||
|
||||
close(uriCh)
|
||||
wg.Wait()
|
||||
|
||||
printFailedLocales(failed)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// printFailedLocales prints sorted list of failed downloads, if any.
|
||||
func printFailedLocales(failed *sync.Map) {
|
||||
keys := []string{}
|
||||
failed.Range(func(k, _ any) bool {
|
||||
s, ok := k.(string)
|
||||
if !ok {
|
||||
panic("unexpected type")
|
||||
}
|
||||
|
||||
keys = append(keys, s)
|
||||
|
||||
return true
|
||||
})
|
||||
|
||||
if len(keys) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
slices.Sort(keys)
|
||||
log.Info("failed locales: %s", strings.Join(keys, " "))
|
||||
}
|
||||
|
||||
// downloadWorker downloads translations by received urls and saves them.
|
||||
// Where failed is a map for storing failed downloads.
|
||||
func downloadWorker(
|
||||
wg *sync.WaitGroup,
|
||||
failed *sync.Map,
|
||||
client *http.Client,
|
||||
uriCh <-chan *url.URL,
|
||||
) {
|
||||
defer wg.Done()
|
||||
|
||||
for uri := range uriCh {
|
||||
q := uri.Query()
|
||||
code := q.Get("language")
|
||||
|
||||
err := saveToFile(client, uri, code)
|
||||
if err != nil {
|
||||
log.Error("download: worker: %s", err)
|
||||
failed.Store(code, struct{}{})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// saveToFile downloads translation by url and saves it to a file, or returns
|
||||
// error.
|
||||
func saveToFile(client *http.Client, uri *url.URL, code string) (err error) {
|
||||
data, err := getTranslation(client, uri.String())
|
||||
if err != nil {
|
||||
log.Info("%s", data)
|
||||
|
||||
return fmt.Errorf("getting translation: %s", err)
|
||||
}
|
||||
|
||||
name := filepath.Join(localesDir, code+".json")
|
||||
err = os.WriteFile(name, data, 0o664)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing file: %s", err)
|
||||
}
|
||||
|
||||
fmt.Println(name)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getTranslation returns received translation data and error. If err is not
|
||||
// nil, data may contain a response from server for inspection.
|
||||
func getTranslation(client *http.Client, url string) (data []byte, err error) {
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("requesting: %w", err)
|
||||
}
|
||||
|
||||
defer log.OnCloserError(resp.Body, log.ERROR)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
|
||||
|
||||
// Go on and download the body for inspection.
|
||||
}
|
||||
|
||||
limitReader, lrErr := aghio.LimitReader(resp.Body, readLimit)
|
||||
if lrErr != nil {
|
||||
// Generally shouldn't happen, since the only error returned by
|
||||
// [aghio.LimitReader] is an argument error.
|
||||
panic(fmt.Errorf("limit reading: %w", lrErr))
|
||||
}
|
||||
|
||||
data, readErr := io.ReadAll(limitReader)
|
||||
|
||||
return data, errors.WithDeferred(err, readErr)
|
||||
}
|
||||
|
||||
// translationURL returns a new url.URL with provided query parameters.
|
||||
func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
|
||||
uri = &url.URL{}
|
||||
*uri = *oldURL
|
||||
|
||||
q := uri.Query()
|
||||
q.Set("format", "json")
|
||||
q.Set("filename", baseFile)
|
||||
q.Set("project", projectID)
|
||||
q.Set("language", string(lang))
|
||||
|
||||
uri.RawQuery = q.Encode()
|
||||
|
||||
return uri
|
||||
}
|
||||
@@ -6,25 +6,16 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghio"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghos"
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/httphdr"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"golang.org/x/exp/maps"
|
||||
"golang.org/x/exp/slices"
|
||||
@@ -38,9 +29,26 @@ const (
|
||||
srcDir = "./client/src"
|
||||
twoskyURI = "https://twosky.int.agrd.dev/api/v1"
|
||||
|
||||
readLimit = 1 * 1024 * 1024
|
||||
readLimit = 1 * 1024 * 1024
|
||||
uploadTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
// blockerLangCodes is the codes of languages which need to be fully translated.
|
||||
var blockerLangCodes = []langCode{
|
||||
"de",
|
||||
"en",
|
||||
"es",
|
||||
"fr",
|
||||
"it",
|
||||
"ja",
|
||||
"ko",
|
||||
"pt-br",
|
||||
"pt-pt",
|
||||
"ru",
|
||||
"zh-cn",
|
||||
"zh-tw",
|
||||
}
|
||||
|
||||
// langCode is a language code.
|
||||
type langCode string
|
||||
|
||||
@@ -62,31 +70,26 @@ func main() {
|
||||
usage("")
|
||||
}
|
||||
|
||||
uriStr := os.Getenv("TWOSKY_URI")
|
||||
if uriStr == "" {
|
||||
uriStr = twoskyURI
|
||||
}
|
||||
|
||||
uri, err := url.Parse(uriStr)
|
||||
conf, err := readTwoskyConfig()
|
||||
check(err)
|
||||
|
||||
projectID := os.Getenv("TWOSKY_PROJECT_ID")
|
||||
if projectID == "" {
|
||||
projectID = defaultProjectID
|
||||
}
|
||||
|
||||
conf, err := readTwoskyConf()
|
||||
check(err)
|
||||
var cli *twoskyClient
|
||||
|
||||
switch os.Args[1] {
|
||||
case "summary":
|
||||
err = summary(conf.Languages)
|
||||
case "download":
|
||||
err = download(uri, projectID, conf.Languages)
|
||||
cli, err = conf.toClient()
|
||||
check(err)
|
||||
|
||||
err = cli.download()
|
||||
case "unused":
|
||||
err = unused(conf.LocalizableFiles[0])
|
||||
case "upload":
|
||||
err = upload(uri, projectID, conf.BaseLangcode)
|
||||
cli, err = conf.toClient()
|
||||
check(err)
|
||||
|
||||
err = cli.upload()
|
||||
case "auto-add":
|
||||
err = autoAdd(conf.LocalizableFiles[0])
|
||||
default:
|
||||
@@ -133,51 +136,133 @@ Commands:
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
// twoskyConf is the configuration structure for localization.
|
||||
type twoskyConf struct {
|
||||
// twoskyConfig is the configuration structure for localization.
|
||||
type twoskyConfig struct {
|
||||
Languages languages `json:"languages"`
|
||||
ProjectID string `json:"project_id"`
|
||||
BaseLangcode langCode `json:"base_locale"`
|
||||
LocalizableFiles []string `json:"localizable_files"`
|
||||
}
|
||||
|
||||
// readTwoskyConf returns configuration.
|
||||
func readTwoskyConf() (t twoskyConf, err error) {
|
||||
defer func() { err = errors.Annotate(err, "parsing twosky conf: %w") }()
|
||||
// readTwoskyConfig returns twosky configuration.
|
||||
func readTwoskyConfig() (t *twoskyConfig, err error) {
|
||||
defer func() { err = errors.Annotate(err, "parsing twosky config: %w") }()
|
||||
|
||||
b, err := os.ReadFile(twoskyConfFile)
|
||||
if err != nil {
|
||||
// Don't wrap the error since it's informative enough as is.
|
||||
return twoskyConf{}, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tsc []twoskyConf
|
||||
var tsc []twoskyConfig
|
||||
err = json.Unmarshal(b, &tsc)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err)
|
||||
|
||||
return twoskyConf{}, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(tsc) == 0 {
|
||||
err = fmt.Errorf("%q is empty", twoskyConfFile)
|
||||
|
||||
return twoskyConf{}, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
conf := tsc[0]
|
||||
|
||||
for _, lang := range conf.Languages {
|
||||
if lang == "" {
|
||||
return twoskyConf{}, errors.Error("language is empty")
|
||||
return nil, errors.Error("language is empty")
|
||||
}
|
||||
}
|
||||
|
||||
if len(conf.LocalizableFiles) == 0 {
|
||||
return twoskyConf{}, errors.Error("no localizable files specified")
|
||||
return nil, errors.Error("no localizable files specified")
|
||||
}
|
||||
|
||||
return conf, nil
|
||||
return &conf, nil
|
||||
}
|
||||
|
||||
// twoskyClient is the twosky client with methods for download and upload
|
||||
// translations.
|
||||
type twoskyClient struct {
|
||||
// uri is the base URL.
|
||||
uri *url.URL
|
||||
|
||||
// projectID is the name of the project.
|
||||
projectID string
|
||||
|
||||
// baseLang is the base language code.
|
||||
baseLang langCode
|
||||
|
||||
// langs is the list of codes of languages to download.
|
||||
langs []langCode
|
||||
}
|
||||
|
||||
// toClient reads values from environment variables or defaults, validates
|
||||
// them, and returns the twosky client.
|
||||
func (t *twoskyConfig) toClient() (cli *twoskyClient, err error) {
|
||||
defer func() { err = errors.Annotate(err, "filling config: %w") }()
|
||||
|
||||
uriStr := os.Getenv("TWOSKY_URI")
|
||||
if uriStr == "" {
|
||||
uriStr = twoskyURI
|
||||
}
|
||||
uri, err := url.Parse(uriStr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
projectID := os.Getenv("TWOSKY_PROJECT_ID")
|
||||
if projectID == "" {
|
||||
projectID = defaultProjectID
|
||||
}
|
||||
|
||||
baseLang := t.BaseLangcode
|
||||
uLangStr := os.Getenv("UPLOAD_LANGUAGE")
|
||||
if uLangStr != "" {
|
||||
baseLang = langCode(uLangStr)
|
||||
}
|
||||
|
||||
langs := maps.Keys(t.Languages)
|
||||
dlLangStr := os.Getenv("DOWNLOAD_LANGUAGES")
|
||||
if dlLangStr == "blocker" {
|
||||
langs = blockerLangCodes
|
||||
} else if dlLangStr != "" {
|
||||
var dlLangs []langCode
|
||||
dlLangs, err = validateLanguageStr(dlLangStr, t.Languages)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
langs = dlLangs
|
||||
}
|
||||
|
||||
return &twoskyClient{
|
||||
uri: uri,
|
||||
projectID: projectID,
|
||||
baseLang: baseLang,
|
||||
langs: langs,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// validateLanguageStr validates languages codes that contain in the str and
|
||||
// returns them or error.
|
||||
func validateLanguageStr(str string, all languages) (langs []langCode, err error) {
|
||||
codes := strings.Fields(str)
|
||||
langs = make([]langCode, 0, len(codes))
|
||||
|
||||
for _, k := range codes {
|
||||
lc := langCode(k)
|
||||
_, ok := all[lc]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("validating languages: unexpected language code %q", k)
|
||||
}
|
||||
|
||||
langs = append(langs, lc)
|
||||
}
|
||||
|
||||
return langs, nil
|
||||
}
|
||||
|
||||
// readLocales reads file with name fn and returns a map, where key is text
|
||||
@@ -227,169 +312,47 @@ func summary(langs languages) (err error) {
|
||||
|
||||
f := float64(len(loc)) * 100 / size
|
||||
|
||||
fmt.Printf("%s\t %6.2f %%\n", lang, f)
|
||||
blocker := ""
|
||||
|
||||
// N is small enough to not raise performance questions.
|
||||
ok := slices.Contains(blockerLangCodes, lang)
|
||||
if ok {
|
||||
blocker = " (blocker)"
|
||||
}
|
||||
|
||||
fmt.Printf("%s\t %6.2f %%%s\n", lang, f, blocker)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// download and save all translations. uri is the base URL. projectID is the
|
||||
// name of the project.
|
||||
func download(uri *url.URL, projectID string, langs languages) (err error) {
|
||||
var numWorker int
|
||||
|
||||
flagSet := flag.NewFlagSet("download", flag.ExitOnError)
|
||||
flagSet.Usage = func() {
|
||||
usage("download command error")
|
||||
}
|
||||
flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")
|
||||
|
||||
err = flagSet.Parse(os.Args[2:])
|
||||
if err != nil {
|
||||
// Don't wrap the error since it's informative enough as is.
|
||||
return err
|
||||
}
|
||||
|
||||
if numWorker < 1 {
|
||||
usage("count must be positive")
|
||||
}
|
||||
|
||||
downloadURI := uri.JoinPath("download")
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
wg := &sync.WaitGroup{}
|
||||
uriCh := make(chan *url.URL, len(langs))
|
||||
|
||||
for i := 0; i < numWorker; i++ {
|
||||
wg.Add(1)
|
||||
go downloadWorker(wg, client, uriCh)
|
||||
}
|
||||
|
||||
for lang := range langs {
|
||||
uri = translationURL(downloadURI, defaultBaseFile, projectID, lang)
|
||||
|
||||
uriCh <- uri
|
||||
}
|
||||
|
||||
close(uriCh)
|
||||
wg.Wait()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// downloadWorker downloads translations by received urls and saves them.
|
||||
func downloadWorker(wg *sync.WaitGroup, client *http.Client, uriCh <-chan *url.URL) {
|
||||
defer wg.Done()
|
||||
|
||||
for uri := range uriCh {
|
||||
data, err := getTranslation(client, uri.String())
|
||||
if err != nil {
|
||||
log.Error("download worker: getting translation: %s", err)
|
||||
log.Info("download worker: error response:\n%s", data)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
q := uri.Query()
|
||||
code := q.Get("language")
|
||||
|
||||
// Fix some TwoSky weirdnesses.
|
||||
//
|
||||
// TODO(a.garipov): Remove when those are fixed.
|
||||
code = strings.ToLower(code)
|
||||
|
||||
name := filepath.Join(localesDir, code+".json")
|
||||
err = os.WriteFile(name, data, 0o664)
|
||||
if err != nil {
|
||||
log.Error("download worker: writing file: %s", err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
fmt.Println(name)
|
||||
}
|
||||
}
|
||||
|
||||
// getTranslation returns received translation data and error. If err is not
|
||||
// nil, data may contain a response from server for inspection.
|
||||
func getTranslation(client *http.Client, url string) (data []byte, err error) {
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("requesting: %w", err)
|
||||
}
|
||||
|
||||
defer log.OnCloserError(resp.Body, log.ERROR)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
|
||||
|
||||
// Go on and download the body for inspection.
|
||||
}
|
||||
|
||||
limitReader, lrErr := aghio.LimitReader(resp.Body, readLimit)
|
||||
if lrErr != nil {
|
||||
// Generally shouldn't happen, since the only error returned by
|
||||
// [aghio.LimitReader] is an argument error.
|
||||
panic(fmt.Errorf("limit reading: %w", lrErr))
|
||||
}
|
||||
|
||||
data, readErr := io.ReadAll(limitReader)
|
||||
|
||||
return data, errors.WithDeferred(err, readErr)
|
||||
}
|
||||
|
||||
// translationURL returns a new url.URL with provided query parameters.
|
||||
func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
|
||||
uri = &url.URL{}
|
||||
*uri = *oldURL
|
||||
|
||||
// Fix some TwoSky weirdnesses.
|
||||
//
|
||||
// TODO(a.garipov): Remove when those are fixed.
|
||||
switch lang {
|
||||
case "si-lk":
|
||||
lang = "si-LK"
|
||||
case "zh-hk":
|
||||
lang = "zh-HK"
|
||||
default:
|
||||
// Go on.
|
||||
}
|
||||
|
||||
q := uri.Query()
|
||||
q.Set("format", "json")
|
||||
q.Set("filename", baseFile)
|
||||
q.Set("project", projectID)
|
||||
q.Set("language", string(lang))
|
||||
|
||||
uri.RawQuery = q.Encode()
|
||||
|
||||
return uri
|
||||
}
|
||||
|
||||
// unused prints unused text labels.
|
||||
func unused(basePath string) (err error) {
|
||||
defer func() { err = errors.Annotate(err, "unused: %w") }()
|
||||
|
||||
baseLoc, err := readLocales(basePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unused: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
locDir := filepath.Clean(localesDir)
|
||||
js, err := findJS(locDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fileNames := []string{}
|
||||
err = filepath.Walk(srcDir, func(name string, info os.FileInfo, err error) error {
|
||||
return findUnused(js, baseLoc)
|
||||
}
|
||||
|
||||
// findJS returns list of JavaScript and JSON files or error.
|
||||
func findJS(locDir string) (fileNames []string, err error) {
|
||||
walkFn := func(name string, _ os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
log.Info("warning: accessing a path %q: %s", name, err)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
if strings.HasPrefix(name, locDir) {
|
||||
return nil
|
||||
}
|
||||
@@ -400,13 +363,14 @@ func unused(basePath string) (err error) {
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("filepath walking %q: %w", srcDir, err)
|
||||
}
|
||||
|
||||
return findUnused(fileNames, baseLoc)
|
||||
err = filepath.Walk(srcDir, walkFn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("filepath walking %q: %w", srcDir, err)
|
||||
}
|
||||
|
||||
return fileNames, nil
|
||||
}
|
||||
|
||||
// findUnused prints unused text labels from fileNames.
|
||||
@@ -445,118 +409,6 @@ func findUnused(fileNames []string, loc locales) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// upload base translation. uri is the base URL. projectID is the name of the
|
||||
// project. baseLang is the base language code.
|
||||
func upload(uri *url.URL, projectID string, baseLang langCode) (err error) {
|
||||
defer func() { err = errors.Annotate(err, "upload: %w") }()
|
||||
|
||||
uploadURI := uri.JoinPath("upload")
|
||||
|
||||
lang := baseLang
|
||||
|
||||
langStr := os.Getenv("UPLOAD_LANGUAGE")
|
||||
if langStr != "" {
|
||||
lang = langCode(langStr)
|
||||
}
|
||||
|
||||
basePath := filepath.Join(localesDir, defaultBaseFile)
|
||||
|
||||
formData := map[string]string{
|
||||
"format": "json",
|
||||
"language": string(lang),
|
||||
"filename": defaultBaseFile,
|
||||
"project": projectID,
|
||||
}
|
||||
|
||||
buf, cType, err := prepareMultipartMsg(formData, basePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("preparing multipart msg: %w", err)
|
||||
}
|
||||
|
||||
err = send(uploadURI.String(), cType, buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sending multipart msg: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// prepareMultipartMsg prepares translation data for upload.
|
||||
func prepareMultipartMsg(
|
||||
formData map[string]string,
|
||||
basePath string,
|
||||
) (buf *bytes.Buffer, cType string, err error) {
|
||||
buf = &bytes.Buffer{}
|
||||
w := multipart.NewWriter(buf)
|
||||
var fw io.Writer
|
||||
|
||||
for k, v := range formData {
|
||||
err = w.WriteField(k, v)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("writing field: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
file, err := os.Open(basePath)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("opening file: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err = errors.WithDeferred(err, file.Close())
|
||||
}()
|
||||
|
||||
h := make(textproto.MIMEHeader)
|
||||
h.Set(httphdr.ContentType, aghhttp.HdrValApplicationJSON)
|
||||
|
||||
d := fmt.Sprintf("form-data; name=%q; filename=%q", "file", defaultBaseFile)
|
||||
h.Set(httphdr.ContentDisposition, d)
|
||||
|
||||
fw, err = w.CreatePart(h)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("creating part: %w", err)
|
||||
}
|
||||
|
||||
_, err = io.Copy(fw, file)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("copying: %w", err)
|
||||
}
|
||||
|
||||
err = w.Close()
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("closing writer: %w", err)
|
||||
}
|
||||
|
||||
return buf, w.FormDataContentType(), nil
|
||||
}
|
||||
|
||||
// send POST request to uriStr.
|
||||
func send(uriStr, cType string, buf *bytes.Buffer) (err error) {
|
||||
var client http.Client
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, uriStr, buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("bad request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set(httphdr.ContentType, cType)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("client post form: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err = errors.WithDeferred(err, resp.Body.Close())
|
||||
}()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// autoAdd adds locales with additions to the git and restores locales with
|
||||
// deletions.
|
||||
func autoAdd(basePath string) (err error) {
|
||||
@@ -572,28 +424,48 @@ func autoAdd(basePath string) (err error) {
|
||||
return errors.Error("base locale contains deletions")
|
||||
}
|
||||
|
||||
var (
|
||||
args []string
|
||||
code int
|
||||
out []byte
|
||||
)
|
||||
|
||||
if len(adds) > 0 {
|
||||
args = append([]string{"add"}, adds...)
|
||||
code, out, err = aghos.RunCommand("git", args...)
|
||||
|
||||
if err != nil || code != 0 {
|
||||
return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err)
|
||||
}
|
||||
err = handleAdds(adds)
|
||||
if err != nil {
|
||||
// Don't wrap the error since it's informative enough as is.
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(dels) > 0 {
|
||||
args = append([]string{"restore"}, dels...)
|
||||
code, out, err = aghos.RunCommand("git", args...)
|
||||
err = handleDels(dels)
|
||||
if err != nil {
|
||||
// Don't wrap the error since it's informative enough as is.
|
||||
return nil
|
||||
}
|
||||
|
||||
if err != nil || code != 0 {
|
||||
return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleAdds adds locales with additions to the git.
|
||||
func handleAdds(locales []string) (err error) {
|
||||
if len(locales) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
args := append([]string{"add"}, locales...)
|
||||
code, out, err := aghos.RunCommand("git", args...)
|
||||
|
||||
if err != nil || code != 0 {
|
||||
return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleDels restores locales with deletions.
|
||||
func handleDels(locales []string) (err error) {
|
||||
if len(locales) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
args := append([]string{"restore"}, locales...)
|
||||
code, out, err := aghos.RunCommand("git", args...)
|
||||
|
||||
if err != nil || code != 0 {
|
||||
return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
120
scripts/translations/upload.go
Normal file
120
scripts/translations/upload.go
Normal file
@@ -0,0 +1,120 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/httphdr"
|
||||
"github.com/AdguardTeam/golibs/mapsutil"
|
||||
)
|
||||
|
||||
// upload base translation.
|
||||
func (c *twoskyClient) upload() (err error) {
|
||||
defer func() { err = errors.Annotate(err, "upload: %w") }()
|
||||
|
||||
uploadURI := c.uri.JoinPath("upload")
|
||||
basePath := filepath.Join(localesDir, defaultBaseFile)
|
||||
|
||||
formData := map[string]string{
|
||||
"format": "json",
|
||||
"language": string(c.baseLang),
|
||||
"filename": defaultBaseFile,
|
||||
"project": c.projectID,
|
||||
}
|
||||
|
||||
buf, cType, err := prepareMultipartMsg(formData, basePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("preparing multipart msg: %w", err)
|
||||
}
|
||||
|
||||
err = send(uploadURI.String(), cType, buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sending multipart msg: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// prepareMultipartMsg prepares translation data for upload.
|
||||
func prepareMultipartMsg(
|
||||
formData map[string]string,
|
||||
basePath string,
|
||||
) (buf *bytes.Buffer, cType string, err error) {
|
||||
buf = &bytes.Buffer{}
|
||||
w := multipart.NewWriter(buf)
|
||||
var fw io.Writer
|
||||
|
||||
err = mapsutil.OrderedRangeError(formData, w.WriteField)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("writing field: %w", err)
|
||||
}
|
||||
|
||||
file, err := os.Open(basePath)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("opening file: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err = errors.WithDeferred(err, file.Close())
|
||||
}()
|
||||
|
||||
h := make(textproto.MIMEHeader)
|
||||
h.Set(httphdr.ContentType, aghhttp.HdrValApplicationJSON)
|
||||
|
||||
d := fmt.Sprintf("form-data; name=%q; filename=%q", "file", defaultBaseFile)
|
||||
h.Set(httphdr.ContentDisposition, d)
|
||||
|
||||
fw, err = w.CreatePart(h)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("creating part: %w", err)
|
||||
}
|
||||
|
||||
_, err = io.Copy(fw, file)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("copying: %w", err)
|
||||
}
|
||||
|
||||
err = w.Close()
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("closing writer: %w", err)
|
||||
}
|
||||
|
||||
return buf, w.FormDataContentType(), nil
|
||||
}
|
||||
|
||||
// send POST request to uriStr.
|
||||
func send(uriStr, cType string, buf *bytes.Buffer) (err error) {
|
||||
client := http.Client{
|
||||
Timeout: uploadTimeout,
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, uriStr, buf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("bad request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set(httphdr.ContentType, cType)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("client post form: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err = errors.WithDeferred(err, resp.Body.Close())
|
||||
}()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user