Pull request: all: update go and backend tools

Closes #2576.
Updates #2275.
Updates #2419.
Updates #2443.

Squashed commit of the following:

commit b1a4809ada298d675de12740051ba26fb9945957
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri May 21 14:01:40 2021 +0300

    all: add --local-frontend, upd docker

commit 619ee7c82f27e3405753003dbec556ffb056d025
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu May 20 15:02:33 2021 +0300

    bamboo-specs: bump docker version

commit 5c2b2fbce80afdcc81fd0cb83674dc3d64facbf1
Merge: 6536b32d 9c60aef6
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu May 20 15:01:47 2021 +0300

    Merge branch 'master' into 2275-upd-go

commit 6536b32dd4580425f7dedde6765463a79b9bd699
Merge: 9bb32bc4 6f7fd33a
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed May 19 20:38:48 2021 +0300

    Merge branch 'master' into 2275-upd-go

commit 9bb32bc4c0ac0f3a97195adc75359e48c9c58897
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed May 19 18:48:50 2021 +0300

    all: fix build, imp err handling

commit 6868eac7f7d2980fb706881f53e72afe5f7c3447
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed May 19 18:09:32 2021 +0300

    all: fix github lint

commit ebbb9c55f32fbd57e34e8b161016aa6b291c097c
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed May 19 17:36:56 2021 +0300

    all: update go and backend tools
This commit is contained in:
Ainar Garipov
2021-05-21 14:55:42 +03:00
parent 9c60aef637
commit c6888326b0
79 changed files with 382 additions and 703 deletions

View File

@@ -3,7 +3,7 @@ package home
import (
"bytes"
"encoding/binary"
"io/ioutil"
"io"
"net"
"net/http"
"os"
@@ -79,7 +79,8 @@ func glGetTokenDate(file string) uint32 {
fileReadCloser, err := aghio.LimitReadCloser(f, MaxFileSize)
if err != nil {
log.Error("LimitReadCloser: %s", err)
log.Error("creating limited reader: %s", err)
return 0
}
defer fileReadCloser.Close()
@@ -87,16 +88,18 @@ func glGetTokenDate(file string) uint32 {
var dateToken uint32
// This use of ReadAll is now safe, because we limited reader.
bs, err := ioutil.ReadAll(fileReadCloser)
bs, err := io.ReadAll(fileReadCloser)
if err != nil {
log.Error("ioutil.ReadAll: %s", err)
log.Error("reading token: %s", err)
return 0
}
buf := bytes.NewBuffer(bs)
err = binary.Read(buf, aghos.NativeEndian, &dateToken)
if err != nil {
log.Error("binary.Read: %s", err)
log.Error("decoding token: %s", err)
return 0
}

View File

@@ -1,8 +1,8 @@
package home
import (
"io/ioutil"
"net/http"
"os"
"testing"
"time"
@@ -23,13 +23,13 @@ func TestAuthGL(t *testing.T) {
data := make([]byte, 4)
aghos.NativeEndian.PutUint32(data, 1)
require.Nil(t, ioutil.WriteFile(glFilePrefix+"test", data, 0o644))
require.Nil(t, os.WriteFile(glFilePrefix+"test", data, 0o644))
assert.False(t, glCheckToken("test"))
data = make([]byte, 4)
aghos.NativeEndian.PutUint32(data, uint32(time.Now().UTC().Unix()+60))
require.Nil(t, ioutil.WriteFile(glFilePrefix+"test", data, 0o644))
require.Nil(t, os.WriteFile(glFilePrefix+"test", data, 0o644))
r, _ := http.NewRequest(http.MethodGet, "http://localhost/", nil)
r.AddCookie(&http.Cookie{Name: glCookieName, Value: "test"})
assert.True(t, glProcessCookie(r))

View File

@@ -3,7 +3,6 @@ package home
import (
"errors"
"fmt"
"io/ioutil"
"net"
"os"
"path/filepath"
@@ -267,7 +266,7 @@ func readConfigFile() ([]byte, error) {
}
configFile := config.getConfigFilename()
d, err := ioutil.ReadFile(configFile)
d, err := os.ReadFile(configFile)
if err != nil {
return nil, fmt.Errorf("couldn't read config file %s: %w", configFile, err)
}

View File

@@ -3,7 +3,7 @@ package home
import (
"encoding/json"
"fmt"
"io/ioutil"
"io"
"net"
"net/http"
"net/url"
@@ -226,7 +226,7 @@ func (f *Filtering) handleFilteringSetURL(w http.ResponseWriter, r *http.Request
func (f *Filtering) handleFilteringSetRules(w http.ResponseWriter, r *http.Request) {
// This use of ReadAll is safe, because request's body is now limited.
body, err := ioutil.ReadAll(r.Body)
body, err := io.ReadAll(r.Body)
if err != nil {
httpError(w, http.StatusBadRequest, "Failed to read request body: %s", err)
return

View File

@@ -4,7 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"io"
"net"
"net/http"
"os"
@@ -220,9 +220,9 @@ func disableDNSStubListener() error {
return fmt.Errorf("os.MkdirAll: %s: %w", dir, err)
}
err = ioutil.WriteFile(resolvedConfPath, []byte(resolvedConfData), 0o644)
err = os.WriteFile(resolvedConfPath, []byte(resolvedConfData), 0o644)
if err != nil {
return fmt.Errorf("ioutil.WriteFile: %s: %w", resolvedConfPath, err)
return fmt.Errorf("os.WriteFile: %s: %w", resolvedConfPath, err)
}
_ = os.Rename(resolvConfPath, resolvConfPath+".backup")
@@ -450,7 +450,7 @@ func (web *Web) handleInstallCheckConfigBeta(w http.ResponseWriter, r *http.Requ
return
}
body := nonBetaReqBody.String()
r.Body = ioutil.NopCloser(strings.NewReader(body))
r.Body = io.NopCloser(strings.NewReader(body))
r.ContentLength = int64(len(body))
web.handleInstallCheckConfig(w, r)
@@ -517,7 +517,7 @@ func (web *Web) handleInstallConfigureBeta(w http.ResponseWriter, r *http.Reques
return
}
body := nonBetaReqBody.String()
r.Body = ioutil.NopCloser(strings.NewReader(body))
r.Body = io.NopCloser(strings.NewReader(body))
r.ContentLength = int64(len(body))
web.handleInstallConfigure(w, r)

View File

@@ -6,7 +6,6 @@ import (
"fmt"
"hash/crc32"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
@@ -542,7 +541,7 @@ func (f *Filtering) updateIntl(filter *filter) (updated bool, err error) {
updated = false
log.Tracef("Downloading update for filter %d from %s", filter.ID, filter.URL)
tmpFile, err := ioutil.TempFile(filepath.Join(Context.getDataDir(), filterDir), "")
tmpFile, err := os.CreateTemp(filepath.Join(Context.getDataDir(), filterDir), "")
if err != nil {
return updated, err
}

View File

@@ -7,7 +7,7 @@ import (
"crypto/x509"
"errors"
"fmt"
"io/ioutil"
"io/fs"
"net"
"net/http"
"net/http/pprof"
@@ -92,7 +92,7 @@ func (c *homeContext) getDataDir() string {
var Context homeContext
// Main is the entry point
func Main() {
func Main(clientBuildFS fs.FS) {
// config can be specified, which reads options from there, but other command line flags have to override config values
// therefore, we must do it manually instead of using a lib
args := loadOptions()
@@ -117,12 +117,12 @@ func Main() {
}()
if args.serviceControlAction != "" {
handleServiceControlAction(args)
handleServiceControlAction(args, clientBuildFS)
return
}
// run the protection
run(args)
run(args, clientBuildFS)
}
func setupContext(args options) {
@@ -227,8 +227,49 @@ func setupConfig(args options) {
}
}
func initWeb(args options, clientBuildFS fs.FS) (web *Web, err error) {
var clientFS, clientBetaFS fs.FS
if args.localFrontend {
log.Info("warning: using local frontend files")
clientFS = os.DirFS("build/static")
clientBetaFS = os.DirFS("build2/static")
} else {
clientFS, err = fs.Sub(clientBuildFS, "build/static")
if err != nil {
return nil, fmt.Errorf("getting embedded client subdir: %w", err)
}
clientBetaFS, err = fs.Sub(clientBuildFS, "build2/static")
if err != nil {
return nil, fmt.Errorf("getting embedded beta client subdir: %w", err)
}
}
webConf := webConfig{
firstRun: Context.firstRun,
BindHost: config.BindHost,
BindPort: config.BindPort,
BetaBindPort: config.BetaBindPort,
ReadTimeout: readTimeout,
ReadHeaderTimeout: readHdrTimeout,
WriteTimeout: writeTimeout,
clientFS: clientFS,
clientBetaFS: clientBetaFS,
}
web = CreateWeb(&webConf)
if web == nil {
return nil, fmt.Errorf("initializing web: %w", err)
}
return web, nil
}
// run performs configurating and starts AdGuard Home.
func run(args options) {
func run(args options, clientBuildFS fs.FS) {
// configure config filename
initConfigFilename(args)
@@ -295,6 +336,7 @@ func run(args options) {
} else {
log.Info("authratelimiter is disabled")
}
Context.auth = InitAuth(
sessFilename,
config.Users,
@@ -311,19 +353,9 @@ func run(args options) {
log.Fatalf("Can't initialize TLS module")
}
webConf := webConfig{
firstRun: Context.firstRun,
BindHost: config.BindHost,
BindPort: config.BindPort,
BetaBindPort: config.BetaBindPort,
ReadTimeout: readTimeout,
ReadHeaderTimeout: readHdrTimeout,
WriteTimeout: writeTimeout,
}
Context.web = CreateWeb(&webConf)
if Context.web == nil {
log.Fatalf("Can't initialize Web module")
Context.web, err = initWeb(args, clientBuildFS)
if err != nil {
log.Fatal(err)
}
Context.subnetDetector, err = aghnet.NewSubnetDetector()
@@ -426,7 +458,7 @@ Please note, that this is crucial for a DNS server to be able to use that port.`
// Write PID to a file
func writePIDFile(fn string) bool {
data := fmt.Sprintf("%d", os.Getpid())
err := ioutil.WriteFile(fn, []byte(data), 0o644)
err := os.WriteFile(fn, []byte(data), 0o644)
if err != nil {
log.Error("Couldn't write PID to file %s: %v", fn, err)
return false

View File

@@ -1,191 +0,0 @@
// +build !race
// TODO(e.burkov): Remove this weird buildtag.
package home
import (
"context"
"encoding/base64"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"testing"
"time"
"github.com/AdguardTeam/dnsproxy/proxyutil"
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
)
const yamlConf = `bind_host: 127.0.0.1
bind_port: 3000
users: []
language: en
rlimit_nofile: 0
web_session_ttl: 720
dns:
bind_host: 127.0.0.1
port: 5354
statistics_interval: 90
querylog_enabled: true
querylog_interval: 90
querylog_size_memory: 0
protection_enabled: true
blocking_mode: null_ip
blocked_response_ttl: 0
ratelimit: 100
ratelimit_whitelist: []
refuse_any: false
bootstrap_dns:
- 1.1.1.1:53
all_servers: false
allowed_clients: []
disallowed_clients: []
blocked_hosts: []
parental_block_host: family-block.dns.adguard.com
safebrowsing_block_host: standard-block.dns.adguard.com
cache_size: 0
upstream_dns:
- https://1.1.1.1/dns-query
filtering_enabled: true
filters_update_interval: 168
parental_sensitivity: 13
parental_enabled: true
safesearch_enabled: false
safebrowsing_enabled: false
safebrowsing_cache_size: 1048576
safesearch_cache_size: 1048576
parental_cache_size: 1048576
cache_time: 30
rewrites: []
blocked_services: []
tls:
enabled: false
server_name: www.example.com
force_https: false
port_https: 443
port_dns_over_tls: 853
allow_unencrypted_doh: true
certificate_chain: ""
private_key: ""
certificate_path: ""
private_key_path: ""
filters:
- enabled: true
url: https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt
name: AdGuard Simplified Domain Names filter
id: 1
- enabled: false
url: https://hosts-file.net/ad_servers.txt
name: hpHosts - Ad and Tracking servers only
id: 2
- enabled: false
url: https://adaway.org/hosts.txt
name: adaway
id: 3
user_rules:
- ""
dhcp:
enabled: false
interface_name: ""
gateway_ip: ""
subnet_mask: ""
range_start: ""
range_end: ""
lease_duration: 86400
icmp_timeout_msec: 1000
clients: []
log_file: ""
verbose: false
schema_version: 5
`
// . Create a configuration file
// . Start AGH instance
// . Check Web server
// . Check DNS server
// . Check DNS server with DOH
// . Wait until the filters are downloaded
// . Stop and cleanup
func TestHome(t *testing.T) {
// Init new context
Context = homeContext{}
dir := t.TempDir()
fn := filepath.Join(dir, "AdGuardHome.yaml")
// Prepare the test config
assert.Nil(t, ioutil.WriteFile(fn, []byte(yamlConf), 0o644))
fn, _ = filepath.Abs(fn)
config = configuration{} // the global variable is dirty because of the previous tests run
args := options{}
args.configFilename = fn
args.workDir = dir
go run(args)
var err error
var resp *http.Response
h := http.Client{}
for i := 0; i != 50; i++ {
resp, err = h.Get("http://127.0.0.1:3000/")
if err == nil && resp.StatusCode != http.StatusNotFound {
break
}
time.Sleep(100 * time.Millisecond)
}
assert.Nilf(t, err, "%s", err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
resp, err = h.Get("http://127.0.0.1:3000/control/status")
assert.Nilf(t, err, "%s", err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
// test DNS over UDP
r, err := upstream.NewResolver("127.0.0.1:5354", upstream.Options{Timeout: 3 * time.Second})
assert.Nil(t, err)
addrs, err := r.LookupIPAddr(context.TODO(), "static.adguard.com")
assert.Nil(t, err)
haveIP := len(addrs) != 0
assert.True(t, haveIP)
// test DNS over HTTP without encryption
req := dns.Msg{}
req.Id = dns.Id()
req.RecursionDesired = true
req.Question = []dns.Question{{Name: "static.adguard.com.", Qtype: dns.TypeA, Qclass: dns.ClassINET}}
buf, err := req.Pack()
assert.Nil(t, err)
requestURL := "http://127.0.0.1:3000/dns-query?dns=" + base64.RawURLEncoding.EncodeToString(buf)
resp, err = http.DefaultClient.Get(requestURL)
assert.Nil(t, err)
body, err := ioutil.ReadAll(resp.Body)
assert.Nil(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
response := dns.Msg{}
err = response.Unpack(body)
assert.Nil(t, err)
addrs = nil
proxyutil.AppendIPAddrs(&addrs, response.Answer)
haveIP = len(addrs) != 0
assert.True(t, haveIP)
for i := 1; ; i++ {
var fi os.FileInfo
fi, err = os.Stat(filepath.Join(dir, "data", "filters", "1.txt"))
if err == nil && fi.Size() != 0 {
break
}
if i == 5 {
assert.True(t, false)
break
}
time.Sleep(1 * time.Second)
}
cleanup(context.Background())
cleanupAlways()
}

View File

@@ -2,7 +2,7 @@ package home
import (
"fmt"
"io/ioutil"
"io"
"net/http"
"strings"
@@ -61,7 +61,7 @@ func handleI18nCurrentLanguage(w http.ResponseWriter, r *http.Request) {
func handleI18nChangeLanguage(w http.ResponseWriter, r *http.Request) {
// This use of ReadAll is safe, because request's body is now limited.
body, err := ioutil.ReadAll(r.Body)
body, err := io.ReadAll(r.Body)
if err != nil {
msg := fmt.Sprintf("failed to read request body: %s", err)
log.Println(msg)

View File

@@ -1,7 +1,7 @@
package home
import (
"io/ioutil"
"io"
"net/http"
"net/http/httptest"
"strings"
@@ -44,7 +44,7 @@ func TestLimitRequestBody(t *testing.T) {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var b []byte
b, *err = ioutil.ReadAll(r.Body)
b, *err = io.ReadAll(r.Body)
_, werr := w.Write(b)
require.Nil(t, werr)
})

View File

@@ -36,6 +36,10 @@ type options struct {
// noEtcHosts flag should be provided when /etc/hosts file shouldn't be
// used.
noEtcHosts bool
// localFrontend forces AdGuard Home to use the frontend files from disk
// rather than the ones that have been compiled into the binary.
localFrontend bool
}
// functions used for their side-effects
@@ -234,6 +238,16 @@ var noEtcHostsArg = arg{
serialize: func(o options) []string { return boolSliceOrNil(o.noEtcHosts) },
}
var localFrontendArg = arg{
description: "Use local frontend directories.",
longName: "local-frontend",
shortName: "",
updateWithValue: nil,
updateNoValue: func(o options) (options, error) { o.localFrontend = true; return o, nil },
effect: nil,
serialize: func(o options) []string { return boolSliceOrNil(o.localFrontend) },
}
func init() {
args = []arg{
configArg,
@@ -247,6 +261,7 @@ func init() {
noCheckUpdateArg,
disableMemoryOptimizationArg,
noEtcHostsArg,
localFrontendArg,
verboseArg,
glinetArg,
versionArg,

View File

@@ -3,7 +3,7 @@ package home
import (
"errors"
"fmt"
"io/ioutil"
"io/fs"
"os"
"runtime"
"strconv"
@@ -28,7 +28,8 @@ const (
// Represents the program that will be launched by a service or daemon
type program struct {
opts options
clientBuildFS fs.FS
opts options
}
// Start should quickly start the program
@@ -36,7 +37,8 @@ func (p *program) Start(s service.Service) error {
// Start should not block. Do the actual work async.
args := p.opts
args.runningAsService = true
go run(args)
go run(args, p.clientBuildFS)
return nil
}
@@ -95,7 +97,7 @@ func sendSigReload() {
}
pidfile := fmt.Sprintf("/var/run/%s.pid", serviceName)
data, err := ioutil.ReadFile(pidfile)
data, err := os.ReadFile(pidfile)
if errors.Is(err, os.ErrNotExist) {
var code int
var psdata string
@@ -142,7 +144,7 @@ func sendSigReload() {
// run - this is a special command that is not supposed to be used directly
// it is specified when we register a service, and it indicates to the app
// that it is being run as a service/daemon.
func handleServiceControlAction(opts options) {
func handleServiceControlAction(opts options, clientBuildFS fs.FS) {
action := opts.serviceControlAction
log.Printf("Service control action: %s", action)
@@ -165,7 +167,10 @@ func handleServiceControlAction(opts options) {
Arguments: serialize(runOpts),
}
configureService(svcConfig)
prg := &program{runOpts}
prg := &program{
clientBuildFS: clientBuildFS,
opts: runOpts,
}
s, err := service.New(prg, svcConfig)
if err != nil {
log.Fatal(err)

View File

@@ -12,7 +12,6 @@ import (
"encoding/pem"
"errors"
"fmt"
"io/ioutil"
"net/http"
"os"
"path/filepath"
@@ -164,7 +163,7 @@ func tlsLoadConfig(tls *tlsConfigSettings, status *tlsConfigStatus) bool {
status.WarningValidation = "certificate data and file can't be set together"
return false
}
tls.CertificateChainData, err = ioutil.ReadFile(tls.CertificatePath)
tls.CertificateChainData, err = os.ReadFile(tls.CertificatePath)
if err != nil {
status.WarningValidation = err.Error()
return false
@@ -177,7 +176,7 @@ func tlsLoadConfig(tls *tlsConfigSettings, status *tlsConfigStatus) bool {
status.WarningValidation = "private key data and file can't be set together"
return false
}
tls.PrivateKeyData, err = ioutil.ReadFile(tls.PrivateKeyPath)
tls.PrivateKeyData, err = os.ReadFile(tls.PrivateKeyPath)
if err != nil {
status.WarningValidation = err.Error()
return false
@@ -574,18 +573,17 @@ func LoadSystemRootCAs() (roots *x509.CertPool) {
}
roots = x509.NewCertPool()
for _, dir := range dirs {
fis, err := ioutil.ReadDir(dir)
dirEnts, err := os.ReadDir(dir)
if errors.Is(err, os.ErrNotExist) {
continue
}
if err != nil {
} else if err != nil {
log.Error("opening directory: %q: %s", dir, err)
}
var rootsAdded bool
for _, fi := range fis {
for _, de := range dirEnts {
var certData []byte
certData, err = ioutil.ReadFile(filepath.Join(dir, fi.Name()))
certData, err = os.ReadFile(filepath.Join(dir, de.Name()))
if err == nil && roots.AppendCertsFromPEM(certData) {
rootsAdded = true
}

View File

@@ -3,6 +3,7 @@ package home
import (
"context"
"crypto/tls"
"io/fs"
"net"
"net/http"
"strconv"
@@ -12,7 +13,6 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/golibs/log"
"github.com/NYTimes/gziphandler"
"github.com/gobuffalo/packr"
)
// HTTP scheme constants.
@@ -33,11 +33,14 @@ const (
)
type webConfig struct {
firstRun bool
BindHost net.IP
BindPort int
BetaBindPort int
PortHTTPS int
firstRun bool
clientFS fs.FS
clientBetaFS fs.FS
// ReadTimeout is an option to pass to http.Server for setting an
// appropriate field.
@@ -85,19 +88,18 @@ func CreateWeb(conf *webConfig) *Web {
w := Web{}
w.conf = conf
// Initialize and run the admin Web interface
box := packr.NewBox("../../build/static")
boxBeta := packr.NewBox("../../build2/static")
clientFS := http.FileServer(http.FS(conf.clientFS))
betaClientFS := http.FileServer(http.FS(conf.clientBetaFS))
// if not configured, redirect / to /install.html, otherwise redirect /install.html to /
Context.mux.Handle("/", withMiddlewares(http.FileServer(box), gziphandler.GzipHandler, optionalAuthHandler, postInstallHandler))
w.handlerBeta = withMiddlewares(http.FileServer(boxBeta), gziphandler.GzipHandler, optionalAuthHandler, postInstallHandler)
Context.mux.Handle("/", withMiddlewares(clientFS, gziphandler.GzipHandler, optionalAuthHandler, postInstallHandler))
w.handlerBeta = withMiddlewares(betaClientFS, gziphandler.GzipHandler, optionalAuthHandler, postInstallHandler)
// add handlers for /install paths, we only need them when we're not configured yet
if conf.firstRun {
log.Info("This is the first launch of AdGuard Home, redirecting everything to /install.html ")
Context.mux.Handle("/install.html", preInstallHandler(http.FileServer(box)))
w.installerBeta = preInstallHandler(http.FileServer(boxBeta))
Context.mux.Handle("/install.html", preInstallHandler(clientFS))
w.installerBeta = preInstallHandler(betaClientFS)
w.registerInstallHandlers()
// This must be removed in API v1.
w.registerBetaInstallHandlers()

View File

@@ -4,7 +4,7 @@ import (
"context"
"encoding/binary"
"fmt"
"io/ioutil"
"io"
"net"
"strings"
"time"
@@ -163,7 +163,7 @@ func (w *Whois) query(ctx context.Context, target, serverAddr string) (string, e
}
// This use of ReadAll is now safe, because we limited the conn Reader.
data, err := ioutil.ReadAll(connReadCloser)
data, err := io.ReadAll(connReadCloser)
if err != nil {
return "", err
}