From b6d00f774bfa73c1890fec12db61c0cd4aa776ed Mon Sep 17 00:00:00 2001 From: Ainar Garipov Date: Mon, 19 Jun 2023 12:21:32 +0300 Subject: [PATCH 1/8] Pull request 1875: next-gocognit Merge in DNS/adguard-home from next-gocognit to master Squashed commit of the following: commit 554458a63b44e71244b4364168e6e9903b880638 Author: Ainar Garipov Date: Fri Jun 16 19:58:14 2023 +0300 all: add tool; typo commit 1b97f93811d1ea826762cfa43fef49ae43e06939 Author: Ainar Garipov Date: Fri Jun 16 19:49:40 2023 +0300 all: add gocognit --- internal/next/cmd/signal.go | 2 +- internal/next/websvc/http.go | 53 +++++++++++++++++++--------------- internal/next/websvc/websvc.go | 2 +- internal/tools/go.mod | 9 +++--- internal/tools/go.sum | 23 +++++++++------ internal/tools/tools.go | 1 + internal/version/version.go | 22 +++++++++----- scripts/make/go-lint.sh | 11 +++++++ scripts/make/go-tools.sh | 2 ++ staticcheck.conf | 7 +++++ 10 files changed, 86 insertions(+), 46 deletions(-) diff --git a/internal/next/cmd/signal.go b/internal/next/cmd/signal.go index 487eabdb..d02b8b5e 100644 --- a/internal/next/cmd/signal.go +++ b/internal/next/cmd/signal.go @@ -54,7 +54,7 @@ func (h *signalHandler) reconfigure() { status := h.shutdown() if status != statusSuccess { - log.Info("sighdlr: reconfiruging: exiting with status %d", status) + log.Info("sighdlr: reconfiguring: exiting with status %d", status) os.Exit(status) } diff --git a/internal/next/websvc/http.go b/internal/next/websvc/http.go index 0720baab..2813d9fe 100644 --- a/internal/next/websvc/http.go +++ b/internal/next/websvc/http.go @@ -78,34 +78,41 @@ func (svc *Service) handlePatchSettingsHTTP(w http.ResponseWriter, r *http.Reque // Launch the new HTTP service in a separate goroutine to let this handler // finish and thus, this server to shutdown. - go func() { - defer cancelUpd() + go svc.relaunch(updCtx, cancelUpd, newConf) +} - updErr := svc.confMgr.UpdateWeb(updCtx, newConf) - if updErr != nil { - writeJSONErrorResponse(w, r, fmt.Errorf("updating: %w", updErr)) +// relaunch updates the web service in the configuration manager and starts it. +// It is intended to be used as a goroutine. +func (svc *Service) relaunch(ctx context.Context, cancel context.CancelFunc, newConf *Config) { + defer log.OnPanic("websvc: relaunching") + + defer cancel() + + err := svc.confMgr.UpdateWeb(ctx, newConf) + if err != nil { + log.Error("websvc: updating web: %s", err) + + return + } + + // TODO(a.garipov): Consider better ways to do this. + const maxUpdDur = 5 * time.Second + updStart := time.Now() + var newSvc agh.ServiceWithConfig[*Config] + for newSvc = svc.confMgr.Web(); newSvc == svc; { + if time.Since(updStart) >= maxUpdDur { + log.Error("websvc: failed to update svc after %s", maxUpdDur) return } - // TODO(a.garipov): Consider better ways to do this. - const maxUpdDur = 10 * time.Second - updStart := time.Now() - var newSvc agh.ServiceWithConfig[*Config] - for newSvc = svc.confMgr.Web(); newSvc == svc; { - if time.Since(updStart) >= maxUpdDur { - log.Error("websvc: failed to update svc after %s", maxUpdDur) + log.Debug("websvc: waiting for new websvc to be configured") - return - } + time.Sleep(100 * time.Millisecond) + } - log.Debug("websvc: waiting for new websvc to be configured") - time.Sleep(1 * time.Second) - } - - updErr = newSvc.Start() - if updErr != nil { - log.Error("websvc: new svc failed to start with error: %s", updErr) - } - }() + err = newSvc.Start() + if err != nil { + log.Error("websvc: new svc failed to start with error: %s", err) + } } diff --git a/internal/next/websvc/websvc.go b/internal/next/websvc/websvc.go index 54a4840f..0165142f 100644 --- a/internal/next/websvc/websvc.go +++ b/internal/next/websvc/websvc.go @@ -139,7 +139,7 @@ func New(c *Config) (svc *Service, err error) { return svc, nil } -// newMux returns a new HTTP request multiplexor for the AdGuard Home web +// newMux returns a new HTTP request multiplexer for the AdGuard Home web // service. func newMux(svc *Service) (mux *httptreemux.ContextMux) { mux = httptreemux.NewContextMux() diff --git a/internal/tools/go.mod b/internal/tools/go.mod index bf6a766d..293e09f1 100644 --- a/internal/tools/go.mod +++ b/internal/tools/go.mod @@ -9,7 +9,8 @@ require ( github.com/kisielk/errcheck v1.6.3 github.com/kyoh86/looppointer v0.2.1 github.com/securego/gosec/v2 v2.16.0 - golang.org/x/tools v0.9.3 + github.com/uudashr/gocognit v1.0.6 + golang.org/x/tools v0.10.0 golang.org/x/vuln v0.1.0 honnef.co/go/tools v0.4.3 mvdan.cc/gofumpt v0.5.0 @@ -26,8 +27,8 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect golang.org/x/exp/typeparams v0.0.0-20230522175609-2e198f4a06a1 // indirect - golang.org/x/mod v0.10.0 // indirect - golang.org/x/sync v0.2.0 // indirect - golang.org/x/sys v0.8.0 // indirect + golang.org/x/mod v0.11.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.9.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/internal/tools/go.sum b/internal/tools/go.sum index 93724bea..58f23dd6 100644 --- a/internal/tools/go.sum +++ b/internal/tools/go.sum @@ -40,6 +40,8 @@ github.com/securego/gosec/v2 v2.16.0 h1:Pi0JKoasQQ3NnoRao/ww/N/XdynIB9NRYYZT5CyO github.com/securego/gosec/v2 v2.16.0/go.mod h1:xvLcVZqUfo4aAQu56TNv7/Ltz6emAOQAEsrZrt7uGlI= github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/uudashr/gocognit v1.0.6 h1:2Cgi6MweCsdB6kpcVQp7EW4U23iBFQWfTXiWlyp842Y= +github.com/uudashr/gocognit v1.0.6/go.mod h1:nAIUuVBnYU7pcninia3BHOvQkpQCeO76Uscky5BOwcY= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -56,20 +58,21 @@ golang.org/x/exp/typeparams v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:AbB0pIl golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -79,8 +82,9 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220702020025-31831981b65f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -92,8 +96,9 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20201007032633-0806396f153e/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= -golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/vuln v0.1.0 h1:9GRdj6wAIkDrsMevuolY+SXERPjQPp2P1ysYA0jpZe0= golang.org/x/vuln v0.1.0/go.mod h1:/YuzZYjGbwB8y19CisAppfyw3uTZnuCz3r+qgx/QRzU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/tools/tools.go b/internal/tools/tools.go index 88f55fa9..a1a473ff 100644 --- a/internal/tools/tools.go +++ b/internal/tools/tools.go @@ -9,6 +9,7 @@ import ( _ "github.com/kisielk/errcheck" _ "github.com/kyoh86/looppointer" _ "github.com/securego/gosec/v2/cmd/gosec" + _ "github.com/uudashr/gocognit/cmd/gocognit" _ "golang.org/x/tools/go/analysis/passes/nilness/cmd/nilness" _ "golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow" _ "golang.org/x/vuln/cmd/govulncheck" diff --git a/internal/version/version.go b/internal/version/version.go index ca78efff..b553d6f8 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -143,14 +143,7 @@ func Verbose() (v string) { runtime.Version(), ) - if committime != "" { - commitTimeUnix, err := strconv.ParseInt(committime, 10, 64) - if err != nil { - stringutil.WriteToBuilder(b, nl, vFmtTimeHdr, fmt.Sprintf("parse error: %s", err)) - } else { - stringutil.WriteToBuilder(b, nl, vFmtTimeHdr, time.Unix(commitTimeUnix, 0).String()) - } - } + writeCommitTime(b) stringutil.WriteToBuilder(b, nl, vFmtGOOSHdr, nl, vFmtGOARCHHdr) if goarm != "" { @@ -179,3 +172,16 @@ func Verbose() (v string) { return b.String() } + +func writeCommitTime(b *strings.Builder) { + if committime == "" { + return + } + + commitTimeUnix, err := strconv.ParseInt(committime, 10, 64) + if err != nil { + stringutil.WriteToBuilder(b, "\n", vFmtTimeHdr, fmt.Sprintf("parse error: %s", err)) + } else { + stringutil.WriteToBuilder(b, "\n", vFmtTimeHdr, time.Unix(commitTimeUnix, 0).String()) + } +} diff --git a/scripts/make/go-lint.sh b/scripts/make/go-lint.sh index 93b7de11..33a10baf 100644 --- a/scripts/make/go-lint.sh +++ b/scripts/make/go-lint.sh @@ -170,6 +170,17 @@ run_linter govulncheck ./... run_linter gocyclo --over 10 . +# TODO(a.garipov): Enable for all. +run_linter gocognit --over 10\ + ./internal/aghalg/\ + ./internal/aghchan/\ + ./internal/aghhttp/\ + ./internal/aghio/\ + ./internal/tools/\ + ./internal/next/\ + ./internal/version/\ + ; + run_linter ineffassign ./... run_linter unparam ./... diff --git a/scripts/make/go-tools.sh b/scripts/make/go-tools.sh index ba512dc3..f9ccb42f 100644 --- a/scripts/make/go-tools.sh +++ b/scripts/make/go-tools.sh @@ -38,6 +38,7 @@ readonly go rm -f\ bin/errcheck\ bin/fieldalignment\ + bin/gocognit\ bin/gocyclo\ bin/gofumpt\ bin/gosec\ @@ -69,6 +70,7 @@ env\ github.com/kisielk/errcheck\ github.com/kyoh86/looppointer/cmd/looppointer\ github.com/securego/gosec/v2/cmd/gosec\ + github.com/uudashr/gocognit/cmd/gocognit\ golang.org/x/tools/go/analysis/passes/fieldalignment/cmd/fieldalignment\ golang.org/x/tools/go/analysis/passes/nilness/cmd/nilness\ golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow\ diff --git a/staticcheck.conf b/staticcheck.conf index 84d08287..3962957c 100644 --- a/staticcheck.conf +++ b/staticcheck.conf @@ -4,13 +4,20 @@ initialisms = [ # # Do not add "PTR" since we use "Ptr" as a suffix. "inherit" +, "ASN" , "DHCP" +, "DNSSEC" + # E.g. SentryDSN. +, "DSN" +, "ECS" , "EDNS" , "MX" , "QUIC" , "RA" +, "RRSIG" , "SDNS" , "SLAAC" +, "SOA" , "SVCB" , "TLD" , "WHOIS" From d26c480d03944653bcf78674724da4ad74f2a502 Mon Sep 17 00:00:00 2001 From: Ainar Garipov Date: Mon, 19 Jun 2023 15:45:01 +0300 Subject: [PATCH 2/8] Pull request 1877: 5913-fix-safesearch-ipv6 Updates #5913. Squashed commit of the following: commit a0ab1320ea22dc1b4e2804ef2d14e0091daa6a1e Author: Ainar Garipov Date: Mon Jun 19 15:23:44 2023 +0300 all: fmt; typo commit 3a2e561c535bbbd2b2eeeaa1a6f423bc123b1a6b Author: Ainar Garipov Date: Mon Jun 19 15:16:28 2023 +0300 all: fix safesearch for ipv6 --- CHANGELOG.md | 8 ++++---- internal/dnsforward/msg.go | 15 ++++++--------- internal/filtering/rewrite/storage.go | 2 +- internal/filtering/safesearch/safesearch.go | 11 +++++++++-- internal/filtering/safesearch/safesearch_test.go | 11 +++++++++++ 5 files changed, 31 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d23c574c..12ac29a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,11 +82,11 @@ In this release, the schema version has changed from 20 to 21. ### Fixed - - DNSCrypt upstream not resetting the client and resolver information on - dialing errors ([#5872]). +- Safe Search not working with `AAAA` queries for Yandex domains ([#5913]). [#951]: https://github.com/AdguardTeam/AdGuardHome/issues/951 [#1577]: https://github.com/AdguardTeam/AdGuardHome/issues/1577 +[#5913]: https://github.com/AdguardTeam/AdGuardHome/issues/5913 @@ -84,7 +84,8 @@ In this release, the schema version has changed from 20 to 21. - Queries with the question-section target `.`, for example `NS .`, are now counted in the statistics and correctly shown in the query log ([#5910]). -- Safe Search not working with `AAAA` queries for Yandex domains ([#5913]). +- Safe Search not working with `AAAA` queries for domains that don't have `AAAA` + records ([#5913]). [#951]: https://github.com/AdguardTeam/AdGuardHome/issues/951 [#1577]: https://github.com/AdguardTeam/AdGuardHome/issues/1577 diff --git a/internal/filtering/safesearch/safesearch.go b/internal/filtering/safesearch/safesearch.go index a1ac6404..9d5b5121 100644 --- a/internal/filtering/safesearch/safesearch.go +++ b/internal/filtering/safesearch/safesearch.go @@ -161,12 +161,8 @@ func (ss *Default) resetEngine( // type check var _ filtering.SafeSearch = (*Default)(nil) -// CheckHost implements the [filtering.SafeSearch] interface for -// *DefaultSafeSearch. -func (ss *Default) CheckHost( - host string, - qtype rules.RRType, -) (res filtering.Result, err error) { +// CheckHost implements the [filtering.SafeSearch] interface for *Default. +func (ss *Default) CheckHost(host string, qtype rules.RRType) (res filtering.Result, err error) { start := time.Now() defer func() { ss.log(log.DEBUG, "lookup for %q finished in %s", host, time.Since(start)) @@ -196,14 +192,10 @@ func (ss *Default) CheckHost( return filtering.Result{}, err } - if fltRes != nil { - res = *fltRes - ss.setCacheResult(host, qtype, res) + res = *fltRes + ss.setCacheResult(host, qtype, res) - return res, nil - } - - return filtering.Result{}, fmt.Errorf("no ip addresses for %q", host) + return res, nil } // searchHost looks up DNS rewrites in the internal DNS filtering engine. @@ -229,7 +221,11 @@ func (ss *Default) searchHost(host string, qtype rules.RRType) (res *rules.DNSRe } // newResult creates Result object from rewrite rule. qtype must be either -// [dns.TypeA] or [dns.TypeAAAA]. +// [dns.TypeA] or [dns.TypeAAAA]. If err is nil, res is never nil, so that the +// empty result is converted into a NODATA response. +// +// TODO(a.garipov): Use the main rewrite result mechanism used in +// [dnsforward.Server.filterDNSRequest]. func (ss *Default) newResult( rewrite *rules.DNSRewrite, qtype rules.RRType, @@ -243,9 +239,10 @@ func (ss *Default) newResult( } if rewrite.RRType == qtype { - ip, ok := rewrite.Value.(net.IP) + v := rewrite.Value + ip, ok := v.(net.IP) if !ok || ip == nil { - return nil, nil + return nil, fmt.Errorf("expected ip rewrite value, got %T(%[1]v)", v) } res.Rules[0].IP = ip @@ -255,13 +252,6 @@ func (ss *Default) newResult( host := rewrite.NewCNAME if host == "" { - // If there is a rewrite, but it's neither a CNAME one nor one matching - // the IP version, then it's a service that only has one type of IP - // record but not the other. Return the empty result to be converted - // into a NODATA response. - // - // TODO(a.garipov): Use the main rewrite result mechanism used in - // [dnsforward.Server.filterDNSRequest]. return res, nil } @@ -269,7 +259,7 @@ func (ss *Default) newResult( ips, err := ss.resolver.LookupIP(context.Background(), qtypeToProto(qtype), host) if err != nil { - return nil, err + return nil, fmt.Errorf("resolving cname: %w", err) } ss.log(log.DEBUG, "resolved %s", ips) @@ -283,11 +273,9 @@ func (ss *Default) newResult( } res.Rules[0].IP = ip - - return res, nil } - return nil, nil + return res, nil } // qtypeToProto returns "ip4" for [dns.TypeA] and "ip6" for [dns.TypeAAAA]. diff --git a/internal/filtering/safesearch/safesearch_test.go b/internal/filtering/safesearch/safesearch_test.go index b0775c60..12860c5d 100644 --- a/internal/filtering/safesearch/safesearch_test.go +++ b/internal/filtering/safesearch/safesearch_test.go @@ -1,6 +1,7 @@ package safesearch_test import ( + "context" "net" "testing" "time" @@ -80,6 +81,14 @@ func TestDefault_CheckHost_yandexAAAA(t *testing.T) { require.NoError(t, err) assert.True(t, res.IsFiltered) + + // TODO(a.garipov): Currently, the safe-search filter returns a single rule + // with a nil IP address. This isn't really necessary and should be changed + // once the TODO in [safesearch.Default.newResult] is resolved. + require.Len(t, res.Rules, 1) + + assert.Nil(t, res.Rules[0].IP) + assert.EqualValues(t, filtering.SafeSearchListID, res.Rules[0].FilterListID) } func TestDefault_CheckHost_google(t *testing.T) { @@ -116,6 +125,56 @@ func TestDefault_CheckHost_google(t *testing.T) { } } +// testResolver is a [filtering.Resolver] for tests. +// +// TODO(a.garipov): Move to aghtest and use everywhere. +type testResolver struct { + OnLookupIP func(ctx context.Context, network, host string) (ips []net.IP, err error) +} + +// type check +var _ filtering.Resolver = (*testResolver)(nil) + +// LookupIP implements the [filtering.Resolver] interface for *testResolver. +func (r *testResolver) LookupIP( + ctx context.Context, + network string, + host string, +) (ips []net.IP, err error) { + return r.OnLookupIP(ctx, network, host) +} + +func TestDefault_CheckHost_duckduckgoAAAA(t *testing.T) { + conf := testConf + conf.CustomResolver = &testResolver{ + OnLookupIP: func(_ context.Context, network, host string) (ips []net.IP, err error) { + assert.Equal(t, "ip6", network) + assert.Equal(t, "safe.duckduckgo.com", host) + + return nil, nil + }, + } + + ss, err := safesearch.NewDefault(conf, "", testCacheSize, testCacheTTL) + require.NoError(t, err) + + // The DuckDuckGo safe-search addresses are resolved through CNAMEs, but + // DuckDuckGo doesn't have a safe-search IPv6 address. The result should be + // the same as the one for Yandex IPv6. That is, a NODATA response. + res, err := ss.CheckHost("www.duckduckgo.com", dns.TypeAAAA) + require.NoError(t, err) + + assert.True(t, res.IsFiltered) + + // TODO(a.garipov): Currently, the safe-search filter returns a single rule + // with a nil IP address. This isn't really necessary and should be changed + // once the TODO in [safesearch.Default.newResult] is resolved. + require.Len(t, res.Rules, 1) + + assert.Nil(t, res.Rules[0].IP) + assert.EqualValues(t, filtering.SafeSearchListID, res.Rules[0].FilterListID) +} + func TestDefault_Update(t *testing.T) { conf := testConf ss, err := safesearch.NewDefault(conf, "", testCacheSize, testCacheTTL) diff --git a/internal/next/changelog.md b/internal/next/changelog.md index 39e1aff0..ac304796 100644 --- a/internal/next/changelog.md +++ b/internal/next/changelog.md @@ -24,7 +24,7 @@ enough. ### Fixed -- Inconsistent application of `--work-dir/-w` ([#2902]). +- Inconsistent application of `--work-dir/-w` ([#2598], [#2902]). - The order of `-v/--verbose` and `--version` being significant ([#2893]). ### Removed @@ -33,6 +33,7 @@ enough. - `--host` and `-p/--port` flags. Use `--web-addr=host:port` to set an address on which to serve the Web UI. `-h` is now an alias for `--help`, see above. +[#2598]: https://github.com/AdguardTeam/AdGuardHome/issues/2598 [#2893]: https://github.com/AdguardTeam/AdGuardHome/issues/2893 [#2902]: https://github.com/AdguardTeam/AdGuardHome/issues/2902 [#5676]: https://github.com/AdguardTeam/AdGuardHome/issues/5676 From 06d465b0d128c1e87432a34c69b65e5f3095a1b6 Mon Sep 17 00:00:00 2001 From: Stanislav Chzhen Date: Wed, 21 Jun 2023 12:53:53 +0300 Subject: [PATCH 6/8] Pull request 1858: AG-22594-imp-whois Merge in DNS/adguard-home from AG-22594-imp-whois to master Squashed commit of the following: commit 093feed53291d02469fb1bd8d99472597ebd5015 Merge: 956d20dc4 ca313521d Author: Stanislav Chzhen Date: Wed Jun 21 12:42:40 2023 +0300 Merge branch 'master' into AG-22594-imp-whois commit 956d20dc473dcec90895b6f618fc56e96e9ff833 Author: Stanislav Chzhen Date: Tue Jun 20 18:30:48 2023 +0300 whois: imp code more commit c771fd9c5e4d90e76d079a0d25ab097ab5652a42 Author: Stanislav Chzhen Date: Tue Jun 20 15:05:45 2023 +0300 whois: imp code commit 21900fd468e10d9aee22149a6312b8596ff39810 Merge: 8dbe132c0 371261b2c Author: Stanislav Chzhen Date: Tue Jun 20 11:34:06 2023 +0300 Merge branch 'master' into AG-22594-imp-whois commit 8dbe132c08d3ad4a63b0d4bdb9d00a5bc25971f4 Author: Stanislav Chzhen Date: Tue Jun 20 11:33:26 2023 +0300 whois: imp code more commit f5e761a260237579c67cbd48f01ea90499bff6b0 Author: Stanislav Chzhen Date: Mon Jun 19 16:04:35 2023 +0300 whois: imp code commit 2780f7e16aacddad8736f83b77ef9bfa1271f8b1 Author: Stanislav Chzhen Date: Fri Jun 16 17:33:47 2023 +0300 all: imp code commit 1fc67016068b745a46b3d0d341ab14f9f5bdc9aa Author: Stanislav Chzhen Date: Fri Jun 16 17:29:19 2023 +0300 whois: imp tests commit 204761870764fb10feea20065d79dee8c321e70b Author: Stanislav Chzhen Date: Fri Jun 16 11:55:37 2023 +0300 all: upd deps commit ded4f59498c5c544277b9c8e249567626547680e Author: Stanislav Chzhen Date: Wed Jun 14 20:43:32 2023 +0300 all: imp tests commit 0eed9834ff9dd94d0788ce69d0bb0521fa725410 Author: Stanislav Chzhen Date: Wed Jun 14 19:31:49 2023 +0300 all: imp code commit 9f867587c8ad87363b8c8b061ead536c1ec59c5d Merge: 504e9484d 681c604c2 Author: Stanislav Chzhen Date: Tue Jun 13 14:20:44 2023 +0300 Merge branch 'master' into AG-22594-imp-whois commit 504e9484dd84ab9d7c84a3f8399993d6422d3b67 Author: Stanislav Chzhen Date: Tue Jun 13 14:18:06 2023 +0300 all: imp cache commit c492abe41ace7ad76fcd4e297c22b910a90fec30 Merge: db36adb9c 826b314f1 Author: Stanislav Chzhen Date: Fri Jun 9 16:06:12 2023 +0300 Merge branch 'master' into AG-22594-imp-whois commit db36adb9c14ce92b3971db0d87ec313d5bcd787e Author: Stanislav Chzhen Date: Fri Jun 9 15:53:33 2023 +0300 all: add todo commit 5cf192de9f93cd0d8521a3a6b4ded7f2bc5e0031 Author: Stanislav Chzhen Date: Thu Jun 8 14:59:26 2023 +0300 all: imp docs commit 021aa3eb5b9476a93b4af5fc90cc9ccf014ca152 Author: Stanislav Chzhen Date: Mon Jun 5 18:35:25 2023 +0300 all: imp naming commit 4626c3a7fa3f2543501806c9fa1a19531547f394 Author: Stanislav Chzhen Date: Fri Jun 2 17:41:00 2023 +0300 all: imp tests commit 1afcc9605ca176e4c7f76a03a2c996cf7d6bde13 Author: Stanislav Chzhen Date: Fri Jun 2 12:44:32 2023 +0300 all: imp docs commit cdd0544ff1a63faed5ced3dae6bfb3b783e45428 Author: Stanislav Chzhen Date: Thu Jun 1 17:21:37 2023 +0300 all: add docs ... and 2 more commits --- go.mod | 14 +- go.sum | 28 ++- internal/home/client.go | 18 +- internal/home/clients.go | 29 +-- internal/home/clients_test.go | 8 +- internal/home/clientshttp.go | 16 +- internal/home/dns.go | 77 ++++++- internal/home/home.go | 4 +- internal/home/whois.go | 259 ----------------------- internal/home/whois_test.go | 152 -------------- internal/querylog/client.go | 22 +- internal/whois/whois.go | 376 ++++++++++++++++++++++++++++++++++ internal/whois/whois_test.go | 155 ++++++++++++++ 13 files changed, 658 insertions(+), 500 deletions(-) delete mode 100644 internal/home/whois.go delete mode 100644 internal/home/whois_test.go create mode 100644 internal/whois/whois.go create mode 100644 internal/whois/whois_test.go diff --git a/go.mod b/go.mod index 2a3c0947..bf2f3e10 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,11 @@ go 1.19 require ( github.com/AdguardTeam/dnsproxy v0.50.2 - github.com/AdguardTeam/golibs v0.13.2 + github.com/AdguardTeam/golibs v0.13.3 github.com/AdguardTeam/urlfilter v0.16.1 github.com/NYTimes/gziphandler v1.1.1 github.com/ameshkov/dnscrypt/v2 v2.2.7 + github.com/bluele/gcache v0.0.2 github.com/digineo/go-ipset/v2 v2.2.1 github.com/dimfeld/httptreemux/v5 v5.5.0 github.com/fsnotify/fsnotify v1.6.0 @@ -27,13 +28,13 @@ require ( github.com/mdlayher/raw v0.1.0 github.com/miekg/dns v1.1.54 github.com/quic-go/quic-go v0.35.1 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.4 github.com/ti-mo/netfilter v0.5.0 go.etcd.io/bbolt v1.3.7 - golang.org/x/crypto v0.9.0 + golang.org/x/crypto v0.10.0 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 - golang.org/x/net v0.10.0 - golang.org/x/sys v0.8.0 + golang.org/x/net v0.11.0 + golang.org/x/sys v0.9.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 howett.net/plist v1.0.0 @@ -44,7 +45,6 @@ require ( github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 // indirect github.com/ameshkov/dnsstamps v1.0.3 // indirect github.com/beefsack/go-rate v0.0.0-20220214233405-116f4ca011a0 // indirect - github.com/bluele/gcache v0.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/golang/mock v1.6.0 // indirect @@ -61,6 +61,6 @@ require ( github.com/u-root/uio v0.0.0-20230305220412-3e8cd9d6bf63 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/sync v0.2.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect golang.org/x/tools v0.9.3 // indirect ) diff --git a/go.sum b/go.sum index ce08ec3e..2f07b778 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ github.com/AdguardTeam/dnsproxy v0.50.2 h1:p1471SsMZ6SMo7T51Olw4aNluahvMwSLMorwx github.com/AdguardTeam/dnsproxy v0.50.2/go.mod h1:CQhZTkqC8X0ID6glrtyaxgqRRdiYfn1gJulC1cZ5Dn8= github.com/AdguardTeam/golibs v0.4.0/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4= github.com/AdguardTeam/golibs v0.10.4/go.mod h1:rSfQRGHIdgfxriDDNgNJ7HmE5zRoURq8R+VdR81Zuzw= -github.com/AdguardTeam/golibs v0.13.2 h1:BPASsyQKmb+b8VnvsNOHp7bKfcZl9Z+Z2UhPjOiupSc= -github.com/AdguardTeam/golibs v0.13.2/go.mod h1:7ylQLv2Lqsc3UW3jHoITynYk6Y1tYtgEMkR09ppfsN8= +github.com/AdguardTeam/golibs v0.13.3 h1:RT3QbzThtaLiFLkIUDS6/hlGEXrh0zYvdf4bd7UWpGo= +github.com/AdguardTeam/golibs v0.13.3/go.mod h1:wkJ6EUsN4np/9Gp7+9QeooY9E2U2WCLJYAioLCzkHsI= github.com/AdguardTeam/gomitmproxy v0.2.0/go.mod h1:Qdv0Mktnzer5zpdpi5rAwixNJzW2FN91LjKJCkVbYGU= github.com/AdguardTeam/urlfilter v0.16.1 h1:ZPi0rjqo8cQf2FVdzo6cqumNoHZx2KPXj2yZa1A5BBw= github.com/AdguardTeam/urlfilter v0.16.1/go.mod h1:46YZDOV1+qtdRDuhZKVPSSp7JWWes0KayqHrKAFBdEI= @@ -113,17 +113,13 @@ github.com/quic-go/quic-go v0.35.1/go.mod h1:+4CVgVppm0FNjpG3UcX8Joi/frKOH7/ciD5 github.com/shirou/gopsutil/v3 v3.21.8 h1:nKct+uP0TV8DjjNiHanKf8SAuub+GNsbrOtM9Nl9biA= github.com/shirou/gopsutil/v3 v3.21.8/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/ti-mo/netfilter v0.2.0/go.mod h1:8GbBGsY/8fxtyIdfwy29JiluNcPK4K7wIT+x42ipqUU= github.com/ti-mo/netfilter v0.5.0 h1:MZmsUw5bFRecOb0AeyjOPxTHg4UxYzyEs0Ek/6Lxoy8= github.com/ti-mo/netfilter v0.5.0/go.mod h1:nt+8B9hx/QpqHr7Hazq+2qMCCA8u2OTkyc/7+U9ARz8= @@ -138,8 +134,8 @@ go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= @@ -156,8 +152,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= @@ -181,16 +177,16 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= diff --git a/internal/home/client.go b/internal/home/client.go index 1aee021e..92c88385 100644 --- a/internal/home/client.go +++ b/internal/home/client.go @@ -7,6 +7,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/filtering" "github.com/AdguardTeam/AdGuardHome/internal/filtering/safesearch" + "github.com/AdguardTeam/AdGuardHome/internal/whois" "github.com/AdguardTeam/dnsproxy/proxy" "github.com/AdguardTeam/golibs/stringutil" ) @@ -127,14 +128,13 @@ func (cs clientSource) MarshalText() (text []byte, err error) { // RuntimeClient is a client information about which has been obtained using the // source described in the Source field. type RuntimeClient struct { - WHOISInfo *RuntimeClientWHOISInfo - Host string - Source clientSource -} + // WHOIS is the filtered WHOIS data of a client. + WHOIS *whois.Info -// RuntimeClientWHOISInfo is the filtered WHOIS data for a runtime client. -type RuntimeClientWHOISInfo struct { - City string `json:"city,omitempty"` - Country string `json:"country,omitempty"` - Orgname string `json:"orgname,omitempty"` + // Host is the host name of a client. + Host string + + // Source is the source from which the information about the client has + // been obtained. + Source clientSource } diff --git a/internal/home/clients.go b/internal/home/clients.go index d2e4194b..6862cf25 100644 --- a/internal/home/clients.go +++ b/internal/home/clients.go @@ -14,6 +14,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/dnsforward" "github.com/AdguardTeam/AdGuardHome/internal/filtering" "github.com/AdguardTeam/AdGuardHome/internal/querylog" + "github.com/AdguardTeam/AdGuardHome/internal/whois" "github.com/AdguardTeam/dnsproxy/proxy" "github.com/AdguardTeam/dnsproxy/upstream" "github.com/AdguardTeam/golibs/errors" @@ -307,18 +308,6 @@ func (clients *clientsContainer) clientSource(ip netip.Addr) (src clientSource) return rc.Source } -func toQueryLogWHOIS(wi *RuntimeClientWHOISInfo) (cw *querylog.ClientWHOIS) { - if wi == nil { - return &querylog.ClientWHOIS{} - } - - return &querylog.ClientWHOIS{ - City: wi.City, - Country: wi.Country, - Orgname: wi.Orgname, - } -} - // findMultiple is a wrapper around Find to make it a valid client finder for // the query log. c is never nil; if no information about the client is found, // it returns an artificial client record by only setting the blocking-related @@ -352,7 +341,7 @@ func (clients *clientsContainer) clientOrArtificial( defer func() { c.Disallowed, c.DisallowedRule = clients.dnsServer.IsBlockedClient(ip, id) if c.WHOIS == nil { - c.WHOIS = &querylog.ClientWHOIS{} + c.WHOIS = &whois.Info{} } }() @@ -369,7 +358,7 @@ func (clients *clientsContainer) clientOrArtificial( if ok { return &querylog.Client{ Name: rc.Host, - WHOIS: toQueryLogWHOIS(rc.WHOISInfo), + WHOIS: rc.WHOIS, }, false } @@ -701,7 +690,7 @@ func (clients *clientsContainer) Update(prev, c *Client) (err error) { } // setWHOISInfo sets the WHOIS information for a client. -func (clients *clientsContainer) setWHOISInfo(ip netip.Addr, wi *RuntimeClientWHOISInfo) { +func (clients *clientsContainer) setWHOISInfo(ip netip.Addr, wi *whois.Info) { clients.lock.Lock() defer clients.lock.Unlock() @@ -713,7 +702,7 @@ func (clients *clientsContainer) setWHOISInfo(ip netip.Addr, wi *RuntimeClientWH rc, ok := clients.ipToRC[ip] if ok { - rc.WHOISInfo = wi + rc.WHOIS = wi log.Debug("clients: set whois info for runtime client %s: %+v", rc.Host, wi) return @@ -725,7 +714,7 @@ func (clients *clientsContainer) setWHOISInfo(ip netip.Addr, wi *RuntimeClientWH Source: ClientSourceWHOIS, } - rc.WHOISInfo = wi + rc.WHOIS = wi clients.ipToRC[ip] = rc @@ -762,9 +751,9 @@ func (clients *clientsContainer) addHostLocked( rc.Source = src } else { rc = &RuntimeClient{ - Host: host, - Source: src, - WHOISInfo: &RuntimeClientWHOISInfo{}, + Host: host, + Source: src, + WHOIS: &whois.Info{}, } clients.ipToRC[ip] = rc diff --git a/internal/home/clients_test.go b/internal/home/clients_test.go index 8361528a..b203415f 100644 --- a/internal/home/clients_test.go +++ b/internal/home/clients_test.go @@ -9,7 +9,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/dhcpd" "github.com/AdguardTeam/AdGuardHome/internal/filtering" - + "github.com/AdguardTeam/AdGuardHome/internal/whois" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -199,7 +199,7 @@ func TestClients(t *testing.T) { func TestClientsWHOIS(t *testing.T) { clients := newClientsContainer() - whois := &RuntimeClientWHOISInfo{ + whois := &whois.Info{ Country: "AU", Orgname: "Example Org", } @@ -210,7 +210,7 @@ func TestClientsWHOIS(t *testing.T) { rc := clients.ipToRC[ip] require.NotNil(t, rc) - assert.Equal(t, rc.WHOISInfo, whois) + assert.Equal(t, rc.WHOIS, whois) }) t.Run("existing_auto-client", func(t *testing.T) { @@ -222,7 +222,7 @@ func TestClientsWHOIS(t *testing.T) { rc := clients.ipToRC[ip] require.NotNil(t, rc) - assert.Equal(t, rc.WHOISInfo, whois) + assert.Equal(t, rc.WHOIS, whois) }) t.Run("can't_set_manually-added", func(t *testing.T) { diff --git a/internal/home/clientshttp.go b/internal/home/clientshttp.go index 6425f941..9eb91341 100644 --- a/internal/home/clientshttp.go +++ b/internal/home/clientshttp.go @@ -9,6 +9,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/aghalg" "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" "github.com/AdguardTeam/AdGuardHome/internal/filtering" + "github.com/AdguardTeam/AdGuardHome/internal/whois" ) // clientJSON is a common structure used by several handlers to deal with @@ -28,7 +29,8 @@ type clientJSON struct { // the allowlist. DisallowedRule *string `json:"disallowed_rule,omitempty"` - WHOISInfo *RuntimeClientWHOISInfo `json:"whois_info,omitempty"` + // WHOIS is the filtered WHOIS data of a client. + WHOIS *whois.Info `json:"whois_info,omitempty"` SafeSearchConf *filtering.SafeSearchConfig `json:"safe_search"` Name string `json:"name"` @@ -51,7 +53,7 @@ type clientJSON struct { } type runtimeClientJSON struct { - WHOISInfo *RuntimeClientWHOISInfo `json:"whois_info"` + WHOIS *whois.Info `json:"whois_info"` IP netip.Addr `json:"ip"` Name string `json:"name"` @@ -78,7 +80,7 @@ func (clients *clientsContainer) handleGetClients(w http.ResponseWriter, r *http for ip, rc := range clients.ipToRC { cj := runtimeClientJSON{ - WHOISInfo: rc.WHOISInfo, + WHOIS: rc.WHOIS, Name: rc.Host, Source: rc.Source, @@ -344,16 +346,16 @@ func (clients *clientsContainer) findRuntime(ip netip.Addr, idStr string) (cj *c IDs: []string{idStr}, Disallowed: &disallowed, DisallowedRule: &rule, - WHOISInfo: &RuntimeClientWHOISInfo{}, + WHOIS: &whois.Info{}, } return cj } cj = &clientJSON{ - Name: rc.Host, - IDs: []string{idStr}, - WHOISInfo: rc.WHOISInfo, + Name: rc.Host, + IDs: []string{idStr}, + WHOIS: rc.WHOIS, } disallowed, rule := clients.dnsServer.IsBlockedClient(ip, idStr) diff --git a/internal/home/dns.go b/internal/home/dns.go index 48b332f2..ebbcb16e 100644 --- a/internal/home/dns.go +++ b/internal/home/dns.go @@ -8,6 +8,7 @@ import ( "net/url" "os" "path/filepath" + "time" "github.com/AdguardTeam/AdGuardHome/internal/aghalg" "github.com/AdguardTeam/AdGuardHome/internal/aghhttp" @@ -17,6 +18,7 @@ import ( "github.com/AdguardTeam/AdGuardHome/internal/filtering" "github.com/AdguardTeam/AdGuardHome/internal/querylog" "github.com/AdguardTeam/AdGuardHome/internal/stats" + "github.com/AdguardTeam/AdGuardHome/internal/whois" "github.com/AdguardTeam/dnsproxy/proxy" "github.com/AdguardTeam/golibs/errors" "github.com/AdguardTeam/golibs/log" @@ -25,7 +27,7 @@ import ( yaml "gopkg.in/yaml.v3" ) -// Default ports. +// Default listening ports. const ( defaultPortDNS = 53 defaultPortHTTP = 80 @@ -169,13 +171,72 @@ func initDNSServer( Context.rdns = NewRDNS(Context.dnsServer, &Context.clients, config.DNS.UsePrivateRDNS) } - if config.Clients.Sources.WHOIS { - Context.whois = initWHOIS(&Context.clients) - } + initWHOIS() return nil } +// initWHOIS initializes the WHOIS. +// +// TODO(s.chzhen): Consider making configurable. +func initWHOIS() { + const ( + // defaultQueueSize is the size of queue of IPs for WHOIS processing. + defaultQueueSize = 255 + + // defaultTimeout is the timeout for WHOIS requests. + defaultTimeout = 5 * time.Second + + // defaultCacheSize is the maximum size of the cache. If it's zero, + // cache size is unlimited. + defaultCacheSize = 10_000 + + // defaultMaxConnReadSize is an upper limit in bytes for reading from + // net.Conn. + defaultMaxConnReadSize = 64 * 1024 + + // defaultMaxRedirects is the maximum redirects count. + defaultMaxRedirects = 5 + + // defaultMaxInfoLen is the maximum length of whois.Info fields. + defaultMaxInfoLen = 250 + + // defaultIPTTL is the Time to Live duration for cached IP addresses. + defaultIPTTL = 1 * time.Hour + ) + + Context.whoisCh = make(chan netip.Addr, defaultQueueSize) + + var w whois.Interface + + if config.Clients.Sources.WHOIS { + w = whois.New(&whois.Config{ + DialContext: customDialContext, + ServerAddr: whois.DefaultServer, + Port: whois.DefaultPort, + Timeout: defaultTimeout, + CacheSize: defaultCacheSize, + MaxConnReadSize: defaultMaxConnReadSize, + MaxRedirects: defaultMaxRedirects, + MaxInfoLen: defaultMaxInfoLen, + CacheTTL: defaultIPTTL, + }) + } else { + w = whois.Empty{} + } + + go func() { + defer log.OnPanic("whois") + + for ip := range Context.whoisCh { + info, changed := w.Process(context.Background(), ip) + if info != nil && changed { + Context.clients.setWHOISInfo(ip, info) + } + } + }() +} + // parseSubnetSet parses a slice of subnets. If the slice is empty, it returns // a subnet set that matches all locally served networks, see // [netutil.IsLocallyServed]. @@ -218,9 +279,7 @@ func onDNSRequest(pctx *proxy.DNSContext) { Context.rdns.Begin(ip) } - if srcs.WHOIS && !netutil.IsSpecialPurposeAddr(ip) { - Context.whois.Begin(ip) - } + Context.whoisCh <- ip } func ipsToTCPAddrs(ips []netip.Addr, port int) (tcpAddrs []*net.TCPAddr) { @@ -463,9 +522,7 @@ func startDNSServer() error { Context.rdns.Begin(ip) } - if srcs.WHOIS && !netutil.IsSpecialPurposeAddr(ip) { - Context.whois.Begin(ip) - } + Context.whoisCh <- ip } return nil diff --git a/internal/home/home.go b/internal/home/home.go index 5f1dd6f2..b00b4721 100644 --- a/internal/home/home.go +++ b/internal/home/home.go @@ -57,7 +57,6 @@ type homeContext struct { queryLog querylog.QueryLog // query log module dnsServer *dnsforward.Server // DNS module rdns *RDNS // rDNS module - whois *WHOIS // WHOIS module dhcpServer dhcpd.Interface // DHCP module auth *Auth // HTTP authentication module filters *filtering.DNSFilter // DNS filtering module @@ -84,6 +83,9 @@ type homeContext struct { client *http.Client appSignalChannel chan os.Signal // Channel for receiving OS signals by the console app + // whoisCh is the channel for receiving IPs for WHOIS processing. + whoisCh chan netip.Addr + // tlsCipherIDs are the ID of the cipher suites that AdGuard Home must use. tlsCipherIDs []uint16 diff --git a/internal/home/whois.go b/internal/home/whois.go deleted file mode 100644 index 9ffee9e0..00000000 --- a/internal/home/whois.go +++ /dev/null @@ -1,259 +0,0 @@ -package home - -import ( - "context" - "encoding/binary" - "fmt" - "io" - "net" - "net/netip" - "strings" - "time" - - "github.com/AdguardTeam/AdGuardHome/internal/aghio" - "github.com/AdguardTeam/golibs/cache" - "github.com/AdguardTeam/golibs/errors" - "github.com/AdguardTeam/golibs/log" - "github.com/AdguardTeam/golibs/stringutil" -) - -const ( - defaultServer = "whois.arin.net" - defaultPort = "43" - maxValueLength = 250 - whoisTTL = 1 * 60 * 60 // 1 hour -) - -// WHOIS - module context -type WHOIS struct { - clients *clientsContainer - ipChan chan netip.Addr - - // dialContext specifies the dial function for creating unencrypted TCP - // connections. - dialContext func(ctx context.Context, network, addr string) (conn net.Conn, err error) - - // Contains IP addresses of clients - // An active IP address is resolved once again after it expires. - // If IP address couldn't be resolved, it stays here for some time to prevent further attempts to resolve the same IP. - ipAddrs cache.Cache - - // TODO(a.garipov): Rewrite to use time.Duration. Like, seriously, why? - timeoutMsec uint -} - -// initWHOIS creates the WHOIS module context. -func initWHOIS(clients *clientsContainer) *WHOIS { - w := WHOIS{ - timeoutMsec: 5000, - clients: clients, - ipAddrs: cache.New(cache.Config{ - EnableLRU: true, - MaxCount: 10000, - }), - dialContext: customDialContext, - ipChan: make(chan netip.Addr, 255), - } - - go w.workerLoop() - - return &w -} - -// If the value is too large - cut it and append "..." -func trimValue(s string) string { - if len(s) <= maxValueLength { - return s - } - return s[:maxValueLength-3] + "..." -} - -// isWHOISComment returns true if the string is empty or is a WHOIS comment. -func isWHOISComment(s string) (ok bool) { - return len(s) == 0 || s[0] == '#' || s[0] == '%' -} - -// strmap is an alias for convenience. -type strmap = map[string]string - -// whoisParse parses a subset of plain-text data from the WHOIS response into -// a string map. -func whoisParse(data string) (m strmap) { - m = strmap{} - - var orgname string - lines := strings.Split(data, "\n") - for _, l := range lines { - if isWHOISComment(l) { - continue - } - - kv := strings.SplitN(l, ":", 2) - if len(kv) != 2 { - continue - } - - k := strings.ToLower(strings.TrimSpace(kv[0])) - v := strings.TrimSpace(kv[1]) - if v == "" { - continue - } - - switch k { - case "orgname", "org-name": - k = "orgname" - v = trimValue(v) - orgname = v - case "city", "country": - v = trimValue(v) - case "descr", "netname": - k = "orgname" - v = stringutil.Coalesce(orgname, v) - orgname = v - case "whois": - k = "whois" - case "referralserver": - k = "whois" - v = strings.TrimPrefix(v, "whois://") - default: - continue - } - - m[k] = v - } - - return m -} - -// MaxConnReadSize is an upper limit in bytes for reading from net.Conn. -const MaxConnReadSize = 64 * 1024 - -// Send request to a server and receive the response -func (w *WHOIS) query(ctx context.Context, target, serverAddr string) (data string, err error) { - addr, _, _ := net.SplitHostPort(serverAddr) - if addr == "whois.arin.net" { - target = "n + " + target - } - - conn, err := w.dialContext(ctx, "tcp", serverAddr) - if err != nil { - return "", err - } - defer func() { err = errors.WithDeferred(err, conn.Close()) }() - - r, err := aghio.LimitReader(conn, MaxConnReadSize) - if err != nil { - return "", err - } - - _ = conn.SetReadDeadline(time.Now().Add(time.Duration(w.timeoutMsec) * time.Millisecond)) - _, err = conn.Write([]byte(target + "\r\n")) - if err != nil { - return "", err - } - - // This use of ReadAll is now safe, because we limited the conn Reader. - var whoisData []byte - whoisData, err = io.ReadAll(r) - if err != nil { - return "", err - } - - return string(whoisData), nil -} - -// Query WHOIS servers (handle redirects) -func (w *WHOIS) queryAll(ctx context.Context, target string) (string, error) { - server := net.JoinHostPort(defaultServer, defaultPort) - const maxRedirects = 5 - for i := 0; i != maxRedirects; i++ { - resp, err := w.query(ctx, target, server) - if err != nil { - return "", err - } - log.Debug("whois: received response (%d bytes) from %s IP:%s", len(resp), server, target) - - m := whoisParse(resp) - redir, ok := m["whois"] - if !ok { - return resp, nil - } - redir = strings.ToLower(redir) - - _, _, err = net.SplitHostPort(redir) - if err != nil { - server = net.JoinHostPort(redir, defaultPort) - } else { - server = redir - } - - log.Debug("whois: redirected to %s IP:%s", redir, target) - } - return "", fmt.Errorf("whois: redirect loop") -} - -// Request WHOIS information -func (w *WHOIS) process(ctx context.Context, ip netip.Addr) (wi *RuntimeClientWHOISInfo) { - resp, err := w.queryAll(ctx, ip.String()) - if err != nil { - log.Debug("whois: error: %s IP:%s", err, ip) - - return nil - } - - log.Debug("whois: IP:%s response: %d bytes", ip, len(resp)) - - m := whoisParse(resp) - - wi = &RuntimeClientWHOISInfo{ - City: m["city"], - Country: m["country"], - Orgname: m["orgname"], - } - - // Don't return an empty struct so that the frontend doesn't get - // confused. - if *wi == (RuntimeClientWHOISInfo{}) { - return nil - } - - return wi -} - -// Begin - begin requesting WHOIS info -func (w *WHOIS) Begin(ip netip.Addr) { - ipBytes := ip.AsSlice() - now := uint64(time.Now().Unix()) - expire := w.ipAddrs.Get(ipBytes) - if len(expire) != 0 { - exp := binary.BigEndian.Uint64(expire) - if exp > now { - return - } - } - - expire = make([]byte, 8) - binary.BigEndian.PutUint64(expire, now+whoisTTL) - _ = w.ipAddrs.Set(ipBytes, expire) - - log.Debug("whois: adding %s", ip) - - select { - case w.ipChan <- ip: - default: - log.Debug("whois: queue is full") - } -} - -// workerLoop processes the IP addresses it got from the channel and associates -// the retrieving WHOIS info with a client. -func (w *WHOIS) workerLoop() { - for ip := range w.ipChan { - info := w.process(context.Background(), ip) - if info == nil { - continue - } - - w.clients.setWHOISInfo(ip, info) - } -} diff --git a/internal/home/whois_test.go b/internal/home/whois_test.go deleted file mode 100644 index f05bd670..00000000 --- a/internal/home/whois_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package home - -import ( - "context" - "io" - "net" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// fakeConn is a mock implementation of net.Conn to simplify testing. -// -// TODO(e.burkov): Search for other places in code where it may be used. Move -// into aghtest then. -type fakeConn struct { - // Conn is embedded here simply to make *fakeConn a net.Conn without - // actually implementing all methods. - net.Conn - data []byte -} - -// Write implements net.Conn interface for *fakeConn. It always returns 0 and a -// nil error without mutating the slice. -func (c *fakeConn) Write(_ []byte) (n int, err error) { - return 0, nil -} - -// Read implements net.Conn interface for *fakeConn. It puts the content of -// c.data field into b up to the b's capacity. -func (c *fakeConn) Read(b []byte) (n int, err error) { - return copy(b, c.data), io.EOF -} - -// Close implements net.Conn interface for *fakeConn. It always returns nil. -func (c *fakeConn) Close() (err error) { - return nil -} - -// SetReadDeadline implements net.Conn interface for *fakeConn. It always -// returns nil. -func (c *fakeConn) SetReadDeadline(_ time.Time) (err error) { - return nil -} - -// fakeDial is a mock implementation of customDialContext to simplify testing. -func (c *fakeConn) fakeDial(ctx context.Context, network, addr string) (conn net.Conn, err error) { - return c, nil -} - -func TestWHOIS(t *testing.T) { - const ( - nl = "\n" - data = `OrgName: FakeOrg LLC` + nl + - `City: Nonreal` + nl + - `Country: Imagiland` + nl - ) - - fc := &fakeConn{ - data: []byte(data), - } - - w := WHOIS{ - timeoutMsec: 5000, - dialContext: fc.fakeDial, - } - resp, err := w.queryAll(context.Background(), "1.2.3.4") - assert.NoError(t, err) - - m := whoisParse(resp) - require.NotEmpty(t, m) - - assert.Equal(t, "FakeOrg LLC", m["orgname"]) - assert.Equal(t, "Imagiland", m["country"]) - assert.Equal(t, "Nonreal", m["city"]) -} - -func TestWHOISParse(t *testing.T) { - const ( - city = "Nonreal" - country = "Imagiland" - orgname = "FakeOrgLLC" - whois = "whois.example.net" - ) - - testCases := []struct { - want strmap - name string - in string - }{{ - want: strmap{}, - name: "empty", - in: ``, - }, { - want: strmap{}, - name: "comments", - in: "%\n#", - }, { - want: strmap{}, - name: "no_colon", - in: "city", - }, { - want: strmap{}, - name: "no_value", - in: "city:", - }, { - want: strmap{"city": city}, - name: "city", - in: `city: ` + city, - }, { - want: strmap{"country": country}, - name: "country", - in: `country: ` + country, - }, { - want: strmap{"orgname": orgname}, - name: "orgname", - in: `orgname: ` + orgname, - }, { - want: strmap{"orgname": orgname}, - name: "orgname_hyphen", - in: `org-name: ` + orgname, - }, { - want: strmap{"orgname": orgname}, - name: "orgname_descr", - in: `descr: ` + orgname, - }, { - want: strmap{"orgname": orgname}, - name: "orgname_netname", - in: `netname: ` + orgname, - }, { - want: strmap{"whois": whois}, - name: "whois", - in: `whois: ` + whois, - }, { - want: strmap{"whois": whois}, - name: "referralserver", - in: `referralserver: whois://` + whois, - }, { - want: strmap{}, - name: "other", - in: `other: value`, - }} - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - got := whoisParse(tc.in) - assert.Equal(t, tc.want, got) - }) - } -} diff --git a/internal/querylog/client.go b/internal/querylog/client.go index f25440cc..b12313d6 100644 --- a/internal/querylog/client.go +++ b/internal/querylog/client.go @@ -1,23 +1,15 @@ package querylog +import "github.com/AdguardTeam/AdGuardHome/internal/whois" + // Client is the information required by the query log to match against clients // during searches. type Client struct { - WHOIS *ClientWHOIS `json:"whois,omitempty"` - Name string `json:"name"` - DisallowedRule string `json:"disallowed_rule"` - Disallowed bool `json:"disallowed"` - IgnoreQueryLog bool `json:"-"` -} - -// ClientWHOIS is the filtered WHOIS data for the client. -// -// TODO(a.garipov): Merge with home.RuntimeClientWHOISInfo after the -// refactoring is done. -type ClientWHOIS struct { - City string `json:"city,omitempty"` - Country string `json:"country,omitempty"` - Orgname string `json:"orgname,omitempty"` + WHOIS *whois.Info `json:"whois,omitempty"` + Name string `json:"name"` + DisallowedRule string `json:"disallowed_rule"` + Disallowed bool `json:"disallowed"` + IgnoreQueryLog bool `json:"-"` } // clientCacheKey is the key by which a cached client information is found. diff --git a/internal/whois/whois.go b/internal/whois/whois.go new file mode 100644 index 00000000..e55bcaf7 --- /dev/null +++ b/internal/whois/whois.go @@ -0,0 +1,376 @@ +// Package whois provides WHOIS functionality. +package whois + +import ( + "bytes" + "context" + "fmt" + "io" + "net" + "net/netip" + "strconv" + "strings" + "time" + + "github.com/AdguardTeam/AdGuardHome/internal/aghio" + "github.com/AdguardTeam/golibs/errors" + "github.com/AdguardTeam/golibs/log" + "github.com/AdguardTeam/golibs/netutil" + "github.com/AdguardTeam/golibs/stringutil" + "github.com/bluele/gcache" +) + +const ( + // DefaultServer is the default WHOIS server. + DefaultServer = "whois.arin.net" + + // DefaultPort is the default port for WHOIS requests. + DefaultPort = 43 +) + +// Interface provides WHOIS functionality. +type Interface interface { + // Process makes WHOIS request and returns WHOIS information or nil. + // changed indicates that Info was updated since last request. + Process(ctx context.Context, ip netip.Addr) (info *Info, changed bool) +} + +// Empty is an empty [Interface] implementation which does nothing. +type Empty struct{} + +// type check +var _ Interface = (*Empty)(nil) + +// Process implements the [Interface] interface for Empty. +func (Empty) Process(_ context.Context, _ netip.Addr) (info *Info, changed bool) { + return nil, false +} + +// Config is the configuration structure for Default. +type Config struct { + // DialContext specifies the dial function for creating unencrypted TCP + // connections. + DialContext func(ctx context.Context, network, addr string) (conn net.Conn, err error) + + // ServerAddr is the address of the WHOIS server. + ServerAddr string + + // Timeout is the timeout for WHOIS requests. + Timeout time.Duration + + // CacheTTL is the Time to Live duration for cached IP addresses. + CacheTTL time.Duration + + // MaxConnReadSize is an upper limit in bytes for reading from net.Conn. + MaxConnReadSize int64 + + // MaxRedirects is the maximum redirects count. + MaxRedirects int + + // MaxInfoLen is the maximum length of Info fields returned by Process. + MaxInfoLen int + + // CacheSize is the maximum size of the cache. It must be greater than + // zero. + CacheSize int + + // Port is the port for WHOIS requests. + Port uint16 +} + +// Default is the default WHOIS information processor. +type Default struct { + // cache is the cache containing IP addresses of clients. An active IP + // address is resolved once again after it expires. If IP address couldn't + // be resolved, it stays here for some time to prevent further attempts to + // resolve the same IP. + cache gcache.Cache + + // dialContext connects to a remote server resolving hostname using our own + // DNS server and unecrypted TCP connection. + dialContext func(ctx context.Context, network, addr string) (conn net.Conn, err error) + + // serverAddr is the address of the WHOIS server. + serverAddr string + + // portStr is the port for WHOIS requests. + portStr string + + // timeout is the timeout for WHOIS requests. + timeout time.Duration + + // cacheTTL is the Time to Live duration for cached IP addresses. + cacheTTL time.Duration + + // maxConnReadSize is an upper limit in bytes for reading from net.Conn. + maxConnReadSize int64 + + // maxRedirects is the maximum redirects count. + maxRedirects int + + // maxInfoLen is the maximum length of Info fields returned by Process. + maxInfoLen int +} + +// New returns a new default WHOIS information processor. conf must not be +// nil. +func New(conf *Config) (w *Default) { + return &Default{ + serverAddr: conf.ServerAddr, + dialContext: conf.DialContext, + timeout: conf.Timeout, + cache: gcache.New(conf.CacheSize).LRU().Build(), + maxConnReadSize: conf.MaxConnReadSize, + maxRedirects: conf.MaxRedirects, + portStr: strconv.Itoa(int(conf.Port)), + maxInfoLen: conf.MaxInfoLen, + cacheTTL: conf.CacheTTL, + } +} + +// trimValue trims s and replaces the last 3 characters of the cut with "..." +// to fit into max. max must be greater than 3. +func trimValue(s string, max int) string { + if len(s) <= max { + return s + } + + return s[:max-3] + "..." +} + +// isWHOISComment returns true if the data is empty or is a WHOIS comment. +func isWHOISComment(data []byte) (ok bool) { + return len(data) == 0 || data[0] == '#' || data[0] == '%' +} + +// whoisParse parses a subset of plain-text data from the WHOIS response into a +// string map. It trims values of the returned map to maxLen. +func whoisParse(data []byte, maxLen int) (info map[string]string) { + info = map[string]string{} + + var orgname string + lines := bytes.Split(data, []byte("\n")) + for _, l := range lines { + if isWHOISComment(l) { + continue + } + + before, after, found := bytes.Cut(l, []byte(":")) + if !found { + continue + } + + key := strings.ToLower(string(before)) + val := strings.TrimSpace(string(after)) + if val == "" { + continue + } + + switch key { + case "orgname", "org-name": + key = "orgname" + val = trimValue(val, maxLen) + orgname = val + case "city", "country": + val = trimValue(val, maxLen) + case "descr", "netname": + key = "orgname" + val = stringutil.Coalesce(orgname, val) + orgname = val + case "whois": + key = "whois" + case "referralserver": + key = "whois" + val = strings.TrimPrefix(val, "whois://") + default: + continue + } + + info[key] = val + } + + return info +} + +// query sends request to a server and returns the response or error. +func (w *Default) query(ctx context.Context, target, serverAddr string) (data []byte, err error) { + addr, _, _ := net.SplitHostPort(serverAddr) + if addr == DefaultServer { + // Display type flags for query. + // + // See https://www.arin.net/resources/registry/whois/rws/api/#nicname-whois-queries. + target = "n + " + target + } + + conn, err := w.dialContext(ctx, "tcp", serverAddr) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return nil, err + } + defer func() { err = errors.WithDeferred(err, conn.Close()) }() + + r, err := aghio.LimitReader(conn, w.maxConnReadSize) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return nil, err + } + + _ = conn.SetReadDeadline(time.Now().Add(w.timeout)) + _, err = io.WriteString(conn, target+"\r\n") + if err != nil { + // Don't wrap the error since it's informative enough as is. + return nil, err + } + + // This use of ReadAll is now safe, because we limited the conn Reader. + data, err = io.ReadAll(r) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return nil, err + } + + return data, nil +} + +// queryAll queries WHOIS server and handles redirects. +func (w *Default) queryAll(ctx context.Context, target string) (info map[string]string, err error) { + server := net.JoinHostPort(w.serverAddr, w.portStr) + var data []byte + + for i := 0; i < w.maxRedirects; i++ { + data, err = w.query(ctx, target, server) + if err != nil { + // Don't wrap the error since it's informative enough as is. + return nil, err + } + + log.Debug("whois: received response (%d bytes) from %q about %q", len(data), server, target) + + info = whoisParse(data, w.maxInfoLen) + redir, ok := info["whois"] + if !ok { + return info, nil + } + + redir = strings.ToLower(redir) + + _, _, err = net.SplitHostPort(redir) + if err != nil { + server = net.JoinHostPort(redir, w.portStr) + } else { + server = redir + } + + log.Debug("whois: redirected to %q about %q", redir, target) + } + + return nil, fmt.Errorf("whois: redirect loop") +} + +// type check +var _ Interface = (*Default)(nil) + +// Process makes WHOIS request and returns WHOIS information or nil. changed +// indicates that Info was updated since last request. +func (w *Default) Process(ctx context.Context, ip netip.Addr) (wi *Info, changed bool) { + if netutil.IsSpecialPurposeAddr(ip) { + return nil, false + } + + wi, expired := w.findInCache(ip) + if wi != nil && !expired { + // Don't return an empty struct so that the frontend doesn't get + // confused. + if (*wi == Info{}) { + return nil, false + } + + return wi, false + } + + var info Info + + defer func() { + item := toCacheItem(info, w.cacheTTL) + err := w.cache.Set(ip, item) + if err != nil { + log.Debug("whois: cache: adding item %q: %s", ip, err) + } + }() + + kv, err := w.queryAll(ctx, ip.String()) + if err != nil { + log.Debug("whois: quering about %q: %s", ip, err) + + return nil, true + } + + info = Info{ + City: kv["city"], + Country: kv["country"], + Orgname: kv["orgname"], + } + + // Don't return an empty struct so that the frontend doesn't get confused. + if (info == Info{}) { + return nil, true + } + + return &info, wi == nil || info != *wi +} + +// findInCache finds Info in the cache. expired indicates that Info is valid. +func (w *Default) findInCache(ip netip.Addr) (wi *Info, expired bool) { + val, err := w.cache.Get(ip) + if err != nil { + if !errors.Is(err, gcache.KeyNotFoundError) { + log.Debug("whois: cache: retrieving info about %q: %s", ip, err) + } + + return nil, false + } + + item, ok := val.(*cacheItem) + if !ok { + log.Debug("whois: cache: %q bad type %T", ip, val) + + return nil, false + } + + return fromCacheItem(item) +} + +// Info is the filtered WHOIS data for a runtime client. +type Info struct { + City string `json:"city,omitempty"` + Country string `json:"country,omitempty"` + Orgname string `json:"orgname,omitempty"` +} + +// cacheItem represents an item that we will store in the cache. +type cacheItem struct { + // expiry is the time when cacheItem will expire. + expiry time.Time + + // info is the WHOIS data for a runtime client. + info *Info +} + +// toCacheItem creates a cached item from a WHOIS info and Time to Live +// duration. +func toCacheItem(info Info, ttl time.Duration) (item *cacheItem) { + return &cacheItem{ + expiry: time.Now().Add(ttl), + info: &info, + } +} + +// fromCacheItem creates a WHOIS info from the cached item. expired indicates +// that WHOIS info is valid. item must not be nil. +func fromCacheItem(item *cacheItem) (info *Info, expired bool) { + if time.Now().After(item.expiry) { + return item.info, true + } + + return item.info, false +} diff --git a/internal/whois/whois_test.go b/internal/whois/whois_test.go new file mode 100644 index 00000000..2fe32255 --- /dev/null +++ b/internal/whois/whois_test.go @@ -0,0 +1,155 @@ +package whois_test + +import ( + "context" + "io" + "net" + "net/netip" + "testing" + "time" + + "github.com/AdguardTeam/AdGuardHome/internal/whois" + "github.com/AdguardTeam/golibs/testutil/fakenet" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDefault_Process(t *testing.T) { + const ( + nl = "\n" + city = "Nonreal" + country = "Imagiland" + orgname = "FakeOrgLLC" + referralserver = "whois.example.net" + ) + + ip := netip.MustParseAddr("1.2.3.4") + + testCases := []struct { + want *whois.Info + name string + data string + }{{ + want: nil, + name: "empty", + data: "", + }, { + want: nil, + name: "comments", + data: "%\n#", + }, { + want: nil, + name: "no_colon", + data: "city", + }, { + want: nil, + name: "no_value", + data: "city:", + }, { + want: &whois.Info{ + City: city, + }, + name: "city", + data: "city: " + city, + }, { + want: &whois.Info{ + Country: country, + }, + name: "country", + data: "country: " + country, + }, { + want: &whois.Info{ + Orgname: orgname, + }, + name: "orgname", + data: "orgname: " + orgname, + }, { + want: &whois.Info{ + Orgname: orgname, + }, + name: "orgname_hyphen", + data: "org-name: " + orgname, + }, { + want: &whois.Info{ + Orgname: orgname, + }, + name: "orgname_descr", + data: "descr: " + orgname, + }, { + want: &whois.Info{ + Orgname: orgname, + }, + name: "orgname_netname", + data: "netname: " + orgname, + }, { + want: &whois.Info{ + City: city, + Country: country, + Orgname: orgname, + }, + name: "full", + data: "OrgName: " + orgname + nl + "City: " + city + nl + "Country: " + country, + }, { + want: nil, + name: "whois", + data: "whois: " + referralserver, + }, { + want: nil, + name: "referralserver", + data: "referralserver: whois://" + referralserver, + }, { + want: nil, + name: "other", + data: "other: value", + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + hit := 0 + + fakeConn := &fakenet.Conn{ + OnRead: func(b []byte) (n int, err error) { + hit++ + + return copy(b, tc.data), io.EOF + }, + OnWrite: func(b []byte) (n int, err error) { + return len(b), nil + }, + OnClose: func() (err error) { + return nil + }, + OnSetReadDeadline: func(t time.Time) (err error) { + return nil + }, + } + + w := whois.New(&whois.Config{ + Timeout: 5 * time.Second, + DialContext: func(_ context.Context, _, addr string) (_ net.Conn, _ error) { + hit = 0 + + return fakeConn, nil + }, + MaxConnReadSize: 1024, + MaxRedirects: 3, + MaxInfoLen: 250, + CacheSize: 100, + CacheTTL: time.Hour, + }) + + got, changed := w.Process(context.Background(), ip) + require.True(t, changed) + + assert.Equal(t, tc.want, got) + assert.Equal(t, 1, hit) + + // From cache. + got, changed = w.Process(context.Background(), ip) + require.False(t, changed) + + assert.Equal(t, tc.want, got) + assert.Equal(t, 1, hit) + }) + } +} From 994906fbd404df0478d70d04664ee021616ad527 Mon Sep 17 00:00:00 2001 From: Ainar Garipov Date: Wed, 21 Jun 2023 17:14:10 +0300 Subject: [PATCH 7/8] Pull request 1884: AG-23334-split-snap Merge in DNS/adguard-home from AG-23334-split-snap to master Squashed commit of the following: commit 5a3d0f105d6930a0868f342821618a2a4acae282 Merge: bba693db6 06d465b0d Author: Ainar Garipov Date: Wed Jun 21 17:10:15 2023 +0300 Merge branch 'master' into AG-23334-split-snap commit bba693db60fc7e154df3bc6bf186292ee9bc4ed5 Author: Ainar Garipov Date: Wed Jun 21 16:50:45 2023 +0300 scripts/snap: fix docs; imp data commit cb4a1d5bed147a41dda69b80b7ae6c3902c26538 Author: Ainar Garipov Date: Wed Jun 21 14:03:48 2023 +0300 all: fix scripts; imp docs commit f88320b16ed7679e151a5358f4ac8e0212b4a827 Author: Ainar Garipov Date: Tue Jun 20 18:52:21 2023 +0300 all: split snap ci --- bamboo-specs/bamboo.yaml | 9 +- bamboo-specs/release.yaml | 58 ----- bamboo-specs/snapcraft.yaml | 213 ++++++++++++++++++ scripts/README.md | 98 ++++++-- scripts/make/build-docker.sh | 27 +-- scripts/make/build-release.sh | 112 +++------ scripts/snap/build.sh | 77 +++++++ scripts/snap/download.sh | 29 +++ scripts/snap/upload.sh | 93 ++++++++ .../gui/adguard-home-web.desktop | 0 .../snap => snap}/gui/adguard-home-web.png | Bin .../snap => snap}/local/adguard-home-web.sh | 0 {scripts/snap => snap}/snap.tmpl.yaml | 0 13 files changed, 538 insertions(+), 178 deletions(-) create mode 100644 bamboo-specs/snapcraft.yaml create mode 100644 scripts/snap/build.sh create mode 100644 scripts/snap/download.sh create mode 100644 scripts/snap/upload.sh rename {scripts/snap => snap}/gui/adguard-home-web.desktop (100%) rename {scripts/snap => snap}/gui/adguard-home-web.png (100%) rename {scripts/snap => snap}/local/adguard-home-web.sh (100%) rename {scripts/snap => snap}/snap.tmpl.yaml (100%) diff --git a/bamboo-specs/bamboo.yaml b/bamboo-specs/bamboo.yaml index 9ff52b59..9eff70ae 100644 --- a/bamboo-specs/bamboo.yaml +++ b/bamboo-specs/bamboo.yaml @@ -1,5 +1,8 @@ ---- -!include test.yaml - --- !include release.yaml + +--- +!include snapcraft.yaml + +--- +!include test.yaml diff --git a/bamboo-specs/release.yaml b/bamboo-specs/release.yaml index 20dbe9b5..0791575c 100644 --- a/bamboo-specs/release.yaml +++ b/bamboo-specs/release.yaml @@ -34,12 +34,6 @@ 'jobs': - 'Publish to static storage' -- 'Publish to Snapstore': - 'manual': false - 'final': false - 'jobs': - - 'Publish to Snapstore' - - 'Publish to GitHub Releases': 'manual': false 'final': false @@ -204,58 +198,6 @@ 'requirements': - 'adg-docker': 'true' -'Publish to Snapstore': - 'docker': - 'image': '${bamboo.dockerGo}' - 'key': 'PTS' - 'other': - 'clean-working-dir': true - 'tasks': - - 'clean' - - 'checkout': - 'repository': 'bamboo-deploy-publisher' - 'path': 'bamboo-deploy-publisher' - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'scripts': - - | - #!/bin/sh - - set -e -f -u -x - - cd ./dist/ - - channel="${bamboo.channel}" - readonly channel - - case "$channel" - in - ('release') - snapchannel='candidate' - ;; - ('beta') - snapchannel='beta' - ;; - ('edge') - snapchannel='edge' - ;; - (*) - echo "invalid channel '$channel'" - exit 1 - ;; - esac - - env\ - SNAPCRAFT_CHANNEL="$snapchannel"\ - SNAPCRAFT_EMAIL="${bamboo.snapcraftEmail}"\ - SNAPCRAFT_STORE_CREDENTIALS="${bamboo.snapcraftMacaroonPassword}"\ - ../bamboo-deploy-publisher/deploy.sh adguard-home-snap - 'final-tasks': - - 'clean' - 'requirements': - - 'adg-docker': 'true' - 'Publish to GitHub Releases': 'key': 'PTGR' 'other': diff --git a/bamboo-specs/snapcraft.yaml b/bamboo-specs/snapcraft.yaml new file mode 100644 index 00000000..383a90ae --- /dev/null +++ b/bamboo-specs/snapcraft.yaml @@ -0,0 +1,213 @@ +--- +# This part of the release build is separate from the one described in +# release.yaml, because the Snapcraft infrastructure is brittle, and timeouts +# during logins and uploads often lead to release blocking. +'version': 2 +'plan': + 'project-key': 'AGH' + 'key': 'AGHSNAP' + 'name': 'AdGuard Home - Build and publish Snapcraft release' +# Make sure to sync any changes with the branch overrides below. +'variables': + 'channel': 'edge' + 'dockerGo': 'adguard/golang-ubuntu:6.7' + 'snapcraftChannel': 'edge' + +'stages': + - 'Download release': + 'manual': false + 'final': false + 'jobs': + - 'Download release' + + - 'Build packages': + 'manual': false + 'final': false + 'jobs': + - 'Build packages' + + - 'Publish to Snapstore': + 'manual': false + 'final': false + 'jobs': + - 'Publish to Snapstore' + +# TODO(a.garipov): Consider using the Artifact Downloader Task if it ever learns +# about plan branches. +'Download release': + 'artifacts': + - 'name': 'i386_binary' + 'pattern': 'AdGuardHome_i386' + 'shared': true + 'required': true + - 'name': 'amd64_binary' + 'pattern': 'AdGuardHome_amd64' + 'shared': true + 'required': true + - 'name': 'armhf_binary' + 'pattern': 'AdGuardHome_armhf' + 'shared': true + 'required': true + - 'name': 'arm64_binary' + 'pattern': 'AdGuardHome_arm64' + 'shared': true + 'required': true + 'docker': + 'image': '${bamboo.dockerGo}' + 'key': 'DR' + 'other': + 'clean-working-dir': true + 'tasks': + - 'checkout': + 'force-clean-build': true + - 'script': + 'interpreter': 'SHELL' + 'scripts': + - | + #!/bin/sh + + set -e -f -u -x + + env\ + CHANNEL="${bamboo.channel}"\ + VERBOSE='1'\ + sh ./scripts/snap/download.sh + 'requirements': + - 'adg-docker': 'true' + +'Build packages': + 'artifact-subscriptions': + - 'artifact': 'i386_binary' + - 'artifact': 'amd64_binary' + - 'artifact': 'armhf_binary' + - 'artifact': 'arm64_binary' + 'artifacts': + - 'name': 'i386_snap' + 'pattern': 'AdGuardHome_i386.snap' + 'shared': true + 'required': true + - 'name': 'amd64_snap' + 'pattern': 'AdGuardHome_amd64.snap' + 'shared': true + 'required': true + - 'name': 'armhf_snap' + 'pattern': 'AdGuardHome_armhf.snap' + 'shared': true + 'required': true + - 'name': 'arm64_snap' + 'pattern': 'AdGuardHome_arm64.snap' + 'shared': true + 'required': true + 'docker': + 'image': '${bamboo.dockerGo}' + 'key': 'BP' + 'other': + 'clean-working-dir': true + 'tasks': + - 'checkout': + 'force-clean-build': true + - 'script': + 'interpreter': 'SHELL' + 'scripts': + - | + #!/bin/sh + + set -e -f -u -x + + env\ + VERBOSE='1'\ + sh ./scripts/snap/build.sh + 'final-tasks': + - 'clean' + 'requirements': + - 'adg-docker': 'true' + +'Publish to Snapstore': + 'artifact-subscriptions': + - 'artifact': 'i386_snap' + - 'artifact': 'amd64_snap' + - 'artifact': 'armhf_snap' + - 'artifact': 'arm64_snap' + 'docker': + 'image': '${bamboo.dockerGo}' + 'key': 'PTS' + 'other': + 'clean-working-dir': true + 'tasks': + - 'checkout': + 'force-clean-build': true + - 'script': + 'interpreter': 'SHELL' + 'scripts': + - | + #!/bin/sh + + set -e -f -u -x + + env\ + SNAPCRAFT_CHANNEL="${bamboo.snapcraftChannel}"\ + SNAPCRAFT_STORE_CREDENTIALS="${bamboo.snapcraftMacaroonPassword}"\ + VERBOSE='1'\ + sh ./scripts/snap/upload.sh + 'final-tasks': + - 'clean' + 'requirements': + - 'adg-docker': 'true' + +'triggers': + # Don't use minute values that end with a zero or a five as these are often + # used in CI and so resources during these minutes can be quite busy. + # + # NOTE: The time is chosen to be exactly one hour after the main release + # build as defined as in release.yaml. + - 'cron': '0 42 14 ? * MON-FRI *' +'branches': + 'create': 'manually' + 'delete': + 'after-deleted-days': 1 + 'after-inactive-days': 30 + 'integration': + 'push-on-success': false + 'merge-from': 'AdGuard Home - Build and publish Snapcraft release' + 'link-to-jira': true + +'notifications': + - 'events': + - 'plan-completed' + 'recipients': + - 'webhook': + 'name': 'Build webhook' + 'url': 'http://prod.jirahub.service.eu.consul/v1/webhook/bamboo?channel=adguard-qa' + +'labels': [] +'other': + 'concurrent-build-plugin': 'system-default' + +'branch-overrides': + # beta-vX.Y branches are the branches into which the commits that are needed + # to release a new patch version are initially cherry-picked. + - '^beta-v[0-9]+\.[0-9]+': + # Build betas on release branches manually. + 'triggers': [] + # Set the default release channel on the release branch to beta, as we may + # need to build a few of these. + 'variables': + 'channel': 'beta' + 'dockerGo': 'adguard/golang-ubuntu:6.7' + 'snapcraftChannel': 'beta' + # release-vX.Y.Z branches are the branches from which the actual final + # release is built. + - '^release-v[0-9]+\.[0-9]+\.[0-9]+': + # Disable integration branches for release branches. + 'branch-config': + 'integration': + 'push-on-success': false + 'merge-from': 'beta-v0.107' + # Build final releases on release branches manually. + 'triggers': [] + # Set the default release channel on the final branch to release, as these + # are the ones that actually get released. + 'variables': + 'channel': 'release' + 'dockerGo': 'adguard/golang-ubuntu:6.7' + 'snapcraftChannel': 'candidate' diff --git a/scripts/README.md b/scripts/README.md index cc823a99..579ee08a 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -2,7 +2,7 @@ ## `hooks/`: Git Hooks - ### Usage + ### Usage Run `make init` from the project root. @@ -10,7 +10,7 @@ Run `make init` from the project root. ## `querylog/`: Query Log Helpers - ### Usage + ### Usage * `npm install`: install dependencies. Run this first. * `npm run anonymize `: read the query log from the `` @@ -26,157 +26,215 @@ don't print anything, and `1`, be verbose. - ### `build-docker.sh`: Build A Multi-Architecture Docker Image + ### `build-docker.sh`: Build A Multi-Architecture Docker Image Required environment: * `CHANNEL`: release channel, see above. + * `COMMIT`: current Git revision. + * `DIST_DIR`: the directory where a release has previously been built. + * `VERSION`: release version. Optional environment: * `DOCKER_IMAGE_NAME`: the name of the resulting Docker container. By default it's `adguardhome-dev`. + * `DOCKER_OUTPUT`: the `--output` parameters. By default they are `type=image,name=${DOCKER_IMAGE_NAME},push=false`. + * `SUDO`: allow users to use `sudo` or `doas` with `docker`. By default none is used. - ### `build-release.sh`: Build A Release For All Platforms + ### `build-release.sh`: Build A Release For All Platforms Required environment: + * `CHANNEL`: release channel, see above. + * `GPG_KEY` and `GPG_KEY_PASSPHRASE`: data for `gpg`. Only required if `SIGN` is `1`. Optional environment: + * `ARCH` and `OS`: space-separated list of architectures and operating systems for which to build a release. For example, to build only for 64-bit ARM and AMD on Linux and Darwin: + ```sh make ARCH='amd64 arm64' OS='darwin linux' … build-release ``` The default value is `''`, which means build everything. - * `BUILD_SNAP`: `0` to not build Snapcraft packages, `1` to build. The - default value is `1`. + * `DIST_DIR`: the directory to build a release into. The default value is `dist`. + * `GO`: set an alternative name for the Go compiler. + * `SIGN`: `0` to not sign the resulting packages, `1` to sign. The default value is `1`. + * `VERBOSE`: `1` to be verbose, `2` to also print environment. This script calls `go-build.sh` with the verbosity level one level lower, so to get verbosity level `2` in `go-build.sh`, set this to `3` when calling `build-release.sh`. + * `VERSION`: release version. Will be set by `version.sh` if it is unset or if it has the default `Makefile` value of `v0.0.0`. - ### `clean.sh`: Cleanup + ### `clean.sh`: Cleanup Optional environment: + * `GO`: set an alternative name for the Go compiler. Required environment: + * `DIST_DIR`: the directory where a release has previously been built. - ### `go-build.sh`: Build The Backend + ### `go-build.sh`: Build The Backend Optional environment: + * `GOARM`: ARM processor options for the Go compiler. + * `GOMIPS`: ARM processor options for the Go compiler. + * `GO`: set an alternative name for the Go compiler. + * `OUT`: output binary name. + * `PARALLELISM`: set the maximum number of concurrently run build commands (that is, compiler, linker, etc.). + * `SOURCE_DATE_EPOCH`: the [standardized][repr] environment variable for the Unix epoch time of the latest commit in the repository. If set, overrides the default obtained from Git. Useful for reproducible builds. + * `VERBOSE`: verbosity level. `1` shows every command that is run and every Go package that is processed. `2` also shows subcommands and environment. The default value is `0`, don't be verbose. + * `VERSION`: release version. Will be set by `version.sh` if it is unset or if it has the default `Makefile` value of `v0.0.0`. Required environment: + * `CHANNEL`: release channel, see above. [repr]: https://reproducible-builds.org/docs/source-date-epoch/ - ### `go-deps.sh`: Install Backend Dependencies + ### `go-deps.sh`: Install Backend Dependencies Optional environment: + * `GO`: set an alternative name for the Go compiler. + * `VERBOSE`: verbosity level. `1` shows every command that is run and every Go package that is processed. `2` also shows subcommands and environment. The default value is `0`, don't be verbose. - ### `go-lint.sh`: Run Backend Static Analyzers + ### `go-lint.sh`: Run Backend Static Analyzers Don't forget to run `make go-tools` once first! Optional environment: + * `EXIT_ON_ERROR`: if set to `0`, don't exit the script after the first encountered error. The default value is `1`. + * `GO`: set an alternative name for the Go compiler. + * `VERBOSE`: verbosity level. `1` shows every command that is run. `2` also shows subcommands. The default value is `0`, don't be verbose. - ### `go-test.sh`: Run Backend Tests + ### `go-test.sh`: Run Backend Tests Optional environment: + * `GO`: set an alternative name for the Go compiler. + * `RACE`: set to `0` to not use the Go race detector. The default value is `1`, use the race detector. + * `TIMEOUT_FLAGS`: set timeout flags for tests. The default value is `--timeout 30s`. + * `VERBOSE`: verbosity level. `1` shows every command that is run and every Go package that is processed. `2` also shows subcommands. The default value is `0`, don't be verbose. - ### `go-tools.sh`: Install Backend Tooling + ### `go-tools.sh`: Install Backend Tooling Installs the Go static analysis and other tools into `${PWD}/bin`. Either add `${PWD}/bin` to your `$PATH` before all other entries, or use the commands directly, or use the commands through `make` (for example, `make go-lint`). Optional environment: + * `GO`: set an alternative name for the Go compiler. - ### `version.sh`: Generate And Print The Current Version + ### `version.sh`: Generate And Print The Current Version Required environment: + * `CHANNEL`: release channel, see above. -## `snap/`: Snap GUI Files +## `snap/`: Snapcraft scripts -App icons (see https://github.com/AdguardTeam/AdGuardHome/pull/1836), Snap -manifest file templates, and helper scripts. + ### `build.sh` + +Builds the Snapcraft packages from the binaries created by `download.sh`. + + ### `download.sh` + +Downloads the binaries to pack them into Snapcraft packages. + +Required environment: + + * `CHANNEL`: release channel, see above. + + ### `upload.sh` + +Uploads the Snapcraft packages created by `build.sh`. + +Required environment: + + * `SNAPCRAFT_CHANNEL`: Snapcraft release channel: `edge`, `beta`, or + `candidate`. + + * `SNAPCRAFT_STORE_CREDENTIALS`: Credentials for Snapcraft store. + +Optional environment: + + * `SNAPCRAFT_CMD`: Overrides the Snapcraft command. Default: `snapcraft`. ## `translations/`: Twosky Integration Script - ### Usage + ### Usage * `go run main.go help`: print usage. @@ -211,7 +269,7 @@ Optional environment: A simple script that downloads and updates the companies DB in the `client` code from [the repo][companiesrepo]. - ### Usage + ### Usage ```sh sh ./scripts/companiesdb/download.sh @@ -231,7 +289,7 @@ Optional environment: * `URL`: the URL of the index file. By default it's `https://adguardteam.github.io/HostlistsRegistry/assets/services.json`. - ### Usage + ### Usage ```sh go run ./scripts/blocked-services/main.go @@ -251,7 +309,7 @@ Optional environment: * `URL`: the URL of the index file. By default it's `https://adguardteam.github.io/HostlistsRegistry/assets/filters.json`. - ### Usage + ### Usage ```sh go run ./scripts/vetted-filters/main.go diff --git a/scripts/make/build-docker.sh b/scripts/make/build-docker.sh index 14fd515b..ab754edc 100644 --- a/scripts/make/build-docker.sh +++ b/scripts/make/build-docker.sh @@ -5,11 +5,12 @@ verbose="${VERBOSE:-0}" if [ "$verbose" -gt '0' ] then set -x - debug_flags='-D' + debug_flags='--debug=1' else set +x - debug_flags='' + debug_flags='--debug=0' fi +readonly debug_flags set -e -f -u @@ -61,21 +62,16 @@ readonly docker_output case "$channel" in ('release') - docker_image_full_name="${docker_image_name}:${version}" - docker_tags="--tag ${docker_image_name}:latest" + docker_tags="--tag=${docker_image_name}:${version},${docker_image_name}:latest" ;; ('beta') - docker_image_full_name="${docker_image_name}:${version}" - docker_tags="--tag ${docker_image_name}:beta" + docker_tags="--tag=${docker_image_name}:${version},${docker_image_name}:beta" ;; ('edge') - # Don't set the version tag when pushing to the edge channel. - docker_image_full_name="${docker_image_name}:edge" - docker_tags='' + docker_tags="--tag=${docker_image_name}:edge" ;; ('development') - docker_image_full_name="${docker_image_name}" - docker_tags='' + docker_tags="--tag=${docker_image_name}" ;; (*) echo "invalid channel '$channel', supported values are\ @@ -83,7 +79,7 @@ in exit 1 ;; esac -readonly docker_image_full_name docker_tags +readonly docker_tags # Copy the binaries into a new directory under new names, so that it's easier to # COPY them later. DO NOT remove the trailing underscores. See file @@ -117,10 +113,8 @@ cp "./docker/web-bind.awk"\ cp "./docker/healthcheck.sh"\ "${dist_docker_scripts}/healthcheck.sh" -# Don't use quotes with $docker_tags and $debug_flags because we want word -# splitting and or an empty space if tags are empty. $sudo_cmd docker\ - $debug_flags\ + "$debug_flags"\ buildx build\ --build-arg BUILD_DATE="$build_date"\ --build-arg DIST_DIR="$dist_dir"\ @@ -128,7 +122,6 @@ $sudo_cmd docker\ --build-arg VERSION="$version"\ --output "$docker_output"\ --platform "$docker_platforms"\ - $docker_tags\ - -t "$docker_image_full_name"\ + "$docker_tags"\ -f ./docker/Dockerfile\ . diff --git a/scripts/make/build-release.sh b/scripts/make/build-release.sh index 2884f568..3e556f97 100644 --- a/scripts/make/build-release.sh +++ b/scripts/make/build-release.sh @@ -78,14 +78,6 @@ else fi readonly oses -snap_enabled="${BUILD_SNAP:-1}" -readonly snap_enabled - -if [ "$snap_enabled" -eq '0' ] -then - log 'snap: disabled' -fi - # Require the gpg key and passphrase to be set if the signing is required. if [ "$sign" -eq '1' ] then @@ -106,7 +98,7 @@ log "checking tools" # Make sure we fail gracefully if one of the tools we need is missing. Use # alternatives when available. use_shasum='0' -for tool in gpg gzip sed sha256sum snapcraft tar zip +for tool in gpg gzip sed sha256sum tar zip do if ! command -v "$tool" > /dev/null then @@ -128,36 +120,36 @@ readonly use_shasum # Data section. Arrange data into space-separated tables for read -r to read. # Use a hyphen for missing values. -# os arch arm mips snap +# os arch arm mips platforms="\ -darwin amd64 - - - -darwin arm64 - - - -freebsd 386 - - - -freebsd amd64 - - - -freebsd arm 5 - - -freebsd arm 6 - - -freebsd arm 7 - - -freebsd arm64 - - - -linux 386 - - i386 -linux amd64 - - amd64 -linux arm 5 - - -linux arm 6 - - -linux arm 7 - armhf -linux arm64 - - arm64 -linux mips - softfloat - -linux mips64 - softfloat - -linux mips64le - softfloat - -linux mipsle - softfloat - -linux ppc64le - - - -openbsd amd64 - - - -openbsd arm64 - - - -windows 386 - - - -windows amd64 - - - -windows arm64 - - -" +darwin amd64 - - +darwin arm64 - - +freebsd 386 - - +freebsd amd64 - - +freebsd arm 5 - +freebsd arm 6 - +freebsd arm 7 - +freebsd arm64 - - +linux 386 - - +linux amd64 - - +linux arm 5 - +linux arm 6 - +linux arm 7 - +linux arm64 - - +linux mips - softfloat +linux mips64 - softfloat +linux mips64le - softfloat +linux mipsle - softfloat +linux ppc64le - - +openbsd amd64 - - +openbsd arm64 - - +windows 386 - - +windows amd64 - - +windows arm64 - -" readonly platforms -# Function build builds the release for one platform. It builds a binary, an -# archive and, if needed, a snap package. +# Function build builds the release for one platform. It builds a binary and an +# archive. build() { # Get the arguments. Here and below, use the "build_" prefix for all # variables local to function build. @@ -167,7 +159,6 @@ build() { build_arch="$4"\ build_arm="$5"\ build_mips="$6"\ - build_snap="$7"\ ; # Use the ".exe" filename extension if we build a Windows release. @@ -229,52 +220,13 @@ build() { esac log "$build_archive" - - # Exit if we don't need to build the Snap package. - if [ "$build_snap" = '-' ] || [ "$snap_enabled" -eq '0' ] - then - return - fi - - # Prepare the Snap build. - build_snap_output="./${dist}/AdGuardHome_${build_snap}.snap" - build_snap_dir="${build_snap_output}.dir" - - # Create the meta subdirectory and copy files there. - mkdir -p "${build_snap_dir}/meta" - cp "$build_output" './scripts/snap/local/adguard-home-web.sh' "$build_snap_dir" - cp -r './scripts/snap/gui' "${build_snap_dir}/meta/" - - # Create a snap.yaml file, setting the values. - sed -e 's/%VERSION%/'"$version"'/'\ - -e 's/%ARCH%/'"$build_snap"'/'\ - ./scripts/snap/snap.tmpl.yaml\ - >"${build_snap_dir}/meta/snap.yaml" - - # TODO(a.garipov): The snapcraft tool will *always* write everything, - # including errors, to stdout. And there doesn't seem to be a way to change - # that. So, save the combined output, but only show it when snapcraft - # actually fails. - set +e - build_snapcraft_output="$( - snapcraft pack "$build_snap_dir" --output "$build_snap_output" 2>&1 - )" - build_snapcraft_exit_code="$?" - set -e - if [ "$build_snapcraft_exit_code" -ne '0' ] - then - log "$build_snapcraft_output" - exit "$build_snapcraft_exit_code" - fi - - log "$build_snap_output" } log "starting builds" # Go over all platforms defined in the space-separated table above, tweak the # values where necessary, and feed to build. -echo "$platforms" | while read -r os arch arm mips snap +echo "$platforms" | while read -r os arch arm mips do # See if the architecture or the OS is in the allowlist. To do so, try # removing everything that matches the pattern (well, a prefix, but that @@ -314,7 +266,7 @@ do ;; esac - build "$dir" "$ar" "$os" "$arch" "$arm" "$mips" "$snap" + build "$dir" "$ar" "$os" "$arch" "$arm" "$mips" done log "packing frontend" @@ -413,14 +365,14 @@ do platform="$f" # Remove the prefix. - platform="${platform#./${dist}/AdGuardHome_}" + platform="${platform#"./${dist}/AdGuardHome_"}" # Remove the filename extensions. platform="${platform%.zip}" platform="${platform%.tar.gz}" # Use the filename's base path. - filename="${f#./${dist}/}" + filename="${f#"./${dist}/"}" if [ "$i" -eq "$ar_files_len" ] then diff --git a/scripts/snap/build.sh b/scripts/snap/build.sh new file mode 100644 index 00000000..53f487a1 --- /dev/null +++ b/scripts/snap/build.sh @@ -0,0 +1,77 @@ +#!/bin/sh + +verbose="${VERBOSE:-0}" + +if [ "$verbose" -gt '0' ] +then + set -x +fi + +set -e -f -u + +# Function log is an echo wrapper that writes to stderr if the caller requested +# verbosity level greater than 0. Otherwise, it does nothing. +# +# TODO(a.garipov): Add to helpers.sh and use more actively in scripts. +log() { + if [ "$verbose" -gt '0' ] + then + # Don't use quotes to get word splitting. + echo "$1" 1>&2 + fi +} + +version="$( ./AdGuardHome_amd64 --version | cut -d ' ' -f 4 )" +if [ "$version" = '' ] +then + log 'empty version from ./AdGuardHome_amd64' + exit 1 +fi +readonly version + +log "version '$version'" + +for arch in\ + 'i386'\ + 'amd64'\ + 'armhf'\ + 'arm64' +do + build_output="./AdGuardHome_${arch}" + snap_output="./AdGuardHome_${arch}.snap" + snap_dir="${snap_output}.dir" + + # Create the meta subdirectory and copy files there. + mkdir -p "${snap_dir}/meta" + cp "$build_output" "${snap_dir}/AdGuardHome" + cp './snap/local/adguard-home-web.sh' "$snap_dir" + cp -r './snap/gui' "${snap_dir}/meta/" + + # Create a snap.yaml file, setting the values. + sed\ + -e 's/%VERSION%/'"$version"'/'\ + -e 's/%ARCH%/'"$arch"'/'\ + ./snap/snap.tmpl.yaml\ + > "${snap_dir}/meta/snap.yaml" + + # TODO(a.garipov): The snapcraft tool will *always* write everything, + # including errors, to stdout. And there doesn't seem to be a way to change + # that. So, save the combined output, but only show it when snapcraft + # actually fails. + set +e + snapcraft_output="$( + snapcraft pack "$snap_dir" --output "$snap_output" 2>&1 + )" + snapcraft_exit_code="$?" + set -e + + if [ "$snapcraft_exit_code" -ne '0' ] + then + log "$snapcraft_output" + exit "$snapcraft_exit_code" + fi + + log "$snap_output" + + rm -f -r "$snap_dir" +done diff --git a/scripts/snap/download.sh b/scripts/snap/download.sh new file mode 100644 index 00000000..7eb03a0e --- /dev/null +++ b/scripts/snap/download.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +verbose="${VERBOSE:-0}" + +if [ "$verbose" -gt '0' ] +then + set -x +fi + +set -e -f -u + +channel="${CHANNEL:?please set CHANNEL}" +readonly channel + +printf '%s %s\n'\ + '386' 'i386'\ + 'amd64' 'amd64'\ + 'armv7' 'armhf'\ + 'arm64' 'arm64' \ +| while read -r arch snap_arch +do + release_url="https://static.adtidy.org/adguardhome/${channel}/AdGuardHome_linux_${arch}.tar.gz" + output="./AdGuardHome_linux_${arch}.tar.gz" + + curl -o "$output" -v "$release_url" + tar -f "$output" -v -x -z + cp ./AdGuardHome/AdGuardHome "./AdGuardHome_${snap_arch}" + rm -f -r "$output" ./AdGuardHome +done diff --git a/scripts/snap/upload.sh b/scripts/snap/upload.sh new file mode 100644 index 00000000..35be0a9e --- /dev/null +++ b/scripts/snap/upload.sh @@ -0,0 +1,93 @@ +#!/bin/sh + +verbose="${VERBOSE:-0}" + +if [ "$verbose" -gt '0' ] +then + set -x +fi + +set -e -f -u + +# Function log is an echo wrapper that writes to stderr if the caller requested +# verbosity level greater than 0. Otherwise, it does nothing. +log() { + if [ "$verbose" -gt '0' ] + then + # Don't use quotes to get word splitting. + echo "$1" 1>&2 + fi +} + +# Do not set a new lowercase variable, because the snapcraft tool expects the +# uppercase form. +if [ "${SNAPCRAFT_STORE_CREDENTIALS:-}" = '' ] +then + log 'please set SNAPCRAFT_STORE_CREDENTIALS' + + exit 1 +fi +export SNAPCRAFT_STORE_CREDENTIALS + +snapcraft_channel="${SNAPCRAFT_CHANNEL:?please set SNAPCRAFT_CHANNEL}" +readonly snapcraft_channel + +# Allow developers to overwrite the command, e.g. for testing. +snapcraft_cmd="${SNAPCRAFT_CMD:-snapcraft}" +readonly snapcraft_cmd + +default_timeout='90s' +kill_timeout='120s' +readonly default_timeout kill_timeout + +for arch in\ + 'i386'\ + 'amd64'\ + 'armhf'\ + 'arm64' +do + snap_file="./AdGuardHome_${arch}.snap" + + # Catch the exit code and the combined output to later inspect it. + set +e + snapcraft_output="$( + # Use timeout(1) to force snapcraft to quit after a certain time. There + # seems to be no environment variable or flag to force this behavior. + timeout\ + --preserve-status\ + -k "$kill_timeout"\ + -v "$default_timeout"\ + "$snapcraft_cmd" upload\ + --release="${snapcraft_channel}"\ + --quiet\ + "${snap_file}"\ + 2>&1 + )" + snapcraft_exit_code="$?" + set -e + + if [ "$snapcraft_exit_code" -eq '0' ] + then + log "successful upload: ${snapcraft_output}" + + continue + fi + + # Skip the ones that were failed by a duplicate upload error. + case "$snapcraft_output" + in + (*'A file with this exact same content has already been uploaded'|\ + 'Error checking upload uniqueness'*) + + log "warning: duplicate upload, skipping" + log "snapcraft upload error: ${snapcraft_output}" + + continue + ;; + (*) + echo "unexpected snapcraft upload error: ${snapcraft_output}" + + return "$snapcraft_exit_code" + ;; + esac +done diff --git a/scripts/snap/gui/adguard-home-web.desktop b/snap/gui/adguard-home-web.desktop similarity index 100% rename from scripts/snap/gui/adguard-home-web.desktop rename to snap/gui/adguard-home-web.desktop diff --git a/scripts/snap/gui/adguard-home-web.png b/snap/gui/adguard-home-web.png similarity index 100% rename from scripts/snap/gui/adguard-home-web.png rename to snap/gui/adguard-home-web.png diff --git a/scripts/snap/local/adguard-home-web.sh b/snap/local/adguard-home-web.sh similarity index 100% rename from scripts/snap/local/adguard-home-web.sh rename to snap/local/adguard-home-web.sh diff --git a/scripts/snap/snap.tmpl.yaml b/snap/snap.tmpl.yaml similarity index 100% rename from scripts/snap/snap.tmpl.yaml rename to snap/snap.tmpl.yaml From 123ca87388315bacc3a16ff4960c7470df936e09 Mon Sep 17 00:00:00 2001 From: Ainar Garipov Date: Wed, 21 Jun 2023 17:47:16 +0300 Subject: [PATCH 8/8] Pull request 1885: AG-23334-fix-snap-plan Merge in DNS/adguard-home from AG-23334-fix-snap-plan to master Squashed commit of the following: commit 5d632d1d63c56911e005d0e772e82a509302e948 Author: Ainar Garipov Date: Wed Jun 21 17:42:36 2023 +0300 bamboo-specs: fix snap build; fmt --- bamboo-specs/release.yaml | 468 ++++++++++++++++++------------------ bamboo-specs/snapcraft.yaml | 2 - bamboo-specs/test.yaml | 94 ++++---- 3 files changed, 281 insertions(+), 283 deletions(-) diff --git a/bamboo-specs/release.yaml b/bamboo-specs/release.yaml index 0791575c..d82b6f2f 100644 --- a/bamboo-specs/release.yaml +++ b/bamboo-specs/release.yaml @@ -1,290 +1,290 @@ --- 'version': 2 'plan': - 'project-key': 'AGH' - 'key': 'AGHBSNAPSPECS' - 'name': 'AdGuard Home - Build and publish release' + 'project-key': 'AGH' + 'key': 'AGHBSNAPSPECS' + 'name': 'AdGuard Home - Build and publish release' # Make sure to sync any changes with the branch overrides below. 'variables': - 'channel': 'edge' - 'dockerGo': 'adguard/golang-ubuntu:6.7' + 'channel': 'edge' + 'dockerGo': 'adguard/golang-ubuntu:6.7' 'stages': -- 'Build frontend': - 'manual': false - 'final': false - 'jobs': - - 'Build frontend' + - 'Build frontend': + 'manual': false + 'final': false + 'jobs': + - 'Build frontend' -- 'Make release': - 'manual': false - 'final': false - 'jobs': - - 'Make release' + - 'Make release': + 'manual': false + 'final': false + 'jobs': + - 'Make release' -- 'Make and publish docker': - 'manual': false - 'final': false - 'jobs': - - 'Make and publish docker' + - 'Make and publish docker': + 'manual': false + 'final': false + 'jobs': + - 'Make and publish docker' -- 'Publish to static storage': - 'manual': false - 'final': false - 'jobs': - - 'Publish to static storage' + - 'Publish to static storage': + 'manual': false + 'final': false + 'jobs': + - 'Publish to static storage' -- 'Publish to GitHub Releases': - 'manual': false - 'final': false - 'jobs': - - 'Publish to GitHub Releases' + - 'Publish to GitHub Releases': + 'manual': false + 'final': false + 'jobs': + - 'Publish to GitHub Releases' 'Build frontend': - 'docker': - 'image': '${bamboo.dockerGo}' - 'volumes': - '${system.YARN_DIR}': '${bamboo.cacheYarn}' - 'key': 'BF' - 'other': - 'clean-working-dir': true - 'tasks': - - 'checkout': - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'scripts': - - | - #!/bin/sh + 'docker': + 'image': '${bamboo.dockerGo}' + 'volumes': + '${system.YARN_DIR}': '${bamboo.cacheYarn}' + 'key': 'BF' + 'other': + 'clean-working-dir': true + 'tasks': + - 'checkout': + 'force-clean-build': true + - 'script': + 'interpreter': 'SHELL' + 'scripts': + - | + #!/bin/sh - set -e -f -u -x + set -e -f -u -x - # Explicitly checkout the revision that we need. - git checkout "${bamboo.repository.revision.number}" + # Explicitly checkout the revision that we need. + git checkout "${bamboo.repository.revision.number}" - make js-deps js-build - 'artifacts': - - 'name': 'AdGuardHome frontend' - 'pattern': 'build*/**' - 'shared': true - 'required': true - 'requirements': - - 'adg-docker': 'true' + make js-deps js-build + 'artifacts': + - 'name': 'AdGuardHome frontend' + 'pattern': 'build/**' + 'shared': true + 'required': true + 'requirements': + - 'adg-docker': 'true' 'Make release': - 'docker': - 'image': '${bamboo.dockerGo}' - 'volumes': - '${system.GO_CACHE_DIR}': '${bamboo.cacheGo}' - '${system.GO_PKG_CACHE_DIR}': '${bamboo.cacheGoPkg}' - 'key': 'MR' - 'other': - 'clean-working-dir': true - 'tasks': - - 'checkout': - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'scripts': - - | - #!/bin/sh + 'docker': + 'image': '${bamboo.dockerGo}' + 'volumes': + '${system.GO_CACHE_DIR}': '${bamboo.cacheGo}' + '${system.GO_PKG_CACHE_DIR}': '${bamboo.cacheGoPkg}' + 'key': 'MR' + 'other': + 'clean-working-dir': true + 'tasks': + - 'checkout': + 'force-clean-build': true + - 'script': + 'interpreter': 'SHELL' + 'scripts': + - | + #!/bin/sh - set -e -f -u -x + set -e -f -u -x - # Explicitly checkout the revision that we need. - git checkout "${bamboo.repository.revision.number}" + # Explicitly checkout the revision that we need. + git checkout "${bamboo.repository.revision.number}" - # Run the build with the specified channel. - echo "${bamboo.gpgSecretKeyPart1}${bamboo.gpgSecretKeyPart2}"\ - | awk '{ gsub(/\\n/, "\n"); print; }'\ - | gpg --import --batch --yes + # Run the build with the specified channel. + echo "${bamboo.gpgSecretKeyPart1}${bamboo.gpgSecretKeyPart2}"\ + | awk '{ gsub(/\\n/, "\n"); print; }'\ + | gpg --import --batch --yes - make\ - CHANNEL=${bamboo.channel}\ - GPG_KEY_PASSPHRASE=${bamboo.gpgPassword}\ - FRONTEND_PREBUILT=1\ - PARALLELISM=1\ - VERBOSE=2\ - build-release - # TODO(a.garipov): Use more fine-grained artifact rules. - 'artifacts': - - 'name': 'AdGuardHome dists' - 'pattern': 'dist/**' - 'shared': true - 'required': true - 'requirements': - - 'adg-docker': 'true' + make\ + CHANNEL=${bamboo.channel}\ + GPG_KEY_PASSPHRASE=${bamboo.gpgPassword}\ + FRONTEND_PREBUILT=1\ + PARALLELISM=1\ + VERBOSE=2\ + build-release + # TODO(a.garipov): Use more fine-grained artifact rules. + 'artifacts': + - 'name': 'AdGuardHome dists' + 'pattern': 'dist/**' + 'shared': true + 'required': true + 'requirements': + - 'adg-docker': 'true' 'Make and publish docker': - 'key': 'MPD' - 'other': - 'clean-working-dir': true - 'tasks': - - 'checkout': - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'scripts': - - | - #!/bin/sh + 'key': 'MPD' + 'other': + 'clean-working-dir': true + 'tasks': + - 'checkout': + 'force-clean-build': true + - 'script': + 'interpreter': 'SHELL' + 'scripts': + - | + #!/bin/sh - set -e -f -u -x + set -e -f -u -x - COMMIT="${bamboo.repository.revision.number}" - export COMMIT - readonly COMMIT + COMMIT="${bamboo.repository.revision.number}" + export COMMIT + readonly COMMIT - # Explicitly checkout the revision that we need. - git checkout "$COMMIT" + # Explicitly checkout the revision that we need. + git checkout "$COMMIT" - # Install Qemu, create builder. - docker version -f '{{ .Server.Experimental }}' - docker buildx rm buildx-builder || : - docker buildx create --name buildx-builder --driver docker-container\ - --use - docker buildx inspect --bootstrap + # Install Qemu, create builder. + docker version -f '{{ .Server.Experimental }}' + docker buildx rm buildx-builder || : + docker buildx create --name buildx-builder --driver docker-container\ + --use + docker buildx inspect --bootstrap - # Login to DockerHub. - docker login -u="${bamboo.dockerHubUsername}"\ - -p="${bamboo.dockerHubPassword}" + # Login to DockerHub. + docker login -u="${bamboo.dockerHubUsername}"\ + -p="${bamboo.dockerHubPassword}" - # Boot the builder. - docker buildx inspect --bootstrap + # Boot the builder. + docker buildx inspect --bootstrap - # Print Docker info. - docker info + # Print Docker info. + docker info - # Prepare and push the build. - env\ - CHANNEL="${bamboo.channel}"\ - DIST_DIR='dist'\ - DOCKER_IMAGE_NAME='adguard/adguardhome'\ - DOCKER_OUTPUT="type=image,name=adguard/adguardhome,push=true"\ - VERBOSE='1'\ - sh ./scripts/make/build-docker.sh - 'environment': - DOCKER_CLI_EXPERIMENTAL=enabled - 'final-tasks': - - 'clean' - 'requirements': - - 'adg-docker': 'true' + # Prepare and push the build. + env\ + CHANNEL="${bamboo.channel}"\ + DIST_DIR='dist'\ + DOCKER_IMAGE_NAME='adguard/adguardhome'\ + DOCKER_OUTPUT="type=image,name=adguard/adguardhome,push=true"\ + VERBOSE='1'\ + sh ./scripts/make/build-docker.sh + 'environment': + DOCKER_CLI_EXPERIMENTAL=enabled + 'final-tasks': + - 'clean' + 'requirements': + - 'adg-docker': 'true' 'Publish to static storage': - 'key': 'PUB' - 'other': - 'clean-working-dir': true - 'tasks': - - 'clean' - - 'checkout': - 'repository': 'bamboo-deploy-publisher' - 'path': 'bamboo-deploy-publisher' - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'scripts': - - | - #!/bin/sh + 'key': 'PUB' + 'other': + 'clean-working-dir': true + 'tasks': + - 'clean' + - 'checkout': + 'repository': 'bamboo-deploy-publisher' + 'path': 'bamboo-deploy-publisher' + 'force-clean-build': true + - 'script': + 'interpreter': 'SHELL' + 'scripts': + - | + #!/bin/sh - set -e -f -u -x + set -e -f -u -x - cd ./dist/ + cd ./dist/ - CHANNEL="${bamboo.channel}" - export CHANNEL + CHANNEL="${bamboo.channel}" + export CHANNEL - ../bamboo-deploy-publisher/deploy.sh adguard-home-"$CHANNEL" - 'final-tasks': - - 'clean' - 'requirements': - - 'adg-docker': 'true' + ../bamboo-deploy-publisher/deploy.sh adguard-home-"$CHANNEL" + 'final-tasks': + - 'clean' + 'requirements': + - 'adg-docker': 'true' 'Publish to GitHub Releases': - 'key': 'PTGR' - 'other': - 'clean-working-dir': true - 'tasks': - - 'clean' - - 'checkout': - 'repository': 'bamboo-deploy-publisher' - 'path': 'bamboo-deploy-publisher' - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'scripts': - - | - #!/bin/sh + 'key': 'PTGR' + 'other': + 'clean-working-dir': true + 'tasks': + - 'clean' + - 'checkout': + 'repository': 'bamboo-deploy-publisher' + 'path': 'bamboo-deploy-publisher' + 'force-clean-build': true + - 'script': + 'interpreter': 'SHELL' + 'scripts': + - | + #!/bin/sh - set -e -f -u -x + set -e -f -u -x - channel="${bamboo.channel}" - readonly channel + channel="${bamboo.channel}" + readonly channel - if [ "$channel" != 'release' ] && [ "${channel}" != 'beta' ] - then - echo "don't publish to GitHub Releases for this channel" + if [ "$channel" != 'release' ] && [ "${channel}" != 'beta' ] + then + echo "don't publish to GitHub Releases for this channel" - exit 0 - fi + exit 0 + fi - cd ./dist/ + cd ./dist/ - env\ - GITHUB_TOKEN="${bamboo.githubPublicRepoPassword}"\ - ../bamboo-deploy-publisher/deploy.sh adguard-home-github - 'final-tasks': - - 'clean' - 'requirements': - - 'adg-docker': 'true' + env\ + GITHUB_TOKEN="${bamboo.githubPublicRepoPassword}"\ + ../bamboo-deploy-publisher/deploy.sh adguard-home-github + 'final-tasks': + - 'clean' + 'requirements': + - 'adg-docker': 'true' 'triggers': -# Don't use minute values that end with a zero or a five as these are often used -# in CI and so resources during these minutes can be quite busy. -- 'cron': '0 42 13 ? * MON-FRI *' + # Don't use minute values that end with a zero or a five as these are often + # used in CI and so resources during these minutes can be quite busy. + - 'cron': '0 42 13 ? * MON-FRI *' 'branches': - 'create': 'manually' - 'delete': - 'after-deleted-days': 1 - 'after-inactive-days': 30 - 'integration': - 'push-on-success': false - 'merge-from': 'AdGuard Home - Build and publish release' - 'link-to-jira': true + 'create': 'manually' + 'delete': + 'after-deleted-days': 1 + 'after-inactive-days': 30 + 'integration': + 'push-on-success': false + 'merge-from': 'AdGuard Home - Build and publish release' + 'link-to-jira': true 'notifications': -- 'events': - - 'plan-completed' - 'recipients': - - 'webhook': - 'name': 'Build webhook' - 'url': 'http://prod.jirahub.service.eu.consul/v1/webhook/bamboo?channel=adguard-qa' + - 'events': + - 'plan-completed' + 'recipients': + - 'webhook': + 'name': 'Build webhook' + 'url': 'http://prod.jirahub.service.eu.consul/v1/webhook/bamboo?channel=adguard-qa' 'labels': [] 'other': - 'concurrent-build-plugin': 'system-default' + 'concurrent-build-plugin': 'system-default' 'branch-overrides': -# beta-vX.Y branches are the branches into which the commits that are needed to -# release a new patch version are initially cherry-picked. -- '^beta-v[0-9]+\.[0-9]+': - # Build betas on release branches manually. - 'triggers': [] - # Set the default release channel on the release branch to beta, as we may - # need to build a few of these. - 'variables': - 'channel': 'beta' - 'dockerGo': 'adguard/golang-ubuntu:6.7' -# release-vX.Y.Z branches are the branches from which the actual final release -# is built. -- '^release-v[0-9]+\.[0-9]+\.[0-9]+': - # Disable integration branches for release branches. - 'branch-config': - 'integration': - 'push-on-success': false - 'merge-from': 'beta-v0.107' - # Build final releases on release branches manually. - 'triggers': [] - # Set the default release channel on the final branch to release, as these - # are the ones that actually get released. - 'variables': - 'channel': 'release' - 'dockerGo': 'adguard/golang-ubuntu:6.7' + # beta-vX.Y branches are the branches into which the commits that are needed + # to release a new patch version are initially cherry-picked. + - '^beta-v[0-9]+\.[0-9]+': + # Build betas on release branches manually. + 'triggers': [] + # Set the default release channel on the release branch to beta, as we may + # need to build a few of these. + 'variables': + 'channel': 'beta' + 'dockerGo': 'adguard/golang-ubuntu:6.7' + # release-vX.Y.Z branches are the branches from which the actual final + # release is built. + - '^release-v[0-9]+\.[0-9]+\.[0-9]+': + # Disable integration branches for release branches. + 'branch-config': + 'integration': + 'push-on-success': false + 'merge-from': 'beta-v0.107' + # Build final releases on release branches manually. + 'triggers': [] + # Set the default release channel on the final branch to release, as these + # are the ones that actually get released. + 'variables': + 'channel': 'release' + 'dockerGo': 'adguard/golang-ubuntu:6.7' diff --git a/bamboo-specs/snapcraft.yaml b/bamboo-specs/snapcraft.yaml index 383a90ae..53efff41 100644 --- a/bamboo-specs/snapcraft.yaml +++ b/bamboo-specs/snapcraft.yaml @@ -117,8 +117,6 @@ env\ VERBOSE='1'\ sh ./scripts/snap/build.sh - 'final-tasks': - - 'clean' 'requirements': - 'adg-docker': 'true' diff --git a/bamboo-specs/test.yaml b/bamboo-specs/test.yaml index abfa4566..e2cc8caf 100644 --- a/bamboo-specs/test.yaml +++ b/bamboo-specs/test.yaml @@ -1,64 +1,64 @@ --- 'version': 2 'plan': - 'project-key': 'AGH' - 'key': 'AHBRTSPECS' - 'name': 'AdGuard Home - Build and run tests' + 'project-key': 'AGH' + 'key': 'AHBRTSPECS' + 'name': 'AdGuard Home - Build and run tests' 'variables': - 'dockerGo': 'adguard/golang-ubuntu:6.7' + 'dockerGo': 'adguard/golang-ubuntu:6.7' 'stages': -- 'Tests': - 'manual': false - 'final': false - 'jobs': - - 'Test' + - 'Tests': + 'manual': false + 'final': false + 'jobs': + - 'Test' 'Test': - 'docker': - 'image': '${bamboo.dockerGo}' - 'volumes': - '${system.YARN_DIR}': '${bamboo.cacheYarn}' - '${system.GO_CACHE_DIR}': '${bamboo.cacheGo}' - '${system.GO_PKG_CACHE_DIR}': '${bamboo.cacheGoPkg}' - 'key': 'TEST' - 'other': - 'clean-working-dir': true - 'tasks': - - 'checkout': - 'force-clean-build': true - - 'script': - 'interpreter': 'SHELL' - 'scripts': - - | - #!/bin/sh + 'docker': + 'image': '${bamboo.dockerGo}' + 'volumes': + '${system.YARN_DIR}': '${bamboo.cacheYarn}' + '${system.GO_CACHE_DIR}': '${bamboo.cacheGo}' + '${system.GO_PKG_CACHE_DIR}': '${bamboo.cacheGoPkg}' + 'key': 'TEST' + 'other': + 'clean-working-dir': true + 'tasks': + - 'checkout': + 'force-clean-build': true + - 'script': + 'interpreter': 'SHELL' + 'scripts': + - | + #!/bin/sh - set -e -f -u -x + set -e -f -u -x - make VERBOSE=1 ci go-tools lint - 'final-tasks': - - 'clean' - 'requirements': - - 'adg-docker': 'true' + make VERBOSE=1 ci go-tools lint + 'final-tasks': + - 'clean' + 'requirements': + - 'adg-docker': 'true' 'branches': - 'create': 'for-pull-request' - 'delete': - 'after-deleted-days': 1 - 'after-inactive-days': 5 - 'integration': - 'push-on-success': false - 'merge-from': 'AdGuard Home - Build and run tests' - 'link-to-jira': true + 'create': 'for-pull-request' + 'delete': + 'after-deleted-days': 1 + 'after-inactive-days': 5 + 'integration': + 'push-on-success': false + 'merge-from': 'AdGuard Home - Build and run tests' + 'link-to-jira': true 'notifications': -- 'events': - - 'plan-status-changed' - 'recipients': - - 'webhook': - 'name': 'Build webhook' - 'url': 'http://prod.jirahub.service.eu.consul/v1/webhook/bamboo' + - 'events': + - 'plan-status-changed' + 'recipients': + - 'webhook': + 'name': 'Build webhook' + 'url': 'http://prod.jirahub.service.eu.consul/v1/webhook/bamboo' 'labels': [] 'other': - 'concurrent-build-plugin': 'system-default' + 'concurrent-build-plugin': 'system-default'