Merge branch 'master' into 4535-list-services

2022-08-25 18:24:07 +03:00
parent 56f78edb97 be1bc76cfa
commit 57c0b1203e
178 changed files with 5867 additions and 3011 deletions
--- a/internal/filtering/blocked.go
+++ b/internal/filtering/blocked.go
@@ -20,8 +20,12 @@ type svc struct {
 // client/src/helpers/constants.js
 // client/src/components/ui/Icons.js
 var serviceRulesArray = []svc{{
-	name:  "whatsapp",
-	rules: []string{"||whatsapp.net^", "||whatsapp.com^"},
+	name: "whatsapp",
+	rules: []string{
+		"||wa.me^",
+		"||whatsapp.com^",
+		"||whatsapp.net^",
+	},
 }, {
 	name: "facebook",
 	rules: []string{
@@ -31,29 +35,43 @@ var serviceRulesArray = []svc{{
 		"||accountkit.com^",
 		"||fb.me^",
 		"||fb.com^",
+		"||fb.gg^",
 		"||fbsbx.com^",
+		"||fbwat.ch^",
 		"||messenger.com^",
 		"||facebookcorewwwi.onion^",
 		"||fbcdn.com^",
 		"||fb.watch^",
 	},
 }, {
-	name:  "twitter",
-	rules: []string{"||twitter.com^", "||twttr.com^", "||t.co^", "||twimg.com^"},
+	name: "twitter",
+	rules: []string{
+		"||t.co^",
+		"||twimg.com^",
+		"||twitter.com^",
+		"||twttr.com^",
+	},
 }, {
 	name: "youtube",
 	rules: []string{
-		"||youtube.com^",
-		"||ytimg.com^",
-		"||youtu.be^",
 		"||googlevideo.com^",
-		"||youtubei.googleapis.com^",
-		"||youtube-nocookie.com^",
+		"||wide-youtube.l.google.com^",
+		"||youtu.be^",
 		"||youtube",
+		"||youtube-nocookie.com^",
+		"||youtube.com^",
+		"||youtubei.googleapis.com^",
+		"||youtubekids.com^",
+		"||ytimg.com^",
 	},
 }, {
-	name:  "twitch",
-	rules: []string{"||twitch.tv^", "||ttvnw.net^", "||jtvnw.net^", "||twitchcdn.net^"},
+	name: "twitch",
+	rules: []string{
+		"||jtvnw.net^",
+		"||ttvnw.net^",
+		"||twitch.tv^",
+		"||twitchcdn.net^",
+	},
 }, {
 	name: "netflix",
 	rules: []string{
@@ -83,20 +101,36 @@ var serviceRulesArray = []svc{{
 		"||discordapp.net^",
 		"||discordapp.com^",
 		"||discord.com^",
+		"||discord.gift",
 		"||discord.media^",
 	},
 }, {
 	name:  "ok",
 	rules: []string{"||ok.ru^"},
 }, {
-	name:  "skype",
-	rules: []string{"||skype.com^", "||skypeassets.com^"},
+	name: "skype",
+	rules: []string{
+		"||edge-skype-com.s-0001.s-msedge.net^",
+		"||skype-edf.akadns.net^",
+		"||skype.com^",
+		"||skypeassets.com^",
+		"||skypedata.akadns.net^",
+	},
 }, {
-	name:  "vk",
-	rules: []string{"||vk.com^", "||userapi.com^", "||vk-cdn.net^", "||vkuservideo.net^"},
+	name: "vk",
+	rules: []string{
+		"||userapi.com^",
+		"||vk-cdn.net^",
+		"||vk.com^",
+		"||vkuservideo.net^",
+	},
 }, {
-	name:  "origin",
-	rules: []string{"||origin.com^", "||signin.ea.com^", "||accounts.ea.com^"},
+	name: "origin",
+	rules: []string{
+		"||accounts.ea.com^",
+		"||origin.com^",
+		"||signin.ea.com^",
+	},
 }, {
 	name: "steam",
 	rules: []string{
@@ -160,6 +194,7 @@ var serviceRulesArray = []svc{{
 		"||amazon.com.br^",
 		"||amazon.co.jp^",
 		"||amazon.com.mx^",
+		"||amazon.com.tr^",
 		"||amazon.co.uk^",
 		"||createspace.com^",
 		"||aws",
@@ -209,47 +244,81 @@ var serviceRulesArray = []svc{{
 		"||toutiaocloud.net^",
 		"||bdurl.com^",
 		"||bytecdn.cn^",
+		"||bytedapm.com^",
 		"||byteimg.com^",
+		"||byteoversea.com^",
 		"||ixigua.com^",
 		"||muscdn.com^",
 		"||bytedance.map.fastly.net^",
 		"||douyin.com^",
 		"||tiktokv.com^",
+		"||toutiaovod.com^",
+		"||douyincdn.com^",
 	},
 }, {
-	name:  "vimeo",
-	rules: []string{"||vimeo.com^", "||vimeocdn.com^", "*vod-adaptive.akamaized.net^"},
+	name: "vimeo",
+	rules: []string{
+		"*vod-adaptive.akamaized.net^",
+		"||vimeo.com^",
+		"||vimeocdn.com^",
+	},
 }, {
-	name:  "pinterest",
-	rules: []string{"||pinterest.*^", "||pinimg.com^"},
+	name: "pinterest",
+	rules: []string{
+		"||pinimg.com^",
+		"||pinterest.*^",
+	},
 }, {
 	name:  "imgur",
 	rules: []string{"||imgur.com^"},
 }, {
-	name:  "dailymotion",
-	rules: []string{"||dailymotion.com^", "||dm-event.net^", "||dmcdn.net^"},
+	name: "dailymotion",
+	rules: []string{
+		"||dailymotion.com^",
+		"||dm-event.net^",
+		"||dmcdn.net^",
+	},
 }, {
 	name: "qq",
 	rules: []string{
 		// Block qq.com and subdomains excluding WeChat's domains.
 		"||qq.com^$denyallow=wx.qq.com|weixin.qq.com",
 		"||qqzaixian.com^",
+		"||qq-video.cdn-go.cn^",
+		"||url.cn^",
 	},
 }, {
-	name:  "wechat",
-	rules: []string{"||wechat.com^", "||weixin.qq.com^", "||wx.qq.com^"},
+	name: "wechat",
+	rules: []string{
+		"||wechat.com^",
+		"||weixin.qq.com.cn^",
+		"||weixin.qq.com^",
+		"||weixinbridge.com^",
+		"||wx.qq.com^",
+	},
 }, {
 	name:  "viber",
 	rules: []string{"||viber.com^"},
 }, {
-	name:  "weibo",
-	rules: []string{"||weibo.com^"},
+	name: "weibo",
+	rules: []string{
+		"||weibo.cn^",
+		"||weibo.com^",
+		"||weibocdn.com^",
+	},
 }, {
-	name:  "9gag",
-	rules: []string{"||9cache.com^", "||9gag.com^"},
+	name: "9gag",
+	rules: []string{
+		"||9cache.com^",
+		"||9gag.com^",
+	},
 }, {
-	name:  "telegram",
-	rules: []string{"||t.me^", "||telegram.me^", "||telegram.org^"},
+	name: "telegram",
+	rules: []string{
+		"||t.me^",
+		"||telegram.me^",
+		"||telegram.org^",
+	},
 }, {
 	name: "disneyplus",
 	rules: []string{
@@ -283,6 +352,17 @@ var serviceRulesArray = []svc{{
 		"||tinder.com^",
 		"||tindersparks.com^",
 	},
+}, {
+	name: "bilibili",
+	rules: []string{
+		"||biliapi.net^",
+		"||bilibili.com^",
+		"||biligame.com^",
+		"||bilivideo.cn^",
+		"||bilivideo.com^",
+		"||dreamcast.hk^",
+		"||hdslb.com^",
+	},
 }}

 // convert array to map
--- a/internal/filtering/dnsrewrite_test.go
+++ b/internal/filtering/dnsrewrite_test.go
@@ -61,22 +61,22 @@ func TestDNSFilter_CheckHostRules_dnsrewrite(t *testing.T) {

 	testCasesA := []struct {
 		name  string
-		want  []interface{}
+		want  []any
 		rcode int
 		dtyp  uint16
 	}{{
 		name:  "a-record",
 		rcode: dns.RcodeSuccess,
-		want:  []interface{}{ipv4p1},
+		want:  []any{ipv4p1},
 		dtyp:  dns.TypeA,
 	}, {
 		name:  "aaaa-record",
-		want:  []interface{}{ipv6p1},
+		want:  []any{ipv6p1},
 		rcode: dns.RcodeSuccess,
 		dtyp:  dns.TypeAAAA,
 	}, {
 		name:  "txt-record",
-		want:  []interface{}{"hello-world"},
+		want:  []any{"hello-world"},
 		rcode: dns.RcodeSuccess,
 		dtyp:  dns.TypeTXT,
 	}, {
@@ -86,22 +86,22 @@ func TestDNSFilter_CheckHostRules_dnsrewrite(t *testing.T) {
 		dtyp:  0,
 	}, {
 		name:  "a-records",
-		want:  []interface{}{ipv4p1, ipv4p2},
+		want:  []any{ipv4p1, ipv4p2},
 		rcode: dns.RcodeSuccess,
 		dtyp:  dns.TypeA,
 	}, {
 		name:  "aaaa-records",
-		want:  []interface{}{ipv6p1, ipv6p2},
+		want:  []any{ipv6p1, ipv6p2},
 		rcode: dns.RcodeSuccess,
 		dtyp:  dns.TypeAAAA,
 	}, {
 		name:  "disable-one",
-		want:  []interface{}{ipv4p2},
+		want:  []any{ipv4p2},
 		rcode: dns.RcodeSuccess,
 		dtyp:  dns.TypeA,
 	}, {
 		name:  "disable-cname",
-		want:  []interface{}{ipv4p1},
+		want:  []any{ipv4p1},
 		rcode: dns.RcodeSuccess,
 		dtyp:  dns.TypeA,
 	}}
--- a/internal/filtering/filtering.go
+++ b/internal/filtering/filtering.go
@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io/fs"
 	"net"
-	"net/http"
 	"os"
 	"runtime"
 	"runtime/debug"
@@ -14,6 +13,7 @@ import (
 	"sync"
 	"sync/atomic"

+	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
 	"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
 	"github.com/AdguardTeam/dnsproxy/upstream"
 	"github.com/AdguardTeam/golibs/cache"
@@ -94,7 +94,7 @@ type Config struct {
 	ConfigModified func() `yaml:"-"`

 	// Register an HTTP handler
-	HTTPRegister func(string, string, func(http.ResponseWriter, *http.Request)) `yaml:"-"`
+	HTTPRegister aghhttp.RegisterFunc `yaml:"-"`

 	// CustomResolver is the resolver used by DNSFilter.
 	CustomResolver Resolver `yaml:"-"`
--- a/internal/filtering/filtering_test.go
+++ b/internal/filtering/filtering_test.go
@@ -21,6 +21,11 @@ func TestMain(m *testing.M) {
 	aghtest.DiscardLogOutput(m)
 }

+const (
+	sbBlocked = "wmconvirus.narod.ru"
+	pcBlocked = "pornhub.com"
+)
+
 var setts = Settings{
 	ProtectionEnabled: true,
 }
@@ -173,43 +178,37 @@ func TestSafeBrowsing(t *testing.T) {

 	d := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil)
 	t.Cleanup(d.Close)
-	const matching = "wmconvirus.narod.ru"
-	d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{
-		Hostname: matching,
-		Block:    true,
-	})
-	d.checkMatch(t, matching)

-	require.Contains(t, logOutput.String(), "SafeBrowsing lookup for "+matching)
+	d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
+	d.checkMatch(t, sbBlocked)

-	d.checkMatch(t, "test."+matching)
+	require.Contains(t, logOutput.String(), fmt.Sprintf("safebrowsing lookup for %q", sbBlocked))
+
+	d.checkMatch(t, "test."+sbBlocked)
 	d.checkMatchEmpty(t, "yandex.ru")
-	d.checkMatchEmpty(t, "pornhub.com")
+	d.checkMatchEmpty(t, pcBlocked)

 	// Cached result.
 	d.safeBrowsingServer = "127.0.0.1"
-	d.checkMatch(t, matching)
-	d.checkMatchEmpty(t, "pornhub.com")
+	d.checkMatch(t, sbBlocked)
+	d.checkMatchEmpty(t, pcBlocked)
 	d.safeBrowsingServer = defaultSafebrowsingServer
 }

 func TestParallelSB(t *testing.T) {
 	d := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil)
 	t.Cleanup(d.Close)
-	const matching = "wmconvirus.narod.ru"
-	d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{
-		Hostname: matching,
-		Block:    true,
-	})
+
+	d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))

 	t.Run("group", func(t *testing.T) {
 		for i := 0; i < 100; i++ {
 			t.Run(fmt.Sprintf("aaa%d", i), func(t *testing.T) {
 				t.Parallel()
-				d.checkMatch(t, matching)
-				d.checkMatch(t, "test."+matching)
+				d.checkMatch(t, sbBlocked)
+				d.checkMatch(t, "test."+sbBlocked)
 				d.checkMatchEmpty(t, "yandex.ru")
-				d.checkMatchEmpty(t, "pornhub.com")
+				d.checkMatchEmpty(t, pcBlocked)
 			})
 		}
 	})
@@ -382,23 +381,19 @@ func TestParentalControl(t *testing.T) {

 	d := newForTest(t, &Config{ParentalEnabled: true}, nil)
 	t.Cleanup(d.Close)
-	const matching = "pornhub.com"
-	d.SetParentalUpstream(&aghtest.TestBlockUpstream{
-		Hostname: matching,
-		Block:    true,
-	})

-	d.checkMatch(t, matching)
-	require.Contains(t, logOutput.String(), "Parental lookup for "+matching)
+	d.SetParentalUpstream(aghtest.NewBlockUpstream(pcBlocked, true))
+	d.checkMatch(t, pcBlocked)
+	require.Contains(t, logOutput.String(), fmt.Sprintf("parental lookup for %q", pcBlocked))

-	d.checkMatch(t, "www."+matching)
+	d.checkMatch(t, "www."+pcBlocked)
 	d.checkMatchEmpty(t, "www.yandex.ru")
 	d.checkMatchEmpty(t, "yandex.ru")
 	d.checkMatchEmpty(t, "api.jquery.com")

 	// Test cached result.
 	d.parentalServer = "127.0.0.1"
-	d.checkMatch(t, matching)
+	d.checkMatch(t, pcBlocked)
 	d.checkMatchEmpty(t, "yandex.ru")
 }

@@ -445,7 +440,7 @@ func TestMatching(t *testing.T) {
 	}, {
 		name:           "sanity",
 		rules:          "||doubleclick.net^",
-		host:           "wmconvirus.narod.ru",
+		host:           sbBlocked,
 		wantIsFiltered: false,
 		wantReason:     NotFilteredNotFound,
 		wantDNSType:    dns.TypeA,
@@ -765,14 +760,9 @@ func TestClientSettings(t *testing.T) {
 		}},
 	)
 	t.Cleanup(d.Close)
-	d.SetParentalUpstream(&aghtest.TestBlockUpstream{
-		Hostname: "pornhub.com",
-		Block:    true,
-	})
-	d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{
-		Hostname: "wmconvirus.narod.ru",
-		Block:    true,
-	})
+
+	d.SetParentalUpstream(aghtest.NewBlockUpstream(pcBlocked, true))
+	d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))

 	type testCase struct {
 		name       string
@@ -787,12 +777,12 @@ func TestClientSettings(t *testing.T) {
 		wantReason: FilteredBlockList,
 	}, {
 		name:       "parental",
-		host:       "pornhub.com",
+		host:       pcBlocked,
 		before:     true,
 		wantReason: FilteredParental,
 	}, {
 		name:       "safebrowsing",
-		host:       "wmconvirus.narod.ru",
+		host:       sbBlocked,
 		before:     false,
 		wantReason: FilteredSafeBrowsing,
 	}, {
@@ -836,33 +826,29 @@ func TestClientSettings(t *testing.T) {
 func BenchmarkSafeBrowsing(b *testing.B) {
 	d := newForTest(b, &Config{SafeBrowsingEnabled: true}, nil)
 	b.Cleanup(d.Close)
-	blocked := "wmconvirus.narod.ru"
-	d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{
-		Hostname: blocked,
-		Block:    true,
-	})
+
+	d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
+
 	for n := 0; n < b.N; n++ {
-		res, err := d.CheckHost(blocked, dns.TypeA, &setts)
+		res, err := d.CheckHost(sbBlocked, dns.TypeA, &setts)
 		require.NoError(b, err)

-		assert.True(b, res.IsFiltered, "Expected hostname %s to match", blocked)
+		assert.Truef(b, res.IsFiltered, "expected hostname %q to match", sbBlocked)
 	}
 }

 func BenchmarkSafeBrowsingParallel(b *testing.B) {
 	d := newForTest(b, &Config{SafeBrowsingEnabled: true}, nil)
 	b.Cleanup(d.Close)
-	blocked := "wmconvirus.narod.ru"
-	d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{
-		Hostname: blocked,
-		Block:    true,
-	})
+
+	d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
+
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			res, err := d.CheckHost(blocked, dns.TypeA, &setts)
+			res, err := d.CheckHost(sbBlocked, dns.TypeA, &setts)
 			require.NoError(b, err)

-			assert.True(b, res.IsFiltered, "Expected hostname %s to match", blocked)
+			assert.Truef(b, res.IsFiltered, "expected hostname %q to match", sbBlocked)
 		}
 	})
 }
--- a/internal/filtering/safebrowsing.go
+++ b/internal/filtering/safebrowsing.go
@@ -24,10 +24,11 @@ import (

 // Safe browsing and parental control methods.

+// TODO(a.garipov): Make configurable.
 const (
 	dnsTimeout                = 3 * time.Second
-	defaultSafebrowsingServer = `https://dns-family.adguard.com/dns-query`
-	defaultParentalServer     = `https://dns-family.adguard.com/dns-query`
+	defaultSafebrowsingServer = `https://family.adguard-dns.com/dns-query`
+	defaultParentalServer     = `https://family.adguard-dns.com/dns-query`
 	sbTXTSuffix               = `sb.dns.adguard.com.`
 	pcTXTSuffix               = `pc.dns.adguard.com.`
 )
@@ -313,7 +314,7 @@ func (d *DNSFilter) checkSafeBrowsing(

 	if log.GetLevel() >= log.DEBUG {
 		timer := log.StartTimer()
-		defer timer.LogElapsed("SafeBrowsing lookup for %s", host)
+		defer timer.LogElapsed("safebrowsing lookup for %q", host)
 	}

 	sctx := &sbCtx{
@@ -347,7 +348,7 @@ func (d *DNSFilter) checkParental(

 	if log.GetLevel() >= log.DEBUG {
 		timer := log.StartTimer()
-		defer timer.LogElapsed("Parental lookup for %s", host)
+		defer timer.LogElapsed("parental lookup for %q", host)
 	}

 	sctx := &sbCtx{
--- a/internal/filtering/safebrowsing_test.go
+++ b/internal/filtering/safebrowsing_test.go
@@ -74,21 +74,20 @@ func TestSafeBrowsingCache(t *testing.T) {
 	c.hashToHost[hash] = "sub.host.com"
 	assert.Equal(t, -1, c.getCached())

-	// match "sub.host.com" from cache,
-	//  but another hash for "nonexisting.com" is not in cache
-	//  which means that we must get data from server for it
+	// Match "sub.host.com" from cache.  Another hash for "host.example" is not
+	// in the cache, so get data for it from the server.
 	c.hashToHost = make(map[[32]byte]string)
 	hash = sha256.Sum256([]byte("sub.host.com"))
 	c.hashToHost[hash] = "sub.host.com"
-	hash = sha256.Sum256([]byte("nonexisting.com"))
-	c.hashToHost[hash] = "nonexisting.com"
+	hash = sha256.Sum256([]byte("host.example"))
+	c.hashToHost[hash] = "host.example"
 	assert.Empty(t, c.getCached())

 	hash = sha256.Sum256([]byte("sub.host.com"))
 	_, ok := c.hashToHost[hash]
 	assert.False(t, ok)

-	hash = sha256.Sum256([]byte("nonexisting.com"))
+	hash = sha256.Sum256([]byte("host.example"))
 	_, ok = c.hashToHost[hash]
 	assert.True(t, ok)

@@ -111,8 +110,7 @@ func TestSBPC_checkErrorUpstream(t *testing.T) {
 	d := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil)
 	t.Cleanup(d.Close)

-	ups := &aghtest.TestErrUpstream{}
-
+	ups := aghtest.NewErrorUpstream()
 	d.SetSafeBrowsingUpstream(ups)
 	d.SetParentalUpstream(ups)

@@ -170,10 +168,16 @@ func TestSBPC(t *testing.T) {

 	for _, tc := range testCases {
 		// Prepare the upstream.
-		ups := &aghtest.TestBlockUpstream{
-			Hostname: hostname,
-			Block:    tc.block,
+		ups := aghtest.NewBlockUpstream(hostname, tc.block)
+
+		var numReq int
+		onExchange := ups.OnExchange
+		ups.OnExchange = func(req *dns.Msg) (resp *dns.Msg, err error) {
+			numReq++
+
+			return onExchange(req)
 		}
+
 		d.SetSafeBrowsingUpstream(ups)
 		d.SetParentalUpstream(ups)

@@ -196,7 +200,7 @@ func TestSBPC(t *testing.T) {
 			assert.Equal(t, hits, tc.testCache.Stats().Hit)

 			// There was one request to an upstream.
-			assert.Equal(t, 1, ups.RequestsCount())
+			assert.Equal(t, 1, numReq)

 			// Now make the same request to check the cache was used.
 			res, err = tc.testFunc(hostname, dns.TypeA, setts)
@@ -214,7 +218,7 @@ func TestSBPC(t *testing.T) {
 			assert.Equal(t, hits+1, tc.testCache.Stats().Hit)

 			// Check that there were no additional requests.
-			assert.Equal(t, 1, ups.RequestsCount())
+			assert.Equal(t, 1, numReq)
 		})

 		purgeCaches(d)