daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pull request: 2574 external tests vol.3

Browse Source 
Merge in DNS/adguard-home from 2574-external-tests-3 to master

Updates #2574.

Squashed commit of the following:

commit 29d429c65dee2621ca503710a7ba9522f14f55f9
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Thu Feb 4 20:06:57 2021 +0300

    all: finally fix spacing

commit 9e3a3be63b74852a7802e3f1832648444b58e4d0
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Thu Feb 4 19:59:09 2021 +0300

    aghtest: polish spacing

commit 8a984159fe813b95b989803f5b8b78d01a41bd39
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Thu Feb 4 18:44:47 2021 +0300

    all: fix linux tests, imp code quality

commit 0c1b42bacba1b23fa847e1fa032579c525b3eaa1
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Thu Feb 4 17:33:12 2021 +0300

    all: mv testutil to aghtest package, imp tests
This commit is contained in:
Eugene Burkov
2021-02-04 20:35:13 +03:00
parent 8aec08727c
commit c9d2436d77
24 changed files with 737 additions and 496 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
internal/dnsfilter/dnsfilter.go
View File

@@ -43,6 +43,12 @@ type RequestFilteringSettings struct {
ServicesRules []ServiceEntry
}
// Resolver is the interface for net.Resolver to simplify testing.
type Resolver interface {
// TODO(e.burkov): Replace with LookupIP after upgrading go to v1.15.
LookupIPAddr(ctx context.Context, host string) (ips []net.IPAddr, err error)
}
// Config allows you to configure DNS filtering with New() or just change variables directly.
type Config struct {
ParentalEnabled bool `yaml:"parental_enabled"`
@@ -69,6 +75,9 @@ type Config struct {
// Register an HTTP handler
HTTPRegister func(string, string, func(http.ResponseWriter, *http.Request)) `yaml:"-"`
// CustomResolver is the resolver used by DNSFilter.
CustomResolver Resolver
}
// LookupStats store stats collected during safebrowsing or parental checks
@@ -92,12 +101,6 @@ type filtersInitializerParams struct {
blockFilters []Filter
}
// Resolver is the interface for net.Resolver to simplify testing.
type Resolver interface {
// TODO(e.burkov): Replace with LookupIP after upgrading go to v1.15.
LookupIPAddr(ctx context.Context, host string) (ips []net.IPAddr, err error)
}
// DNSFilter matches hostnames and DNS requests against filtering rules.
type DNSFilter struct {
rulesStorage *filterlist.RuleStorage
@@ -796,6 +799,7 @@ func InitModule() {
// New creates properly initialized DNS Filter that is ready to be used.
func New(c *Config, blockFilters []Filter) *DNSFilter {
var resolver Resolver = net.DefaultResolver
if c != nil {
cacheConf := cache.Config{
EnableLRU: true,
@@ -815,10 +819,14 @@ func New(c *Config, blockFilters []Filter) *DNSFilter {
cacheConf.MaxSize = c.ParentalCacheSize
gctx.parentalCache = cache.New(cacheConf)
}
if c.CustomResolver != nil {
resolver = c.CustomResolver
}
}
d := &DNSFilter{
resolver: net.DefaultResolver,
resolver: resolver,
}
err := d.initSecurityServices()

103
internal/dnsfilter/dnsfilter_test.go
View File

@@ -3,12 +3,11 @@ package dnsfilter
import (
"bytes"
"context"
"crypto/sha256"
"fmt"
"net"
"testing"
"github.com/AdguardTeam/AdGuardHome/internal/testutil"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/golibs/cache"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/urlfilter/rules"
@@ -17,7 +16,7 @@ import (
)
func TestMain(m *testing.M) {
testutil.DiscardLogOutput(m)
aghtest.DiscardLogOutput(m)
}
var setts RequestFilteringSettings
@@ -37,7 +36,9 @@ func purgeCaches() {
}
func newForTest(c *Config, filters []Filter) *DNSFilter {
setts = RequestFilteringSettings{}
setts = RequestFilteringSettings{
FilteringEnabled: true,
}
setts.FilteringEnabled = true
if c != nil {
c.SafeBrowsingCacheSize = 10000
@@ -149,16 +150,16 @@ func TestEtcHostsMatching(t *testing.T) {
func TestSafeBrowsing(t *testing.T) {
logOutput := &bytes.Buffer{}
testutil.ReplaceLogWriter(t, logOutput)
testutil.ReplaceLogLevel(t, log.DEBUG)
aghtest.ReplaceLogWriter(t, logOutput)
aghtest.ReplaceLogLevel(t, log.DEBUG)
d := newForTest(&Config{SafeBrowsingEnabled: true}, nil)
t.Cleanup(d.Close)
matching := "wmconvirus.narod.ru"
d.safeBrowsingUpstream = &testSbUpstream{
hostname: matching,
block: true,
}
d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{
Hostname: matching,
Block: true,
})
d.checkMatch(t, matching)
assert.Contains(t, logOutput.String(), "SafeBrowsing lookup for "+matching)
@@ -178,10 +179,10 @@ func TestParallelSB(t *testing.T) {
d := newForTest(&Config{SafeBrowsingEnabled: true}, nil)
t.Cleanup(d.Close)
matching := "wmconvirus.narod.ru"
d.safeBrowsingUpstream = &testSbUpstream{
hostname: matching,
block: true,
}
d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{
Hostname: matching,
Block: true,
})
t.Run("group", func(t *testing.T) {
for i := 0; i < 100; i++ {
@@ -228,26 +229,12 @@ func TestCheckHostSafeSearchYandex(t *testing.T) {
}
}
// testResolver is a Resolver for tests.
type testResolver struct{}
// LookupIP implements Resolver interface for *testResolver.
func (r *testResolver) LookupIPAddr(_ context.Context, host string) (ips []net.IPAddr, err error) {
hash := sha256.Sum256([]byte(host))
addrs := []net.IPAddr{{
IP: net.IP(hash[:4]),
Zone: "somezone",
}, {
IP: net.IP(hash[4:20]),
Zone: "somezone",
}}
return addrs, nil
}
func TestCheckHostSafeSearchGoogle(t *testing.T) {
d := newForTest(&Config{SafeSearchEnabled: true}, nil)
d := newForTest(&Config{
SafeSearchEnabled: true,
CustomResolver: &aghtest.TestResolver{},
}, nil)
t.Cleanup(d.Close)
d.resolver = &testResolver{}
// Check host for each domain.
for _, host := range []string{
@@ -299,12 +286,12 @@ func TestSafeSearchCacheYandex(t *testing.T) {
}
func TestSafeSearchCacheGoogle(t *testing.T) {
d := newForTest(nil, nil)
resolver := &aghtest.TestResolver{}
d := newForTest(&Config{
CustomResolver: resolver,
}, nil)
t.Cleanup(d.Close)
resolver := &testResolver{}
d.resolver = resolver
domain := "www.google.ru"
res, err := d.CheckHost(domain, dns.TypeA, &setts)
assert.Nil(t, err)
@@ -350,16 +337,16 @@ func TestSafeSearchCacheGoogle(t *testing.T) {
func TestParentalControl(t *testing.T) {
logOutput := &bytes.Buffer{}
testutil.ReplaceLogWriter(t, logOutput)
testutil.ReplaceLogLevel(t, log.DEBUG)
aghtest.ReplaceLogWriter(t, logOutput)
aghtest.ReplaceLogLevel(t, log.DEBUG)
d := newForTest(&Config{ParentalEnabled: true}, nil)
t.Cleanup(d.Close)
matching := "pornhub.com"
d.parentalUpstream = &testSbUpstream{
hostname: matching,
block: true,
}
d.SetParentalUpstream(&aghtest.TestBlockUpstream{
Hostname: matching,
Block: true,
})
d.checkMatch(t, matching)
assert.Contains(t, logOutput.String(), "Parental lookup for "+matching)
@@ -733,14 +720,14 @@ func TestClientSettings(t *testing.T) {
}},
)
t.Cleanup(d.Close)
d.parentalUpstream = &testSbUpstream{
hostname: "pornhub.com",
block: true,
}
d.safeBrowsingUpstream = &testSbUpstream{
hostname: "wmconvirus.narod.ru",
block: true,
}
d.SetParentalUpstream(&aghtest.TestBlockUpstream{
Hostname: "pornhub.com",
Block: true,
})
d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{
Hostname: "wmconvirus.narod.ru",
Block: true,
})
type testCase struct {
name string
@@ -801,10 +788,10 @@ func BenchmarkSafeBrowsing(b *testing.B) {
d := newForTest(&Config{SafeBrowsingEnabled: true}, nil)
b.Cleanup(d.Close)
blocked := "wmconvirus.narod.ru"
d.safeBrowsingUpstream = &testSbUpstream{
hostname: blocked,
block: true,
}
d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{
Hostname: blocked,
Block: true,
})
for n := 0; n < b.N; n++ {
res, err := d.CheckHost(blocked, dns.TypeA, &setts)
assert.Nilf(b, err, "Error while matching host %s: %s", blocked, err)
@@ -816,10 +803,10 @@ func BenchmarkSafeBrowsingParallel(b *testing.B) {
d := newForTest(&Config{SafeBrowsingEnabled: true}, nil)
b.Cleanup(d.Close)
blocked := "wmconvirus.narod.ru"
d.safeBrowsingUpstream = &testSbUpstream{
hostname: blocked,
block: true,
}
d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{
Hostname: blocked,
Block: true,
})
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
res, err := d.CheckHost(blocked, dns.TypeA, &setts)

22
internal/dnsfilter/safebrowsing.go
View File

@@ -30,6 +30,20 @@ const (
pcTXTSuffix = `pc.dns.adguard.com.`
)
// SetParentalUpstream sets the parental upstream for *DNSFilter.
//
// TODO(e.burkov): Remove this in v1 API to forbid the direct access.
func (d *DNSFilter) SetParentalUpstream(u upstream.Upstream) {
d.parentalUpstream = u
}
// SetSafeBrowsingUpstream sets the safe browsing upstream for *DNSFilter.
//
// TODO(e.burkov): Remove this in v1 API to forbid the direct access.
func (d *DNSFilter) SetSafeBrowsingUpstream(u upstream.Upstream) {
d.safeBrowsingUpstream = u
}
func (d *DNSFilter) initSecurityServices() error {
var err error
d.safeBrowsingServer = defaultSafebrowsingServer
@@ -44,15 +58,17 @@ func (d *DNSFilter) initSecurityServices() error {
},
}
d.parentalUpstream, err = upstream.AddressToUpstream(d.parentalServer, opts)
parUps, err := upstream.AddressToUpstream(d.parentalServer, opts)
if err != nil {
return fmt.Errorf("converting parental server: %w", err)
}
d.SetParentalUpstream(parUps)
d.safeBrowsingUpstream, err = upstream.AddressToUpstream(d.safeBrowsingServer, opts)
sbUps, err := upstream.AddressToUpstream(d.safeBrowsingServer, opts)
if err != nil {
return fmt.Errorf("converting safe browsing server: %w", err)
}
d.SetSafeBrowsingUpstream(sbUps)
return nil
}
@@ -227,7 +243,7 @@ func (c *sbCtx) processTXT(resp *dns.Msg) (bool, [][]byte) {
func (c *sbCtx) storeCache(hashes [][]byte) {
sort.Slice(hashes, func(a, b int) bool {
return bytes.Compare(hashes[a], hashes[b]) < 0
return bytes.Compare(hashes[a], hashes[b]) == -1
})
var curData []byte

204
internal/dnsfilter/safebrowsing_test.go
View File

@@ -2,14 +2,11 @@ package dnsfilter
import (
"crypto/sha256"
"encoding/hex"
"strings"
"sync"
"testing"
"github.com/AdguardTeam/AdGuardHome/internal/agherr"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/golibs/cache"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
)
@@ -108,27 +105,14 @@ func TestSafeBrowsingCache(t *testing.T) {
assert.Empty(t, c.getCached())
}
// testErrUpstream implements upstream.Upstream interface for replacing real
// upstream in tests.
type testErrUpstream struct{}
// Exchange always returns nil Msg and non-nil error.
func (teu *testErrUpstream) Exchange(*dns.Msg) (*dns.Msg, error) {
return nil, agherr.Error("bad")
}
func (teu *testErrUpstream) Address() string {
return ""
}
func TestSBPC_checkErrorUpstream(t *testing.T) {
d := newForTest(&Config{SafeBrowsingEnabled: true}, nil)
t.Cleanup(d.Close)
ups := &testErrUpstream{}
ups := &aghtest.TestErrUpstream{}
d.safeBrowsingUpstream = ups
d.parentalUpstream = ups
d.SetSafeBrowsingUpstream(ups)
d.SetParentalUpstream(ups)
_, err := d.checkSafeBrowsing("smthng.com")
assert.NotNil(t, err)
@@ -137,122 +121,86 @@ func TestSBPC_checkErrorUpstream(t *testing.T) {
assert.NotNil(t, err)
}
// testSbUpstream implements upstream.Upstream interface for replacing real
// upstream in tests.
type testSbUpstream struct {
hostname string
block bool
requestsCount int
counterLock sync.RWMutex
}
// Exchange returns a message depending on the upstream settings (hostname, block)
func (u *testSbUpstream) Exchange(r *dns.Msg) (*dns.Msg, error) {
u.counterLock.Lock()
u.requestsCount++
u.counterLock.Unlock()
hash := sha256.Sum256([]byte(u.hostname))
prefix := hash[0:2]
hashToReturn := hex.EncodeToString(prefix) + strings.Repeat("ab", 28)
if u.block {
hashToReturn = hex.EncodeToString(hash[:])
}
m := &dns.Msg{}
m.Answer = []dns.RR{
&dns.TXT{
Hdr: dns.RR_Header{
Name: r.Question[0].Name,
},
Txt: []string{
hashToReturn,
},
},
}
return m, nil
}
func (u *testSbUpstream) Address() string {
return ""
}
func TestSBPC_sbValidResponse(t *testing.T) {
func TestSBPC(t *testing.T) {
d := newForTest(&Config{SafeBrowsingEnabled: true}, nil)
t.Cleanup(d.Close)
ups := &testSbUpstream{}
d.safeBrowsingUpstream = ups
d.parentalUpstream = ups
const hostname = "example.org"
// Prepare the upstream
ups.hostname = "example.org"
ups.block = false
ups.requestsCount = 0
testCases := []struct {
name string
block bool
testFunc func(string) (Result, error)
testCache cache.Cache
}{{
name: "sb_no_block",
block: false,
testFunc: d.checkSafeBrowsing,
testCache: gctx.safebrowsingCache,
}, {
name: "sb_block",
block: true,
testFunc: d.checkSafeBrowsing,
testCache: gctx.safebrowsingCache,
}, {
name: "pc_no_block",
block: false,
testFunc: d.checkParental,
testCache: gctx.parentalCache,
}, {
name: "pc_block",
block: true,
testFunc: d.checkParental,
testCache: gctx.parentalCache,
}}
// First - check that the request is not blocked
res, err := d.checkSafeBrowsing("example.org")
assert.Nil(t, err)
assert.False(t, res.IsFiltered)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
// Prepare the upstream.
ups := &aghtest.TestBlockUpstream{
Hostname: hostname,
Block: tc.block,
}
d.SetSafeBrowsingUpstream(ups)
d.SetParentalUpstream(ups)
// Check the cache state, check that the response is now cached
assert.Equal(t, 1, gctx.safebrowsingCache.Stats().Count)
assert.Equal(t, 0, gctx.safebrowsingCache.Stats().Hit)
// Firstly, check the request blocking.
hits := 0
res, err := tc.testFunc(hostname)
assert.Nil(t, err)
if tc.block {
assert.True(t, res.IsFiltered)
assert.Len(t, res.Rules, 1)
hits++
} else {
assert.False(t, res.IsFiltered)
}
// There was one request to an upstream
assert.Equal(t, 1, ups.requestsCount)
// Check the cache state, check the response is now cached.
assert.Equal(t, 1, tc.testCache.Stats().Count)
assert.Equal(t, hits, tc.testCache.Stats().Hit)
// Now make the same request to check that the cache was used
res, err = d.checkSafeBrowsing("example.org")
assert.Nil(t, err)
assert.False(t, res.IsFiltered)
// There was one request to an upstream.
assert.Equal(t, 1, ups.RequestsCount())
// Check the cache state, it should've been used
assert.Equal(t, 1, gctx.safebrowsingCache.Stats().Count)
assert.Equal(t, 1, gctx.safebrowsingCache.Stats().Hit)
// Now make the same request to check the cache was used.
res, err = tc.testFunc(hostname)
assert.Nil(t, err)
if tc.block {
assert.True(t, res.IsFiltered)
assert.Len(t, res.Rules, 1)
} else {
assert.False(t, res.IsFiltered)
}
// Check that there were no additional requests
assert.Equal(t, 1, ups.requestsCount)
}
func TestSBPC_pcBlockedResponse(t *testing.T) {
d := newForTest(&Config{SafeBrowsingEnabled: true}, nil)
t.Cleanup(d.Close)
ups := &testSbUpstream{}
d.safeBrowsingUpstream = ups
d.parentalUpstream = ups
// Prepare the upstream
// Make sure that the upstream will return a response that matches the queried domain
ups.hostname = "example.com"
ups.block = true
ups.requestsCount = 0
// Make a lookup
res, err := d.checkParental("example.com")
assert.Nil(t, err)
assert.True(t, res.IsFiltered)
assert.Len(t, res.Rules, 1)
// Check the cache state, check that the response is now cached
assert.Equal(t, 1, gctx.parentalCache.Stats().Count)
assert.Equal(t, 1, gctx.parentalCache.Stats().Hit)
// There was one request to an upstream
assert.Equal(t, 1, ups.requestsCount)
// Make a second lookup for the same domain
res, err = d.checkParental("example.com")
assert.Nil(t, err)
assert.True(t, res.IsFiltered)
assert.Len(t, res.Rules, 1)
// Check the cache state, it should've been used
assert.Equal(t, 1, gctx.parentalCache.Stats().Count)
assert.Equal(t, 2, gctx.parentalCache.Stats().Hit)
// Check that there were no additional requests
assert.Equal(t, 1, ups.requestsCount)
// Check the cache state, it should've been used.
assert.Equal(t, 1, tc.testCache.Stats().Count)
assert.Equal(t, hits+1, tc.testCache.Stats().Hit)
// Check that there were no additional requests.
assert.Equal(t, 1, ups.RequestsCount())
purgeCaches()
})
}
}