all: sync with master; upd chlog

This commit is contained in:
Ainar Garipov
2023-06-07 20:04:01 +03:00
parent 7030c7c24c
commit c65700923a
76 changed files with 2998 additions and 1909 deletions

View File

@@ -18,8 +18,6 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/cache"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/mathutil"
@@ -75,6 +73,12 @@ type Resolver interface {
// Config allows you to configure DNS filtering with New() or just change variables directly.
type Config struct {
// SafeBrowsingChecker is the safe browsing hash-prefix checker.
SafeBrowsingChecker Checker `yaml:"-"`
// ParentControl is the parental control hash-prefix checker.
ParentalControlChecker Checker `yaml:"-"`
// enabled is used to be returned within Settings.
//
// It is of type uint32 to be accessed by atomic.
@@ -158,8 +162,22 @@ type hostChecker struct {
name string
}
// Checker is used for safe browsing or parental control hash-prefix filtering.
type Checker interface {
// Check returns true if request for the host should be blocked.
Check(host string) (block bool, err error)
}
// DNSFilter matches hostnames and DNS requests against filtering rules.
type DNSFilter struct {
safeSearch SafeSearch
// safeBrowsingChecker is the safe browsing hash-prefix checker.
safeBrowsingChecker Checker
// parentalControl is the parental control hash-prefix checker.
parentalControlChecker Checker
rulesStorage *filterlist.RuleStorage
filteringEngine *urlfilter.DNSEngine
@@ -168,14 +186,6 @@ type DNSFilter struct {
engineLock sync.RWMutex
parentalServer string // access via methods
safeBrowsingServer string // access via methods
parentalUpstream upstream.Upstream
safeBrowsingUpstream upstream.Upstream
safebrowsingCache cache.Cache
parentalCache cache.Cache
Config // for direct access by library users, even a = assignment
// confLock protects Config.
confLock sync.RWMutex
@@ -192,7 +202,6 @@ type DNSFilter struct {
// TODO(e.burkov): Don't use regexp for such a simple text processing task.
filterTitleRegexp *regexp.Regexp
safeSearch SafeSearch
hostCheckers []hostChecker
}
@@ -940,19 +949,12 @@ func InitModule() {
// be non-nil.
func New(c *Config, blockFilters []Filter) (d *DNSFilter, err error) {
d = &DNSFilter{
refreshLock: &sync.Mutex{},
filterTitleRegexp: regexp.MustCompile(`^! Title: +(.*)$`),
refreshLock: &sync.Mutex{},
filterTitleRegexp: regexp.MustCompile(`^! Title: +(.*)$`),
safeBrowsingChecker: c.SafeBrowsingChecker,
parentalControlChecker: c.ParentalControlChecker,
}
d.safebrowsingCache = cache.New(cache.Config{
EnableLRU: true,
MaxSize: c.SafeBrowsingCacheSize,
})
d.parentalCache = cache.New(cache.Config{
EnableLRU: true,
MaxSize: c.ParentalCacheSize,
})
d.safeSearch = c.SafeSearch
d.hostCheckers = []hostChecker{{
@@ -977,11 +979,6 @@ func New(c *Config, blockFilters []Filter) (d *DNSFilter, err error) {
defer func() { err = errors.Annotate(err, "filtering: %w") }()
err = d.initSecurityServices()
if err != nil {
return nil, fmt.Errorf("initializing services: %s", err)
}
d.Config = *c
d.filtersMu = &sync.RWMutex{}
@@ -1038,3 +1035,69 @@ func (d *DNSFilter) Start() {
// So for now we just start this periodic task from here.
go d.periodicallyRefreshFilters()
}
// Safe browsing and parental control methods.
// TODO(a.garipov): Unify with checkParental.
func (d *DNSFilter) checkSafeBrowsing(
host string,
_ uint16,
setts *Settings,
) (res Result, err error) {
if !setts.ProtectionEnabled || !setts.SafeBrowsingEnabled {
return Result{}, nil
}
if log.GetLevel() >= log.DEBUG {
timer := log.StartTimer()
defer timer.LogElapsed("safebrowsing lookup for %q", host)
}
res = Result{
Rules: []*ResultRule{{
Text: "adguard-malware-shavar",
FilterListID: SafeBrowsingListID,
}},
Reason: FilteredSafeBrowsing,
IsFiltered: true,
}
block, err := d.safeBrowsingChecker.Check(host)
if !block || err != nil {
return Result{}, err
}
return res, nil
}
// TODO(a.garipov): Unify with checkSafeBrowsing.
func (d *DNSFilter) checkParental(
host string,
_ uint16,
setts *Settings,
) (res Result, err error) {
if !setts.ProtectionEnabled || !setts.ParentalEnabled {
return Result{}, nil
}
if log.GetLevel() >= log.DEBUG {
timer := log.StartTimer()
defer timer.LogElapsed("parental lookup for %q", host)
}
res = Result{
Rules: []*ResultRule{{
Text: "parental CATEGORY_BLACKLISTED",
FilterListID: ParentalListID,
}},
Reason: FilteredParental,
IsFiltered: true,
}
block, err := d.parentalControlChecker.Check(host)
if !block || err != nil {
return Result{}, err
}
return res, nil
}

View File

@@ -7,7 +7,7 @@ import (
"testing"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/golibs/cache"
"github.com/AdguardTeam/AdGuardHome/internal/filtering/hashprefix"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/testutil"
"github.com/AdguardTeam/urlfilter/rules"
@@ -27,17 +27,6 @@ const (
// Helpers.
func purgeCaches(d *DNSFilter) {
for _, c := range []cache.Cache{
d.safebrowsingCache,
d.parentalCache,
} {
if c != nil {
c.Clear()
}
}
}
func newForTest(t testing.TB, c *Config, filters []Filter) (f *DNSFilter, setts *Settings) {
setts = &Settings{
ProtectionEnabled: true,
@@ -58,11 +47,17 @@ func newForTest(t testing.TB, c *Config, filters []Filter) (f *DNSFilter, setts
f, err := New(c, filters)
require.NoError(t, err)
purgeCaches(f)
return f, setts
}
func newChecker(host string) Checker {
return hashprefix.New(&hashprefix.Config{
CacheTime: 10,
CacheSize: 100000,
Upstream: aghtest.NewBlockUpstream(host, true),
})
}
func (d *DNSFilter) checkMatch(t *testing.T, hostname string, setts *Settings) {
t.Helper()
@@ -175,10 +170,14 @@ func TestSafeBrowsing(t *testing.T) {
aghtest.ReplaceLogWriter(t, logOutput)
aghtest.ReplaceLogLevel(t, log.DEBUG)
d, setts := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil)
sbChecker := newChecker(sbBlocked)
d, setts := newForTest(t, &Config{
SafeBrowsingEnabled: true,
SafeBrowsingChecker: sbChecker,
}, nil)
t.Cleanup(d.Close)
d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
d.checkMatch(t, sbBlocked, setts)
require.Contains(t, logOutput.String(), fmt.Sprintf("safebrowsing lookup for %q", sbBlocked))
@@ -188,18 +187,17 @@ func TestSafeBrowsing(t *testing.T) {
d.checkMatchEmpty(t, pcBlocked, setts)
// Cached result.
d.safeBrowsingServer = "127.0.0.1"
d.checkMatch(t, sbBlocked, setts)
d.checkMatchEmpty(t, pcBlocked, setts)
d.safeBrowsingServer = defaultSafebrowsingServer
}
func TestParallelSB(t *testing.T) {
d, setts := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil)
d, setts := newForTest(t, &Config{
SafeBrowsingEnabled: true,
SafeBrowsingChecker: newChecker(sbBlocked),
}, nil)
t.Cleanup(d.Close)
d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
t.Run("group", func(t *testing.T) {
for i := 0; i < 100; i++ {
t.Run(fmt.Sprintf("aaa%d", i), func(t *testing.T) {
@@ -220,10 +218,12 @@ func TestParentalControl(t *testing.T) {
aghtest.ReplaceLogWriter(t, logOutput)
aghtest.ReplaceLogLevel(t, log.DEBUG)
d, setts := newForTest(t, &Config{ParentalEnabled: true}, nil)
d, setts := newForTest(t, &Config{
ParentalEnabled: true,
ParentalControlChecker: newChecker(pcBlocked),
}, nil)
t.Cleanup(d.Close)
d.SetParentalUpstream(aghtest.NewBlockUpstream(pcBlocked, true))
d.checkMatch(t, pcBlocked, setts)
require.Contains(t, logOutput.String(), fmt.Sprintf("parental lookup for %q", pcBlocked))
@@ -233,7 +233,6 @@ func TestParentalControl(t *testing.T) {
d.checkMatchEmpty(t, "api.jquery.com", setts)
// Test cached result.
d.parentalServer = "127.0.0.1"
d.checkMatch(t, pcBlocked, setts)
d.checkMatchEmpty(t, "yandex.ru", setts)
}
@@ -593,8 +592,10 @@ func applyClientSettings(setts *Settings) {
func TestClientSettings(t *testing.T) {
d, setts := newForTest(t,
&Config{
ParentalEnabled: true,
SafeBrowsingEnabled: false,
ParentalEnabled: true,
SafeBrowsingEnabled: false,
SafeBrowsingChecker: newChecker(sbBlocked),
ParentalControlChecker: newChecker(pcBlocked),
},
[]Filter{{
ID: 0, Data: []byte("||example.org^\n"),
@@ -602,9 +603,6 @@ func TestClientSettings(t *testing.T) {
)
t.Cleanup(d.Close)
d.SetParentalUpstream(aghtest.NewBlockUpstream(pcBlocked, true))
d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
type testCase struct {
name string
host string
@@ -665,11 +663,12 @@ func TestClientSettings(t *testing.T) {
// Benchmarks.
func BenchmarkSafeBrowsing(b *testing.B) {
d, setts := newForTest(b, &Config{SafeBrowsingEnabled: true}, nil)
d, setts := newForTest(b, &Config{
SafeBrowsingEnabled: true,
SafeBrowsingChecker: newChecker(sbBlocked),
}, nil)
b.Cleanup(d.Close)
d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
for n := 0; n < b.N; n++ {
res, err := d.CheckHost(sbBlocked, dns.TypeA, setts)
require.NoError(b, err)
@@ -679,11 +678,12 @@ func BenchmarkSafeBrowsing(b *testing.B) {
}
func BenchmarkSafeBrowsingParallel(b *testing.B) {
d, setts := newForTest(b, &Config{SafeBrowsingEnabled: true}, nil)
d, setts := newForTest(b, &Config{
SafeBrowsingEnabled: true,
SafeBrowsingChecker: newChecker(sbBlocked),
}, nil)
b.Cleanup(d.Close)
d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
res, err := d.CheckHost(sbBlocked, dns.TypeA, setts)

View File

@@ -0,0 +1,130 @@
package hashprefix
import (
"encoding/binary"
"time"
"github.com/AdguardTeam/golibs/log"
)
// expirySize is the size of expiry in cacheItem.
const expirySize = 8
// cacheItem represents an item that we will store in the cache.
type cacheItem struct {
// expiry is the time when cacheItem will expire.
expiry time.Time
// hashes is the hashed hostnames.
hashes []hostnameHash
}
// toCacheItem decodes cacheItem from data. data must be at least equal to
// expiry size.
func toCacheItem(data []byte) *cacheItem {
t := time.Unix(int64(binary.BigEndian.Uint64(data)), 0)
data = data[expirySize:]
hashes := make([]hostnameHash, len(data)/hashSize)
for i := 0; i < len(data); i += hashSize {
var hash hostnameHash
copy(hash[:], data[i:i+hashSize])
hashes = append(hashes, hash)
}
return &cacheItem{
expiry: t,
hashes: hashes,
}
}
// fromCacheItem encodes cacheItem into data.
func fromCacheItem(item *cacheItem) (data []byte) {
data = make([]byte, len(item.hashes)*hashSize+expirySize)
expiry := item.expiry.Unix()
binary.BigEndian.PutUint64(data[:expirySize], uint64(expiry))
for _, v := range item.hashes {
// nolint:looppointer // The subsilce is used for a copy.
data = append(data, v[:]...)
}
return data
}
// findInCache finds hashes in the cache. If nothing found returns list of
// hashes, prefixes of which will be sent to upstream.
func (c *Checker) findInCache(
hashes []hostnameHash,
) (found, blocked bool, hashesToRequest []hostnameHash) {
now := time.Now()
i := 0
for _, hash := range hashes {
// nolint:looppointer // The subsilce is used for a safe cache lookup.
data := c.cache.Get(hash[:prefixLen])
if data == nil {
hashes[i] = hash
i++
continue
}
item := toCacheItem(data)
if now.After(item.expiry) {
hashes[i] = hash
i++
continue
}
if ok := findMatch(hashes, item.hashes); ok {
return true, true, nil
}
}
if i == 0 {
return true, false, nil
}
return false, false, hashes[:i]
}
// storeInCache caches hashes.
func (c *Checker) storeInCache(hashesToRequest, respHashes []hostnameHash) {
hashToStore := make(map[prefix][]hostnameHash)
for _, hash := range respHashes {
var pref prefix
// nolint:looppointer // The subsilce is used for a copy.
copy(pref[:], hash[:])
hashToStore[pref] = append(hashToStore[pref], hash)
}
for pref, hash := range hashToStore {
// nolint:looppointer // The subsilce is used for a safe cache lookup.
c.setCache(pref[:], hash)
}
for _, hash := range hashesToRequest {
// nolint:looppointer // The subsilce is used for a safe cache lookup.
pref := hash[:prefixLen]
val := c.cache.Get(pref)
if val == nil {
c.setCache(pref, nil)
}
}
}
// setCache stores hash in cache.
func (c *Checker) setCache(pref []byte, hashes []hostnameHash) {
item := &cacheItem{
expiry: time.Now().Add(c.cacheTime),
hashes: hashes,
}
c.cache.Set(pref, fromCacheItem(item))
log.Debug("%s: stored in cache: %v", c.svc, pref)
}

View File

@@ -0,0 +1,245 @@
// Package hashprefix used for safe browsing and parent control.
package hashprefix
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"strings"
"time"
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/cache"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/miekg/dns"
"golang.org/x/exp/slices"
"golang.org/x/net/publicsuffix"
)
const (
// prefixLen is the length of the hash prefix of the filtered hostname.
prefixLen = 2
// hashSize is the size of hashed hostname.
hashSize = sha256.Size
// hexSize is the size of hexadecimal representation of hashed hostname.
hexSize = hashSize * 2
)
// prefix is the type of the SHA256 hash prefix used to match against the
// domain-name database.
type prefix [prefixLen]byte
// hostnameHash is the hashed hostname.
//
// TODO(s.chzhen): Split into prefix and suffix.
type hostnameHash [hashSize]byte
// findMatch returns true if one of the a hostnames matches one of the b.
func findMatch(a, b []hostnameHash) (matched bool) {
for _, hash := range a {
if slices.Contains(b, hash) {
return true
}
}
return false
}
// Config is the configuration structure for safe browsing and parental
// control.
type Config struct {
// Upstream is the upstream DNS server.
Upstream upstream.Upstream
// ServiceName is the name of the service.
ServiceName string
// TXTSuffix is the TXT suffix for DNS request.
TXTSuffix string
// CacheTime is the time period to store hash.
CacheTime time.Duration
// CacheSize is the maximum size of the cache. If it's zero, cache size is
// unlimited.
CacheSize uint
}
type Checker struct {
// upstream is the upstream DNS server.
upstream upstream.Upstream
// cache stores hostname hashes.
cache cache.Cache
// svc is the name of the service.
svc string
// txtSuffix is the TXT suffix for DNS request.
txtSuffix string
// cacheTime is the time period to store hash.
cacheTime time.Duration
}
// New returns Checker.
func New(conf *Config) (c *Checker) {
return &Checker{
upstream: conf.Upstream,
cache: cache.New(cache.Config{
EnableLRU: true,
MaxSize: conf.CacheSize,
}),
svc: conf.ServiceName,
txtSuffix: conf.TXTSuffix,
cacheTime: conf.CacheTime,
}
}
// Check returns true if request for the host should be blocked.
func (c *Checker) Check(host string) (ok bool, err error) {
hashes := hostnameToHashes(host)
found, blocked, hashesToRequest := c.findInCache(hashes)
if found {
log.Debug("%s: found %q in cache, blocked: %t", c.svc, host, blocked)
return blocked, nil
}
question := c.getQuestion(hashesToRequest)
log.Debug("%s: checking %s: %s", c.svc, host, question)
req := (&dns.Msg{}).SetQuestion(question, dns.TypeTXT)
resp, err := c.upstream.Exchange(req)
if err != nil {
return false, fmt.Errorf("getting hashes: %w", err)
}
matched, receivedHashes := c.processAnswer(hashesToRequest, resp, host)
c.storeInCache(hashesToRequest, receivedHashes)
return matched, nil
}
// hostnameToHashes returns hashes that should be checked by the hash prefix
// filter.
func hostnameToHashes(host string) (hashes []hostnameHash) {
// subDomainNum defines how many labels should be hashed to match against a
// hash prefix filter.
const subDomainNum = 4
pubSuf, icann := publicsuffix.PublicSuffix(host)
if !icann {
// Check the full private domain space.
pubSuf = ""
}
nDots := 0
i := strings.LastIndexFunc(host, func(r rune) (ok bool) {
if r == '.' {
nDots++
}
return nDots == subDomainNum
})
if i != -1 {
host = host[i+1:]
}
sub := netutil.Subdomains(host)
for _, s := range sub {
if s == pubSuf {
break
}
sum := sha256.Sum256([]byte(s))
hashes = append(hashes, sum)
}
return hashes
}
// getQuestion combines hexadecimal encoded prefixes of hashed hostnames into
// string.
func (c *Checker) getQuestion(hashes []hostnameHash) (q string) {
b := &strings.Builder{}
for _, hash := range hashes {
// nolint:looppointer // The subsilce is used for safe hex encoding.
stringutil.WriteToBuilder(b, hex.EncodeToString(hash[:prefixLen]), ".")
}
stringutil.WriteToBuilder(b, c.txtSuffix)
return b.String()
}
// processAnswer returns true if DNS response matches the hash, and received
// hashed hostnames from the upstream.
func (c *Checker) processAnswer(
hashesToRequest []hostnameHash,
resp *dns.Msg,
host string,
) (matched bool, receivedHashes []hostnameHash) {
txtCount := 0
for _, a := range resp.Answer {
txt, ok := a.(*dns.TXT)
if !ok {
continue
}
txtCount++
receivedHashes = c.appendHashesFromTXT(receivedHashes, txt, host)
}
log.Debug("%s: received answer for %s with %d TXT count", c.svc, host, txtCount)
matched = findMatch(hashesToRequest, receivedHashes)
if matched {
log.Debug("%s: matched %s", c.svc, host)
return true, receivedHashes
}
return false, receivedHashes
}
// appendHashesFromTXT appends received hashed hostnames.
func (c *Checker) appendHashesFromTXT(
hashes []hostnameHash,
txt *dns.TXT,
host string,
) (receivedHashes []hostnameHash) {
log.Debug("%s: received hashes for %s: %v", c.svc, host, txt.Txt)
for _, t := range txt.Txt {
if len(t) != hexSize {
log.Debug("%s: wrong hex size %d for %s %s", c.svc, len(t), host, t)
continue
}
buf, err := hex.DecodeString(t)
if err != nil {
log.Debug("%s: decoding hex string %s: %s", c.svc, t, err)
continue
}
var hash hostnameHash
copy(hash[:], buf)
hashes = append(hashes, hash)
}
return hashes
}

View File

@@ -0,0 +1,248 @@
package hashprefix
import (
"crypto/sha256"
"encoding/hex"
"strings"
"testing"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/golibs/cache"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/exp/slices"
)
const (
cacheTime = 10 * time.Minute
cacheSize = 10000
)
func TestChcker_getQuestion(t *testing.T) {
const suf = "sb.dns.adguard.com."
// test hostnameToHashes()
hashes := hostnameToHashes("1.2.3.sub.host.com")
assert.Len(t, hashes, 3)
hash := sha256.Sum256([]byte("3.sub.host.com"))
hexPref1 := hex.EncodeToString(hash[:prefixLen])
assert.True(t, slices.Contains(hashes, hash))
hash = sha256.Sum256([]byte("sub.host.com"))
hexPref2 := hex.EncodeToString(hash[:prefixLen])
assert.True(t, slices.Contains(hashes, hash))
hash = sha256.Sum256([]byte("host.com"))
hexPref3 := hex.EncodeToString(hash[:prefixLen])
assert.True(t, slices.Contains(hashes, hash))
hash = sha256.Sum256([]byte("com"))
assert.False(t, slices.Contains(hashes, hash))
c := &Checker{
svc: "SafeBrowsing",
txtSuffix: suf,
}
q := c.getQuestion(hashes)
assert.Contains(t, q, hexPref1)
assert.Contains(t, q, hexPref2)
assert.Contains(t, q, hexPref3)
assert.True(t, strings.HasSuffix(q, suf))
}
func TestHostnameToHashes(t *testing.T) {
testCases := []struct {
name string
host string
wantLen int
}{{
name: "basic",
host: "example.com",
wantLen: 1,
}, {
name: "sub_basic",
host: "www.example.com",
wantLen: 2,
}, {
name: "private_domain",
host: "foo.co.uk",
wantLen: 1,
}, {
name: "sub_private_domain",
host: "bar.foo.co.uk",
wantLen: 2,
}, {
name: "private_domain_v2",
host: "foo.blogspot.co.uk",
wantLen: 4,
}, {
name: "sub_private_domain_v2",
host: "bar.foo.blogspot.co.uk",
wantLen: 4,
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
hashes := hostnameToHashes(tc.host)
assert.Len(t, hashes, tc.wantLen)
})
}
}
func TestChecker_storeInCache(t *testing.T) {
c := &Checker{
svc: "SafeBrowsing",
cacheTime: cacheTime,
}
conf := cache.Config{}
c.cache = cache.New(conf)
// store in cache hashes for "3.sub.host.com" and "host.com"
// and empty data for hash-prefix for "sub.host.com"
hashes := []hostnameHash{}
hash := sha256.Sum256([]byte("sub.host.com"))
hashes = append(hashes, hash)
var hashesArray []hostnameHash
hash4 := sha256.Sum256([]byte("3.sub.host.com"))
hashesArray = append(hashesArray, hash4)
hash2 := sha256.Sum256([]byte("host.com"))
hashesArray = append(hashesArray, hash2)
c.storeInCache(hashes, hashesArray)
// match "3.sub.host.com" or "host.com" from cache
hashes = []hostnameHash{}
hash = sha256.Sum256([]byte("3.sub.host.com"))
hashes = append(hashes, hash)
hash = sha256.Sum256([]byte("sub.host.com"))
hashes = append(hashes, hash)
hash = sha256.Sum256([]byte("host.com"))
hashes = append(hashes, hash)
found, blocked, _ := c.findInCache(hashes)
assert.True(t, found)
assert.True(t, blocked)
// match "sub.host.com" from cache
hashes = []hostnameHash{}
hash = sha256.Sum256([]byte("sub.host.com"))
hashes = append(hashes, hash)
found, blocked, _ = c.findInCache(hashes)
assert.True(t, found)
assert.False(t, blocked)
// Match "sub.host.com" from cache. Another hash for "host.example" is not
// in the cache, so get data for it from the server.
hashes = []hostnameHash{}
hash = sha256.Sum256([]byte("sub.host.com"))
hashes = append(hashes, hash)
hash = sha256.Sum256([]byte("host.example"))
hashes = append(hashes, hash)
found, _, hashesToRequest := c.findInCache(hashes)
assert.False(t, found)
hash = sha256.Sum256([]byte("sub.host.com"))
ok := slices.Contains(hashesToRequest, hash)
assert.False(t, ok)
hash = sha256.Sum256([]byte("host.example"))
ok = slices.Contains(hashesToRequest, hash)
assert.True(t, ok)
c = &Checker{
svc: "SafeBrowsing",
cacheTime: cacheTime,
}
c.cache = cache.New(cache.Config{})
hashes = []hostnameHash{}
hash = sha256.Sum256([]byte("sub.host.com"))
hashes = append(hashes, hash)
c.cache.Set(hash[:prefixLen], make([]byte, expirySize+hashSize))
found, _, _ = c.findInCache(hashes)
assert.False(t, found)
}
func TestChecker_Check(t *testing.T) {
const hostname = "example.org"
testCases := []struct {
name string
wantBlock bool
}{{
name: "sb_no_block",
wantBlock: false,
}, {
name: "sb_block",
wantBlock: true,
}, {
name: "pc_no_block",
wantBlock: false,
}, {
name: "pc_block",
wantBlock: true,
}}
for _, tc := range testCases {
c := New(&Config{
CacheTime: cacheTime,
CacheSize: cacheSize,
})
// Prepare the upstream.
ups := aghtest.NewBlockUpstream(hostname, tc.wantBlock)
var numReq int
onExchange := ups.OnExchange
ups.OnExchange = func(req *dns.Msg) (resp *dns.Msg, err error) {
numReq++
return onExchange(req)
}
c.upstream = ups
t.Run(tc.name, func(t *testing.T) {
// Firstly, check the request blocking.
hits := 0
res := false
res, err := c.Check(hostname)
require.NoError(t, err)
if tc.wantBlock {
assert.True(t, res)
hits++
} else {
require.False(t, res)
}
// Check the cache state, check the response is now cached.
assert.Equal(t, 1, c.cache.Stats().Count)
assert.Equal(t, hits, c.cache.Stats().Hit)
// There was one request to an upstream.
assert.Equal(t, 1, numReq)
// Now make the same request to check the cache was used.
res, err = c.Check(hostname)
require.NoError(t, err)
if tc.wantBlock {
assert.True(t, res)
} else {
require.False(t, res)
}
// Check the cache state, it should've been used.
assert.Equal(t, 1, c.cache.Stats().Count)
assert.Equal(t, hits+1, c.cache.Stats().Hit)
// Check that there were no additional requests.
assert.Equal(t, 1, numReq)
})
}
}

View File

@@ -8,6 +8,7 @@ import (
"net/url"
"os"
"path/filepath"
"sync"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
@@ -458,6 +459,80 @@ func (d *DNSFilter) handleCheckHost(w http.ResponseWriter, r *http.Request) {
_ = aghhttp.WriteJSONResponse(w, r, resp)
}
// setProtectedBool sets the value of a boolean pointer under a lock. l must
// protect the value under ptr.
//
// TODO(e.burkov): Make it generic?
func setProtectedBool(mu *sync.RWMutex, ptr *bool, val bool) {
mu.Lock()
defer mu.Unlock()
*ptr = val
}
// protectedBool gets the value of a boolean pointer under a read lock. l must
// protect the value under ptr.
//
// TODO(e.burkov): Make it generic?
func protectedBool(mu *sync.RWMutex, ptr *bool) (val bool) {
mu.RLock()
defer mu.RUnlock()
return *ptr
}
// handleSafeBrowsingEnable is the handler for the POST
// /control/safebrowsing/enable HTTP API.
func (d *DNSFilter) handleSafeBrowsingEnable(w http.ResponseWriter, r *http.Request) {
setProtectedBool(&d.confLock, &d.Config.SafeBrowsingEnabled, true)
d.Config.ConfigModified()
}
// handleSafeBrowsingDisable is the handler for the POST
// /control/safebrowsing/disable HTTP API.
func (d *DNSFilter) handleSafeBrowsingDisable(w http.ResponseWriter, r *http.Request) {
setProtectedBool(&d.confLock, &d.Config.SafeBrowsingEnabled, false)
d.Config.ConfigModified()
}
// handleSafeBrowsingStatus is the handler for the GET
// /control/safebrowsing/status HTTP API.
func (d *DNSFilter) handleSafeBrowsingStatus(w http.ResponseWriter, r *http.Request) {
resp := &struct {
Enabled bool `json:"enabled"`
}{
Enabled: protectedBool(&d.confLock, &d.Config.SafeBrowsingEnabled),
}
_ = aghhttp.WriteJSONResponse(w, r, resp)
}
// handleParentalEnable is the handler for the POST /control/parental/enable
// HTTP API.
func (d *DNSFilter) handleParentalEnable(w http.ResponseWriter, r *http.Request) {
setProtectedBool(&d.confLock, &d.Config.ParentalEnabled, true)
d.Config.ConfigModified()
}
// handleParentalDisable is the handler for the POST /control/parental/disable
// HTTP API.
func (d *DNSFilter) handleParentalDisable(w http.ResponseWriter, r *http.Request) {
setProtectedBool(&d.confLock, &d.Config.ParentalEnabled, false)
d.Config.ConfigModified()
}
// handleParentalStatus is the handler for the GET /control/parental/status
// HTTP API.
func (d *DNSFilter) handleParentalStatus(w http.ResponseWriter, r *http.Request) {
resp := &struct {
Enabled bool `json:"enabled"`
}{
Enabled: protectedBool(&d.confLock, &d.Config.ParentalEnabled),
}
_ = aghhttp.WriteJSONResponse(w, r, resp)
}
// RegisterFilteringHandlers - register handlers
func (d *DNSFilter) RegisterFilteringHandlers() {
registerHTTP := d.HTTPRegister

View File

@@ -8,6 +8,7 @@ import (
"testing"
"time"
"github.com/AdguardTeam/golibs/testutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -136,3 +137,171 @@ func TestDNSFilter_handleFilteringSetURL(t *testing.T) {
})
}
}
func TestDNSFilter_handleSafeBrowsingStatus(t *testing.T) {
const (
testTimeout = time.Second
statusURL = "/control/safebrowsing/status"
)
confModCh := make(chan struct{})
filtersDir := t.TempDir()
testCases := []struct {
name string
url string
enabled bool
wantStatus assert.BoolAssertionFunc
}{{
name: "enable_off",
url: "/control/safebrowsing/enable",
enabled: false,
wantStatus: assert.True,
}, {
name: "enable_on",
url: "/control/safebrowsing/enable",
enabled: true,
wantStatus: assert.True,
}, {
name: "disable_on",
url: "/control/safebrowsing/disable",
enabled: true,
wantStatus: assert.False,
}, {
name: "disable_off",
url: "/control/safebrowsing/disable",
enabled: false,
wantStatus: assert.False,
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
handlers := make(map[string]http.Handler)
d, err := New(&Config{
ConfigModified: func() {
testutil.RequireSend(testutil.PanicT{}, confModCh, struct{}{}, testTimeout)
},
DataDir: filtersDir,
HTTPRegister: func(_, url string, handler http.HandlerFunc) {
handlers[url] = handler
},
SafeBrowsingEnabled: tc.enabled,
}, nil)
require.NoError(t, err)
t.Cleanup(d.Close)
d.RegisterFilteringHandlers()
require.NotEmpty(t, handlers)
require.Contains(t, handlers, statusURL)
r := httptest.NewRequest(http.MethodPost, tc.url, nil)
w := httptest.NewRecorder()
go handlers[tc.url].ServeHTTP(w, r)
testutil.RequireReceive(t, confModCh, testTimeout)
r = httptest.NewRequest(http.MethodGet, statusURL, nil)
w = httptest.NewRecorder()
handlers[statusURL].ServeHTTP(w, r)
require.Equal(t, http.StatusOK, w.Code)
status := struct {
Enabled bool `json:"enabled"`
}{
Enabled: false,
}
err = json.NewDecoder(w.Body).Decode(&status)
require.NoError(t, err)
tc.wantStatus(t, status.Enabled)
})
}
}
func TestDNSFilter_handleParentalStatus(t *testing.T) {
const (
testTimeout = time.Second
statusURL = "/control/parental/status"
)
confModCh := make(chan struct{})
filtersDir := t.TempDir()
testCases := []struct {
name string
url string
enabled bool
wantStatus assert.BoolAssertionFunc
}{{
name: "enable_off",
url: "/control/parental/enable",
enabled: false,
wantStatus: assert.True,
}, {
name: "enable_on",
url: "/control/parental/enable",
enabled: true,
wantStatus: assert.True,
}, {
name: "disable_on",
url: "/control/parental/disable",
enabled: true,
wantStatus: assert.False,
}, {
name: "disable_off",
url: "/control/parental/disable",
enabled: false,
wantStatus: assert.False,
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
handlers := make(map[string]http.Handler)
d, err := New(&Config{
ConfigModified: func() {
testutil.RequireSend(testutil.PanicT{}, confModCh, struct{}{}, testTimeout)
},
DataDir: filtersDir,
HTTPRegister: func(_, url string, handler http.HandlerFunc) {
handlers[url] = handler
},
ParentalEnabled: tc.enabled,
}, nil)
require.NoError(t, err)
t.Cleanup(d.Close)
d.RegisterFilteringHandlers()
require.NotEmpty(t, handlers)
require.Contains(t, handlers, statusURL)
r := httptest.NewRequest(http.MethodPost, tc.url, nil)
w := httptest.NewRecorder()
go handlers[tc.url].ServeHTTP(w, r)
testutil.RequireReceive(t, confModCh, testTimeout)
r = httptest.NewRequest(http.MethodGet, statusURL, nil)
w = httptest.NewRecorder()
handlers[statusURL].ServeHTTP(w, r)
require.Equal(t, http.StatusOK, w.Code)
status := struct {
Enabled bool `json:"enabled"`
}{
Enabled: false,
}
err = json.NewDecoder(w.Body).Decode(&status)
require.NoError(t, err)
tc.wantStatus(t, status.Enabled)
})
}
}

View File

@@ -1,433 +0,0 @@
package filtering
import (
"bytes"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"fmt"
"net"
"net/http"
"strings"
"sync"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/cache"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/miekg/dns"
"golang.org/x/exp/slices"
"golang.org/x/net/publicsuffix"
)
// Safe browsing and parental control methods.
// TODO(a.garipov): Make configurable.
const (
dnsTimeout = 3 * time.Second
defaultSafebrowsingServer = `https://family.adguard-dns.com/dns-query`
defaultParentalServer = `https://family.adguard-dns.com/dns-query`
sbTXTSuffix = `sb.dns.adguard.com.`
pcTXTSuffix = `pc.dns.adguard.com.`
)
// SetParentalUpstream sets the parental upstream for *DNSFilter.
//
// TODO(e.burkov): Remove this in v1 API to forbid the direct access.
func (d *DNSFilter) SetParentalUpstream(u upstream.Upstream) {
d.parentalUpstream = u
}
// SetSafeBrowsingUpstream sets the safe browsing upstream for *DNSFilter.
//
// TODO(e.burkov): Remove this in v1 API to forbid the direct access.
func (d *DNSFilter) SetSafeBrowsingUpstream(u upstream.Upstream) {
d.safeBrowsingUpstream = u
}
func (d *DNSFilter) initSecurityServices() error {
var err error
d.safeBrowsingServer = defaultSafebrowsingServer
d.parentalServer = defaultParentalServer
opts := &upstream.Options{
Timeout: dnsTimeout,
ServerIPAddrs: []net.IP{
{94, 140, 14, 15},
{94, 140, 15, 16},
net.ParseIP("2a10:50c0::bad1:ff"),
net.ParseIP("2a10:50c0::bad2:ff"),
},
}
parUps, err := upstream.AddressToUpstream(d.parentalServer, opts)
if err != nil {
return fmt.Errorf("converting parental server: %w", err)
}
d.SetParentalUpstream(parUps)
sbUps, err := upstream.AddressToUpstream(d.safeBrowsingServer, opts)
if err != nil {
return fmt.Errorf("converting safe browsing server: %w", err)
}
d.SetSafeBrowsingUpstream(sbUps)
return nil
}
/*
expire byte[4]
hash byte[32]
...
*/
func (c *sbCtx) setCache(prefix, hashes []byte) {
d := make([]byte, 4+len(hashes))
expire := uint(time.Now().Unix()) + c.cacheTime*60
binary.BigEndian.PutUint32(d[:4], uint32(expire))
copy(d[4:], hashes)
c.cache.Set(prefix, d)
log.Debug("%s: stored in cache: %v", c.svc, prefix)
}
// findInHash returns 32-byte hash if it's found in hashToHost.
func (c *sbCtx) findInHash(val []byte) (hash32 [32]byte, found bool) {
for i := 4; i < len(val); i += 32 {
hash := val[i : i+32]
copy(hash32[:], hash[0:32])
_, found = c.hashToHost[hash32]
if found {
return hash32, found
}
}
return [32]byte{}, false
}
func (c *sbCtx) getCached() int {
now := time.Now().Unix()
hashesToRequest := map[[32]byte]string{}
for k, v := range c.hashToHost {
// nolint:looppointer // The subsilce is used for a safe cache lookup.
val := c.cache.Get(k[0:2])
if val == nil || now >= int64(binary.BigEndian.Uint32(val)) {
hashesToRequest[k] = v
continue
}
if hash32, found := c.findInHash(val); found {
log.Debug("%s: found in cache: %s: blocked by %v", c.svc, c.host, hash32)
return 1
}
}
if len(hashesToRequest) == 0 {
log.Debug("%s: found in cache: %s: not blocked", c.svc, c.host)
return -1
}
c.hashToHost = hashesToRequest
return 0
}
type sbCtx struct {
host string
svc string
hashToHost map[[32]byte]string
cache cache.Cache
cacheTime uint
}
func hostnameToHashes(host string) map[[32]byte]string {
hashes := map[[32]byte]string{}
tld, icann := publicsuffix.PublicSuffix(host)
if !icann {
// private suffixes like cloudfront.net
tld = ""
}
curhost := host
nDots := 0
for i := len(curhost) - 1; i >= 0; i-- {
if curhost[i] == '.' {
nDots++
if nDots == 4 {
curhost = curhost[i+1:] // "xxx.a.b.c.d" -> "a.b.c.d"
break
}
}
}
for {
if curhost == "" {
// we've reached end of string
break
}
if tld != "" && curhost == tld {
// we've reached the TLD, don't hash it
break
}
sum := sha256.Sum256([]byte(curhost))
hashes[sum] = curhost
pos := strings.IndexByte(curhost, byte('.'))
if pos < 0 {
break
}
curhost = curhost[pos+1:]
}
return hashes
}
// convert hash array to string
func (c *sbCtx) getQuestion() string {
b := &strings.Builder{}
for hash := range c.hashToHost {
// nolint:looppointer // The subsilce is used for safe hex encoding.
stringutil.WriteToBuilder(b, hex.EncodeToString(hash[0:2]), ".")
}
if c.svc == "SafeBrowsing" {
stringutil.WriteToBuilder(b, sbTXTSuffix)
return b.String()
}
stringutil.WriteToBuilder(b, pcTXTSuffix)
return b.String()
}
// Find the target hash in TXT response
func (c *sbCtx) processTXT(resp *dns.Msg) (bool, [][]byte) {
matched := false
hashes := [][]byte{}
for _, a := range resp.Answer {
txt, ok := a.(*dns.TXT)
if !ok {
continue
}
log.Debug("%s: received hashes for %s: %v", c.svc, c.host, txt.Txt)
for _, t := range txt.Txt {
if len(t) != 32*2 {
continue
}
hash, err := hex.DecodeString(t)
if err != nil {
continue
}
hashes = append(hashes, hash)
if !matched {
var hash32 [32]byte
copy(hash32[:], hash)
var hashHost string
hashHost, ok = c.hashToHost[hash32]
if ok {
log.Debug("%s: matched %s by %s/%s", c.svc, c.host, hashHost, t)
matched = true
}
}
}
}
return matched, hashes
}
func (c *sbCtx) storeCache(hashes [][]byte) {
slices.SortFunc(hashes, func(a, b []byte) (sortsBefore bool) {
return bytes.Compare(a, b) == -1
})
var curData []byte
var prevPrefix []byte
for i, hash := range hashes {
// nolint:looppointer // The subsilce is used for a safe comparison.
if !bytes.Equal(hash[0:2], prevPrefix) {
if i != 0 {
c.setCache(prevPrefix, curData)
curData = nil
}
prevPrefix = hashes[i][0:2]
}
curData = append(curData, hash...)
}
if len(prevPrefix) != 0 {
c.setCache(prevPrefix, curData)
}
for hash := range c.hashToHost {
// nolint:looppointer // The subsilce is used for a safe cache lookup.
prefix := hash[0:2]
val := c.cache.Get(prefix)
if val == nil {
c.setCache(prefix, nil)
}
}
}
func check(c *sbCtx, r Result, u upstream.Upstream) (Result, error) {
c.hashToHost = hostnameToHashes(c.host)
switch c.getCached() {
case -1:
return Result{}, nil
case 1:
return r, nil
}
question := c.getQuestion()
log.Tracef("%s: checking %s: %s", c.svc, c.host, question)
req := (&dns.Msg{}).SetQuestion(question, dns.TypeTXT)
resp, err := u.Exchange(req)
if err != nil {
return Result{}, err
}
matched, receivedHashes := c.processTXT(resp)
c.storeCache(receivedHashes)
if matched {
return r, nil
}
return Result{}, nil
}
// TODO(a.garipov): Unify with checkParental.
func (d *DNSFilter) checkSafeBrowsing(
host string,
_ uint16,
setts *Settings,
) (res Result, err error) {
if !setts.ProtectionEnabled || !setts.SafeBrowsingEnabled {
return Result{}, nil
}
if log.GetLevel() >= log.DEBUG {
timer := log.StartTimer()
defer timer.LogElapsed("safebrowsing lookup for %q", host)
}
sctx := &sbCtx{
host: host,
svc: "SafeBrowsing",
cache: d.safebrowsingCache,
cacheTime: d.Config.CacheTime,
}
res = Result{
Rules: []*ResultRule{{
Text: "adguard-malware-shavar",
FilterListID: SafeBrowsingListID,
}},
Reason: FilteredSafeBrowsing,
IsFiltered: true,
}
return check(sctx, res, d.safeBrowsingUpstream)
}
// TODO(a.garipov): Unify with checkSafeBrowsing.
func (d *DNSFilter) checkParental(
host string,
_ uint16,
setts *Settings,
) (res Result, err error) {
if !setts.ProtectionEnabled || !setts.ParentalEnabled {
return Result{}, nil
}
if log.GetLevel() >= log.DEBUG {
timer := log.StartTimer()
defer timer.LogElapsed("parental lookup for %q", host)
}
sctx := &sbCtx{
host: host,
svc: "Parental",
cache: d.parentalCache,
cacheTime: d.Config.CacheTime,
}
res = Result{
Rules: []*ResultRule{{
Text: "parental CATEGORY_BLACKLISTED",
FilterListID: ParentalListID,
}},
Reason: FilteredParental,
IsFiltered: true,
}
return check(sctx, res, d.parentalUpstream)
}
// setProtectedBool sets the value of a boolean pointer under a lock. l must
// protect the value under ptr.
//
// TODO(e.burkov): Make it generic?
func setProtectedBool(mu *sync.RWMutex, ptr *bool, val bool) {
mu.Lock()
defer mu.Unlock()
*ptr = val
}
// protectedBool gets the value of a boolean pointer under a read lock. l must
// protect the value under ptr.
//
// TODO(e.burkov): Make it generic?
func protectedBool(mu *sync.RWMutex, ptr *bool) (val bool) {
mu.RLock()
defer mu.RUnlock()
return *ptr
}
func (d *DNSFilter) handleSafeBrowsingEnable(w http.ResponseWriter, r *http.Request) {
setProtectedBool(&d.confLock, &d.Config.SafeBrowsingEnabled, true)
d.Config.ConfigModified()
}
func (d *DNSFilter) handleSafeBrowsingDisable(w http.ResponseWriter, r *http.Request) {
setProtectedBool(&d.confLock, &d.Config.SafeBrowsingEnabled, false)
d.Config.ConfigModified()
}
func (d *DNSFilter) handleSafeBrowsingStatus(w http.ResponseWriter, r *http.Request) {
resp := &struct {
Enabled bool `json:"enabled"`
}{
Enabled: protectedBool(&d.confLock, &d.Config.SafeBrowsingEnabled),
}
_ = aghhttp.WriteJSONResponse(w, r, resp)
}
func (d *DNSFilter) handleParentalEnable(w http.ResponseWriter, r *http.Request) {
setProtectedBool(&d.confLock, &d.Config.ParentalEnabled, true)
d.Config.ConfigModified()
}
func (d *DNSFilter) handleParentalDisable(w http.ResponseWriter, r *http.Request) {
setProtectedBool(&d.confLock, &d.Config.ParentalEnabled, false)
d.Config.ConfigModified()
}
func (d *DNSFilter) handleParentalStatus(w http.ResponseWriter, r *http.Request) {
resp := &struct {
Enabled bool `json:"enabled"`
}{
Enabled: protectedBool(&d.confLock, &d.Config.ParentalEnabled),
}
_ = aghhttp.WriteJSONResponse(w, r, resp)
}

View File

@@ -1,226 +0,0 @@
package filtering
import (
"crypto/sha256"
"strings"
"testing"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/golibs/cache"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSafeBrowsingHash(t *testing.T) {
// test hostnameToHashes()
hashes := hostnameToHashes("1.2.3.sub.host.com")
assert.Len(t, hashes, 3)
_, ok := hashes[sha256.Sum256([]byte("3.sub.host.com"))]
assert.True(t, ok)
_, ok = hashes[sha256.Sum256([]byte("sub.host.com"))]
assert.True(t, ok)
_, ok = hashes[sha256.Sum256([]byte("host.com"))]
assert.True(t, ok)
_, ok = hashes[sha256.Sum256([]byte("com"))]
assert.False(t, ok)
c := &sbCtx{
svc: "SafeBrowsing",
hashToHost: hashes,
}
q := c.getQuestion()
assert.Contains(t, q, "7a1b.")
assert.Contains(t, q, "af5a.")
assert.Contains(t, q, "eb11.")
assert.True(t, strings.HasSuffix(q, "sb.dns.adguard.com."))
}
func TestSafeBrowsingCache(t *testing.T) {
c := &sbCtx{
svc: "SafeBrowsing",
cacheTime: 100,
}
conf := cache.Config{}
c.cache = cache.New(conf)
// store in cache hashes for "3.sub.host.com" and "host.com"
// and empty data for hash-prefix for "sub.host.com"
hash := sha256.Sum256([]byte("sub.host.com"))
c.hashToHost = make(map[[32]byte]string)
c.hashToHost[hash] = "sub.host.com"
var hashesArray [][]byte
hash4 := sha256.Sum256([]byte("3.sub.host.com"))
hashesArray = append(hashesArray, hash4[:])
hash2 := sha256.Sum256([]byte("host.com"))
hashesArray = append(hashesArray, hash2[:])
c.storeCache(hashesArray)
// match "3.sub.host.com" or "host.com" from cache
c.hashToHost = make(map[[32]byte]string)
hash = sha256.Sum256([]byte("3.sub.host.com"))
c.hashToHost[hash] = "3.sub.host.com"
hash = sha256.Sum256([]byte("sub.host.com"))
c.hashToHost[hash] = "sub.host.com"
hash = sha256.Sum256([]byte("host.com"))
c.hashToHost[hash] = "host.com"
assert.Equal(t, 1, c.getCached())
// match "sub.host.com" from cache
c.hashToHost = make(map[[32]byte]string)
hash = sha256.Sum256([]byte("sub.host.com"))
c.hashToHost[hash] = "sub.host.com"
assert.Equal(t, -1, c.getCached())
// Match "sub.host.com" from cache. Another hash for "host.example" is not
// in the cache, so get data for it from the server.
c.hashToHost = make(map[[32]byte]string)
hash = sha256.Sum256([]byte("sub.host.com"))
c.hashToHost[hash] = "sub.host.com"
hash = sha256.Sum256([]byte("host.example"))
c.hashToHost[hash] = "host.example"
assert.Empty(t, c.getCached())
hash = sha256.Sum256([]byte("sub.host.com"))
_, ok := c.hashToHost[hash]
assert.False(t, ok)
hash = sha256.Sum256([]byte("host.example"))
_, ok = c.hashToHost[hash]
assert.True(t, ok)
c = &sbCtx{
svc: "SafeBrowsing",
cacheTime: 100,
}
conf = cache.Config{}
c.cache = cache.New(conf)
hash = sha256.Sum256([]byte("sub.host.com"))
c.hashToHost = make(map[[32]byte]string)
c.hashToHost[hash] = "sub.host.com"
c.cache.Set(hash[0:2], make([]byte, 32))
assert.Empty(t, c.getCached())
}
func TestSBPC_checkErrorUpstream(t *testing.T) {
d, _ := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil)
t.Cleanup(d.Close)
ups := aghtest.NewErrorUpstream()
d.SetSafeBrowsingUpstream(ups)
d.SetParentalUpstream(ups)
setts := &Settings{
ProtectionEnabled: true,
SafeBrowsingEnabled: true,
ParentalEnabled: true,
}
_, err := d.checkSafeBrowsing("smthng.com", dns.TypeA, setts)
assert.Error(t, err)
_, err = d.checkParental("smthng.com", dns.TypeA, setts)
assert.Error(t, err)
}
func TestSBPC(t *testing.T) {
d, _ := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil)
t.Cleanup(d.Close)
const hostname = "example.org"
setts := &Settings{
ProtectionEnabled: true,
SafeBrowsingEnabled: true,
ParentalEnabled: true,
}
testCases := []struct {
testCache cache.Cache
testFunc func(host string, _ uint16, _ *Settings) (res Result, err error)
name string
block bool
}{{
testCache: d.safebrowsingCache,
testFunc: d.checkSafeBrowsing,
name: "sb_no_block",
block: false,
}, {
testCache: d.safebrowsingCache,
testFunc: d.checkSafeBrowsing,
name: "sb_block",
block: true,
}, {
testCache: d.parentalCache,
testFunc: d.checkParental,
name: "pc_no_block",
block: false,
}, {
testCache: d.parentalCache,
testFunc: d.checkParental,
name: "pc_block",
block: true,
}}
for _, tc := range testCases {
// Prepare the upstream.
ups := aghtest.NewBlockUpstream(hostname, tc.block)
var numReq int
onExchange := ups.OnExchange
ups.OnExchange = func(req *dns.Msg) (resp *dns.Msg, err error) {
numReq++
return onExchange(req)
}
d.SetSafeBrowsingUpstream(ups)
d.SetParentalUpstream(ups)
t.Run(tc.name, func(t *testing.T) {
// Firstly, check the request blocking.
hits := 0
res, err := tc.testFunc(hostname, dns.TypeA, setts)
require.NoError(t, err)
if tc.block {
assert.True(t, res.IsFiltered)
require.Len(t, res.Rules, 1)
hits++
} else {
require.False(t, res.IsFiltered)
}
// Check the cache state, check the response is now cached.
assert.Equal(t, 1, tc.testCache.Stats().Count)
assert.Equal(t, hits, tc.testCache.Stats().Hit)
// There was one request to an upstream.
assert.Equal(t, 1, numReq)
// Now make the same request to check the cache was used.
res, err = tc.testFunc(hostname, dns.TypeA, setts)
require.NoError(t, err)
if tc.block {
assert.True(t, res.IsFiltered)
require.Len(t, res.Rules, 1)
} else {
require.False(t, res.IsFiltered)
}
// Check the cache state, it should've been used.
assert.Equal(t, 1, tc.testCache.Stats().Count)
assert.Equal(t, hits+1, tc.testCache.Stats().Hit)
// Check that there were no additional requests.
assert.Equal(t, 1, numReq)
})
purgeCaches(d)
}
}

View File

@@ -12,6 +12,14 @@ type blockedService struct {
// blockedServices contains raw blocked service data.
var blockedServices = []blockedService{{
ID: "500px",
Name: "500px",
IconSVG: []byte("<svg xmlns=\"http://www.w3.org/2000/svg\" fill=\"currentColor\" viewBox=\"0 0 50 50\"><path d=\"M 5 14 L 2.5 26 L 6.800781 26 C 6.800781 26 7.699219 24.300781 10.199219 24.300781 C 12.699219 24.300781 14 26.199219 14 28.300781 C 14 30.402344 12.5 32.800781 10.199219 32.800781 C 7.898438 32.800781 6.5 30.398438 6.5 29 L 2 29 C 2 30.199219 3 36 10.199219 36 C 15.15625 36 17.417969 33.121094 18.015625 31.898438 C 19.386719 34.34375 21.992188 36 24.984375 36 C 27.253906 36 29.777344 34.808594 32.5 32.453125 C 35.222656 34.808594 37.746094 36 40.015625 36 C 44.417969 36 48 32.410156 48 28 C 48 23.589844 44.417969 20 40.015625 20 C 37.746094 20 35.222656 21.191406 32.5 23.546875 C 29.777344 21.191406 27.253906 20 24.984375 20 C 21.832031 20 19.105469 21.847656 17.8125 24.511719 C 17.113281 23.382813 15.414063 21 11.902344 21 C 8.101563 21 7.300781 22.597656 7.300781 22.597656 C 7.300781 22.597656 7.699219 21.300781 8.300781 18 L 17 18 L 17 14 Z M 24.984375 25 C 25.453125 25 26.800781 25.226563 29.230469 27.328125 L 30.011719 28 L 29.230469 28.671875 C 26.800781 30.773438 25.453125 31 24.984375 31 C 23.339844 31 22 29.652344 22 28 C 22 26.347656 23.339844 25 24.984375 25 Z M 40.015625 25 C 41.660156 25 43 26.347656 43 28 C 43 29.652344 41.660156 31 40.015625 31 C 39.546875 31 38.199219 30.773438 35.769531 28.671875 L 34.988281 28 L 35.769531 27.328125 C 38.199219 25.226563 39.546875 25 40.015625 25 Z\"/></svg>"),
Rules: []string{
"||500px.com^",
"||500px.org^",
},
}, {
ID: "9gag",
Name: "9GAG",
IconSVG: []byte("<svg xmlns=\"http://www.w3.org/2000/svg\" fill=\"currentColor\" viewBox=\"0 0 50 50\"><path d=\"M 44 14 C 44 13.644531 43.8125 13.316406 43.507813 13.136719 C 40.453125 11.347656 28.46875 4.847656 25.535156 3.136719 C 25.222656 2.957031 24.839844 2.957031 24.527344 3.136719 C 21.128906 5.117188 10.089844 11.621094 7.496094 13.136719 C 7.1875 13.316406 7 13.644531 7 14 L 7 20 C 7 20.378906 7.214844 20.722656 7.550781 20.894531 C 7.660156 20.949219 18.597656 26.453125 24.5 29.867188 C 24.8125 30.046875 25.195313 30.046875 25.507813 29.863281 C 27.269531 28.828125 29.117188 27.859375 30.902344 26.921875 C 32.253906 26.214844 33.636719 25.488281 35.003906 24.722656 C 35.007813 26.820313 35.003906 29.296875 35 30.40625 L 25 35.859375 L 14.480469 30.121094 C 14.144531 29.9375 13.730469 29.964844 13.417969 30.1875 L 6.417969 35.1875 C 6.140625 35.386719 5.980469 35.714844 6.003906 36.054688 C 6.023438 36.398438 6.214844 36.707031 6.515625 36.871094 L 24.542969 46.871094 C 24.695313 46.957031 24.859375 47 25.027344 47 C 25.195313 47 25.363281 46.957031 25.515625 46.875 L 43.484375 36.875 C 43.804688 36.695313 44 36.363281 44 36 C 44 36 43.992188 21.011719 44 14 Z M 25 20 L 18 16 L 25 12 L 32 16 Z\" /></svg>"),
@@ -1180,6 +1188,18 @@ var blockedServices = []blockedService{{
"||zuckerberg.com^",
"||zuckerberg.net^",
},
}, {
ID: "flickr",
Name: "Flickr",
IconSVG: []byte("<svg xmlns=\"http://www.w3.org/2000/svg\" fill=\"currentColor\" viewBox=\"0 0 50 50\"><path d=\"M 9 4 C 6.2504839 4 4 6.2504839 4 9 L 4 41 C 4 43.749516 6.2504839 46 9 46 L 41 46 C 43.749516 46 46 43.749516 46 41 L 46 9 C 46 6.2504839 43.749516 4 41 4 L 9 4 z M 9 6 L 41 6 C 42.668484 6 44 7.3315161 44 9 L 44 41 C 44 42.668484 42.668484 44 41 44 L 9 44 C 7.3315161 44 6 42.668484 6 41 L 6 9 C 6 7.3315161 7.3315161 6 9 6 z M 16 17 C 11.59 17 8 20.59 8 25 C 8 29.41 11.59 33 16 33 C 20.41 33 24 29.41 24 25 C 24 20.59 20.41 17 16 17 z M 34 17 C 29.59 17 26 20.59 26 25 C 26 29.41 29.59 33 34 33 C 38.41 33 42 29.41 42 25 C 42 20.59 38.41 17 34 17 z\"/></svg>"),
Rules: []string{
"||flic.kr^",
"||flickr.com^",
"||flickr.net^",
"||flickrprints.com^",
"||flickrpro.com^",
"||staticflickr.com^",
},
}, {
ID: "gog",
Name: "GOG",
@@ -1325,6 +1345,13 @@ var blockedServices = []blockedService{{
"||kakao.com^",
"||kgslb.com^",
},
}, {
ID: "kik",
Name: "Kik",
IconSVG: []byte("<svg xmlns=\"http://www.w3.org/2000/svg\" fill=\"currentColor\" viewBox=\"0 0 50 50\"><path d=\"M 3.5039062 12 C 1.9347705 11.994817 0.87857579 12.97636 0.4453125 13.849609 C 0.01204921 14.722858 0 15.564453 0 15.564453 A 1.0001 1.0001 0 0 0 0 15.59375 L 0 35 A 1.0001 1.0001 0 0 0 0.00390625 35.078125 C 0.00390625 35.078125 0.05696144 35.828363 0.5390625 36.554688 C 1.0211636 37.281011 2.0459252 38.004441 3.5019531 38.001953 C 4.8916439 38.000053 5.8837351 37.273604 6.3769531 36.578125 C 6.8701712 35.882646 6.9863281 35.166016 6.9863281 35.166016 A 1.0001 1.0001 0 0 0 7 35 L 7 31.802734 L 10.167969 36.554688 L 10.130859 36.494141 C 10.511831 37.164615 11.143097 37.525465 11.742188 37.730469 C 12.341278 37.935473 12.950104 38.001953 13.5 38.001953 C 15.411725 38.001953 17 36.431487 17 34.5 C 17 34.056649 16.90825 34.03442 16.851562 33.912109 C 16.794882 33.789799 16.730864 33.671331 16.654297 33.537109 C 16.501163 33.268666 16.298339 32.944015 16.058594 32.572266 C 15.579103 31.828767 14.950355 30.90254 14.322266 29.992188 C 13.310206 28.525308 12.655222 27.610988 12.300781 27.113281 L 14.707031 24.707031 A 1.0001 1.0001 0 0 0 14.738281 24.673828 C 14.738281 24.673828 15.354706 24.012223 15.748047 23.042969 C 16.141388 22.073714 16.298687 20.56089 15.259766 19.349609 C 14.281705 18.208994 12.842689 18.141009 11.925781 18.416016 C 11.008874 18.691022 10.371094 19.222656 10.371094 19.222656 A 1.0001 1.0001 0 0 0 10.292969 19.292969 L 6.9980469 22.587891 L 6.9921875 15.646484 A 1.0001 1.0001 0 0 0 6.9902344 15.580078 C 6.9902344 15.580078 6.9441634 14.743069 6.5058594 13.875 C 6.0675579 13.006938 5.0412971 12.005313 3.5039062 12 z M 30.503906 12 C 28.93477 11.9948 27.878577 12.97636 27.445312 13.849609 C 27.012049 14.722858 27 15.564453 27 15.564453 A 1.0001 1.0001 0 0 0 27 15.59375 L 27 35 A 1.0001 1.0001 0 0 0 27.003906 35.078125 C 27.003906 35.078125 27.056966 35.828363 27.539062 36.554688 C 28.021165 37.281011 29.045925 38.004441 30.501953 38.001953 C 31.891644 38.000053 32.883735 37.273604 33.376953 36.578125 C 33.870171 35.882646 33.986328 35.166016 33.986328 35.166016 A 1.0001 1.0001 0 0 0 34 35 L 34 31.802734 L 37.167969 36.554688 L 37.130859 36.494141 C 37.511831 37.164615 38.143096 37.525465 38.742188 37.730469 C 39.341277 37.935473 39.950104 38.001953 40.5 38.001953 C 42.411725 38.001953 44 36.431487 44 34.5 C 44 34.056649 43.908251 34.03442 43.851562 33.912109 C 43.794882 33.789799 43.730864 33.671331 43.654297 33.537109 C 43.501163 33.268666 43.298339 32.944015 43.058594 32.572266 C 42.579103 31.828767 41.950355 30.90254 41.322266 29.992188 C 40.310206 28.525308 39.655222 27.610988 39.300781 27.113281 L 41.707031 24.707031 A 1.0001 1.0001 0 0 0 41.738281 24.673828 C 41.738281 24.673828 42.354706 24.012223 42.748047 23.042969 C 43.141388 22.073714 43.298687 20.56089 42.259766 19.349609 C 41.281705 18.208994 39.842689 18.141009 38.925781 18.416016 C 38.008874 18.691022 37.371094 19.222656 37.371094 19.222656 A 1.0001 1.0001 0 0 0 37.292969 19.292969 L 33.998047 22.587891 L 33.992188 15.646484 A 1.0001 1.0001 0 0 0 33.990234 15.580078 C 33.990234 15.580078 33.944164 14.743069 33.505859 13.875 C 33.067647 13.006938 32.041297 12.005313 30.503906 12 z M 21.507812 18 C 19.85324 17.98686 18.785557 19.124468 18.382812 20.09375 C 18.181441 20.578391 18.090615 21.031738 18.044922 21.375 C 18.022072 21.546631 18.011459 21.69063 18.005859 21.796875 C 18.000252 21.90312 18 22.065333 18 21.984375 L 17.982422 34.998047 A 1.0001 1.0001 0 0 0 17.990234 35.134766 C 17.990234 35.134766 18.085674 35.862804 18.576172 36.568359 C 19.06667 37.273915 20.071581 37.997467 21.486328 38 C 22.885358 38.0026 23.885897 37.278643 24.380859 36.580078 C 24.875822 35.881513 24.986328 35.160156 24.986328 35.160156 A 1.0001 1.0001 0 0 0 25 35 L 25 21.996094 C 25 21.996094 25.02572 21.084043 24.625 20.117188 C 24.224283 19.150332 23.164841 18.013078 21.507812 18 z M 46.5 24 C 44.578848 24 43 25.578848 43 27.5 C 43 29.421152 44.578848 31 46.5 31 C 48.421152 31 50 29.421152 50 27.5 C 50 25.578848 48.421152 24 46.5 24 z M 46.5 26 C 47.340272 26 48 26.659728 48 27.5 C 48 28.340272 47.340272 29 46.5 29 C 45.659728 29 45 28.340272 45 27.5 C 45 26.659728 45.659728 26 46.5 26 z\"/></svg>"),
Rules: []string{
"||kik.com^",
},
}, {
ID: "lazada",
Name: "Lazada",
@@ -1385,6 +1412,7 @@ var blockedServices = []blockedService{{
Rules: []string{
"||aus.social^",
"||awscommunity.social^",
"||climatejustice.social^",
"||cyberplace.social^",
"||defcon.social^",
"||det.social^",
@@ -1442,13 +1470,13 @@ var blockedServices = []blockedService{{
"||mstdn.plus^",
"||mstdn.social^",
"||muenchen.social^",
"||muenster.im^",
"||newsie.social^",
"||noc.social^",
"||norden.social^",
"||nrw.social^",
"||o3o.ca^",
"||ohai.social^",
"||pewtix.com^",
"||piaille.fr^",
"||pol.social^",
"||ravenation.club^",
@@ -1480,7 +1508,6 @@ var blockedServices = []blockedService{{
"||union.place^",
"||universeodon.com^",
"||urbanists.social^",
"||wien.rocks^",
"||wxw.moe^",
},
}, {
@@ -1827,6 +1854,13 @@ var blockedServices = []blockedService{{
"||tx.me^",
"||usercontent.dev^",
},
}, {
ID: "tidal",
Name: "Tidal",
IconSVG: []byte("<svg xmlns=\"http://www.w3.org/2000/svg\" fill=\"currentColor\" viewBox=\"0 0 50 50\"><path d=\"M 9 12 C 8.7615 12 8.5237969 12.091437 8.3417969 12.273438 L 1.2734375 19.341797 C 0.9094375 19.705797 0.9094375 20.294203 1.2734375 20.658203 L 8.3417969 27.726562 C 8.7057969 28.090563 9.2942031 28.090563 9.6582031 27.726562 L 16.726562 20.658203 C 16.908563 20.476203 17 20.2385 17 20 C 17 19.7615 16.908563 19.523797 16.726562 19.341797 L 9.6582031 12.273438 C 9.4762031 12.091437 9.2385 12 9 12 z M 17 20 C 17 20.2385 17.091438 20.476203 17.273438 20.658203 L 24.341797 27.726562 C 24.523797 27.908563 24.7615 28 25 28 C 25.2385 28 25.476203 27.908563 25.658203 27.726562 L 32.726562 20.658203 C 32.908563 20.476203 33 20.2385 33 20 C 33 19.7615 32.908563 19.523797 32.726562 19.341797 L 25.658203 12.273438 C 25.294203 11.909437 24.705797 11.909437 24.341797 12.273438 L 17.273438 19.341797 C 17.091437 19.523797 17 19.7615 17 20 z M 33 20 C 33 20.2385 33.091437 20.476203 33.273438 20.658203 L 40.341797 27.726562 C 40.705797 28.090563 41.294203 28.090563 41.658203 27.726562 L 48.726562 20.658203 C 49.090563 20.294203 49.090563 19.705797 48.726562 19.341797 L 41.658203 12.273438 C 41.294203 11.909437 40.705797 11.909437 40.341797 12.273438 L 33.273438 19.341797 C 33.091437 19.523797 33 19.7615 33 20 z M 25 28 C 24.7615 28 24.523797 28.091437 24.341797 28.273438 L 17.273438 35.341797 C 16.909437 35.705797 16.909437 36.294203 17.273438 36.658203 L 24.341797 43.726562 C 24.705797 44.090562 25.294203 44.090562 25.658203 43.726562 L 32.726562 36.658203 C 33.090563 36.294203 33.090563 35.705797 32.726562 35.341797 L 25.658203 28.273438 C 25.476203 28.091437 25.2385 28 25 28 z\"/></svg>"),
Rules: []string{
"||tidal.com^",
},
}, {
ID: "tiktok",
Name: "TikTok",