Pull request: all: less annoying pkg names
Merge in DNS/adguard-home from imp-naming to master Squashed commit of the following: commit d9e75c37af9a738384393735c141a41406d22eeb Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Thu May 13 15:52:14 2021 +0300 all: less annoying pkg names
This commit is contained in:
Ainar Garipov
70
internal/filtering/README.md
Normal file
70
internal/filtering/README.md
Normal file
@@ -0,0 +1,70 @@
|
||||
# AdGuard Home's DNS filtering go library
|
||||
|
||||
Example use:
|
||||
```bash
|
||||
[ -z "$GOPATH" ] && export GOPATH=$HOME/go
|
||||
go get -d github.com/AdguardTeam/AdGuardHome/filtering
|
||||
```
|
||||
|
||||
Create file filter.go
|
||||
```filter.go
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/AdguardTeam/AdGuardHome/filtering"
|
||||
"log"
|
||||
)
|
||||
|
||||
func main() {
|
||||
filter := filtering.New()
|
||||
filter.AddRule("||dou*ck.net^")
|
||||
host := "www.doubleclick.net"
|
||||
res, err := filter.CheckHost(host)
|
||||
if err != nil {
|
||||
// temporary failure
|
||||
log.Fatalf("Failed to check host %q: %s", host, err)
|
||||
}
|
||||
if res.IsFiltered {
|
||||
log.Printf("Host %s is filtered, reason - %q, matched rule: %q", host, res.Reason, res.Rule)
|
||||
} else {
|
||||
log.Printf("Host %s is not filtered, reason - %q", host, res.Reason)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
And then run it:
|
||||
```bash
|
||||
go run filter.go
|
||||
```
|
||||
|
||||
You will get:
|
||||
```
|
||||
2000/01/01 00:00:00 Host www.doubleclick.net is filtered, reason - 'FilteredBlackList', matched rule: '||dou*ck.net^'
|
||||
```
|
||||
|
||||
You can also enable checking against AdGuard's SafeBrowsing:
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/AdguardTeam/AdGuardHome/filtering"
|
||||
"log"
|
||||
)
|
||||
|
||||
func main() {
|
||||
filter := filtering.New()
|
||||
filter.EnableSafeBrowsing()
|
||||
host := "wmconvirus.narod.ru" // hostname for testing safebrowsing
|
||||
res, err := filter.CheckHost(host)
|
||||
if err != nil {
|
||||
// temporary failure
|
||||
log.Fatalf("Failed to check host %q: %s", host, err)
|
||||
}
|
||||
if res.IsFiltered {
|
||||
log.Printf("Host %s is filtered, reason - %q, matched rule: %q", host, res.Reason, res.Rule)
|
||||
} else {
|
||||
log.Printf("Host %s is not filtered, reason - %q", host, res.Reason)
|
||||
}
|
||||
}
|
||||
```
|
||||
313
internal/filtering/blocked.go
Normal file
313
internal/filtering/blocked.go
Normal file
@@ -0,0 +1,313 @@
|
||||
package filtering
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/urlfilter/rules"
|
||||
)
|
||||
|
||||
var serviceRules map[string][]*rules.NetworkRule // service name -> filtering rules
|
||||
|
||||
type svc struct {
|
||||
name string
|
||||
rules []string
|
||||
}
|
||||
|
||||
// Keep in sync with:
|
||||
// client/src/helpers/constants.js
|
||||
// client/src/components/ui/Icons.js
|
||||
var serviceRulesArray = []svc{
|
||||
{"whatsapp", []string{"||whatsapp.net^", "||whatsapp.com^"}},
|
||||
{"facebook", []string{
|
||||
"||facebook.com^",
|
||||
"||facebook.net^",
|
||||
"||fbcdn.net^",
|
||||
"||accountkit.com^",
|
||||
"||fb.me^",
|
||||
"||fb.com^",
|
||||
"||fbsbx.com^",
|
||||
"||messenger.com^",
|
||||
"||facebookcorewwwi.onion^",
|
||||
"||fbcdn.com^",
|
||||
}},
|
||||
{"twitter", []string{"||twitter.com^", "||twttr.com^", "||t.co^", "||twimg.com^"}},
|
||||
{"youtube", []string{
|
||||
"||youtube.com^",
|
||||
"||ytimg.com^",
|
||||
"||youtu.be^",
|
||||
"||googlevideo.com^",
|
||||
"||youtubei.googleapis.com^",
|
||||
"||youtube-nocookie.com^",
|
||||
"||youtube",
|
||||
}},
|
||||
{"twitch", []string{"||twitch.tv^", "||ttvnw.net^", "||jtvnw.net^", "||twitchcdn.net^"}},
|
||||
{"netflix", []string{"||nflxext.com^", "||netflix.com^", "||nflximg.net^", "||nflxvideo.net^", "||nflxso.net^"}},
|
||||
{"instagram", []string{"||instagram.com^", "||cdninstagram.com^"}},
|
||||
{"snapchat", []string{
|
||||
"||snapchat.com^",
|
||||
"||sc-cdn.net^",
|
||||
"||snap-dev.net^",
|
||||
"||snapkit.co",
|
||||
"||snapads.com^",
|
||||
"||impala-media-production.s3.amazonaws.com^",
|
||||
}},
|
||||
{"discord", []string{"||discord.gg^", "||discordapp.net^", "||discordapp.com^", "||discord.com^", "||discord.media^"}},
|
||||
{"ok", []string{"||ok.ru^"}},
|
||||
{"skype", []string{"||skype.com^", "||skypeassets.com^"}},
|
||||
{"vk", []string{"||vk.com^", "||userapi.com^", "||vk-cdn.net^", "||vkuservideo.net^"}},
|
||||
{"origin", []string{"||origin.com^", "||signin.ea.com^", "||accounts.ea.com^"}},
|
||||
{"steam", []string{
|
||||
"||steam.com^",
|
||||
"||steampowered.com^",
|
||||
"||steamcommunity.com^",
|
||||
"||steamstatic.com^",
|
||||
"||steamstore-a.akamaihd.net^",
|
||||
"||steamcdn-a.akamaihd.net^",
|
||||
}},
|
||||
{"epic_games", []string{"||epicgames.com^", "||easyanticheat.net^", "||easy.ac^", "||eac-cdn.com^"}},
|
||||
{"reddit", []string{"||reddit.com^", "||redditstatic.com^", "||redditmedia.com^", "||redd.it^"}},
|
||||
{"mail_ru", []string{"||mail.ru^"}},
|
||||
{"cloudflare", []string{
|
||||
"||cloudflare.com^",
|
||||
"||cloudflare-dns.com^",
|
||||
"||cloudflare.net^",
|
||||
"||cloudflareinsights.com^",
|
||||
"||cloudflarestream.com^",
|
||||
"||cloudflareresolve.com^",
|
||||
"||cloudflareclient.com^",
|
||||
"||cloudflarebolt.com^",
|
||||
"||cloudflarestatus.com^",
|
||||
"||cloudflare.cn^",
|
||||
"||one.one^",
|
||||
"||warp.plus^",
|
||||
"||1.1.1.1^",
|
||||
"||dns4torpnlfs2ifuz2s2yf3fc7rdmsbhm6rw75euj35pac6ap25zgqad.onion^",
|
||||
}},
|
||||
{"amazon", []string{
|
||||
"||amazon.com^",
|
||||
"||media-amazon.com^",
|
||||
"||primevideo.com^",
|
||||
"||amazontrust.com^",
|
||||
"||images-amazon.com^",
|
||||
"||ssl-images-amazon.com^",
|
||||
"||amazonpay.com^",
|
||||
"||amazonpay.in^",
|
||||
"||amazon-adsystem.com^",
|
||||
"||a2z.com^",
|
||||
"||amazon.ae^",
|
||||
"||amazon.ca^",
|
||||
"||amazon.cn^",
|
||||
"||amazon.de^",
|
||||
"||amazon.es^",
|
||||
"||amazon.fr^",
|
||||
"||amazon.in^",
|
||||
"||amazon.it^",
|
||||
"||amazon.nl^",
|
||||
"||amazon.com.au^",
|
||||
"||amazon.com.br^",
|
||||
"||amazon.co.jp^",
|
||||
"||amazon.com.mx^",
|
||||
"||amazon.co.uk^",
|
||||
"||createspace.com^",
|
||||
"||aws",
|
||||
}},
|
||||
{"ebay", []string{
|
||||
"||ebay.com^",
|
||||
"||ebayimg.com^",
|
||||
"||ebaystatic.com^",
|
||||
"||ebaycdn.net^",
|
||||
"||ebayinc.com^",
|
||||
"||ebay.at^",
|
||||
"||ebay.be^",
|
||||
"||ebay.ca^",
|
||||
"||ebay.ch^",
|
||||
"||ebay.cn^",
|
||||
"||ebay.de^",
|
||||
"||ebay.es^",
|
||||
"||ebay.fr^",
|
||||
"||ebay.ie^",
|
||||
"||ebay.in^",
|
||||
"||ebay.it^",
|
||||
"||ebay.ph^",
|
||||
"||ebay.pl^",
|
||||
"||ebay.nl^",
|
||||
"||ebay.com.au^",
|
||||
"||ebay.com.cn^",
|
||||
"||ebay.com.hk^",
|
||||
"||ebay.com.my^",
|
||||
"||ebay.com.sg^",
|
||||
"||ebay.co.uk^",
|
||||
}},
|
||||
{"tiktok", []string{
|
||||
"||tiktok.com^",
|
||||
"||tiktokcdn.com^",
|
||||
"||musical.ly^",
|
||||
"||snssdk.com^",
|
||||
"||amemv.com^",
|
||||
"||toutiao.com^",
|
||||
"||ixigua.com^",
|
||||
"||pstatp.com^",
|
||||
"||ixiguavideo.com^",
|
||||
"||toutiaocloud.com^",
|
||||
"||toutiaocloud.net^",
|
||||
"||bdurl.com^",
|
||||
"||bytecdn.cn^",
|
||||
"||byteimg.com^",
|
||||
"||ixigua.com^",
|
||||
"||muscdn.com^",
|
||||
"||bytedance.map.fastly.net^",
|
||||
"||douyin.com^",
|
||||
"||tiktokv.com^",
|
||||
}},
|
||||
{"vimeo", []string{
|
||||
"||vimeo.com^",
|
||||
"||vimeocdn.com^",
|
||||
"*vod-adaptive.akamaized.net^",
|
||||
}},
|
||||
{"pinterest", []string{
|
||||
"||pinterest.*^",
|
||||
"||pinimg.com^",
|
||||
}},
|
||||
{"imgur", []string{
|
||||
"||imgur.com^",
|
||||
}},
|
||||
{"dailymotion", []string{
|
||||
"||dailymotion.com^",
|
||||
"||dm-event.net^",
|
||||
"||dmcdn.net^",
|
||||
}},
|
||||
{"qq", []string{
|
||||
// block qq.com and subdomains excluding WeChat domains
|
||||
"^(?!weixin|wx)([^.]+\\.)?qq\\.com$",
|
||||
"||qqzaixian.com^",
|
||||
}},
|
||||
{"wechat", []string{
|
||||
"||wechat.com^",
|
||||
"||weixin.qq.com^",
|
||||
"||wx.qq.com^",
|
||||
}},
|
||||
{"viber", []string{
|
||||
"||viber.com^",
|
||||
}},
|
||||
{"weibo", []string{
|
||||
"||weibo.com^",
|
||||
}},
|
||||
{"9gag", []string{
|
||||
"||9cache.com^",
|
||||
"||gag.com^",
|
||||
}},
|
||||
{"telegram", []string{
|
||||
"||t.me^",
|
||||
"||telegram.me^",
|
||||
"||telegram.org^",
|
||||
}},
|
||||
{"disneyplus", []string{
|
||||
"||disney-plus.net^",
|
||||
"||disneyplus.com^",
|
||||
}},
|
||||
{"hulu", []string{
|
||||
"||hulu.com^",
|
||||
}},
|
||||
{"spotify", []string{
|
||||
"/_spotify-connect._tcp.local/",
|
||||
"||spotify.com^",
|
||||
"||scdn.co^",
|
||||
"||spotify.com.edgesuite.net^",
|
||||
"||spotify.map.fastly.net^",
|
||||
"||spotify.map.fastlylb.net^",
|
||||
"||spotifycdn.net^",
|
||||
"||audio-ak-spotify-com.akamaized.net^",
|
||||
"||audio4-ak-spotify-com.akamaized.net^",
|
||||
"||heads-ak-spotify-com.akamaized.net^",
|
||||
"||heads4-ak-spotify-com.akamaized.net^",
|
||||
}},
|
||||
{"tinder", []string{
|
||||
"||gotinder.com^",
|
||||
"||tinder.com^",
|
||||
"||tindersparks.com^",
|
||||
}},
|
||||
}
|
||||
|
||||
// convert array to map
|
||||
func initBlockedServices() {
|
||||
serviceRules = make(map[string][]*rules.NetworkRule)
|
||||
for _, s := range serviceRulesArray {
|
||||
netRules := []*rules.NetworkRule{}
|
||||
for _, text := range s.rules {
|
||||
rule, err := rules.NewNetworkRule(text, 0)
|
||||
if err != nil {
|
||||
log.Error("rules.NewNetworkRule: %s rule: %s", err, text)
|
||||
continue
|
||||
}
|
||||
netRules = append(netRules, rule)
|
||||
}
|
||||
serviceRules[s.name] = netRules
|
||||
}
|
||||
}
|
||||
|
||||
// BlockedSvcKnown - return TRUE if a blocked service name is known
|
||||
func BlockedSvcKnown(s string) bool {
|
||||
_, ok := serviceRules[s]
|
||||
return ok
|
||||
}
|
||||
|
||||
// ApplyBlockedServices - set blocked services settings for this DNS request
|
||||
func (d *DNSFilter) ApplyBlockedServices(setts *Settings, list []string, global bool) {
|
||||
setts.ServicesRules = []ServiceEntry{}
|
||||
if global {
|
||||
d.confLock.RLock()
|
||||
defer d.confLock.RUnlock()
|
||||
list = d.Config.BlockedServices
|
||||
}
|
||||
for _, name := range list {
|
||||
rules, ok := serviceRules[name]
|
||||
|
||||
if !ok {
|
||||
log.Error("unknown service name: %s", name)
|
||||
continue
|
||||
}
|
||||
|
||||
s := ServiceEntry{}
|
||||
s.Name = name
|
||||
s.Rules = rules
|
||||
setts.ServicesRules = append(setts.ServicesRules, s)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DNSFilter) handleBlockedServicesList(w http.ResponseWriter, r *http.Request) {
|
||||
d.confLock.RLock()
|
||||
list := d.Config.BlockedServices
|
||||
d.confLock.RUnlock()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
err := json.NewEncoder(w).Encode(list)
|
||||
if err != nil {
|
||||
httpError(r, w, http.StatusInternalServerError, "json.Encode: %s", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DNSFilter) handleBlockedServicesSet(w http.ResponseWriter, r *http.Request) {
|
||||
list := []string{}
|
||||
err := json.NewDecoder(r.Body).Decode(&list)
|
||||
if err != nil {
|
||||
httpError(r, w, http.StatusBadRequest, "json.Decode: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
d.confLock.Lock()
|
||||
d.Config.BlockedServices = list
|
||||
d.confLock.Unlock()
|
||||
|
||||
log.Debug("Updated blocked services list: %d", len(list))
|
||||
|
||||
d.ConfigModified()
|
||||
}
|
||||
|
||||
// registerBlockedServicesHandlers - register HTTP handlers
|
||||
func (d *DNSFilter) registerBlockedServicesHandlers() {
|
||||
d.Config.HTTPRegister(http.MethodGet, "/control/blocked_services/list", d.handleBlockedServicesList)
|
||||
d.Config.HTTPRegister(http.MethodPost, "/control/blocked_services/set", d.handleBlockedServicesSet)
|
||||
}
|
||||
39
internal/filtering/blocked_test.go
Normal file
39
internal/filtering/blocked_test.go
Normal file
@@ -0,0 +1,39 @@
|
||||
// +build ignore
|
||||
|
||||
//go:build ignore
|
||||
|
||||
package filtering
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// This is a simple tool that takes a list of services and prints them to the output.
|
||||
// It is supposed to be used to update:
|
||||
// client/src/helpers/constants.js
|
||||
// client/src/components/ui/Icons.js
|
||||
//
|
||||
// Usage:
|
||||
// 1. go run ./internal/filtering/blocked_test.go
|
||||
// 2. Use the output to replace `SERVICES` array in "client/src/helpers/constants.js".
|
||||
// 3. You'll need to enter services names manually.
|
||||
// 4. Don't forget to add missing icons to "client/src/components/ui/Icons.js".
|
||||
//
|
||||
// TODO(ameshkov): Rework generator: have a JSON file with all the metadata we need
|
||||
// then use this JSON file to generate JS and Go code
|
||||
func TestGenServicesArray(t *testing.T) {
|
||||
services := make([]svc, len(serviceRulesArray))
|
||||
copy(services, serviceRulesArray)
|
||||
|
||||
sort.Slice(services, func(i, j int) bool {
|
||||
return services[i].name < services[j].name
|
||||
})
|
||||
|
||||
fmt.Println("export const SERVICES = [")
|
||||
for _, s := range services {
|
||||
fmt.Printf(" {\n id: '%s',\n name: '%s',\n },\n", s.name, s.name)
|
||||
}
|
||||
fmt.Println("];")
|
||||
}
|
||||
80
internal/filtering/dnsrewrite.go
Normal file
80
internal/filtering/dnsrewrite.go
Normal file
@@ -0,0 +1,80 @@
|
||||
package filtering
|
||||
|
||||
import (
|
||||
"github.com/AdguardTeam/urlfilter/rules"
|
||||
"github.com/miekg/dns"
|
||||
)
|
||||
|
||||
// DNSRewriteResult is the result of application of $dnsrewrite rules.
|
||||
type DNSRewriteResult struct {
|
||||
Response DNSRewriteResultResponse `json:",omitempty"`
|
||||
RCode rules.RCode `json:",omitempty"`
|
||||
}
|
||||
|
||||
// DNSRewriteResultResponse is the collection of DNS response records
|
||||
// the server returns.
|
||||
type DNSRewriteResultResponse map[rules.RRType][]rules.RRValue
|
||||
|
||||
// processDNSRewrites processes DNS rewrite rules in dnsr. It returns
|
||||
// an empty result if dnsr is empty. Otherwise, the result will have
|
||||
// either CanonName or DNSRewriteResult set.
|
||||
func (d *DNSFilter) processDNSRewrites(dnsr []*rules.NetworkRule) (res Result) {
|
||||
if len(dnsr) == 0 {
|
||||
return Result{}
|
||||
}
|
||||
|
||||
var rules []*ResultRule
|
||||
dnsrr := &DNSRewriteResult{
|
||||
Response: DNSRewriteResultResponse{},
|
||||
}
|
||||
|
||||
for _, nr := range dnsr {
|
||||
dr := nr.DNSRewrite
|
||||
if dr.NewCNAME != "" {
|
||||
// NewCNAME rules have a higher priority than
|
||||
// the other rules.
|
||||
rules = []*ResultRule{{
|
||||
FilterListID: int64(nr.GetFilterListID()),
|
||||
Text: nr.RuleText,
|
||||
}}
|
||||
|
||||
return Result{
|
||||
Reason: RewrittenRule,
|
||||
Rules: rules,
|
||||
CanonName: dr.NewCNAME,
|
||||
}
|
||||
}
|
||||
|
||||
switch dr.RCode {
|
||||
case dns.RcodeSuccess:
|
||||
dnsrr.RCode = dr.RCode
|
||||
dnsrr.Response[dr.RRType] = append(dnsrr.Response[dr.RRType], dr.Value)
|
||||
rules = append(rules, &ResultRule{
|
||||
FilterListID: int64(nr.GetFilterListID()),
|
||||
Text: nr.RuleText,
|
||||
})
|
||||
default:
|
||||
// RcodeRefused and other such codes have higher
|
||||
// priority. Return immediately.
|
||||
rules = []*ResultRule{{
|
||||
FilterListID: int64(nr.GetFilterListID()),
|
||||
Text: nr.RuleText,
|
||||
}}
|
||||
dnsrr = &DNSRewriteResult{
|
||||
RCode: dr.RCode,
|
||||
}
|
||||
|
||||
return Result{
|
||||
Reason: RewrittenRule,
|
||||
Rules: rules,
|
||||
DNSRewriteResult: dnsrr,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Result{
|
||||
Reason: RewrittenRule,
|
||||
Rules: rules,
|
||||
DNSRewriteResult: dnsrr,
|
||||
}
|
||||
}
|
||||
155
internal/filtering/dnsrewrite_test.go
Normal file
155
internal/filtering/dnsrewrite_test.go
Normal file
@@ -0,0 +1,155 @@
|
||||
package filtering
|
||||
|
||||
import (
|
||||
"net"
|
||||
"path"
|
||||
"testing"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDNSFilter_CheckHostRules_dnsrewrite(t *testing.T) {
|
||||
const text = `
|
||||
|cname^$dnsrewrite=new-cname
|
||||
|
||||
|a-record^$dnsrewrite=127.0.0.1
|
||||
|
||||
|aaaa-record^$dnsrewrite=::1
|
||||
|
||||
|txt-record^$dnsrewrite=NOERROR;TXT;hello-world
|
||||
|
||||
|refused^$dnsrewrite=REFUSED
|
||||
|
||||
|a-records^$dnsrewrite=127.0.0.1
|
||||
|a-records^$dnsrewrite=127.0.0.2
|
||||
|
||||
|aaaa-records^$dnsrewrite=::1
|
||||
|aaaa-records^$dnsrewrite=::2
|
||||
|
||||
|disable-one^$dnsrewrite=127.0.0.1
|
||||
|disable-one^$dnsrewrite=127.0.0.2
|
||||
@@||disable-one^$dnsrewrite=127.0.0.1
|
||||
|
||||
|disable-cname^$dnsrewrite=127.0.0.1
|
||||
|disable-cname^$dnsrewrite=new-cname
|
||||
@@||disable-cname^$dnsrewrite=new-cname
|
||||
|
||||
|disable-cname-many^$dnsrewrite=127.0.0.1
|
||||
|disable-cname-many^$dnsrewrite=new-cname-1
|
||||
|disable-cname-many^$dnsrewrite=new-cname-2
|
||||
@@||disable-cname-many^$dnsrewrite=new-cname-1
|
||||
|
||||
|disable-all^$dnsrewrite=127.0.0.1
|
||||
|disable-all^$dnsrewrite=127.0.0.2
|
||||
@@||disable-all^$dnsrewrite
|
||||
`
|
||||
|
||||
f := newForTest(nil, []Filter{{ID: 0, Data: []byte(text)}})
|
||||
setts := &Settings{
|
||||
FilteringEnabled: true,
|
||||
}
|
||||
|
||||
ipv4p1 := net.IPv4(127, 0, 0, 1)
|
||||
ipv4p2 := net.IPv4(127, 0, 0, 2)
|
||||
ipv6p1 := net.IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}
|
||||
ipv6p2 := net.IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}
|
||||
|
||||
testCasesA := []struct {
|
||||
name string
|
||||
dtyp uint16
|
||||
rcode int
|
||||
want []interface{}
|
||||
}{{
|
||||
name: "a-record",
|
||||
dtyp: dns.TypeA,
|
||||
rcode: dns.RcodeSuccess,
|
||||
want: []interface{}{ipv4p1},
|
||||
}, {
|
||||
name: "aaaa-record",
|
||||
dtyp: dns.TypeAAAA,
|
||||
rcode: dns.RcodeSuccess,
|
||||
want: []interface{}{ipv6p1},
|
||||
}, {
|
||||
name: "txt-record",
|
||||
dtyp: dns.TypeTXT,
|
||||
rcode: dns.RcodeSuccess,
|
||||
want: []interface{}{"hello-world"},
|
||||
}, {
|
||||
name: "refused",
|
||||
rcode: dns.RcodeRefused,
|
||||
}, {
|
||||
name: "a-records",
|
||||
dtyp: dns.TypeA,
|
||||
rcode: dns.RcodeSuccess,
|
||||
want: []interface{}{ipv4p1, ipv4p2},
|
||||
}, {
|
||||
name: "aaaa-records",
|
||||
dtyp: dns.TypeAAAA,
|
||||
rcode: dns.RcodeSuccess,
|
||||
want: []interface{}{ipv6p1, ipv6p2},
|
||||
}, {
|
||||
name: "disable-one",
|
||||
dtyp: dns.TypeA,
|
||||
rcode: dns.RcodeSuccess,
|
||||
want: []interface{}{ipv4p2},
|
||||
}, {
|
||||
name: "disable-cname",
|
||||
dtyp: dns.TypeA,
|
||||
rcode: dns.RcodeSuccess,
|
||||
want: []interface{}{ipv4p1},
|
||||
}}
|
||||
|
||||
for _, tc := range testCasesA {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
host := path.Base(tc.name)
|
||||
|
||||
res, err := f.CheckHostRules(host, tc.dtyp, setts)
|
||||
require.Nil(t, err)
|
||||
|
||||
dnsrr := res.DNSRewriteResult
|
||||
require.NotNil(t, dnsrr)
|
||||
assert.Equal(t, tc.rcode, dnsrr.RCode)
|
||||
|
||||
if tc.rcode == dns.RcodeRefused {
|
||||
return
|
||||
}
|
||||
|
||||
ipVals := dnsrr.Response[tc.dtyp]
|
||||
require.Len(t, ipVals, len(tc.want))
|
||||
for i, val := range tc.want {
|
||||
require.Equal(t, val, ipVals[i])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("cname", func(t *testing.T) {
|
||||
dtyp := dns.TypeA
|
||||
host := path.Base(t.Name())
|
||||
|
||||
res, err := f.CheckHostRules(host, dtyp, setts)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, "new-cname", res.CanonName)
|
||||
})
|
||||
|
||||
t.Run("disable-cname-many", func(t *testing.T) {
|
||||
dtyp := dns.TypeA
|
||||
host := path.Base(t.Name())
|
||||
|
||||
res, err := f.CheckHostRules(host, dtyp, setts)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(t, "new-cname-2", res.CanonName)
|
||||
assert.Nil(t, res.DNSRewriteResult)
|
||||
})
|
||||
|
||||
t.Run("disable-all", func(t *testing.T) {
|
||||
dtyp := dns.TypeA
|
||||
host := path.Base(t.Name())
|
||||
|
||||
res, err := f.CheckHostRules(host, dtyp, setts)
|
||||
require.Nil(t, err)
|
||||
assert.Empty(t, res.CanonName)
|
||||
assert.Empty(t, res.Rules)
|
||||
})
|
||||
}
|
||||
897
internal/filtering/filtering.go
Normal file
897
internal/filtering/filtering.go
Normal file
@@ -0,0 +1,897 @@
|
||||
// Package filtering implements a DNS request and response filter.
|
||||
package filtering
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghstrings"
|
||||
"github.com/AdguardTeam/dnsproxy/upstream"
|
||||
"github.com/AdguardTeam/golibs/cache"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/urlfilter"
|
||||
"github.com/AdguardTeam/urlfilter/filterlist"
|
||||
"github.com/AdguardTeam/urlfilter/rules"
|
||||
"github.com/miekg/dns"
|
||||
)
|
||||
|
||||
// ServiceEntry - blocked service array element
|
||||
type ServiceEntry struct {
|
||||
Name string
|
||||
Rules []*rules.NetworkRule
|
||||
}
|
||||
|
||||
// Settings are custom filtering settings for a client.
|
||||
type Settings struct {
|
||||
ClientName string
|
||||
ClientIP net.IP
|
||||
ClientTags []string
|
||||
|
||||
ServicesRules []ServiceEntry
|
||||
|
||||
FilteringEnabled bool
|
||||
SafeSearchEnabled bool
|
||||
SafeBrowsingEnabled bool
|
||||
ParentalEnabled bool
|
||||
}
|
||||
|
||||
// Resolver is the interface for net.Resolver to simplify testing.
|
||||
type Resolver interface {
|
||||
LookupIP(ctx context.Context, network, host string) (ips []net.IP, err error)
|
||||
}
|
||||
|
||||
// Config allows you to configure DNS filtering with New() or just change variables directly.
|
||||
type Config struct {
|
||||
ParentalEnabled bool `yaml:"parental_enabled"`
|
||||
SafeSearchEnabled bool `yaml:"safesearch_enabled"`
|
||||
SafeBrowsingEnabled bool `yaml:"safebrowsing_enabled"`
|
||||
|
||||
SafeBrowsingCacheSize uint `yaml:"safebrowsing_cache_size"` // (in bytes)
|
||||
SafeSearchCacheSize uint `yaml:"safesearch_cache_size"` // (in bytes)
|
||||
ParentalCacheSize uint `yaml:"parental_cache_size"` // (in bytes)
|
||||
CacheTime uint `yaml:"cache_time"` // Element's TTL (in minutes)
|
||||
|
||||
Rewrites []RewriteEntry `yaml:"rewrites"`
|
||||
|
||||
// Names of services to block (globally).
|
||||
// Per-client settings can override this configuration.
|
||||
BlockedServices []string `yaml:"blocked_services"`
|
||||
|
||||
// EtcHosts is a container of IP-hostname pairs taken from the operating
|
||||
// system configuration files (e.g. /etc/hosts).
|
||||
EtcHosts *aghnet.EtcHostsContainer `yaml:"-"`
|
||||
|
||||
// Called when the configuration is changed by HTTP request
|
||||
ConfigModified func() `yaml:"-"`
|
||||
|
||||
// Register an HTTP handler
|
||||
HTTPRegister func(string, string, func(http.ResponseWriter, *http.Request)) `yaml:"-"`
|
||||
|
||||
// CustomResolver is the resolver used by DNSFilter.
|
||||
CustomResolver Resolver `yaml:"-"`
|
||||
}
|
||||
|
||||
// LookupStats store stats collected during safebrowsing or parental checks
|
||||
type LookupStats struct {
|
||||
Requests uint64 // number of HTTP requests that were sent
|
||||
CacheHits uint64 // number of lookups that didn't need HTTP requests
|
||||
Pending int64 // number of currently pending HTTP requests
|
||||
PendingMax int64 // maximum number of pending HTTP requests
|
||||
}
|
||||
|
||||
// Stats store LookupStats for safebrowsing, parental and safesearch
|
||||
type Stats struct {
|
||||
Safebrowsing LookupStats
|
||||
Parental LookupStats
|
||||
Safesearch LookupStats
|
||||
}
|
||||
|
||||
// Parameters to pass to filters-initializer goroutine
|
||||
type filtersInitializerParams struct {
|
||||
allowFilters []Filter
|
||||
blockFilters []Filter
|
||||
}
|
||||
|
||||
type hostChecker struct {
|
||||
check func(host string, qtype uint16, setts *Settings) (res Result, err error)
|
||||
name string
|
||||
}
|
||||
|
||||
// DNSFilter matches hostnames and DNS requests against filtering rules.
|
||||
type DNSFilter struct {
|
||||
rulesStorage *filterlist.RuleStorage
|
||||
filteringEngine *urlfilter.DNSEngine
|
||||
rulesStorageAllow *filterlist.RuleStorage
|
||||
filteringEngineAllow *urlfilter.DNSEngine
|
||||
engineLock sync.RWMutex
|
||||
|
||||
parentalServer string // access via methods
|
||||
safeBrowsingServer string // access via methods
|
||||
parentalUpstream upstream.Upstream
|
||||
safeBrowsingUpstream upstream.Upstream
|
||||
|
||||
Config // for direct access by library users, even a = assignment
|
||||
confLock sync.RWMutex
|
||||
|
||||
// Channel for passing data to filters-initializer goroutine
|
||||
filtersInitializerChan chan filtersInitializerParams
|
||||
filtersInitializerLock sync.Mutex
|
||||
|
||||
// resolver only looks up the IP address of the host while safe search.
|
||||
//
|
||||
// TODO(e.burkov): Use upstream that configured in dnsforward instead.
|
||||
resolver Resolver
|
||||
|
||||
hostCheckers []hostChecker
|
||||
}
|
||||
|
||||
// Filter represents a filter list
|
||||
type Filter struct {
|
||||
ID int64 // auto-assigned when filter is added (see nextFilterID)
|
||||
Data []byte `yaml:"-"` // List of rules divided by '\n'
|
||||
FilePath string `yaml:"-"` // Path to a filtering rules file
|
||||
}
|
||||
|
||||
// Reason holds an enum detailing why it was filtered or not filtered
|
||||
type Reason int
|
||||
|
||||
const (
|
||||
// reasons for not filtering
|
||||
|
||||
// NotFilteredNotFound - host was not find in any checks, default value for result
|
||||
NotFilteredNotFound Reason = iota
|
||||
// NotFilteredAllowList - the host is explicitly allowed
|
||||
NotFilteredAllowList
|
||||
// NotFilteredError is returned when there was an error during
|
||||
// checking. Reserved, currently unused.
|
||||
NotFilteredError
|
||||
|
||||
// reasons for filtering
|
||||
|
||||
// FilteredBlockList - the host was matched to be advertising host
|
||||
FilteredBlockList
|
||||
// FilteredSafeBrowsing - the host was matched to be malicious/phishing
|
||||
FilteredSafeBrowsing
|
||||
// FilteredParental - the host was matched to be outside of parental control settings
|
||||
FilteredParental
|
||||
// FilteredInvalid - the request was invalid and was not processed
|
||||
FilteredInvalid
|
||||
// FilteredSafeSearch - the host was replaced with safesearch variant
|
||||
FilteredSafeSearch
|
||||
// FilteredBlockedService - the host is blocked by "blocked services" settings
|
||||
FilteredBlockedService
|
||||
|
||||
// Rewritten is returned when there was a rewrite by a legacy DNS
|
||||
// rewrite rule.
|
||||
Rewritten
|
||||
|
||||
// RewrittenAutoHosts is returned when there was a rewrite by autohosts
|
||||
// rules (/etc/hosts and so on).
|
||||
RewrittenAutoHosts
|
||||
|
||||
// RewrittenRule is returned when a $dnsrewrite filter rule was applied.
|
||||
//
|
||||
// TODO(a.garipov): Remove Rewritten and RewrittenAutoHosts by merging
|
||||
// their functionality into RewrittenRule.
|
||||
//
|
||||
// See https://github.com/AdguardTeam/AdGuardHome/issues/2499.
|
||||
RewrittenRule
|
||||
)
|
||||
|
||||
// TODO(a.garipov): Resync with actual code names or replace completely
|
||||
// in HTTP API v1.
|
||||
var reasonNames = []string{
|
||||
NotFilteredNotFound: "NotFilteredNotFound",
|
||||
NotFilteredAllowList: "NotFilteredWhiteList",
|
||||
NotFilteredError: "NotFilteredError",
|
||||
|
||||
FilteredBlockList: "FilteredBlackList",
|
||||
FilteredSafeBrowsing: "FilteredSafeBrowsing",
|
||||
FilteredParental: "FilteredParental",
|
||||
FilteredInvalid: "FilteredInvalid",
|
||||
FilteredSafeSearch: "FilteredSafeSearch",
|
||||
FilteredBlockedService: "FilteredBlockedService",
|
||||
|
||||
Rewritten: "Rewrite",
|
||||
RewrittenAutoHosts: "RewriteEtcHosts",
|
||||
RewrittenRule: "RewriteRule",
|
||||
}
|
||||
|
||||
func (r Reason) String() string {
|
||||
if r < 0 || int(r) >= len(reasonNames) {
|
||||
return ""
|
||||
}
|
||||
|
||||
return reasonNames[r]
|
||||
}
|
||||
|
||||
// In returns true if reasons include r.
|
||||
func (r Reason) In(reasons ...Reason) bool {
|
||||
for _, reason := range reasons {
|
||||
if r == reason {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// GetConfig - get configuration
|
||||
func (d *DNSFilter) GetConfig() Settings {
|
||||
c := Settings{}
|
||||
// d.confLock.RLock()
|
||||
c.SafeSearchEnabled = d.Config.SafeSearchEnabled
|
||||
c.SafeBrowsingEnabled = d.Config.SafeBrowsingEnabled
|
||||
c.ParentalEnabled = d.Config.ParentalEnabled
|
||||
// d.confLock.RUnlock()
|
||||
return c
|
||||
}
|
||||
|
||||
// WriteDiskConfig - write configuration
|
||||
func (d *DNSFilter) WriteDiskConfig(c *Config) {
|
||||
d.confLock.Lock()
|
||||
*c = d.Config
|
||||
c.Rewrites = rewriteArrayDup(d.Config.Rewrites)
|
||||
// BlockedServices
|
||||
d.confLock.Unlock()
|
||||
}
|
||||
|
||||
// SetFilters - set new filters (synchronously or asynchronously)
|
||||
// When filters are set asynchronously, the old filters continue working until the new filters are ready.
|
||||
// In this case the caller must ensure that the old filter files are intact.
|
||||
func (d *DNSFilter) SetFilters(blockFilters, allowFilters []Filter, async bool) error {
|
||||
if async {
|
||||
params := filtersInitializerParams{
|
||||
allowFilters: allowFilters,
|
||||
blockFilters: blockFilters,
|
||||
}
|
||||
|
||||
d.filtersInitializerLock.Lock() // prevent multiple writers from adding more than 1 task
|
||||
// remove all pending tasks
|
||||
stop := false
|
||||
for !stop {
|
||||
select {
|
||||
case <-d.filtersInitializerChan:
|
||||
//
|
||||
default:
|
||||
stop = true
|
||||
}
|
||||
}
|
||||
|
||||
d.filtersInitializerChan <- params
|
||||
d.filtersInitializerLock.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
err := d.initFiltering(allowFilters, blockFilters)
|
||||
if err != nil {
|
||||
log.Error("Can't initialize filtering subsystem: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Starts initializing new filters by signal from channel
|
||||
func (d *DNSFilter) filtersInitializer() {
|
||||
for {
|
||||
params := <-d.filtersInitializerChan
|
||||
err := d.initFiltering(params.allowFilters, params.blockFilters)
|
||||
if err != nil {
|
||||
log.Error("Can't initialize filtering subsystem: %s", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Close - close the object
|
||||
func (d *DNSFilter) Close() {
|
||||
d.engineLock.Lock()
|
||||
defer d.engineLock.Unlock()
|
||||
d.reset()
|
||||
}
|
||||
|
||||
func (d *DNSFilter) reset() {
|
||||
var err error
|
||||
|
||||
if d.rulesStorage != nil {
|
||||
err = d.rulesStorage.Close()
|
||||
if err != nil {
|
||||
log.Error("filtering: rulesStorage.Close: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
if d.rulesStorageAllow != nil {
|
||||
err = d.rulesStorageAllow.Close()
|
||||
if err != nil {
|
||||
log.Error("filtering: rulesStorageAllow.Close: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type dnsFilterContext struct {
|
||||
safebrowsingCache cache.Cache
|
||||
parentalCache cache.Cache
|
||||
safeSearchCache cache.Cache
|
||||
}
|
||||
|
||||
var gctx dnsFilterContext
|
||||
|
||||
// ResultRule contains information about applied rules.
|
||||
type ResultRule struct {
|
||||
// FilterListID is the ID of the rule's filter list.
|
||||
FilterListID int64 `json:",omitempty"`
|
||||
// Text is the text of the rule.
|
||||
Text string `json:",omitempty"`
|
||||
// IP is the host IP. It is nil unless the rule uses the
|
||||
// /etc/hosts syntax or the reason is FilteredSafeSearch.
|
||||
IP net.IP `json:",omitempty"`
|
||||
}
|
||||
|
||||
// Result contains the result of a request check.
|
||||
//
|
||||
// All fields transitively have omitempty tags so that the query log
|
||||
// doesn't become too large.
|
||||
//
|
||||
// TODO(a.garipov): Clarify relationships between fields. Perhaps
|
||||
// replace with a sum type or an interface?
|
||||
type Result struct {
|
||||
// IsFiltered is true if the request is filtered.
|
||||
IsFiltered bool `json:",omitempty"`
|
||||
|
||||
// Reason is the reason for blocking or unblocking the request.
|
||||
Reason Reason `json:",omitempty"`
|
||||
|
||||
// Rules are applied rules. If Rules are not empty, each rule
|
||||
// is not nil.
|
||||
Rules []*ResultRule `json:",omitempty"`
|
||||
|
||||
// ReverseHosts is the reverse lookup rewrite result. It is
|
||||
// empty unless Reason is set to RewrittenAutoHosts.
|
||||
ReverseHosts []string `json:",omitempty"`
|
||||
|
||||
// IPList is the lookup rewrite result. It is empty unless
|
||||
// Reason is set to RewrittenAutoHosts or Rewritten.
|
||||
IPList []net.IP `json:",omitempty"`
|
||||
|
||||
// CanonName is the CNAME value from the lookup rewrite result.
|
||||
// It is empty unless Reason is set to Rewritten or RewrittenRule.
|
||||
CanonName string `json:",omitempty"`
|
||||
|
||||
// ServiceName is the name of the blocked service. It is empty
|
||||
// unless Reason is set to FilteredBlockedService.
|
||||
ServiceName string `json:",omitempty"`
|
||||
|
||||
// DNSRewriteResult is the $dnsrewrite filter rule result.
|
||||
DNSRewriteResult *DNSRewriteResult `json:",omitempty"`
|
||||
}
|
||||
|
||||
// Matched returns true if any match at all was found regardless of
|
||||
// whether it was filtered or not.
|
||||
func (r Reason) Matched() bool {
|
||||
return r != NotFilteredNotFound
|
||||
}
|
||||
|
||||
// CheckHostRules tries to match the host against filtering rules only.
|
||||
func (d *DNSFilter) CheckHostRules(host string, qtype uint16, setts *Settings) (Result, error) {
|
||||
if !setts.FilteringEnabled {
|
||||
return Result{}, nil
|
||||
}
|
||||
|
||||
return d.matchHost(host, qtype, setts)
|
||||
}
|
||||
|
||||
// CheckHost tries to match the host against filtering rules, then safebrowsing
|
||||
// and parental control rules, if they are enabled.
|
||||
func (d *DNSFilter) CheckHost(
|
||||
host string,
|
||||
qtype uint16,
|
||||
setts *Settings,
|
||||
) (res Result, err error) {
|
||||
// Sometimes clients try to resolve ".", which is a request to get root
|
||||
// servers.
|
||||
if host == "" {
|
||||
return Result{Reason: NotFilteredNotFound}, nil
|
||||
}
|
||||
|
||||
host = strings.ToLower(host)
|
||||
|
||||
res = d.processRewrites(host, qtype)
|
||||
if res.Reason == Rewritten {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
for _, hc := range d.hostCheckers {
|
||||
res, err = hc.check(host, qtype, setts)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("%s: %w", hc.name, err)
|
||||
}
|
||||
|
||||
if res.Reason.Matched() {
|
||||
return res, nil
|
||||
}
|
||||
}
|
||||
|
||||
return Result{}, nil
|
||||
}
|
||||
|
||||
// checkEtcHosts compares the host against our /etc/hosts table. The err is
|
||||
// always nil, it is only there to make this a valid hostChecker function.
|
||||
func (d *DNSFilter) checkEtcHosts(
|
||||
host string,
|
||||
qtype uint16,
|
||||
_ *Settings,
|
||||
) (res Result, err error) {
|
||||
if d.Config.EtcHosts == nil {
|
||||
return Result{}, nil
|
||||
}
|
||||
|
||||
ips := d.Config.EtcHosts.Process(host, qtype)
|
||||
if ips != nil {
|
||||
res = Result{
|
||||
Reason: RewrittenAutoHosts,
|
||||
IPList: ips,
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
revHosts := d.Config.EtcHosts.ProcessReverse(host, qtype)
|
||||
if len(revHosts) != 0 {
|
||||
res = Result{
|
||||
Reason: RewrittenAutoHosts,
|
||||
}
|
||||
|
||||
// TODO(a.garipov): Optimize this with a buffer.
|
||||
res.ReverseHosts = make([]string, len(revHosts))
|
||||
for i := range revHosts {
|
||||
res.ReverseHosts[i] = revHosts[i] + "."
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
return Result{}, nil
|
||||
}
|
||||
|
||||
// Process rewrites table
|
||||
// . Find CNAME for a domain name (exact match or by wildcard)
|
||||
// . if found and CNAME equals to domain name - this is an exception; exit
|
||||
// . if found, set domain name to canonical name
|
||||
// . repeat for the new domain name (Note: we return only the last CNAME)
|
||||
// . Find A or AAAA record for a domain name (exact match or by wildcard)
|
||||
// . if found, set IP addresses (IPv4 or IPv6 depending on qtype) in Result.IPList array
|
||||
func (d *DNSFilter) processRewrites(host string, qtype uint16) (res Result) {
|
||||
d.confLock.RLock()
|
||||
defer d.confLock.RUnlock()
|
||||
|
||||
rr := findRewrites(d.Rewrites, host)
|
||||
if len(rr) != 0 {
|
||||
res.Reason = Rewritten
|
||||
}
|
||||
|
||||
cnames := aghstrings.NewSet()
|
||||
origHost := host
|
||||
for len(rr) != 0 && rr[0].Type == dns.TypeCNAME {
|
||||
log.Debug("rewrite: CNAME for %s is %s", host, rr[0].Answer)
|
||||
|
||||
if host == rr[0].Answer { // "host == CNAME" is an exception
|
||||
res.Reason = NotFilteredNotFound
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
host = rr[0].Answer
|
||||
if cnames.Has(host) {
|
||||
log.Info("rewrite: breaking CNAME redirection loop: %s. Question: %s", host, origHost)
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
cnames.Add(host)
|
||||
res.CanonName = rr[0].Answer
|
||||
rr = findRewrites(d.Rewrites, host)
|
||||
}
|
||||
|
||||
for _, r := range rr {
|
||||
if (r.Type == dns.TypeA && qtype == dns.TypeA) ||
|
||||
(r.Type == dns.TypeAAAA && qtype == dns.TypeAAAA) {
|
||||
|
||||
if r.IP == nil { // IP exception
|
||||
res.Reason = 0
|
||||
return res
|
||||
}
|
||||
|
||||
res.IPList = append(res.IPList, r.IP)
|
||||
log.Debug("rewrite: A/AAAA for %s is %s", host, r.IP)
|
||||
}
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
// matchBlockedServicesRules checks the host against the blocked services rules
|
||||
// in settings, if any. The err is always nil, it is only there to make this
|
||||
// a valid hostChecker function.
|
||||
func matchBlockedServicesRules(
|
||||
host string,
|
||||
_ uint16,
|
||||
setts *Settings,
|
||||
) (res Result, err error) {
|
||||
svcs := setts.ServicesRules
|
||||
if len(svcs) == 0 {
|
||||
return Result{}, nil
|
||||
}
|
||||
|
||||
req := rules.NewRequestForHostname(host)
|
||||
for _, s := range svcs {
|
||||
for _, rule := range s.Rules {
|
||||
if rule.Match(req) {
|
||||
res.Reason = FilteredBlockedService
|
||||
res.IsFiltered = true
|
||||
res.ServiceName = s.Name
|
||||
|
||||
ruleText := rule.Text()
|
||||
res.Rules = []*ResultRule{{
|
||||
FilterListID: int64(rule.GetFilterListID()),
|
||||
Text: ruleText,
|
||||
}}
|
||||
|
||||
log.Debug("blocked services: matched rule: %s host: %s service: %s",
|
||||
ruleText, host, s.Name)
|
||||
|
||||
return res, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
//
|
||||
// Adding rule and matching against the rules
|
||||
//
|
||||
|
||||
// fileExists returns true if file exists.
|
||||
func fileExists(fn string) bool {
|
||||
_, err := os.Stat(fn)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func createFilteringEngine(filters []Filter) (*filterlist.RuleStorage, *urlfilter.DNSEngine, error) {
|
||||
listArray := []filterlist.RuleList{}
|
||||
for _, f := range filters {
|
||||
var list filterlist.RuleList
|
||||
|
||||
if f.ID == 0 {
|
||||
list = &filterlist.StringRuleList{
|
||||
ID: 0,
|
||||
RulesText: string(f.Data),
|
||||
IgnoreCosmetic: true,
|
||||
}
|
||||
} else if !fileExists(f.FilePath) {
|
||||
list = &filterlist.StringRuleList{
|
||||
ID: int(f.ID),
|
||||
IgnoreCosmetic: true,
|
||||
}
|
||||
} else if runtime.GOOS == "windows" {
|
||||
// On Windows we don't pass a file to urlfilter because
|
||||
// it's difficult to update this file while it's being
|
||||
// used.
|
||||
data, err := os.ReadFile(f.FilePath)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("reading filter content: %w", err)
|
||||
}
|
||||
|
||||
list = &filterlist.StringRuleList{
|
||||
ID: int(f.ID),
|
||||
RulesText: string(data),
|
||||
IgnoreCosmetic: true,
|
||||
}
|
||||
} else {
|
||||
var err error
|
||||
list, err = filterlist.NewFileRuleList(int(f.ID), f.FilePath, true)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("filterlist.NewFileRuleList(): %s: %w", f.FilePath, err)
|
||||
}
|
||||
}
|
||||
listArray = append(listArray, list)
|
||||
}
|
||||
|
||||
rulesStorage, err := filterlist.NewRuleStorage(listArray)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("filterlist.NewRuleStorage(): %w", err)
|
||||
}
|
||||
filteringEngine := urlfilter.NewDNSEngine(rulesStorage)
|
||||
return rulesStorage, filteringEngine, nil
|
||||
}
|
||||
|
||||
// Initialize urlfilter objects.
|
||||
func (d *DNSFilter) initFiltering(allowFilters, blockFilters []Filter) error {
|
||||
rulesStorage, filteringEngine, err := createFilteringEngine(blockFilters)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rulesStorageAllow, filteringEngineAllow, err := createFilteringEngine(allowFilters)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
d.engineLock.Lock()
|
||||
d.reset()
|
||||
d.rulesStorage = rulesStorage
|
||||
d.filteringEngine = filteringEngine
|
||||
d.rulesStorageAllow = rulesStorageAllow
|
||||
d.filteringEngineAllow = filteringEngineAllow
|
||||
d.engineLock.Unlock()
|
||||
|
||||
// Make sure that the OS reclaims memory as soon as possible
|
||||
debug.FreeOSMemory()
|
||||
log.Debug("initialized filtering engine")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// matchHostProcessAllowList processes the allowlist logic of host
|
||||
// matching.
|
||||
func (d *DNSFilter) matchHostProcessAllowList(host string, dnsres urlfilter.DNSResult) (res Result, err error) {
|
||||
var rule rules.Rule
|
||||
if dnsres.NetworkRule != nil {
|
||||
rule = dnsres.NetworkRule
|
||||
} else if len(dnsres.HostRulesV4) > 0 {
|
||||
rule = dnsres.HostRulesV4[0]
|
||||
} else if len(dnsres.HostRulesV6) > 0 {
|
||||
rule = dnsres.HostRulesV6[0]
|
||||
}
|
||||
|
||||
if rule == nil {
|
||||
return Result{}, fmt.Errorf("invalid dns result: rules are empty")
|
||||
}
|
||||
|
||||
log.Debug("Filtering: found allowlist rule for host %q: %q list_id: %d",
|
||||
host, rule.Text(), rule.GetFilterListID())
|
||||
|
||||
return makeResult(rule, NotFilteredAllowList), nil
|
||||
}
|
||||
|
||||
// matchHostProcessDNSResult processes the matched DNS filtering result.
|
||||
func (d *DNSFilter) matchHostProcessDNSResult(
|
||||
qtype uint16,
|
||||
dnsres urlfilter.DNSResult,
|
||||
) (res Result) {
|
||||
if dnsres.NetworkRule != nil {
|
||||
reason := FilteredBlockList
|
||||
if dnsres.NetworkRule.Whitelist {
|
||||
reason = NotFilteredAllowList
|
||||
}
|
||||
|
||||
return makeResult(dnsres.NetworkRule, reason)
|
||||
}
|
||||
|
||||
if qtype == dns.TypeA && dnsres.HostRulesV4 != nil {
|
||||
rule := dnsres.HostRulesV4[0]
|
||||
res = makeResult(rule, FilteredBlockList)
|
||||
res.Rules[0].IP = rule.IP.To4()
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
if qtype == dns.TypeAAAA && dnsres.HostRulesV6 != nil {
|
||||
rule := dnsres.HostRulesV6[0]
|
||||
res = makeResult(rule, FilteredBlockList)
|
||||
res.Rules[0].IP = rule.IP.To16()
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
if dnsres.HostRulesV4 != nil || dnsres.HostRulesV6 != nil {
|
||||
// Question type doesn't match the host rules. Return the first
|
||||
// matched host rule, but without an IP address.
|
||||
var rule rules.Rule
|
||||
if dnsres.HostRulesV4 != nil {
|
||||
rule = dnsres.HostRulesV4[0]
|
||||
} else if dnsres.HostRulesV6 != nil {
|
||||
rule = dnsres.HostRulesV6[0]
|
||||
}
|
||||
|
||||
res = makeResult(rule, FilteredBlockList)
|
||||
res.Rules[0].IP = net.IP{}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
return Result{}
|
||||
}
|
||||
|
||||
// matchHost is a low-level way to check only if hostname is filtered by rules,
|
||||
// skipping expensive safebrowsing and parental lookups.
|
||||
func (d *DNSFilter) matchHost(
|
||||
host string,
|
||||
qtype uint16,
|
||||
setts *Settings,
|
||||
) (res Result, err error) {
|
||||
if !setts.FilteringEnabled {
|
||||
return Result{}, nil
|
||||
}
|
||||
|
||||
d.engineLock.RLock()
|
||||
// Keep in mind that this lock must be held no just when calling Match()
|
||||
// but also while using the rules returned by it.
|
||||
defer d.engineLock.RUnlock()
|
||||
|
||||
ureq := urlfilter.DNSRequest{
|
||||
Hostname: host,
|
||||
SortedClientTags: setts.ClientTags,
|
||||
// TODO(e.burkov): Wait for urlfilter update to pass net.IP.
|
||||
ClientIP: setts.ClientIP.String(),
|
||||
ClientName: setts.ClientName,
|
||||
DNSType: qtype,
|
||||
}
|
||||
|
||||
if d.filteringEngineAllow != nil {
|
||||
dnsres, ok := d.filteringEngineAllow.MatchRequest(ureq)
|
||||
if ok {
|
||||
return d.matchHostProcessAllowList(host, dnsres)
|
||||
}
|
||||
}
|
||||
|
||||
if d.filteringEngine == nil {
|
||||
return Result{}, nil
|
||||
}
|
||||
|
||||
dnsres, ok := d.filteringEngine.MatchRequest(ureq)
|
||||
|
||||
// Check DNS rewrites first, because the API there is a bit awkward.
|
||||
if dnsr := dnsres.DNSRewrites(); len(dnsr) > 0 {
|
||||
res = d.processDNSRewrites(dnsr)
|
||||
if res.Reason == RewrittenRule && res.CanonName == host {
|
||||
// A rewrite of a host to itself. Go on and try
|
||||
// matching other things.
|
||||
} else {
|
||||
return res, nil
|
||||
}
|
||||
} else if !ok {
|
||||
return Result{}, nil
|
||||
}
|
||||
|
||||
res = d.matchHostProcessDNSResult(qtype, dnsres)
|
||||
if len(res.Rules) > 0 {
|
||||
r := res.Rules[0]
|
||||
log.Debug(
|
||||
"filtering: found rule %q for host %q, filter list id: %d",
|
||||
r.Text,
|
||||
host,
|
||||
r.FilterListID,
|
||||
)
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// makeResult returns a properly constructed Result.
|
||||
func makeResult(rule rules.Rule, reason Reason) Result {
|
||||
res := Result{
|
||||
Reason: reason,
|
||||
Rules: []*ResultRule{{
|
||||
FilterListID: int64(rule.GetFilterListID()),
|
||||
Text: rule.Text(),
|
||||
}},
|
||||
}
|
||||
|
||||
if reason == FilteredBlockList {
|
||||
res.IsFiltered = true
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
// InitModule manually initializes blocked services map.
|
||||
func InitModule() {
|
||||
initBlockedServices()
|
||||
}
|
||||
|
||||
// New creates properly initialized DNS Filter that is ready to be used.
|
||||
func New(c *Config, blockFilters []Filter) *DNSFilter {
|
||||
var resolver Resolver = net.DefaultResolver
|
||||
if c != nil {
|
||||
cacheConf := cache.Config{
|
||||
EnableLRU: true,
|
||||
}
|
||||
|
||||
if gctx.safebrowsingCache == nil {
|
||||
cacheConf.MaxSize = c.SafeBrowsingCacheSize
|
||||
gctx.safebrowsingCache = cache.New(cacheConf)
|
||||
}
|
||||
|
||||
if gctx.safeSearchCache == nil {
|
||||
cacheConf.MaxSize = c.SafeSearchCacheSize
|
||||
gctx.safeSearchCache = cache.New(cacheConf)
|
||||
}
|
||||
|
||||
if gctx.parentalCache == nil {
|
||||
cacheConf.MaxSize = c.ParentalCacheSize
|
||||
gctx.parentalCache = cache.New(cacheConf)
|
||||
}
|
||||
|
||||
if c.CustomResolver != nil {
|
||||
resolver = c.CustomResolver
|
||||
}
|
||||
}
|
||||
|
||||
d := &DNSFilter{
|
||||
resolver: resolver,
|
||||
}
|
||||
|
||||
d.hostCheckers = []hostChecker{{
|
||||
check: d.checkEtcHosts,
|
||||
name: "etchosts",
|
||||
}, {
|
||||
check: d.matchHost,
|
||||
name: "filtering",
|
||||
}, {
|
||||
check: matchBlockedServicesRules,
|
||||
name: "blocked services",
|
||||
}, {
|
||||
check: d.checkSafeBrowsing,
|
||||
name: "safe browsing",
|
||||
}, {
|
||||
check: d.checkParental,
|
||||
name: "parental",
|
||||
}, {
|
||||
check: d.checkSafeSearch,
|
||||
name: "safe search",
|
||||
}}
|
||||
|
||||
err := d.initSecurityServices()
|
||||
if err != nil {
|
||||
log.Error("filtering: initialize services: %s", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
if c != nil {
|
||||
d.Config = *c
|
||||
d.prepareRewrites()
|
||||
}
|
||||
|
||||
bsvcs := []string{}
|
||||
for _, s := range d.BlockedServices {
|
||||
if !BlockedSvcKnown(s) {
|
||||
log.Debug("skipping unknown blocked-service %q", s)
|
||||
continue
|
||||
}
|
||||
bsvcs = append(bsvcs, s)
|
||||
}
|
||||
d.BlockedServices = bsvcs
|
||||
|
||||
if blockFilters != nil {
|
||||
err = d.initFiltering(nil, blockFilters)
|
||||
if err != nil {
|
||||
log.Error("Can't initialize filtering subsystem: %s", err)
|
||||
d.Close()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return d
|
||||
}
|
||||
|
||||
// Start - start the module:
|
||||
// . start async filtering initializer goroutine
|
||||
// . register web handlers
|
||||
func (d *DNSFilter) Start() {
|
||||
d.filtersInitializerChan = make(chan filtersInitializerParams, 1)
|
||||
go d.filtersInitializer()
|
||||
|
||||
if d.Config.HTTPRegister != nil { // for tests
|
||||
d.registerSecurityHandlers()
|
||||
d.registerRewritesHandlers()
|
||||
d.registerBlockedServicesHandlers()
|
||||
}
|
||||
}
|
||||
841
internal/filtering/filtering_test.go
Normal file
841
internal/filtering/filtering_test.go
Normal file
@@ -0,0 +1,841 @@
|
||||
package filtering
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
|
||||
"github.com/AdguardTeam/golibs/cache"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/urlfilter/rules"
|
||||
"github.com/miekg/dns"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
aghtest.DiscardLogOutput(m)
|
||||
}
|
||||
|
||||
var setts Settings
|
||||
|
||||
// Helpers.
|
||||
|
||||
func purgeCaches() {
|
||||
for _, c := range []cache.Cache{
|
||||
gctx.safebrowsingCache,
|
||||
gctx.parentalCache,
|
||||
gctx.safeSearchCache,
|
||||
} {
|
||||
if c != nil {
|
||||
c.Clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newForTest(c *Config, filters []Filter) *DNSFilter {
|
||||
setts = Settings{
|
||||
FilteringEnabled: true,
|
||||
}
|
||||
setts.FilteringEnabled = true
|
||||
if c != nil {
|
||||
c.SafeBrowsingCacheSize = 10000
|
||||
c.ParentalCacheSize = 10000
|
||||
c.SafeSearchCacheSize = 1000
|
||||
c.CacheTime = 30
|
||||
setts.SafeSearchEnabled = c.SafeSearchEnabled
|
||||
setts.SafeBrowsingEnabled = c.SafeBrowsingEnabled
|
||||
setts.ParentalEnabled = c.ParentalEnabled
|
||||
}
|
||||
d := New(c, filters)
|
||||
purgeCaches()
|
||||
return d
|
||||
}
|
||||
|
||||
func (d *DNSFilter) checkMatch(t *testing.T, hostname string) {
|
||||
t.Helper()
|
||||
|
||||
res, err := d.CheckHost(hostname, dns.TypeA, &setts)
|
||||
require.Nilf(t, err, "Error while matching host %s: %s", hostname, err)
|
||||
assert.Truef(t, res.IsFiltered, "Expected hostname %s to match", hostname)
|
||||
}
|
||||
|
||||
func (d *DNSFilter) checkMatchIP(t *testing.T, hostname, ip string, qtype uint16) {
|
||||
t.Helper()
|
||||
|
||||
res, err := d.CheckHost(hostname, qtype, &setts)
|
||||
require.Nilf(t, err, "Error while matching host %s: %s", hostname, err)
|
||||
assert.Truef(t, res.IsFiltered, "Expected hostname %s to match", hostname)
|
||||
|
||||
require.NotEmpty(t, res.Rules, "Expected result to have rules")
|
||||
r := res.Rules[0]
|
||||
require.NotNilf(t, r.IP, "Expected ip %s to match, actual: %v", ip, r.IP)
|
||||
assert.Equalf(t, ip, r.IP.String(), "Expected ip %s to match, actual: %v", ip, r.IP)
|
||||
}
|
||||
|
||||
func (d *DNSFilter) checkMatchEmpty(t *testing.T, hostname string) {
|
||||
t.Helper()
|
||||
|
||||
res, err := d.CheckHost(hostname, dns.TypeA, &setts)
|
||||
require.Nilf(t, err, "Error while matching host %s: %s", hostname, err)
|
||||
assert.Falsef(t, res.IsFiltered, "Expected hostname %s to not match", hostname)
|
||||
}
|
||||
|
||||
func TestEtcHostsMatching(t *testing.T) {
|
||||
addr := "216.239.38.120"
|
||||
addr6 := "::1"
|
||||
text := fmt.Sprintf(` %s google.com www.google.com # enforce google's safesearch
|
||||
%s ipv6.com
|
||||
0.0.0.0 block.com
|
||||
0.0.0.1 host2
|
||||
0.0.0.2 host2
|
||||
::1 host2
|
||||
`,
|
||||
addr, addr6)
|
||||
filters := []Filter{{
|
||||
ID: 0, Data: []byte(text),
|
||||
}}
|
||||
d := newForTest(nil, filters)
|
||||
t.Cleanup(d.Close)
|
||||
|
||||
d.checkMatchIP(t, "google.com", addr, dns.TypeA)
|
||||
d.checkMatchIP(t, "www.google.com", addr, dns.TypeA)
|
||||
d.checkMatchEmpty(t, "subdomain.google.com")
|
||||
d.checkMatchEmpty(t, "example.org")
|
||||
|
||||
// IPv4 match.
|
||||
d.checkMatchIP(t, "block.com", "0.0.0.0", dns.TypeA)
|
||||
|
||||
// Empty IPv6.
|
||||
res, err := d.CheckHost("block.com", dns.TypeAAAA, &setts)
|
||||
require.Nil(t, err)
|
||||
assert.True(t, res.IsFiltered)
|
||||
|
||||
require.Len(t, res.Rules, 1)
|
||||
assert.Equal(t, "0.0.0.0 block.com", res.Rules[0].Text)
|
||||
assert.Empty(t, res.Rules[0].IP)
|
||||
|
||||
// IPv6 match.
|
||||
d.checkMatchIP(t, "ipv6.com", addr6, dns.TypeAAAA)
|
||||
|
||||
// Empty IPv4.
|
||||
res, err = d.CheckHost("ipv6.com", dns.TypeA, &setts)
|
||||
require.Nil(t, err)
|
||||
assert.True(t, res.IsFiltered)
|
||||
|
||||
require.Len(t, res.Rules, 1)
|
||||
assert.Equal(t, "::1 ipv6.com", res.Rules[0].Text)
|
||||
assert.Empty(t, res.Rules[0].IP)
|
||||
|
||||
// Two IPv4, the first one returned.
|
||||
res, err = d.CheckHost("host2", dns.TypeA, &setts)
|
||||
require.Nil(t, err)
|
||||
assert.True(t, res.IsFiltered)
|
||||
|
||||
require.Len(t, res.Rules, 1)
|
||||
assert.Equal(t, res.Rules[0].IP, net.IP{0, 0, 0, 1})
|
||||
|
||||
// One IPv6 address.
|
||||
res, err = d.CheckHost("host2", dns.TypeAAAA, &setts)
|
||||
require.Nil(t, err)
|
||||
assert.True(t, res.IsFiltered)
|
||||
|
||||
require.Len(t, res.Rules, 1)
|
||||
assert.Equal(t, res.Rules[0].IP, net.IPv6loopback)
|
||||
}
|
||||
|
||||
// Safe Browsing.
|
||||
|
||||
func TestSafeBrowsing(t *testing.T) {
|
||||
logOutput := &bytes.Buffer{}
|
||||
aghtest.ReplaceLogWriter(t, logOutput)
|
||||
aghtest.ReplaceLogLevel(t, log.DEBUG)
|
||||
|
||||
d := newForTest(&Config{SafeBrowsingEnabled: true}, nil)
|
||||
t.Cleanup(d.Close)
|
||||
const matching = "wmconvirus.narod.ru"
|
||||
d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{
|
||||
Hostname: matching,
|
||||
Block: true,
|
||||
})
|
||||
d.checkMatch(t, matching)
|
||||
|
||||
require.Contains(t, logOutput.String(), "SafeBrowsing lookup for "+matching)
|
||||
|
||||
d.checkMatch(t, "test."+matching)
|
||||
d.checkMatchEmpty(t, "yandex.ru")
|
||||
d.checkMatchEmpty(t, "pornhub.com")
|
||||
|
||||
// Cached result.
|
||||
d.safeBrowsingServer = "127.0.0.1"
|
||||
d.checkMatch(t, matching)
|
||||
d.checkMatchEmpty(t, "pornhub.com")
|
||||
d.safeBrowsingServer = defaultSafebrowsingServer
|
||||
}
|
||||
|
||||
func TestParallelSB(t *testing.T) {
|
||||
d := newForTest(&Config{SafeBrowsingEnabled: true}, nil)
|
||||
t.Cleanup(d.Close)
|
||||
const matching = "wmconvirus.narod.ru"
|
||||
d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{
|
||||
Hostname: matching,
|
||||
Block: true,
|
||||
})
|
||||
|
||||
t.Run("group", func(t *testing.T) {
|
||||
for i := 0; i < 100; i++ {
|
||||
t.Run(fmt.Sprintf("aaa%d", i), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
d.checkMatch(t, matching)
|
||||
d.checkMatch(t, "test."+matching)
|
||||
d.checkMatchEmpty(t, "yandex.ru")
|
||||
d.checkMatchEmpty(t, "pornhub.com")
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Safe Search.
|
||||
|
||||
func TestSafeSearch(t *testing.T) {
|
||||
d := newForTest(&Config{SafeSearchEnabled: true}, nil)
|
||||
t.Cleanup(d.Close)
|
||||
val, ok := d.SafeSearchDomain("www.google.com")
|
||||
require.True(t, ok, "Expected safesearch to find result for www.google.com")
|
||||
assert.Equal(t, "forcesafesearch.google.com", val, "Expected safesearch for google.com to be forcesafesearch.google.com")
|
||||
}
|
||||
|
||||
func TestCheckHostSafeSearchYandex(t *testing.T) {
|
||||
d := newForTest(&Config{SafeSearchEnabled: true}, nil)
|
||||
t.Cleanup(d.Close)
|
||||
|
||||
yandexIP := net.IPv4(213, 180, 193, 56)
|
||||
|
||||
// Check host for each domain.
|
||||
for _, host := range []string{
|
||||
"yAndeX.ru",
|
||||
"YANdex.COM",
|
||||
"yandex.ua",
|
||||
"yandex.by",
|
||||
"yandex.kz",
|
||||
"www.yandex.com",
|
||||
} {
|
||||
t.Run(strings.ToLower(host), func(t *testing.T) {
|
||||
res, err := d.CheckHost(host, dns.TypeA, &setts)
|
||||
require.Nil(t, err)
|
||||
assert.True(t, res.IsFiltered)
|
||||
|
||||
require.Len(t, res.Rules, 1)
|
||||
assert.Equal(t, yandexIP, res.Rules[0].IP)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckHostSafeSearchGoogle(t *testing.T) {
|
||||
resolver := &aghtest.TestResolver{}
|
||||
d := newForTest(&Config{
|
||||
SafeSearchEnabled: true,
|
||||
CustomResolver: resolver,
|
||||
}, nil)
|
||||
t.Cleanup(d.Close)
|
||||
|
||||
ip, _ := resolver.HostToIPs("forcesafesearch.google.com")
|
||||
|
||||
// Check host for each domain.
|
||||
for _, host := range []string{
|
||||
"www.google.com",
|
||||
"www.google.im",
|
||||
"www.google.co.in",
|
||||
"www.google.iq",
|
||||
"www.google.is",
|
||||
"www.google.it",
|
||||
"www.google.je",
|
||||
} {
|
||||
t.Run(host, func(t *testing.T) {
|
||||
res, err := d.CheckHost(host, dns.TypeA, &setts)
|
||||
require.Nil(t, err)
|
||||
assert.True(t, res.IsFiltered)
|
||||
require.Len(t, res.Rules, 1)
|
||||
assert.Equal(t, ip, res.Rules[0].IP)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeSearchCacheYandex(t *testing.T) {
|
||||
d := newForTest(nil, nil)
|
||||
t.Cleanup(d.Close)
|
||||
const domain = "yandex.ru"
|
||||
|
||||
// Check host with disabled safesearch.
|
||||
res, err := d.CheckHost(domain, dns.TypeA, &setts)
|
||||
require.Nil(t, err)
|
||||
assert.False(t, res.IsFiltered)
|
||||
require.Empty(t, res.Rules)
|
||||
|
||||
yandexIP := net.IPv4(213, 180, 193, 56)
|
||||
|
||||
d = newForTest(&Config{SafeSearchEnabled: true}, nil)
|
||||
t.Cleanup(d.Close)
|
||||
|
||||
res, err = d.CheckHost(domain, dns.TypeA, &setts)
|
||||
require.Nilf(t, err, "CheckHost for safesearh domain %s failed cause %s", domain, err)
|
||||
|
||||
// For yandex we already know valid IP.
|
||||
require.Len(t, res.Rules, 1)
|
||||
assert.Equal(t, res.Rules[0].IP, yandexIP)
|
||||
|
||||
// Check cache.
|
||||
cachedValue, isFound := getCachedResult(gctx.safeSearchCache, domain)
|
||||
require.True(t, isFound)
|
||||
require.Len(t, cachedValue.Rules, 1)
|
||||
assert.Equal(t, cachedValue.Rules[0].IP, yandexIP)
|
||||
}
|
||||
|
||||
func TestSafeSearchCacheGoogle(t *testing.T) {
|
||||
resolver := &aghtest.TestResolver{}
|
||||
d := newForTest(&Config{
|
||||
CustomResolver: resolver,
|
||||
}, nil)
|
||||
t.Cleanup(d.Close)
|
||||
|
||||
const domain = "www.google.ru"
|
||||
res, err := d.CheckHost(domain, dns.TypeA, &setts)
|
||||
require.Nil(t, err)
|
||||
assert.False(t, res.IsFiltered)
|
||||
require.Empty(t, res.Rules)
|
||||
|
||||
d = newForTest(&Config{SafeSearchEnabled: true}, nil)
|
||||
t.Cleanup(d.Close)
|
||||
d.resolver = resolver
|
||||
|
||||
// Lookup for safesearch domain.
|
||||
safeDomain, ok := d.SafeSearchDomain(domain)
|
||||
require.Truef(t, ok, "Failed to get safesearch domain for %s", domain)
|
||||
|
||||
ips, err := resolver.LookupIP(context.Background(), "ip", safeDomain)
|
||||
require.Nilf(t, err, "Failed to lookup for %s", safeDomain)
|
||||
|
||||
var ip net.IP
|
||||
for _, foundIP := range ips {
|
||||
if foundIP.To4() != nil {
|
||||
ip = foundIP
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
res, err = d.CheckHost(domain, dns.TypeA, &setts)
|
||||
require.Nil(t, err)
|
||||
require.Len(t, res.Rules, 1)
|
||||
assert.True(t, res.Rules[0].IP.Equal(ip))
|
||||
|
||||
// Check cache.
|
||||
cachedValue, isFound := getCachedResult(gctx.safeSearchCache, domain)
|
||||
require.True(t, isFound)
|
||||
require.Len(t, cachedValue.Rules, 1)
|
||||
assert.True(t, cachedValue.Rules[0].IP.Equal(ip))
|
||||
}
|
||||
|
||||
// Parental.
|
||||
|
||||
func TestParentalControl(t *testing.T) {
|
||||
logOutput := &bytes.Buffer{}
|
||||
aghtest.ReplaceLogWriter(t, logOutput)
|
||||
aghtest.ReplaceLogLevel(t, log.DEBUG)
|
||||
|
||||
d := newForTest(&Config{ParentalEnabled: true}, nil)
|
||||
t.Cleanup(d.Close)
|
||||
const matching = "pornhub.com"
|
||||
d.SetParentalUpstream(&aghtest.TestBlockUpstream{
|
||||
Hostname: matching,
|
||||
Block: true,
|
||||
})
|
||||
|
||||
d.checkMatch(t, matching)
|
||||
require.Contains(t, logOutput.String(), "Parental lookup for "+matching)
|
||||
d.checkMatch(t, "www."+matching)
|
||||
d.checkMatchEmpty(t, "www.yandex.ru")
|
||||
d.checkMatchEmpty(t, "yandex.ru")
|
||||
d.checkMatchEmpty(t, "api.jquery.com")
|
||||
|
||||
// Test cached result.
|
||||
d.parentalServer = "127.0.0.1"
|
||||
d.checkMatch(t, matching)
|
||||
d.checkMatchEmpty(t, "yandex.ru")
|
||||
}
|
||||
|
||||
// Filtering.
|
||||
|
||||
func TestMatching(t *testing.T) {
|
||||
const nl = "\n"
|
||||
const (
|
||||
blockingRules = `||example.org^` + nl
|
||||
allowlistRules = `||example.org^` + nl + `@@||test.example.org` + nl
|
||||
importantRules = `@@||example.org^` + nl + `||test.example.org^$important` + nl
|
||||
regexRules = `/example\.org/` + nl + `@@||test.example.org^` + nl
|
||||
maskRules = `test*.example.org^` + nl + `exam*.com` + nl
|
||||
dnstypeRules = `||example.org^$dnstype=AAAA` + nl + `@@||test.example.org^` + nl
|
||||
)
|
||||
testCases := []struct {
|
||||
name string
|
||||
rules string
|
||||
host string
|
||||
wantReason Reason
|
||||
wantIsFiltered bool
|
||||
wantDNSType uint16
|
||||
}{{
|
||||
name: "sanity",
|
||||
rules: "||doubleclick.net^",
|
||||
host: "www.doubleclick.net",
|
||||
wantIsFiltered: true,
|
||||
wantReason: FilteredBlockList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "sanity",
|
||||
rules: "||doubleclick.net^",
|
||||
host: "nodoubleclick.net",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredNotFound,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "sanity",
|
||||
rules: "||doubleclick.net^",
|
||||
host: "doubleclick.net.ru",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredNotFound,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "sanity",
|
||||
rules: "||doubleclick.net^",
|
||||
host: "wmconvirus.narod.ru",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredNotFound,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "blocking",
|
||||
rules: blockingRules,
|
||||
host: "example.org",
|
||||
wantIsFiltered: true,
|
||||
wantReason: FilteredBlockList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "blocking",
|
||||
rules: blockingRules,
|
||||
host: "test.example.org",
|
||||
wantIsFiltered: true,
|
||||
wantReason: FilteredBlockList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "blocking",
|
||||
rules: blockingRules,
|
||||
host: "test.test.example.org",
|
||||
wantIsFiltered: true,
|
||||
wantReason: FilteredBlockList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "blocking",
|
||||
rules: blockingRules,
|
||||
host: "testexample.org",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredNotFound,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "blocking",
|
||||
rules: blockingRules,
|
||||
host: "onemoreexample.org",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredNotFound,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "allowlist",
|
||||
rules: allowlistRules,
|
||||
host: "example.org",
|
||||
wantIsFiltered: true,
|
||||
wantReason: FilteredBlockList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "allowlist",
|
||||
rules: allowlistRules,
|
||||
host: "test.example.org",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredAllowList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "allowlist",
|
||||
rules: allowlistRules,
|
||||
host: "test.test.example.org",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredAllowList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "allowlist",
|
||||
rules: allowlistRules,
|
||||
host: "testexample.org",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredNotFound,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "allowlist",
|
||||
rules: allowlistRules,
|
||||
host: "onemoreexample.org",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredNotFound,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "important",
|
||||
rules: importantRules,
|
||||
host: "example.org",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredAllowList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "important",
|
||||
rules: importantRules,
|
||||
host: "test.example.org",
|
||||
wantIsFiltered: true,
|
||||
wantReason: FilteredBlockList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "important",
|
||||
rules: importantRules,
|
||||
host: "test.test.example.org",
|
||||
wantIsFiltered: true,
|
||||
wantReason: FilteredBlockList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "important",
|
||||
rules: importantRules,
|
||||
host: "testexample.org",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredNotFound,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "important",
|
||||
rules: importantRules,
|
||||
host: "onemoreexample.org",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredNotFound,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "regex",
|
||||
rules: regexRules,
|
||||
host: "example.org",
|
||||
wantIsFiltered: true,
|
||||
wantReason: FilteredBlockList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "regex",
|
||||
rules: regexRules,
|
||||
host: "test.example.org",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredAllowList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "regex",
|
||||
rules: regexRules,
|
||||
host: "test.test.example.org",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredAllowList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "regex",
|
||||
rules: regexRules,
|
||||
host: "testexample.org",
|
||||
wantIsFiltered: true,
|
||||
wantReason: FilteredBlockList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "regex",
|
||||
rules: regexRules,
|
||||
host: "onemoreexample.org",
|
||||
wantIsFiltered: true,
|
||||
wantReason: FilteredBlockList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "mask",
|
||||
rules: maskRules,
|
||||
host: "test.example.org",
|
||||
wantIsFiltered: true,
|
||||
wantReason: FilteredBlockList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "mask",
|
||||
rules: maskRules,
|
||||
host: "test2.example.org",
|
||||
wantIsFiltered: true,
|
||||
wantReason: FilteredBlockList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "mask",
|
||||
rules: maskRules,
|
||||
host: "example.com",
|
||||
wantIsFiltered: true,
|
||||
wantReason: FilteredBlockList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "mask",
|
||||
rules: maskRules,
|
||||
host: "exampleeee.com",
|
||||
wantIsFiltered: true,
|
||||
wantReason: FilteredBlockList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "mask",
|
||||
rules: maskRules,
|
||||
host: "onemoreexamsite.com",
|
||||
wantIsFiltered: true,
|
||||
wantReason: FilteredBlockList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "mask",
|
||||
rules: maskRules,
|
||||
host: "example.org",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredNotFound,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "mask",
|
||||
rules: maskRules,
|
||||
host: "testexample.org",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredNotFound,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "mask",
|
||||
rules: maskRules,
|
||||
host: "example.co.uk",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredNotFound,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "dnstype",
|
||||
rules: dnstypeRules,
|
||||
host: "onemoreexample.org",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredNotFound,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "dnstype",
|
||||
rules: dnstypeRules,
|
||||
host: "example.org",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredNotFound,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "dnstype",
|
||||
rules: dnstypeRules,
|
||||
host: "example.org",
|
||||
wantIsFiltered: true,
|
||||
wantReason: FilteredBlockList,
|
||||
wantDNSType: dns.TypeAAAA,
|
||||
}, {
|
||||
name: "dnstype",
|
||||
rules: dnstypeRules,
|
||||
host: "test.example.org",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredAllowList,
|
||||
wantDNSType: dns.TypeA,
|
||||
}, {
|
||||
name: "dnstype",
|
||||
rules: dnstypeRules,
|
||||
host: "test.example.org",
|
||||
wantIsFiltered: false,
|
||||
wantReason: NotFilteredAllowList,
|
||||
wantDNSType: dns.TypeAAAA,
|
||||
}}
|
||||
for _, tc := range testCases {
|
||||
t.Run(fmt.Sprintf("%s-%s", tc.name, tc.host), func(t *testing.T) {
|
||||
filters := []Filter{{ID: 0, Data: []byte(tc.rules)}}
|
||||
d := newForTest(nil, filters)
|
||||
t.Cleanup(d.Close)
|
||||
|
||||
res, err := d.CheckHost(tc.host, tc.wantDNSType, &setts)
|
||||
require.Nilf(t, err, "Error while matching host %s: %s", tc.host, err)
|
||||
assert.Equalf(t, tc.wantIsFiltered, res.IsFiltered, "Hostname %s has wrong result (%v must be %v)", tc.host, res.IsFiltered, tc.wantIsFiltered)
|
||||
assert.Equalf(t, tc.wantReason, res.Reason, "Hostname %s has wrong reason (%v must be %v)", tc.host, res.Reason, tc.wantReason)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhitelist(t *testing.T) {
|
||||
rules := `||host1^
|
||||
||host2^
|
||||
`
|
||||
filters := []Filter{{
|
||||
ID: 0, Data: []byte(rules),
|
||||
}}
|
||||
|
||||
whiteRules := `||host1^
|
||||
||host3^
|
||||
`
|
||||
whiteFilters := []Filter{{
|
||||
ID: 0, Data: []byte(whiteRules),
|
||||
}}
|
||||
d := newForTest(nil, filters)
|
||||
|
||||
require.Nil(t, d.SetFilters(filters, whiteFilters, false))
|
||||
t.Cleanup(d.Close)
|
||||
|
||||
// Matched by white filter.
|
||||
res, err := d.CheckHost("host1", dns.TypeA, &setts)
|
||||
require.Nil(t, err)
|
||||
assert.False(t, res.IsFiltered)
|
||||
assert.Equal(t, res.Reason, NotFilteredAllowList)
|
||||
require.Len(t, res.Rules, 1)
|
||||
assert.Equal(t, "||host1^", res.Rules[0].Text)
|
||||
|
||||
// Not matched by white filter, but matched by block filter.
|
||||
res, err = d.CheckHost("host2", dns.TypeA, &setts)
|
||||
require.Nil(t, err)
|
||||
assert.True(t, res.IsFiltered)
|
||||
assert.Equal(t, res.Reason, FilteredBlockList)
|
||||
require.Len(t, res.Rules, 1)
|
||||
assert.Equal(t, "||host2^", res.Rules[0].Text)
|
||||
}
|
||||
|
||||
// Client Settings.
|
||||
|
||||
func applyClientSettings(setts *Settings) {
|
||||
setts.FilteringEnabled = false
|
||||
setts.ParentalEnabled = false
|
||||
setts.SafeBrowsingEnabled = true
|
||||
|
||||
rule, _ := rules.NewNetworkRule("||facebook.com^", 0)
|
||||
s := ServiceEntry{}
|
||||
s.Name = "facebook"
|
||||
s.Rules = []*rules.NetworkRule{rule}
|
||||
setts.ServicesRules = append(setts.ServicesRules, s)
|
||||
}
|
||||
|
||||
func TestClientSettings(t *testing.T) {
|
||||
d := newForTest(
|
||||
&Config{
|
||||
ParentalEnabled: true,
|
||||
SafeBrowsingEnabled: false,
|
||||
},
|
||||
[]Filter{{
|
||||
ID: 0, Data: []byte("||example.org^\n"),
|
||||
}},
|
||||
)
|
||||
t.Cleanup(d.Close)
|
||||
d.SetParentalUpstream(&aghtest.TestBlockUpstream{
|
||||
Hostname: "pornhub.com",
|
||||
Block: true,
|
||||
})
|
||||
d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{
|
||||
Hostname: "wmconvirus.narod.ru",
|
||||
Block: true,
|
||||
})
|
||||
|
||||
type testCase struct {
|
||||
name string
|
||||
host string
|
||||
before bool
|
||||
wantReason Reason
|
||||
}
|
||||
testCases := []testCase{{
|
||||
name: "filters",
|
||||
host: "example.org",
|
||||
before: true,
|
||||
wantReason: FilteredBlockList,
|
||||
}, {
|
||||
name: "parental",
|
||||
host: "pornhub.com",
|
||||
before: true,
|
||||
wantReason: FilteredParental,
|
||||
}, {
|
||||
name: "safebrowsing",
|
||||
host: "wmconvirus.narod.ru",
|
||||
before: false,
|
||||
wantReason: FilteredSafeBrowsing,
|
||||
}, {
|
||||
name: "additional_rules",
|
||||
host: "facebook.com",
|
||||
before: false,
|
||||
wantReason: FilteredBlockedService,
|
||||
}}
|
||||
|
||||
makeTester := func(tc testCase, before bool) func(t *testing.T) {
|
||||
return func(t *testing.T) {
|
||||
r, _ := d.CheckHost(tc.host, dns.TypeA, &setts)
|
||||
if before {
|
||||
assert.True(t, r.IsFiltered)
|
||||
assert.Equal(t, tc.wantReason, r.Reason)
|
||||
} else {
|
||||
assert.False(t, r.IsFiltered)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check behaviour without any per-client settings, then apply per-client
|
||||
// settings and check behaviour once again.
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, makeTester(tc, tc.before))
|
||||
}
|
||||
|
||||
applyClientSettings(&setts)
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, makeTester(tc, !tc.before))
|
||||
}
|
||||
}
|
||||
|
||||
// Benchmarks.
|
||||
|
||||
func BenchmarkSafeBrowsing(b *testing.B) {
|
||||
d := newForTest(&Config{SafeBrowsingEnabled: true}, nil)
|
||||
b.Cleanup(d.Close)
|
||||
blocked := "wmconvirus.narod.ru"
|
||||
d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{
|
||||
Hostname: blocked,
|
||||
Block: true,
|
||||
})
|
||||
for n := 0; n < b.N; n++ {
|
||||
res, err := d.CheckHost(blocked, dns.TypeA, &setts)
|
||||
require.Nilf(b, err, "Error while matching host %s: %s", blocked, err)
|
||||
assert.True(b, res.IsFiltered, "Expected hostname %s to match", blocked)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSafeBrowsingParallel(b *testing.B) {
|
||||
d := newForTest(&Config{SafeBrowsingEnabled: true}, nil)
|
||||
b.Cleanup(d.Close)
|
||||
blocked := "wmconvirus.narod.ru"
|
||||
d.SetSafeBrowsingUpstream(&aghtest.TestBlockUpstream{
|
||||
Hostname: blocked,
|
||||
Block: true,
|
||||
})
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
res, err := d.CheckHost(blocked, dns.TypeA, &setts)
|
||||
require.Nilf(b, err, "Error while matching host %s: %s", blocked, err)
|
||||
assert.True(b, res.IsFiltered, "Expected hostname %s to match", blocked)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkSafeSearch(b *testing.B) {
|
||||
d := newForTest(&Config{SafeSearchEnabled: true}, nil)
|
||||
b.Cleanup(d.Close)
|
||||
for n := 0; n < b.N; n++ {
|
||||
val, ok := d.SafeSearchDomain("www.google.com")
|
||||
require.True(b, ok, "Expected safesearch to find result for www.google.com")
|
||||
assert.Equal(b, "forcesafesearch.google.com", val, "Expected safesearch for google.com to be forcesafesearch.google.com")
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSafeSearchParallel(b *testing.B) {
|
||||
d := newForTest(&Config{SafeSearchEnabled: true}, nil)
|
||||
b.Cleanup(d.Close)
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
val, ok := d.SafeSearchDomain("www.google.com")
|
||||
require.True(b, ok, "Expected safesearch to find result for www.google.com")
|
||||
assert.Equal(b, "forcesafesearch.google.com", val, "Expected safesearch for google.com to be forcesafesearch.google.com")
|
||||
}
|
||||
})
|
||||
}
|
||||
231
internal/filtering/rewrites.go
Normal file
231
internal/filtering/rewrites.go
Normal file
@@ -0,0 +1,231 @@
|
||||
// DNS Rewrites
|
||||
|
||||
package filtering
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/miekg/dns"
|
||||
)
|
||||
|
||||
// RewriteEntry is a rewrite array element
|
||||
type RewriteEntry struct {
|
||||
Domain string `yaml:"domain"`
|
||||
Answer string `yaml:"answer"` // IP address or canonical name
|
||||
Type uint16 `yaml:"-"` // DNS record type: CNAME, A or AAAA
|
||||
IP net.IP `yaml:"-"` // Parsed IP address (if Type is A or AAAA)
|
||||
}
|
||||
|
||||
func (r *RewriteEntry) equals(b RewriteEntry) bool {
|
||||
return r.Domain == b.Domain && r.Answer == b.Answer
|
||||
}
|
||||
|
||||
func isWildcard(host string) bool {
|
||||
return len(host) >= 2 &&
|
||||
host[0] == '*' && host[1] == '.'
|
||||
}
|
||||
|
||||
// Return TRUE of host name matches a wildcard pattern
|
||||
func matchDomainWildcard(host, wildcard string) bool {
|
||||
return isWildcard(wildcard) &&
|
||||
strings.HasSuffix(host, wildcard[1:])
|
||||
}
|
||||
|
||||
type rewritesArray []RewriteEntry
|
||||
|
||||
func (a rewritesArray) Len() int { return len(a) }
|
||||
|
||||
func (a rewritesArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
|
||||
// Priority:
|
||||
// . CNAME < A/AAAA;
|
||||
// . exact < wildcard;
|
||||
// . higher level wildcard < lower level wildcard
|
||||
func (a rewritesArray) Less(i, j int) bool {
|
||||
if a[i].Type == dns.TypeCNAME && a[j].Type != dns.TypeCNAME {
|
||||
return true
|
||||
} else if a[i].Type != dns.TypeCNAME && a[j].Type == dns.TypeCNAME {
|
||||
return false
|
||||
}
|
||||
|
||||
if isWildcard(a[i].Domain) {
|
||||
if !isWildcard(a[j].Domain) {
|
||||
return false
|
||||
}
|
||||
} else {
|
||||
if isWildcard(a[j].Domain) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// both are wildcards
|
||||
return len(a[i].Domain) > len(a[j].Domain)
|
||||
}
|
||||
|
||||
// Prepare entry for use
|
||||
func (r *RewriteEntry) prepare() {
|
||||
if r.Answer == "AAAA" {
|
||||
r.IP = nil
|
||||
r.Type = dns.TypeAAAA
|
||||
return
|
||||
} else if r.Answer == "A" {
|
||||
r.IP = nil
|
||||
r.Type = dns.TypeA
|
||||
return
|
||||
}
|
||||
|
||||
ip := net.ParseIP(r.Answer)
|
||||
if ip == nil {
|
||||
r.Type = dns.TypeCNAME
|
||||
return
|
||||
}
|
||||
|
||||
r.IP = ip
|
||||
r.Type = dns.TypeAAAA
|
||||
|
||||
ip4 := ip.To4()
|
||||
if ip4 != nil {
|
||||
r.IP = ip4
|
||||
r.Type = dns.TypeA
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DNSFilter) prepareRewrites() {
|
||||
for i := range d.Rewrites {
|
||||
d.Rewrites[i].prepare()
|
||||
}
|
||||
}
|
||||
|
||||
// Get the list of matched rewrite entries.
|
||||
// Priority: CNAME, A/AAAA; exact, wildcard.
|
||||
// If matched exactly, don't return wildcard entries.
|
||||
// If matched by several wildcards, select the more specific one
|
||||
func findRewrites(a []RewriteEntry, host string) []RewriteEntry {
|
||||
rr := rewritesArray{}
|
||||
for _, r := range a {
|
||||
if r.Domain != host {
|
||||
if !matchDomainWildcard(host, r.Domain) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
rr = append(rr, r)
|
||||
}
|
||||
|
||||
if len(rr) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
sort.Sort(rr)
|
||||
|
||||
for i, r := range rr {
|
||||
if isWildcard(r.Domain) {
|
||||
// Don't use rr[:0], because we need to return at least
|
||||
// one item here.
|
||||
rr = rr[:max(1, i)]
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return rr
|
||||
}
|
||||
|
||||
func max(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
func rewriteArrayDup(a []RewriteEntry) []RewriteEntry {
|
||||
a2 := make([]RewriteEntry, len(a))
|
||||
copy(a2, a)
|
||||
return a2
|
||||
}
|
||||
|
||||
type rewriteEntryJSON struct {
|
||||
Domain string `json:"domain"`
|
||||
Answer string `json:"answer"`
|
||||
}
|
||||
|
||||
func (d *DNSFilter) handleRewriteList(w http.ResponseWriter, r *http.Request) {
|
||||
arr := []*rewriteEntryJSON{}
|
||||
|
||||
d.confLock.Lock()
|
||||
for _, ent := range d.Config.Rewrites {
|
||||
jsent := rewriteEntryJSON{
|
||||
Domain: ent.Domain,
|
||||
Answer: ent.Answer,
|
||||
}
|
||||
arr = append(arr, &jsent)
|
||||
}
|
||||
d.confLock.Unlock()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
err := json.NewEncoder(w).Encode(arr)
|
||||
if err != nil {
|
||||
httpError(r, w, http.StatusInternalServerError, "json.Encode: %s", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DNSFilter) handleRewriteAdd(w http.ResponseWriter, r *http.Request) {
|
||||
jsent := rewriteEntryJSON{}
|
||||
err := json.NewDecoder(r.Body).Decode(&jsent)
|
||||
if err != nil {
|
||||
httpError(r, w, http.StatusBadRequest, "json.Decode: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
ent := RewriteEntry{
|
||||
Domain: jsent.Domain,
|
||||
Answer: jsent.Answer,
|
||||
}
|
||||
ent.prepare()
|
||||
d.confLock.Lock()
|
||||
d.Config.Rewrites = append(d.Config.Rewrites, ent)
|
||||
d.confLock.Unlock()
|
||||
log.Debug("Rewrites: added element: %s -> %s [%d]",
|
||||
ent.Domain, ent.Answer, len(d.Config.Rewrites))
|
||||
|
||||
d.Config.ConfigModified()
|
||||
}
|
||||
|
||||
func (d *DNSFilter) handleRewriteDelete(w http.ResponseWriter, r *http.Request) {
|
||||
jsent := rewriteEntryJSON{}
|
||||
err := json.NewDecoder(r.Body).Decode(&jsent)
|
||||
if err != nil {
|
||||
httpError(r, w, http.StatusBadRequest, "json.Decode: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
entDel := RewriteEntry{
|
||||
Domain: jsent.Domain,
|
||||
Answer: jsent.Answer,
|
||||
}
|
||||
arr := []RewriteEntry{}
|
||||
d.confLock.Lock()
|
||||
for _, ent := range d.Config.Rewrites {
|
||||
if ent.equals(entDel) {
|
||||
log.Debug("Rewrites: removed element: %s -> %s", ent.Domain, ent.Answer)
|
||||
continue
|
||||
}
|
||||
arr = append(arr, ent)
|
||||
}
|
||||
d.Config.Rewrites = arr
|
||||
d.confLock.Unlock()
|
||||
|
||||
d.Config.ConfigModified()
|
||||
}
|
||||
|
||||
func (d *DNSFilter) registerRewritesHandlers() {
|
||||
d.Config.HTTPRegister(http.MethodGet, "/control/rewrite/list", d.handleRewriteList)
|
||||
d.Config.HTTPRegister(http.MethodPost, "/control/rewrite/add", d.handleRewriteAdd)
|
||||
d.Config.HTTPRegister(http.MethodPost, "/control/rewrite/delete", d.handleRewriteDelete)
|
||||
}
|
||||
302
internal/filtering/rewrites_test.go
Normal file
302
internal/filtering/rewrites_test.go
Normal file
@@ -0,0 +1,302 @@
|
||||
package filtering
|
||||
|
||||
import (
|
||||
"net"
|
||||
"testing"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TODO(e.burkov): All the tests in this file may and should me merged together.
|
||||
|
||||
func TestRewrites(t *testing.T) {
|
||||
d := newForTest(nil, nil)
|
||||
t.Cleanup(d.Close)
|
||||
|
||||
d.Rewrites = []RewriteEntry{{
|
||||
// This one and below are about CNAME, A and AAAA.
|
||||
Domain: "somecname",
|
||||
Answer: "somehost.com",
|
||||
}, {
|
||||
Domain: "somehost.com",
|
||||
Answer: "0.0.0.0",
|
||||
}, {
|
||||
Domain: "host.com",
|
||||
Answer: "1.2.3.4",
|
||||
}, {
|
||||
Domain: "host.com",
|
||||
Answer: "1.2.3.5",
|
||||
}, {
|
||||
Domain: "host.com",
|
||||
Answer: "1:2:3::4",
|
||||
}, {
|
||||
Domain: "www.host.com",
|
||||
Answer: "host.com",
|
||||
}, {
|
||||
// This one is a wildcard.
|
||||
Domain: "*.host.com",
|
||||
Answer: "1.2.3.5",
|
||||
}, {
|
||||
// This one and below are about wildcard overriding.
|
||||
Domain: "a.host.com",
|
||||
Answer: "1.2.3.4",
|
||||
}, {
|
||||
// This one is about CNAME and wildcard interacting.
|
||||
Domain: "*.host2.com",
|
||||
Answer: "host.com",
|
||||
}, {
|
||||
// This one and below are about 2 level CNAME.
|
||||
Domain: "b.host.com",
|
||||
Answer: "somecname",
|
||||
}, {
|
||||
// This one and below are about 2 level CNAME and wildcard.
|
||||
Domain: "b.host3.com",
|
||||
Answer: "a.host3.com",
|
||||
}, {
|
||||
Domain: "a.host3.com",
|
||||
Answer: "x.host.com",
|
||||
}}
|
||||
d.prepareRewrites()
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
host string
|
||||
dtyp uint16
|
||||
wantCName string
|
||||
wantVals []net.IP
|
||||
}{{
|
||||
name: "not_filtered_not_found",
|
||||
host: "hoost.com",
|
||||
dtyp: dns.TypeA,
|
||||
}, {
|
||||
name: "rewritten_a",
|
||||
host: "www.host.com",
|
||||
dtyp: dns.TypeA,
|
||||
wantCName: "host.com",
|
||||
wantVals: []net.IP{{1, 2, 3, 4}, {1, 2, 3, 5}},
|
||||
}, {
|
||||
name: "rewritten_aaaa",
|
||||
host: "www.host.com",
|
||||
dtyp: dns.TypeAAAA,
|
||||
wantCName: "host.com",
|
||||
wantVals: []net.IP{net.ParseIP("1:2:3::4")},
|
||||
}, {
|
||||
name: "wildcard_match",
|
||||
host: "abc.host.com",
|
||||
dtyp: dns.TypeA,
|
||||
wantVals: []net.IP{{1, 2, 3, 5}},
|
||||
}, {
|
||||
name: "wildcard_override",
|
||||
host: "a.host.com",
|
||||
dtyp: dns.TypeA,
|
||||
wantVals: []net.IP{{1, 2, 3, 4}},
|
||||
}, {
|
||||
name: "wildcard_cname_interaction",
|
||||
host: "www.host2.com",
|
||||
dtyp: dns.TypeA,
|
||||
wantCName: "host.com",
|
||||
wantVals: []net.IP{{1, 2, 3, 4}, {1, 2, 3, 5}},
|
||||
}, {
|
||||
name: "two_cnames",
|
||||
host: "b.host.com",
|
||||
dtyp: dns.TypeA,
|
||||
wantCName: "somehost.com",
|
||||
wantVals: []net.IP{{0, 0, 0, 0}},
|
||||
}, {
|
||||
name: "two_cnames_and_wildcard",
|
||||
host: "b.host3.com",
|
||||
dtyp: dns.TypeA,
|
||||
wantCName: "x.host.com",
|
||||
wantVals: []net.IP{{1, 2, 3, 5}},
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
valsNum := len(tc.wantVals)
|
||||
|
||||
r := d.processRewrites(tc.host, tc.dtyp)
|
||||
if valsNum == 0 {
|
||||
assert.Equal(t, NotFilteredNotFound, r.Reason)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
require.Equal(t, Rewritten, r.Reason)
|
||||
if tc.wantCName != "" {
|
||||
assert.Equal(t, tc.wantCName, r.CanonName)
|
||||
}
|
||||
|
||||
require.Len(t, r.IPList, valsNum)
|
||||
for i, ip := range tc.wantVals {
|
||||
assert.Equal(t, ip, r.IPList[i])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewritesLevels(t *testing.T) {
|
||||
d := newForTest(nil, nil)
|
||||
t.Cleanup(d.Close)
|
||||
// Exact host, wildcard L2, wildcard L3.
|
||||
d.Rewrites = []RewriteEntry{{
|
||||
Domain: "host.com",
|
||||
Answer: "1.1.1.1",
|
||||
}, {
|
||||
Domain: "*.host.com",
|
||||
Answer: "2.2.2.2",
|
||||
}, {
|
||||
Domain: "*.sub.host.com",
|
||||
Answer: "3.3.3.3",
|
||||
}}
|
||||
d.prepareRewrites()
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
host string
|
||||
want net.IP
|
||||
}{{
|
||||
name: "exact_match",
|
||||
host: "host.com",
|
||||
want: net.IP{1, 1, 1, 1},
|
||||
}, {
|
||||
name: "l2_match",
|
||||
host: "sub.host.com",
|
||||
want: net.IP{2, 2, 2, 2},
|
||||
}, {
|
||||
name: "l3_match",
|
||||
host: "my.sub.host.com",
|
||||
want: net.IP{3, 3, 3, 3},
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := d.processRewrites(tc.host, dns.TypeA)
|
||||
assert.Equal(t, Rewritten, r.Reason)
|
||||
require.Len(t, r.IPList, 1)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewritesExceptionCNAME(t *testing.T) {
|
||||
d := newForTest(nil, nil)
|
||||
t.Cleanup(d.Close)
|
||||
// Wildcard and exception for a sub-domain.
|
||||
d.Rewrites = []RewriteEntry{{
|
||||
Domain: "*.host.com",
|
||||
Answer: "2.2.2.2",
|
||||
}, {
|
||||
Domain: "sub.host.com",
|
||||
Answer: "sub.host.com",
|
||||
}, {
|
||||
Domain: "*.sub.host.com",
|
||||
Answer: "*.sub.host.com",
|
||||
}}
|
||||
d.prepareRewrites()
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
host string
|
||||
want net.IP
|
||||
}{{
|
||||
name: "match_sub-domain",
|
||||
host: "my.host.com",
|
||||
want: net.IP{2, 2, 2, 2},
|
||||
}, {
|
||||
name: "exception_cname",
|
||||
host: "sub.host.com",
|
||||
}, {
|
||||
name: "exception_wildcard",
|
||||
host: "my.sub.host.com",
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := d.processRewrites(tc.host, dns.TypeA)
|
||||
if tc.want == nil {
|
||||
assert.Equal(t, NotFilteredNotFound, r.Reason)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
assert.Equal(t, Rewritten, r.Reason)
|
||||
require.Len(t, r.IPList, 1)
|
||||
assert.True(t, tc.want.Equal(r.IPList[0]))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewritesExceptionIP(t *testing.T) {
|
||||
d := newForTest(nil, nil)
|
||||
t.Cleanup(d.Close)
|
||||
// Exception for AAAA record.
|
||||
d.Rewrites = []RewriteEntry{{
|
||||
Domain: "host.com",
|
||||
Answer: "1.2.3.4",
|
||||
}, {
|
||||
Domain: "host.com",
|
||||
Answer: "AAAA",
|
||||
}, {
|
||||
Domain: "host2.com",
|
||||
Answer: "::1",
|
||||
}, {
|
||||
Domain: "host2.com",
|
||||
Answer: "A",
|
||||
}, {
|
||||
Domain: "host3.com",
|
||||
Answer: "A",
|
||||
}}
|
||||
d.prepareRewrites()
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
host string
|
||||
dtyp uint16
|
||||
want []net.IP
|
||||
}{{
|
||||
name: "match_A",
|
||||
host: "host.com",
|
||||
dtyp: dns.TypeA,
|
||||
want: []net.IP{{1, 2, 3, 4}},
|
||||
}, {
|
||||
name: "exception_AAAA_host.com",
|
||||
host: "host.com",
|
||||
dtyp: dns.TypeAAAA,
|
||||
}, {
|
||||
name: "exception_A_host2.com",
|
||||
host: "host2.com",
|
||||
dtyp: dns.TypeA,
|
||||
}, {
|
||||
name: "match_AAAA_host2.com",
|
||||
host: "host2.com",
|
||||
dtyp: dns.TypeAAAA,
|
||||
want: []net.IP{net.ParseIP("::1")},
|
||||
}, {
|
||||
name: "exception_A_host3.com",
|
||||
host: "host3.com",
|
||||
dtyp: dns.TypeA,
|
||||
}, {
|
||||
name: "match_AAAA_host3.com",
|
||||
host: "host3.com",
|
||||
dtyp: dns.TypeAAAA,
|
||||
want: []net.IP{},
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name+"_"+tc.host, func(t *testing.T) {
|
||||
r := d.processRewrites(tc.host, tc.dtyp)
|
||||
if tc.want == nil {
|
||||
assert.Equal(t, NotFilteredNotFound, r.Reason)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
assert.Equal(t, Rewritten, r.Reason)
|
||||
require.Len(t, r.IPList, len(tc.want))
|
||||
for _, ip := range tc.want {
|
||||
assert.True(t, ip.Equal(r.IPList[0]))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
433
internal/filtering/safebrowsing.go
Normal file
433
internal/filtering/safebrowsing.go
Normal file
@@ -0,0 +1,433 @@
|
||||
package filtering
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghstrings"
|
||||
"github.com/AdguardTeam/dnsproxy/upstream"
|
||||
"github.com/AdguardTeam/golibs/cache"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/miekg/dns"
|
||||
"golang.org/x/net/publicsuffix"
|
||||
)
|
||||
|
||||
// Safe browsing and parental control methods.
|
||||
|
||||
const (
|
||||
dnsTimeout = 3 * time.Second
|
||||
defaultSafebrowsingServer = `https://dns-family.adguard.com/dns-query`
|
||||
defaultParentalServer = `https://dns-family.adguard.com/dns-query`
|
||||
sbTXTSuffix = `sb.dns.adguard.com.`
|
||||
pcTXTSuffix = `pc.dns.adguard.com.`
|
||||
)
|
||||
|
||||
// SetParentalUpstream sets the parental upstream for *DNSFilter.
|
||||
//
|
||||
// TODO(e.burkov): Remove this in v1 API to forbid the direct access.
|
||||
func (d *DNSFilter) SetParentalUpstream(u upstream.Upstream) {
|
||||
d.parentalUpstream = u
|
||||
}
|
||||
|
||||
// SetSafeBrowsingUpstream sets the safe browsing upstream for *DNSFilter.
|
||||
//
|
||||
// TODO(e.burkov): Remove this in v1 API to forbid the direct access.
|
||||
func (d *DNSFilter) SetSafeBrowsingUpstream(u upstream.Upstream) {
|
||||
d.safeBrowsingUpstream = u
|
||||
}
|
||||
|
||||
func (d *DNSFilter) initSecurityServices() error {
|
||||
var err error
|
||||
d.safeBrowsingServer = defaultSafebrowsingServer
|
||||
d.parentalServer = defaultParentalServer
|
||||
opts := upstream.Options{
|
||||
Timeout: dnsTimeout,
|
||||
ServerIPAddrs: []net.IP{
|
||||
{94, 140, 14, 15},
|
||||
{94, 140, 15, 16},
|
||||
net.ParseIP("2a10:50c0::bad1:ff"),
|
||||
net.ParseIP("2a10:50c0::bad2:ff"),
|
||||
},
|
||||
}
|
||||
|
||||
parUps, err := upstream.AddressToUpstream(d.parentalServer, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("converting parental server: %w", err)
|
||||
}
|
||||
d.SetParentalUpstream(parUps)
|
||||
|
||||
sbUps, err := upstream.AddressToUpstream(d.safeBrowsingServer, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("converting safe browsing server: %w", err)
|
||||
}
|
||||
d.SetSafeBrowsingUpstream(sbUps)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
expire byte[4]
|
||||
hash byte[32]
|
||||
...
|
||||
*/
|
||||
func (c *sbCtx) setCache(prefix, hashes []byte) {
|
||||
d := make([]byte, 4+len(hashes))
|
||||
expire := uint(time.Now().Unix()) + c.cacheTime*60
|
||||
binary.BigEndian.PutUint32(d[:4], uint32(expire))
|
||||
copy(d[4:], hashes)
|
||||
c.cache.Set(prefix, d)
|
||||
log.Debug("%s: stored in cache: %v", c.svc, prefix)
|
||||
}
|
||||
|
||||
// findInHash returns 32-byte hash if it's found in hashToHost.
|
||||
func (c *sbCtx) findInHash(val []byte) (hash32 [32]byte, found bool) {
|
||||
for i := 4; i < len(val); i += 32 {
|
||||
hash := val[i : i+32]
|
||||
|
||||
copy(hash32[:], hash[0:32])
|
||||
|
||||
_, found = c.hashToHost[hash32]
|
||||
if found {
|
||||
return hash32, found
|
||||
}
|
||||
}
|
||||
|
||||
return [32]byte{}, false
|
||||
}
|
||||
|
||||
func (c *sbCtx) getCached() int {
|
||||
now := time.Now().Unix()
|
||||
hashesToRequest := map[[32]byte]string{}
|
||||
for k, v := range c.hashToHost {
|
||||
key := k[0:2]
|
||||
val := c.cache.Get(key)
|
||||
if val == nil || now >= int64(binary.BigEndian.Uint32(val)) {
|
||||
hashesToRequest[k] = v
|
||||
continue
|
||||
}
|
||||
if hash32, found := c.findInHash(val); found {
|
||||
log.Debug("%s: found in cache: %s: blocked by %v", c.svc, c.host, hash32)
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
if len(hashesToRequest) == 0 {
|
||||
log.Debug("%s: found in cache: %s: not blocked", c.svc, c.host)
|
||||
return -1
|
||||
}
|
||||
|
||||
c.hashToHost = hashesToRequest
|
||||
return 0
|
||||
}
|
||||
|
||||
type sbCtx struct {
|
||||
host string
|
||||
svc string
|
||||
hashToHost map[[32]byte]string
|
||||
cache cache.Cache
|
||||
cacheTime uint
|
||||
}
|
||||
|
||||
func hostnameToHashes(host string) map[[32]byte]string {
|
||||
hashes := map[[32]byte]string{}
|
||||
tld, icann := publicsuffix.PublicSuffix(host)
|
||||
if !icann {
|
||||
// private suffixes like cloudfront.net
|
||||
tld = ""
|
||||
}
|
||||
curhost := host
|
||||
|
||||
nDots := 0
|
||||
for i := len(curhost) - 1; i >= 0; i-- {
|
||||
if curhost[i] == '.' {
|
||||
nDots++
|
||||
if nDots == 4 {
|
||||
curhost = curhost[i+1:] // "xxx.a.b.c.d" -> "a.b.c.d"
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
if curhost == "" {
|
||||
// we've reached end of string
|
||||
break
|
||||
}
|
||||
if tld != "" && curhost == tld {
|
||||
// we've reached the TLD, don't hash it
|
||||
break
|
||||
}
|
||||
|
||||
sum := sha256.Sum256([]byte(curhost))
|
||||
hashes[sum] = curhost
|
||||
|
||||
pos := strings.IndexByte(curhost, byte('.'))
|
||||
if pos < 0 {
|
||||
break
|
||||
}
|
||||
curhost = curhost[pos+1:]
|
||||
}
|
||||
return hashes
|
||||
}
|
||||
|
||||
// convert hash array to string
|
||||
func (c *sbCtx) getQuestion() string {
|
||||
b := &strings.Builder{}
|
||||
|
||||
for hash := range c.hashToHost {
|
||||
// TODO(e.burkov, a.garipov): Find out and document why exactly
|
||||
// this slice.
|
||||
aghstrings.WriteToBuilder(b, hex.EncodeToString(hash[0:2]), ".")
|
||||
}
|
||||
|
||||
if c.svc == "SafeBrowsing" {
|
||||
aghstrings.WriteToBuilder(b, sbTXTSuffix)
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
aghstrings.WriteToBuilder(b, pcTXTSuffix)
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Find the target hash in TXT response
|
||||
func (c *sbCtx) processTXT(resp *dns.Msg) (bool, [][]byte) {
|
||||
matched := false
|
||||
hashes := [][]byte{}
|
||||
for _, a := range resp.Answer {
|
||||
txt, ok := a.(*dns.TXT)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
log.Debug("%s: received hashes for %s: %v", c.svc, c.host, txt.Txt)
|
||||
|
||||
for _, t := range txt.Txt {
|
||||
if len(t) != 32*2 {
|
||||
continue
|
||||
}
|
||||
hash, err := hex.DecodeString(t)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
hashes = append(hashes, hash)
|
||||
|
||||
if !matched {
|
||||
var hash32 [32]byte
|
||||
copy(hash32[:], hash)
|
||||
|
||||
var hashHost string
|
||||
hashHost, ok = c.hashToHost[hash32]
|
||||
if ok {
|
||||
log.Debug("%s: matched %s by %s/%s", c.svc, c.host, hashHost, t)
|
||||
matched = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return matched, hashes
|
||||
}
|
||||
|
||||
func (c *sbCtx) storeCache(hashes [][]byte) {
|
||||
sort.Slice(hashes, func(a, b int) bool {
|
||||
return bytes.Compare(hashes[a], hashes[b]) == -1
|
||||
})
|
||||
|
||||
var curData []byte
|
||||
var prevPrefix []byte
|
||||
for i, hash := range hashes {
|
||||
prefix := hash[0:2]
|
||||
if !bytes.Equal(prefix, prevPrefix) {
|
||||
if i != 0 {
|
||||
c.setCache(prevPrefix, curData)
|
||||
curData = nil
|
||||
}
|
||||
prevPrefix = hashes[i][0:2]
|
||||
}
|
||||
curData = append(curData, hash...)
|
||||
}
|
||||
|
||||
if len(prevPrefix) != 0 {
|
||||
c.setCache(prevPrefix, curData)
|
||||
}
|
||||
|
||||
for hash := range c.hashToHost {
|
||||
prefix := hash[0:2]
|
||||
val := c.cache.Get(prefix)
|
||||
if val == nil {
|
||||
c.setCache(prefix, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func check(c *sbCtx, r Result, u upstream.Upstream) (Result, error) {
|
||||
c.hashToHost = hostnameToHashes(c.host)
|
||||
switch c.getCached() {
|
||||
case -1:
|
||||
return Result{}, nil
|
||||
case 1:
|
||||
return r, nil
|
||||
}
|
||||
|
||||
question := c.getQuestion()
|
||||
|
||||
log.Tracef("%s: checking %s: %s", c.svc, c.host, question)
|
||||
req := (&dns.Msg{}).SetQuestion(question, dns.TypeTXT)
|
||||
|
||||
resp, err := u.Exchange(req)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
matched, receivedHashes := c.processTXT(resp)
|
||||
|
||||
c.storeCache(receivedHashes)
|
||||
if matched {
|
||||
return r, nil
|
||||
}
|
||||
|
||||
return Result{}, nil
|
||||
}
|
||||
|
||||
// TODO(a.garipov): Unify with checkParental.
|
||||
func (d *DNSFilter) checkSafeBrowsing(
|
||||
host string,
|
||||
_ uint16,
|
||||
setts *Settings,
|
||||
) (res Result, err error) {
|
||||
if !setts.SafeBrowsingEnabled {
|
||||
return Result{}, nil
|
||||
}
|
||||
|
||||
if log.GetLevel() >= log.DEBUG {
|
||||
timer := log.StartTimer()
|
||||
defer timer.LogElapsed("SafeBrowsing lookup for %s", host)
|
||||
}
|
||||
|
||||
sctx := &sbCtx{
|
||||
host: host,
|
||||
svc: "SafeBrowsing",
|
||||
cache: gctx.safebrowsingCache,
|
||||
cacheTime: d.Config.CacheTime,
|
||||
}
|
||||
|
||||
res = Result{
|
||||
IsFiltered: true,
|
||||
Reason: FilteredSafeBrowsing,
|
||||
Rules: []*ResultRule{{
|
||||
Text: "adguard-malware-shavar",
|
||||
}},
|
||||
}
|
||||
|
||||
return check(sctx, res, d.safeBrowsingUpstream)
|
||||
}
|
||||
|
||||
// TODO(a.garipov): Unify with checkSafeBrowsing.
|
||||
func (d *DNSFilter) checkParental(
|
||||
host string,
|
||||
_ uint16,
|
||||
setts *Settings,
|
||||
) (res Result, err error) {
|
||||
if !setts.ParentalEnabled {
|
||||
return Result{}, nil
|
||||
}
|
||||
|
||||
if log.GetLevel() >= log.DEBUG {
|
||||
timer := log.StartTimer()
|
||||
defer timer.LogElapsed("Parental lookup for %s", host)
|
||||
}
|
||||
|
||||
sctx := &sbCtx{
|
||||
host: host,
|
||||
svc: "Parental",
|
||||
cache: gctx.parentalCache,
|
||||
cacheTime: d.Config.CacheTime,
|
||||
}
|
||||
|
||||
res = Result{
|
||||
IsFiltered: true,
|
||||
Reason: FilteredParental,
|
||||
Rules: []*ResultRule{{
|
||||
Text: "parental CATEGORY_BLACKLISTED",
|
||||
}},
|
||||
}
|
||||
|
||||
return check(sctx, res, d.parentalUpstream)
|
||||
}
|
||||
|
||||
func httpError(r *http.Request, w http.ResponseWriter, code int, format string, args ...interface{}) {
|
||||
text := fmt.Sprintf(format, args...)
|
||||
log.Info("DNSFilter: %s %s: %s", r.Method, r.URL, text)
|
||||
http.Error(w, text, code)
|
||||
}
|
||||
|
||||
func (d *DNSFilter) handleSafeBrowsingEnable(w http.ResponseWriter, r *http.Request) {
|
||||
d.Config.SafeBrowsingEnabled = true
|
||||
d.Config.ConfigModified()
|
||||
}
|
||||
|
||||
func (d *DNSFilter) handleSafeBrowsingDisable(w http.ResponseWriter, r *http.Request) {
|
||||
d.Config.SafeBrowsingEnabled = false
|
||||
d.Config.ConfigModified()
|
||||
}
|
||||
|
||||
func (d *DNSFilter) handleSafeBrowsingStatus(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
err := json.NewEncoder(w).Encode(&struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
}{
|
||||
Enabled: d.Config.SafeBrowsingEnabled,
|
||||
})
|
||||
if err != nil {
|
||||
httpError(r, w, http.StatusInternalServerError, "Unable to write response json: %s", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DNSFilter) handleParentalEnable(w http.ResponseWriter, r *http.Request) {
|
||||
d.Config.ParentalEnabled = true
|
||||
d.Config.ConfigModified()
|
||||
}
|
||||
|
||||
func (d *DNSFilter) handleParentalDisable(w http.ResponseWriter, r *http.Request) {
|
||||
d.Config.ParentalEnabled = false
|
||||
d.Config.ConfigModified()
|
||||
}
|
||||
|
||||
func (d *DNSFilter) handleParentalStatus(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
err := json.NewEncoder(w).Encode(&struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
}{
|
||||
Enabled: d.Config.ParentalEnabled,
|
||||
})
|
||||
if err != nil {
|
||||
httpError(r, w, http.StatusInternalServerError, "Unable to write response json: %s", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DNSFilter) registerSecurityHandlers() {
|
||||
d.Config.HTTPRegister(http.MethodPost, "/control/safebrowsing/enable", d.handleSafeBrowsingEnable)
|
||||
d.Config.HTTPRegister(http.MethodPost, "/control/safebrowsing/disable", d.handleSafeBrowsingDisable)
|
||||
d.Config.HTTPRegister(http.MethodGet, "/control/safebrowsing/status", d.handleSafeBrowsingStatus)
|
||||
|
||||
d.Config.HTTPRegister(http.MethodPost, "/control/parental/enable", d.handleParentalEnable)
|
||||
d.Config.HTTPRegister(http.MethodPost, "/control/parental/disable", d.handleParentalDisable)
|
||||
d.Config.HTTPRegister(http.MethodGet, "/control/parental/status", d.handleParentalStatus)
|
||||
|
||||
d.Config.HTTPRegister(http.MethodPost, "/control/safesearch/enable", d.handleSafeSearchEnable)
|
||||
d.Config.HTTPRegister(http.MethodPost, "/control/safesearch/disable", d.handleSafeSearchDisable)
|
||||
d.Config.HTTPRegister(http.MethodGet, "/control/safesearch/status", d.handleSafeSearchStatus)
|
||||
}
|
||||
220
internal/filtering/safebrowsing_test.go
Normal file
220
internal/filtering/safebrowsing_test.go
Normal file
@@ -0,0 +1,220 @@
|
||||
package filtering
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
|
||||
"github.com/AdguardTeam/golibs/cache"
|
||||
"github.com/miekg/dns"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSafeBrowsingHash(t *testing.T) {
|
||||
// test hostnameToHashes()
|
||||
hashes := hostnameToHashes("1.2.3.sub.host.com")
|
||||
assert.Len(t, hashes, 3)
|
||||
_, ok := hashes[sha256.Sum256([]byte("3.sub.host.com"))]
|
||||
assert.True(t, ok)
|
||||
_, ok = hashes[sha256.Sum256([]byte("sub.host.com"))]
|
||||
assert.True(t, ok)
|
||||
_, ok = hashes[sha256.Sum256([]byte("host.com"))]
|
||||
assert.True(t, ok)
|
||||
_, ok = hashes[sha256.Sum256([]byte("com"))]
|
||||
assert.False(t, ok)
|
||||
|
||||
c := &sbCtx{
|
||||
svc: "SafeBrowsing",
|
||||
hashToHost: hashes,
|
||||
}
|
||||
|
||||
q := c.getQuestion()
|
||||
|
||||
assert.Contains(t, q, "7a1b.")
|
||||
assert.Contains(t, q, "af5a.")
|
||||
assert.Contains(t, q, "eb11.")
|
||||
assert.True(t, strings.HasSuffix(q, "sb.dns.adguard.com."))
|
||||
}
|
||||
|
||||
func TestSafeBrowsingCache(t *testing.T) {
|
||||
c := &sbCtx{
|
||||
svc: "SafeBrowsing",
|
||||
cacheTime: 100,
|
||||
}
|
||||
conf := cache.Config{}
|
||||
c.cache = cache.New(conf)
|
||||
|
||||
// store in cache hashes for "3.sub.host.com" and "host.com"
|
||||
// and empty data for hash-prefix for "sub.host.com"
|
||||
hash := sha256.Sum256([]byte("sub.host.com"))
|
||||
c.hashToHost = make(map[[32]byte]string)
|
||||
c.hashToHost[hash] = "sub.host.com"
|
||||
var hashesArray [][]byte
|
||||
hash4 := sha256.Sum256([]byte("3.sub.host.com"))
|
||||
hashesArray = append(hashesArray, hash4[:])
|
||||
hash2 := sha256.Sum256([]byte("host.com"))
|
||||
hashesArray = append(hashesArray, hash2[:])
|
||||
c.storeCache(hashesArray)
|
||||
|
||||
// match "3.sub.host.com" or "host.com" from cache
|
||||
c.hashToHost = make(map[[32]byte]string)
|
||||
hash = sha256.Sum256([]byte("3.sub.host.com"))
|
||||
c.hashToHost[hash] = "3.sub.host.com"
|
||||
hash = sha256.Sum256([]byte("sub.host.com"))
|
||||
c.hashToHost[hash] = "sub.host.com"
|
||||
hash = sha256.Sum256([]byte("host.com"))
|
||||
c.hashToHost[hash] = "host.com"
|
||||
assert.Equal(t, 1, c.getCached())
|
||||
|
||||
// match "sub.host.com" from cache
|
||||
c.hashToHost = make(map[[32]byte]string)
|
||||
hash = sha256.Sum256([]byte("sub.host.com"))
|
||||
c.hashToHost[hash] = "sub.host.com"
|
||||
assert.Equal(t, -1, c.getCached())
|
||||
|
||||
// match "sub.host.com" from cache,
|
||||
// but another hash for "nonexisting.com" is not in cache
|
||||
// which means that we must get data from server for it
|
||||
c.hashToHost = make(map[[32]byte]string)
|
||||
hash = sha256.Sum256([]byte("sub.host.com"))
|
||||
c.hashToHost[hash] = "sub.host.com"
|
||||
hash = sha256.Sum256([]byte("nonexisting.com"))
|
||||
c.hashToHost[hash] = "nonexisting.com"
|
||||
assert.Empty(t, c.getCached())
|
||||
|
||||
hash = sha256.Sum256([]byte("sub.host.com"))
|
||||
_, ok := c.hashToHost[hash]
|
||||
assert.False(t, ok)
|
||||
|
||||
hash = sha256.Sum256([]byte("nonexisting.com"))
|
||||
_, ok = c.hashToHost[hash]
|
||||
assert.True(t, ok)
|
||||
|
||||
c = &sbCtx{
|
||||
svc: "SafeBrowsing",
|
||||
cacheTime: 100,
|
||||
}
|
||||
conf = cache.Config{}
|
||||
c.cache = cache.New(conf)
|
||||
|
||||
hash = sha256.Sum256([]byte("sub.host.com"))
|
||||
c.hashToHost = make(map[[32]byte]string)
|
||||
c.hashToHost[hash] = "sub.host.com"
|
||||
|
||||
c.cache.Set(hash[0:2], make([]byte, 32))
|
||||
assert.Empty(t, c.getCached())
|
||||
}
|
||||
|
||||
func TestSBPC_checkErrorUpstream(t *testing.T) {
|
||||
d := newForTest(&Config{SafeBrowsingEnabled: true}, nil)
|
||||
t.Cleanup(d.Close)
|
||||
|
||||
ups := &aghtest.TestErrUpstream{}
|
||||
|
||||
d.SetSafeBrowsingUpstream(ups)
|
||||
d.SetParentalUpstream(ups)
|
||||
|
||||
setts := &Settings{
|
||||
SafeBrowsingEnabled: true,
|
||||
ParentalEnabled: true,
|
||||
}
|
||||
|
||||
_, err := d.checkSafeBrowsing("smthng.com", dns.TypeA, setts)
|
||||
assert.Error(t, err)
|
||||
|
||||
_, err = d.checkParental("smthng.com", dns.TypeA, setts)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestSBPC(t *testing.T) {
|
||||
d := newForTest(&Config{SafeBrowsingEnabled: true}, nil)
|
||||
t.Cleanup(d.Close)
|
||||
|
||||
const hostname = "example.org"
|
||||
|
||||
setts := &Settings{
|
||||
SafeBrowsingEnabled: true,
|
||||
ParentalEnabled: true,
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
block bool
|
||||
testFunc func(host string, _ uint16, _ *Settings) (res Result, err error)
|
||||
testCache cache.Cache
|
||||
}{{
|
||||
name: "sb_no_block",
|
||||
block: false,
|
||||
testFunc: d.checkSafeBrowsing,
|
||||
testCache: gctx.safebrowsingCache,
|
||||
}, {
|
||||
name: "sb_block",
|
||||
block: true,
|
||||
testFunc: d.checkSafeBrowsing,
|
||||
testCache: gctx.safebrowsingCache,
|
||||
}, {
|
||||
name: "pc_no_block",
|
||||
block: false,
|
||||
testFunc: d.checkParental,
|
||||
testCache: gctx.parentalCache,
|
||||
}, {
|
||||
name: "pc_block",
|
||||
block: true,
|
||||
testFunc: d.checkParental,
|
||||
testCache: gctx.parentalCache,
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
// Prepare the upstream.
|
||||
ups := &aghtest.TestBlockUpstream{
|
||||
Hostname: hostname,
|
||||
Block: tc.block,
|
||||
}
|
||||
d.SetSafeBrowsingUpstream(ups)
|
||||
d.SetParentalUpstream(ups)
|
||||
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// Firstly, check the request blocking.
|
||||
hits := 0
|
||||
res, err := tc.testFunc(hostname, dns.TypeA, setts)
|
||||
require.NoError(t, err)
|
||||
|
||||
if tc.block {
|
||||
assert.True(t, res.IsFiltered)
|
||||
require.Len(t, res.Rules, 1)
|
||||
hits++
|
||||
} else {
|
||||
require.False(t, res.IsFiltered)
|
||||
}
|
||||
|
||||
// Check the cache state, check the response is now cached.
|
||||
assert.Equal(t, 1, tc.testCache.Stats().Count)
|
||||
assert.Equal(t, hits, tc.testCache.Stats().Hit)
|
||||
|
||||
// There was one request to an upstream.
|
||||
assert.Equal(t, 1, ups.RequestsCount())
|
||||
|
||||
// Now make the same request to check the cache was used.
|
||||
res, err = tc.testFunc(hostname, dns.TypeA, setts)
|
||||
require.NoError(t, err)
|
||||
|
||||
if tc.block {
|
||||
assert.True(t, res.IsFiltered)
|
||||
require.Len(t, res.Rules, 1)
|
||||
} else {
|
||||
require.False(t, res.IsFiltered)
|
||||
}
|
||||
|
||||
// Check the cache state, it should've been used.
|
||||
assert.Equal(t, 1, tc.testCache.Stats().Count)
|
||||
assert.Equal(t, hits+1, tc.testCache.Stats().Hit)
|
||||
|
||||
// Check that there were no additional requests.
|
||||
assert.Equal(t, 1, ups.RequestsCount())
|
||||
})
|
||||
|
||||
purgeCaches()
|
||||
}
|
||||
}
|
||||
373
internal/filtering/safesearch.go
Normal file
373
internal/filtering/safesearch.go
Normal file
@@ -0,0 +1,373 @@
|
||||
package filtering
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/gob"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/golibs/cache"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
)
|
||||
|
||||
/*
|
||||
expire byte[4]
|
||||
res Result
|
||||
*/
|
||||
func (d *DNSFilter) setCacheResult(cache cache.Cache, host string, res Result) int {
|
||||
var buf bytes.Buffer
|
||||
|
||||
expire := uint(time.Now().Unix()) + d.Config.CacheTime*60
|
||||
exp := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(exp, uint32(expire))
|
||||
_, _ = buf.Write(exp)
|
||||
|
||||
enc := gob.NewEncoder(&buf)
|
||||
err := enc.Encode(res)
|
||||
if err != nil {
|
||||
log.Error("gob.Encode(): %s", err)
|
||||
return 0
|
||||
}
|
||||
val := buf.Bytes()
|
||||
_ = cache.Set([]byte(host), val)
|
||||
return len(val)
|
||||
}
|
||||
|
||||
func getCachedResult(cache cache.Cache, host string) (Result, bool) {
|
||||
data := cache.Get([]byte(host))
|
||||
if data == nil {
|
||||
return Result{}, false
|
||||
}
|
||||
|
||||
exp := int(binary.BigEndian.Uint32(data[:4]))
|
||||
if exp <= int(time.Now().Unix()) {
|
||||
cache.Del([]byte(host))
|
||||
return Result{}, false
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
buf.Write(data[4:])
|
||||
dec := gob.NewDecoder(&buf)
|
||||
r := Result{}
|
||||
err := dec.Decode(&r)
|
||||
if err != nil {
|
||||
log.Debug("gob.Decode(): %s", err)
|
||||
return Result{}, false
|
||||
}
|
||||
|
||||
return r, true
|
||||
}
|
||||
|
||||
// SafeSearchDomain returns replacement address for search engine
|
||||
func (d *DNSFilter) SafeSearchDomain(host string) (string, bool) {
|
||||
val, ok := safeSearchDomains[host]
|
||||
return val, ok
|
||||
}
|
||||
|
||||
func (d *DNSFilter) checkSafeSearch(
|
||||
host string,
|
||||
_ uint16,
|
||||
setts *Settings,
|
||||
) (res Result, err error) {
|
||||
if !setts.SafeSearchEnabled {
|
||||
return Result{}, nil
|
||||
}
|
||||
|
||||
if log.GetLevel() >= log.DEBUG {
|
||||
timer := log.StartTimer()
|
||||
defer timer.LogElapsed("SafeSearch: lookup for %s", host)
|
||||
}
|
||||
|
||||
// Check cache. Return cached result if it was found
|
||||
cachedValue, isFound := getCachedResult(gctx.safeSearchCache, host)
|
||||
if isFound {
|
||||
// atomic.AddUint64(&gctx.stats.Safesearch.CacheHits, 1)
|
||||
log.Tracef("SafeSearch: found in cache: %s", host)
|
||||
return cachedValue, nil
|
||||
}
|
||||
|
||||
safeHost, ok := d.SafeSearchDomain(host)
|
||||
if !ok {
|
||||
return Result{}, nil
|
||||
}
|
||||
|
||||
res = Result{
|
||||
IsFiltered: true,
|
||||
Reason: FilteredSafeSearch,
|
||||
Rules: []*ResultRule{{}},
|
||||
}
|
||||
|
||||
if ip := net.ParseIP(safeHost); ip != nil {
|
||||
res.Rules[0].IP = ip
|
||||
valLen := d.setCacheResult(gctx.safeSearchCache, host, res)
|
||||
log.Debug("SafeSearch: stored in cache: %s (%d bytes)", host, valLen)
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
ips, err := d.resolver.LookupIP(context.Background(), "ip", safeHost)
|
||||
if err != nil {
|
||||
log.Tracef("SafeSearchDomain for %s was found but failed to lookup for %s cause %s", host, safeHost, err)
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
for _, ip := range ips {
|
||||
if ip = ip.To4(); ip == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
res.Rules[0].IP = ip
|
||||
|
||||
l := d.setCacheResult(gctx.safeSearchCache, host, res)
|
||||
log.Debug("SafeSearch: stored in cache: %s (%d bytes)", host, l)
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
return Result{}, fmt.Errorf("no ipv4 addresses in safe search response for %s", safeHost)
|
||||
}
|
||||
|
||||
func (d *DNSFilter) handleSafeSearchEnable(w http.ResponseWriter, r *http.Request) {
|
||||
d.Config.SafeSearchEnabled = true
|
||||
d.Config.ConfigModified()
|
||||
}
|
||||
|
||||
func (d *DNSFilter) handleSafeSearchDisable(w http.ResponseWriter, r *http.Request) {
|
||||
d.Config.SafeSearchEnabled = false
|
||||
d.Config.ConfigModified()
|
||||
}
|
||||
|
||||
func (d *DNSFilter) handleSafeSearchStatus(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
err := json.NewEncoder(w).Encode(&struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
}{
|
||||
Enabled: d.Config.SafeSearchEnabled,
|
||||
})
|
||||
if err != nil {
|
||||
httpError(r, w, http.StatusInternalServerError, "Unable to write response json: %s", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
var safeSearchDomains = map[string]string{
|
||||
"yandex.com": "213.180.193.56",
|
||||
"yandex.ru": "213.180.193.56",
|
||||
"yandex.ua": "213.180.193.56",
|
||||
"yandex.by": "213.180.193.56",
|
||||
"yandex.kz": "213.180.193.56",
|
||||
"www.yandex.com": "213.180.193.56",
|
||||
"www.yandex.ru": "213.180.193.56",
|
||||
"www.yandex.ua": "213.180.193.56",
|
||||
"www.yandex.by": "213.180.193.56",
|
||||
"www.yandex.kz": "213.180.193.56",
|
||||
|
||||
"www.bing.com": "strict.bing.com",
|
||||
|
||||
"duckduckgo.com": "safe.duckduckgo.com",
|
||||
"www.duckduckgo.com": "safe.duckduckgo.com",
|
||||
"start.duckduckgo.com": "safe.duckduckgo.com",
|
||||
|
||||
"www.google.com": "forcesafesearch.google.com",
|
||||
"www.google.ad": "forcesafesearch.google.com",
|
||||
"www.google.ae": "forcesafesearch.google.com",
|
||||
"www.google.com.af": "forcesafesearch.google.com",
|
||||
"www.google.com.ag": "forcesafesearch.google.com",
|
||||
"www.google.com.ai": "forcesafesearch.google.com",
|
||||
"www.google.al": "forcesafesearch.google.com",
|
||||
"www.google.am": "forcesafesearch.google.com",
|
||||
"www.google.co.ao": "forcesafesearch.google.com",
|
||||
"www.google.com.ar": "forcesafesearch.google.com",
|
||||
"www.google.as": "forcesafesearch.google.com",
|
||||
"www.google.at": "forcesafesearch.google.com",
|
||||
"www.google.com.au": "forcesafesearch.google.com",
|
||||
"www.google.az": "forcesafesearch.google.com",
|
||||
"www.google.ba": "forcesafesearch.google.com",
|
||||
"www.google.com.bd": "forcesafesearch.google.com",
|
||||
"www.google.be": "forcesafesearch.google.com",
|
||||
"www.google.bf": "forcesafesearch.google.com",
|
||||
"www.google.bg": "forcesafesearch.google.com",
|
||||
"www.google.com.bh": "forcesafesearch.google.com",
|
||||
"www.google.bi": "forcesafesearch.google.com",
|
||||
"www.google.bj": "forcesafesearch.google.com",
|
||||
"www.google.com.bn": "forcesafesearch.google.com",
|
||||
"www.google.com.bo": "forcesafesearch.google.com",
|
||||
"www.google.com.br": "forcesafesearch.google.com",
|
||||
"www.google.bs": "forcesafesearch.google.com",
|
||||
"www.google.bt": "forcesafesearch.google.com",
|
||||
"www.google.co.bw": "forcesafesearch.google.com",
|
||||
"www.google.by": "forcesafesearch.google.com",
|
||||
"www.google.com.bz": "forcesafesearch.google.com",
|
||||
"www.google.ca": "forcesafesearch.google.com",
|
||||
"www.google.cd": "forcesafesearch.google.com",
|
||||
"www.google.cf": "forcesafesearch.google.com",
|
||||
"www.google.cg": "forcesafesearch.google.com",
|
||||
"www.google.ch": "forcesafesearch.google.com",
|
||||
"www.google.ci": "forcesafesearch.google.com",
|
||||
"www.google.co.ck": "forcesafesearch.google.com",
|
||||
"www.google.cl": "forcesafesearch.google.com",
|
||||
"www.google.cm": "forcesafesearch.google.com",
|
||||
"www.google.cn": "forcesafesearch.google.com",
|
||||
"www.google.com.co": "forcesafesearch.google.com",
|
||||
"www.google.co.cr": "forcesafesearch.google.com",
|
||||
"www.google.com.cu": "forcesafesearch.google.com",
|
||||
"www.google.cv": "forcesafesearch.google.com",
|
||||
"www.google.com.cy": "forcesafesearch.google.com",
|
||||
"www.google.cz": "forcesafesearch.google.com",
|
||||
"www.google.de": "forcesafesearch.google.com",
|
||||
"www.google.dj": "forcesafesearch.google.com",
|
||||
"www.google.dk": "forcesafesearch.google.com",
|
||||
"www.google.dm": "forcesafesearch.google.com",
|
||||
"www.google.com.do": "forcesafesearch.google.com",
|
||||
"www.google.dz": "forcesafesearch.google.com",
|
||||
"www.google.com.ec": "forcesafesearch.google.com",
|
||||
"www.google.ee": "forcesafesearch.google.com",
|
||||
"www.google.com.eg": "forcesafesearch.google.com",
|
||||
"www.google.es": "forcesafesearch.google.com",
|
||||
"www.google.com.et": "forcesafesearch.google.com",
|
||||
"www.google.fi": "forcesafesearch.google.com",
|
||||
"www.google.com.fj": "forcesafesearch.google.com",
|
||||
"www.google.fm": "forcesafesearch.google.com",
|
||||
"www.google.fr": "forcesafesearch.google.com",
|
||||
"www.google.ga": "forcesafesearch.google.com",
|
||||
"www.google.ge": "forcesafesearch.google.com",
|
||||
"www.google.gg": "forcesafesearch.google.com",
|
||||
"www.google.com.gh": "forcesafesearch.google.com",
|
||||
"www.google.com.gi": "forcesafesearch.google.com",
|
||||
"www.google.gl": "forcesafesearch.google.com",
|
||||
"www.google.gm": "forcesafesearch.google.com",
|
||||
"www.google.gp": "forcesafesearch.google.com",
|
||||
"www.google.gr": "forcesafesearch.google.com",
|
||||
"www.google.com.gt": "forcesafesearch.google.com",
|
||||
"www.google.gy": "forcesafesearch.google.com",
|
||||
"www.google.com.hk": "forcesafesearch.google.com",
|
||||
"www.google.hn": "forcesafesearch.google.com",
|
||||
"www.google.hr": "forcesafesearch.google.com",
|
||||
"www.google.ht": "forcesafesearch.google.com",
|
||||
"www.google.hu": "forcesafesearch.google.com",
|
||||
"www.google.co.id": "forcesafesearch.google.com",
|
||||
"www.google.ie": "forcesafesearch.google.com",
|
||||
"www.google.co.il": "forcesafesearch.google.com",
|
||||
"www.google.im": "forcesafesearch.google.com",
|
||||
"www.google.co.in": "forcesafesearch.google.com",
|
||||
"www.google.iq": "forcesafesearch.google.com",
|
||||
"www.google.is": "forcesafesearch.google.com",
|
||||
"www.google.it": "forcesafesearch.google.com",
|
||||
"www.google.je": "forcesafesearch.google.com",
|
||||
"www.google.com.jm": "forcesafesearch.google.com",
|
||||
"www.google.jo": "forcesafesearch.google.com",
|
||||
"www.google.co.jp": "forcesafesearch.google.com",
|
||||
"www.google.co.ke": "forcesafesearch.google.com",
|
||||
"www.google.com.kh": "forcesafesearch.google.com",
|
||||
"www.google.ki": "forcesafesearch.google.com",
|
||||
"www.google.kg": "forcesafesearch.google.com",
|
||||
"www.google.co.kr": "forcesafesearch.google.com",
|
||||
"www.google.com.kw": "forcesafesearch.google.com",
|
||||
"www.google.kz": "forcesafesearch.google.com",
|
||||
"www.google.la": "forcesafesearch.google.com",
|
||||
"www.google.com.lb": "forcesafesearch.google.com",
|
||||
"www.google.li": "forcesafesearch.google.com",
|
||||
"www.google.lk": "forcesafesearch.google.com",
|
||||
"www.google.co.ls": "forcesafesearch.google.com",
|
||||
"www.google.lt": "forcesafesearch.google.com",
|
||||
"www.google.lu": "forcesafesearch.google.com",
|
||||
"www.google.lv": "forcesafesearch.google.com",
|
||||
"www.google.com.ly": "forcesafesearch.google.com",
|
||||
"www.google.co.ma": "forcesafesearch.google.com",
|
||||
"www.google.md": "forcesafesearch.google.com",
|
||||
"www.google.me": "forcesafesearch.google.com",
|
||||
"www.google.mg": "forcesafesearch.google.com",
|
||||
"www.google.mk": "forcesafesearch.google.com",
|
||||
"www.google.ml": "forcesafesearch.google.com",
|
||||
"www.google.com.mm": "forcesafesearch.google.com",
|
||||
"www.google.mn": "forcesafesearch.google.com",
|
||||
"www.google.ms": "forcesafesearch.google.com",
|
||||
"www.google.com.mt": "forcesafesearch.google.com",
|
||||
"www.google.mu": "forcesafesearch.google.com",
|
||||
"www.google.mv": "forcesafesearch.google.com",
|
||||
"www.google.mw": "forcesafesearch.google.com",
|
||||
"www.google.com.mx": "forcesafesearch.google.com",
|
||||
"www.google.com.my": "forcesafesearch.google.com",
|
||||
"www.google.co.mz": "forcesafesearch.google.com",
|
||||
"www.google.com.na": "forcesafesearch.google.com",
|
||||
"www.google.com.nf": "forcesafesearch.google.com",
|
||||
"www.google.com.ng": "forcesafesearch.google.com",
|
||||
"www.google.com.ni": "forcesafesearch.google.com",
|
||||
"www.google.ne": "forcesafesearch.google.com",
|
||||
"www.google.nl": "forcesafesearch.google.com",
|
||||
"www.google.no": "forcesafesearch.google.com",
|
||||
"www.google.com.np": "forcesafesearch.google.com",
|
||||
"www.google.nr": "forcesafesearch.google.com",
|
||||
"www.google.nu": "forcesafesearch.google.com",
|
||||
"www.google.co.nz": "forcesafesearch.google.com",
|
||||
"www.google.com.om": "forcesafesearch.google.com",
|
||||
"www.google.com.pa": "forcesafesearch.google.com",
|
||||
"www.google.com.pe": "forcesafesearch.google.com",
|
||||
"www.google.com.pg": "forcesafesearch.google.com",
|
||||
"www.google.com.ph": "forcesafesearch.google.com",
|
||||
"www.google.com.pk": "forcesafesearch.google.com",
|
||||
"www.google.pl": "forcesafesearch.google.com",
|
||||
"www.google.pn": "forcesafesearch.google.com",
|
||||
"www.google.com.pr": "forcesafesearch.google.com",
|
||||
"www.google.ps": "forcesafesearch.google.com",
|
||||
"www.google.pt": "forcesafesearch.google.com",
|
||||
"www.google.com.py": "forcesafesearch.google.com",
|
||||
"www.google.com.qa": "forcesafesearch.google.com",
|
||||
"www.google.ro": "forcesafesearch.google.com",
|
||||
"www.google.ru": "forcesafesearch.google.com",
|
||||
"www.google.rw": "forcesafesearch.google.com",
|
||||
"www.google.com.sa": "forcesafesearch.google.com",
|
||||
"www.google.com.sb": "forcesafesearch.google.com",
|
||||
"www.google.sc": "forcesafesearch.google.com",
|
||||
"www.google.se": "forcesafesearch.google.com",
|
||||
"www.google.com.sg": "forcesafesearch.google.com",
|
||||
"www.google.sh": "forcesafesearch.google.com",
|
||||
"www.google.si": "forcesafesearch.google.com",
|
||||
"www.google.sk": "forcesafesearch.google.com",
|
||||
"www.google.com.sl": "forcesafesearch.google.com",
|
||||
"www.google.sn": "forcesafesearch.google.com",
|
||||
"www.google.so": "forcesafesearch.google.com",
|
||||
"www.google.sm": "forcesafesearch.google.com",
|
||||
"www.google.sr": "forcesafesearch.google.com",
|
||||
"www.google.st": "forcesafesearch.google.com",
|
||||
"www.google.com.sv": "forcesafesearch.google.com",
|
||||
"www.google.td": "forcesafesearch.google.com",
|
||||
"www.google.tg": "forcesafesearch.google.com",
|
||||
"www.google.co.th": "forcesafesearch.google.com",
|
||||
"www.google.com.tj": "forcesafesearch.google.com",
|
||||
"www.google.tk": "forcesafesearch.google.com",
|
||||
"www.google.tl": "forcesafesearch.google.com",
|
||||
"www.google.tm": "forcesafesearch.google.com",
|
||||
"www.google.tn": "forcesafesearch.google.com",
|
||||
"www.google.to": "forcesafesearch.google.com",
|
||||
"www.google.com.tr": "forcesafesearch.google.com",
|
||||
"www.google.tt": "forcesafesearch.google.com",
|
||||
"www.google.com.tw": "forcesafesearch.google.com",
|
||||
"www.google.co.tz": "forcesafesearch.google.com",
|
||||
"www.google.com.ua": "forcesafesearch.google.com",
|
||||
"www.google.co.ug": "forcesafesearch.google.com",
|
||||
"www.google.co.uk": "forcesafesearch.google.com",
|
||||
"www.google.com.uy": "forcesafesearch.google.com",
|
||||
"www.google.co.uz": "forcesafesearch.google.com",
|
||||
"www.google.com.vc": "forcesafesearch.google.com",
|
||||
"www.google.co.ve": "forcesafesearch.google.com",
|
||||
"www.google.vg": "forcesafesearch.google.com",
|
||||
"www.google.co.vi": "forcesafesearch.google.com",
|
||||
"www.google.com.vn": "forcesafesearch.google.com",
|
||||
"www.google.vu": "forcesafesearch.google.com",
|
||||
"www.google.ws": "forcesafesearch.google.com",
|
||||
"www.google.rs": "forcesafesearch.google.com",
|
||||
|
||||
"www.youtube.com": "restrictmoderate.youtube.com",
|
||||
"m.youtube.com": "restrictmoderate.youtube.com",
|
||||
"youtubei.googleapis.com": "restrictmoderate.youtube.com",
|
||||
"youtube.googleapis.com": "restrictmoderate.youtube.com",
|
||||
"www.youtube-nocookie.com": "restrictmoderate.youtube.com",
|
||||
|
||||
"pixabay.com": "safesearch.pixabay.com",
|
||||
}
|
||||
12812
internal/filtering/tests/dns.txt
Normal file
12812
internal/filtering/tests/dns.txt
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user