Pull request: * all: move internal Go packages to internal/

Merge in DNS/adguard-home from 2234-move-to-internal to master

Squashed commit of the following:

commit d26a288cabeac86f9483fab307677b1027c78524
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Fri Oct 30 12:44:18 2020 +0300

    * all: move internal Go packages to internal/

    Closes #2234.

internal/dnsfilter/README.md

@@ -0,0 +1,70 @@
+# AdGuard Home's DNS filtering go library
+
+Example use:
+```bash
+[ -z "$GOPATH" ] && export GOPATH=$HOME/go
+go get -d github.com/AdguardTeam/AdGuardHome/dnsfilter
+```
+
+Create file filter.go
+```filter.go
+package main
+
+import (
+    "github.com/AdguardTeam/AdGuardHome/dnsfilter"
+    "log"
+)
+
+func main() {
+    filter := dnsfilter.New()
+    filter.AddRule("||dou*ck.net^")
+    host := "www.doubleclick.net"
+    res, err := filter.CheckHost(host)
+    if err != nil {
+        // temporary failure
+        log.Fatalf("Failed to check host '%s': %s", host, err)
+    }
+    if res.IsFiltered {
+        log.Printf("Host %s is filtered, reason - '%s', matched rule: '%s'", host, res.Reason, res.Rule)
+    } else {
+        log.Printf("Host %s is not filtered, reason - '%s'", host, res.Reason)
+    }
+}
+```
+
+And then run it:
+```bash
+go run filter.go
+```
+
+You will get:
+```
+2000/01/01 00:00:00 Host www.doubleclick.net is filtered, reason - 'FilteredBlackList', matched rule: '||dou*ck.net^'
+```
+
+You can also enable checking against AdGuard's SafeBrowsing:
+
+```go
+package main
+
+import (
+    "github.com/AdguardTeam/AdGuardHome/dnsfilter"
+    "log"
+)
+
+func main() {
+    filter := dnsfilter.New()
+    filter.EnableSafeBrowsing()
+    host := "wmconvirus.narod.ru" // hostname for testing safebrowsing
+    res, err := filter.CheckHost(host)
+    if err != nil {
+        // temporary failure
+        log.Fatalf("Failed to check host '%s': %s", host, err)
+    }
+    if res.IsFiltered {
+        log.Printf("Host %s is filtered, reason - '%s', matched rule: '%s'", host, res.Reason, res.Rule)
+    } else {
+        log.Printf("Host %s is not filtered, reason - '%s'", host, res.Reason)
+    }
+}
+```

internal/dnsfilter/blocked_services.go

@@ -0,0 +1,247 @@
+package dnsfilter
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/AdguardTeam/golibs/log"
+	"github.com/AdguardTeam/urlfilter/rules"
+)
+
+var serviceRules map[string][]*rules.NetworkRule // service name -> filtering rules
+
+type svc struct {
+	name  string
+	rules []string
+}
+
+// Keep in sync with:
+// client/src/helpers/constants.js
+// client/src/components/ui/Icons.js
+var serviceRulesArray = []svc{
+	{"whatsapp", []string{"||whatsapp.net^", "||whatsapp.com^"}},
+	{"facebook", []string{
+		"||facebook.com^",
+		"||facebook.net^",
+		"||fbcdn.net^",
+		"||accountkit.com^",
+		"||fb.me^",
+		"||fb.com^",
+		"||fbsbx.com^",
+		"||messenger.com^",
+		"||facebookcorewwwi.onion^",
+		"||fbcdn.com^",
+	}},
+	{"twitter", []string{"||twitter.com^", "||twttr.com^", "||t.co^", "||twimg.com^"}},
+	{"youtube", []string{
+		"||youtube.com^",
+		"||ytimg.com^",
+		"||youtu.be^",
+		"||googlevideo.com^",
+		"||youtubei.googleapis.com^",
+		"||youtube-nocookie.com^",
+		"||youtube",
+	}},
+	{"twitch", []string{"||twitch.tv^", "||ttvnw.net^", "||jtvnw.net^", "||twitchcdn.net^"}},
+	{"netflix", []string{"||nflxext.com^", "||netflix.com^", "||nflximg.net^", "||nflxvideo.net^", "||nflxso.net^"}},
+	{"instagram", []string{"||instagram.com^", "||cdninstagram.com^"}},
+	{"snapchat", []string{
+		"||snapchat.com^",
+		"||sc-cdn.net^",
+		"||snap-dev.net^",
+		"||snapkit.co",
+		"||snapads.com^",
+		"||impala-media-production.s3.amazonaws.com^",
+	}},
+	{"discord", []string{"||discord.gg^", "||discordapp.net^", "||discordapp.com^", "||discord.com^", "||discord.media^"}},
+	{"ok", []string{"||ok.ru^"}},
+	{"skype", []string{"||skype.com^", "||skypeassets.com^"}},
+	{"vk", []string{"||vk.com^", "||userapi.com^", "||vk-cdn.net^", "||vkuservideo.net^"}},
+	{"origin", []string{"||origin.com^", "||signin.ea.com^", "||accounts.ea.com^"}},
+	{"steam", []string{
+		"||steam.com^",
+		"||steampowered.com^",
+		"||steamcommunity.com^",
+		"||steamstatic.com^",
+		"||steamstore-a.akamaihd.net^",
+		"||steamcdn-a.akamaihd.net^",
+	}},
+	{"epic_games", []string{"||epicgames.com^", "||easyanticheat.net^", "||easy.ac^", "||eac-cdn.com^"}},
+	{"reddit", []string{"||reddit.com^", "||redditstatic.com^", "||redditmedia.com^", "||redd.it^"}},
+	{"mail_ru", []string{"||mail.ru^"}},
+	{"cloudflare", []string{
+		"||cloudflare.com^",
+		"||cloudflare-dns.com^",
+		"||cloudflare.net^",
+		"||cloudflareinsights.com^",
+		"||cloudflarestream.com^",
+		"||cloudflareresolve.com^",
+		"||cloudflareclient.com^",
+		"||cloudflarebolt.com^",
+		"||cloudflarestatus.com^",
+		"||cloudflare.cn^",
+		"||one.one^",
+		"||warp.plus^",
+		"||1.1.1.1^",
+		"||dns4torpnlfs2ifuz2s2yf3fc7rdmsbhm6rw75euj35pac6ap25zgqad.onion^",
+	}},
+	{"amazon", []string{
+		"||amazon.com^",
+		"||media-amazon.com^",
+		"||primevideo.com^",
+		"||amazontrust.com^",
+		"||images-amazon.com^",
+		"||ssl-images-amazon.com^",
+		"||amazonpay.com^",
+		"||amazonpay.in^",
+		"||amazon-adsystem.com^",
+		"||a2z.com^",
+		"||amazon.ae^",
+		"||amazon.ca^",
+		"||amazon.cn^",
+		"||amazon.de^",
+		"||amazon.es^",
+		"||amazon.fr^",
+		"||amazon.in^",
+		"||amazon.it^",
+		"||amazon.nl^",
+		"||amazon.com.au^",
+		"||amazon.com.br^",
+		"||amazon.co.jp^",
+		"||amazon.com.mx^",
+		"||amazon.co.uk^",
+		"||createspace.com^",
+		"||aws",
+	}},
+	{"ebay", []string{
+		"||ebay.com^",
+		"||ebayimg.com^",
+		"||ebaystatic.com^",
+		"||ebaycdn.net^",
+		"||ebayinc.com^",
+		"||ebay.at^",
+		"||ebay.be^",
+		"||ebay.ca^",
+		"||ebay.ch^",
+		"||ebay.cn^",
+		"||ebay.de^",
+		"||ebay.es^",
+		"||ebay.fr^",
+		"||ebay.ie^",
+		"||ebay.in^",
+		"||ebay.it^",
+		"||ebay.ph^",
+		"||ebay.pl^",
+		"||ebay.nl^",
+		"||ebay.com.au^",
+		"||ebay.com.cn^",
+		"||ebay.com.hk^",
+		"||ebay.com.my^",
+		"||ebay.com.sg^",
+		"||ebay.co.uk^",
+	}},
+	{"tiktok", []string{
+		"||tiktok.com^",
+		"||tiktokcdn.com^",
+		"||musical.ly^",
+		"||snssdk.com^",
+		"||amemv.com^",
+		"||toutiao.com^",
+		"||ixigua.com^",
+		"||pstatp.com^",
+		"||ixiguavideo.com^",
+		"||toutiaocloud.com^",
+		"||toutiaocloud.net^",
+		"||bdurl.com^",
+		"||bytecdn.cn^",
+		"||byteimg.com^",
+		"||ixigua.com^",
+		"||muscdn.com^",
+		"||bytedance.map.fastly.net^",
+		"||douyin.com^",
+		"||tiktokv.com^",
+	}},
+	{"qq", []string{"||qq.com^", "||qqzaixian.com^"}},
+}
+
+// convert array to map
+func initBlockedServices() {
+	serviceRules = make(map[string][]*rules.NetworkRule)
+	for _, s := range serviceRulesArray {
+		netRules := []*rules.NetworkRule{}
+		for _, text := range s.rules {
+			rule, err := rules.NewNetworkRule(text, 0)
+			if err != nil {
+				log.Error("rules.NewNetworkRule: %s  rule: %s", err, text)
+				continue
+			}
+			netRules = append(netRules, rule)
+		}
+		serviceRules[s.name] = netRules
+	}
+}
+
+// BlockedSvcKnown - return TRUE if a blocked service name is known
+func BlockedSvcKnown(s string) bool {
+	_, ok := serviceRules[s]
+	return ok
+}
+
+// ApplyBlockedServices - set blocked services settings for this DNS request
+func (d *Dnsfilter) ApplyBlockedServices(setts *RequestFilteringSettings, list []string, global bool) {
+	setts.ServicesRules = []ServiceEntry{}
+	if global {
+		d.confLock.RLock()
+		defer d.confLock.RUnlock()
+		list = d.Config.BlockedServices
+	}
+	for _, name := range list {
+		rules, ok := serviceRules[name]
+
+		if !ok {
+			log.Error("unknown service name: %s", name)
+			continue
+		}
+
+		s := ServiceEntry{}
+		s.Name = name
+		s.Rules = rules
+		setts.ServicesRules = append(setts.ServicesRules, s)
+	}
+}
+
+func (d *Dnsfilter) handleBlockedServicesList(w http.ResponseWriter, r *http.Request) {
+	d.confLock.RLock()
+	list := d.Config.BlockedServices
+	d.confLock.RUnlock()
+
+	w.Header().Set("Content-Type", "application/json")
+	err := json.NewEncoder(w).Encode(list)
+	if err != nil {
+		httpError(r, w, http.StatusInternalServerError, "json.Encode: %s", err)
+		return
+	}
+}
+
+func (d *Dnsfilter) handleBlockedServicesSet(w http.ResponseWriter, r *http.Request) {
+	list := []string{}
+	err := json.NewDecoder(r.Body).Decode(&list)
+	if err != nil {
+		httpError(r, w, http.StatusBadRequest, "json.Decode: %s", err)
+		return
+	}
+
+	d.confLock.Lock()
+	d.Config.BlockedServices = list
+	d.confLock.Unlock()
+
+	log.Debug("Updated blocked services list: %d", len(list))
+
+	d.ConfigModified()
+}
+
+// registerBlockedServicesHandlers - register HTTP handlers
+func (d *Dnsfilter) registerBlockedServicesHandlers() {
+	d.Config.HTTPRegister("GET", "/control/blocked_services/list", d.handleBlockedServicesList)
+	d.Config.HTTPRegister("POST", "/control/blocked_services/set", d.handleBlockedServicesSet)
+}

internal/dnsfilter/dnsfilter.go

@@ -0,0 +1,756 @@
+package dnsfilter
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"runtime"
+	"runtime/debug"
+	"strings"
+	"sync"
+
+	"github.com/AdguardTeam/AdGuardHome/internal/util"
+	"github.com/AdguardTeam/dnsproxy/upstream"
+	"github.com/AdguardTeam/golibs/cache"
+	"github.com/AdguardTeam/golibs/log"
+	"github.com/AdguardTeam/urlfilter"
+	"github.com/AdguardTeam/urlfilter/filterlist"
+	"github.com/AdguardTeam/urlfilter/rules"
+	"github.com/miekg/dns"
+)
+
+// ServiceEntry - blocked service array element
+type ServiceEntry struct {
+	Name  string
+	Rules []*rules.NetworkRule
+}
+
+// RequestFilteringSettings is custom filtering settings
+type RequestFilteringSettings struct {
+	FilteringEnabled    bool
+	SafeSearchEnabled   bool
+	SafeBrowsingEnabled bool
+	ParentalEnabled     bool
+
+	ClientName string
+	ClientIP   string
+	ClientTags []string
+
+	ServicesRules []ServiceEntry
+}
+
+// Config allows you to configure DNS filtering with New() or just change variables directly.
+type Config struct {
+	ParentalEnabled     bool   `yaml:"parental_enabled"`
+	SafeSearchEnabled   bool   `yaml:"safesearch_enabled"`
+	SafeBrowsingEnabled bool   `yaml:"safebrowsing_enabled"`
+	ResolverAddress     string `yaml:"-"` // DNS server address
+
+	SafeBrowsingCacheSize uint `yaml:"safebrowsing_cache_size"` // (in bytes)
+	SafeSearchCacheSize   uint `yaml:"safesearch_cache_size"`   // (in bytes)
+	ParentalCacheSize     uint `yaml:"parental_cache_size"`     // (in bytes)
+	CacheTime             uint `yaml:"cache_time"`              // Element's TTL (in minutes)
+
+	Rewrites []RewriteEntry `yaml:"rewrites"`
+
+	// Names of services to block (globally).
+	// Per-client settings can override this configuration.
+	BlockedServices []string `yaml:"blocked_services"`
+
+	// IP-hostname pairs taken from system configuration (e.g. /etc/hosts) files
+	AutoHosts *util.AutoHosts `yaml:"-"`
+
+	// Called when the configuration is changed by HTTP request
+	ConfigModified func() `yaml:"-"`
+
+	// Register an HTTP handler
+	HTTPRegister func(string, string, func(http.ResponseWriter, *http.Request)) `yaml:"-"`
+}
+
+// LookupStats store stats collected during safebrowsing or parental checks
+type LookupStats struct {
+	Requests   uint64 // number of HTTP requests that were sent
+	CacheHits  uint64 // number of lookups that didn't need HTTP requests
+	Pending    int64  // number of currently pending HTTP requests
+	PendingMax int64  // maximum number of pending HTTP requests
+}
+
+// Stats store LookupStats for safebrowsing, parental and safesearch
+type Stats struct {
+	Safebrowsing LookupStats
+	Parental     LookupStats
+	Safesearch   LookupStats
+}
+
+// Parameters to pass to filters-initializer goroutine
+type filtersInitializerParams struct {
+	allowFilters []Filter
+	blockFilters []Filter
+}
+
+// Dnsfilter holds added rules and performs hostname matches against the rules
+type Dnsfilter struct {
+	rulesStorage         *filterlist.RuleStorage
+	filteringEngine      *urlfilter.DNSEngine
+	rulesStorageWhite    *filterlist.RuleStorage
+	filteringEngineWhite *urlfilter.DNSEngine
+	engineLock           sync.RWMutex
+
+	parentalServer       string // access via methods
+	safeBrowsingServer   string // access via methods
+	parentalUpstream     upstream.Upstream
+	safeBrowsingUpstream upstream.Upstream
+
+	Config   // for direct access by library users, even a = assignment
+	confLock sync.RWMutex
+
+	// Channel for passing data to filters-initializer goroutine
+	filtersInitializerChan chan filtersInitializerParams
+	filtersInitializerLock sync.Mutex
+}
+
+// Filter represents a filter list
+type Filter struct {
+	ID       int64  // auto-assigned when filter is added (see nextFilterID)
+	Data     []byte `yaml:"-"` // List of rules divided by '\n'
+	FilePath string `yaml:"-"` // Path to a filtering rules file
+}
+
+// Reason holds an enum detailing why it was filtered or not filtered
+type Reason int
+
+const (
+	// reasons for not filtering
+
+	// NotFilteredNotFound - host was not find in any checks, default value for result
+	NotFilteredNotFound Reason = iota
+	// NotFilteredWhiteList - the host is explicitly whitelisted
+	NotFilteredWhiteList
+	// NotFilteredError - there was a transitive error during check
+	NotFilteredError
+
+	// reasons for filtering
+
+	// FilteredBlackList - the host was matched to be advertising host
+	FilteredBlackList
+	// FilteredSafeBrowsing - the host was matched to be malicious/phishing
+	FilteredSafeBrowsing
+	// FilteredParental - the host was matched to be outside of parental control settings
+	FilteredParental
+	// FilteredInvalid - the request was invalid and was not processed
+	FilteredInvalid
+	// FilteredSafeSearch - the host was replaced with safesearch variant
+	FilteredSafeSearch
+	// FilteredBlockedService - the host is blocked by "blocked services" settings
+	FilteredBlockedService
+
+	// ReasonRewrite - rewrite rule was applied
+	ReasonRewrite
+
+	// RewriteEtcHosts - rewrite by /etc/hosts rule
+	RewriteEtcHosts
+)
+
+var reasonNames = []string{
+	"NotFilteredNotFound",
+	"NotFilteredWhiteList",
+	"NotFilteredError",
+
+	"FilteredBlackList",
+	"FilteredSafeBrowsing",
+	"FilteredParental",
+	"FilteredInvalid",
+	"FilteredSafeSearch",
+	"FilteredBlockedService",
+
+	"Rewrite",
+	"RewriteEtcHosts",
+}
+
+func (r Reason) String() string {
+	if uint(r) >= uint(len(reasonNames)) {
+		return ""
+	}
+	return reasonNames[r]
+}
+
+// GetConfig - get configuration
+func (d *Dnsfilter) GetConfig() RequestFilteringSettings {
+	c := RequestFilteringSettings{}
+	// d.confLock.RLock()
+	c.SafeSearchEnabled = d.Config.SafeSearchEnabled
+	c.SafeBrowsingEnabled = d.Config.SafeBrowsingEnabled
+	c.ParentalEnabled = d.Config.ParentalEnabled
+	// d.confLock.RUnlock()
+	return c
+}
+
+// WriteDiskConfig - write configuration
+func (d *Dnsfilter) WriteDiskConfig(c *Config) {
+	d.confLock.Lock()
+	*c = d.Config
+	c.Rewrites = rewriteArrayDup(d.Config.Rewrites)
+	// BlockedServices
+	d.confLock.Unlock()
+}
+
+// SetFilters - set new filters (synchronously or asynchronously)
+// When filters are set asynchronously, the old filters continue working until the new filters are ready.
+//  In this case the caller must ensure that the old filter files are intact.
+func (d *Dnsfilter) SetFilters(blockFilters []Filter, allowFilters []Filter, async bool) error {
+	if async {
+		params := filtersInitializerParams{
+			allowFilters: allowFilters,
+			blockFilters: blockFilters,
+		}
+
+		d.filtersInitializerLock.Lock() // prevent multiple writers from adding more than 1 task
+		// remove all pending tasks
+		stop := false
+		for !stop {
+			select {
+			case <-d.filtersInitializerChan:
+				//
+			default:
+				stop = true
+			}
+		}
+
+		d.filtersInitializerChan <- params
+		d.filtersInitializerLock.Unlock()
+		return nil
+	}
+
+	err := d.initFiltering(allowFilters, blockFilters)
+	if err != nil {
+		log.Error("Can't initialize filtering subsystem: %s", err)
+		return err
+	}
+
+	return nil
+}
+
+// Starts initializing new filters by signal from channel
+func (d *Dnsfilter) filtersInitializer() {
+	for {
+		params := <-d.filtersInitializerChan
+		err := d.initFiltering(params.allowFilters, params.blockFilters)
+		if err != nil {
+			log.Error("Can't initialize filtering subsystem: %s", err)
+			continue
+		}
+	}
+}
+
+// Close - close the object
+func (d *Dnsfilter) Close() {
+	d.engineLock.Lock()
+	defer d.engineLock.Unlock()
+	d.reset()
+}
+
+func (d *Dnsfilter) reset() {
+	if d.rulesStorage != nil {
+		_ = d.rulesStorage.Close()
+	}
+	if d.rulesStorageWhite != nil {
+		d.rulesStorageWhite.Close()
+	}
+}
+
+type dnsFilterContext struct {
+	stats             Stats
+	safebrowsingCache cache.Cache
+	parentalCache     cache.Cache
+	safeSearchCache   cache.Cache
+}
+
+var gctx dnsFilterContext // global dnsfilter context
+
+// Result holds state of hostname check
+type Result struct {
+	IsFiltered bool   `json:",omitempty"` // True if the host name is filtered
+	Reason     Reason `json:",omitempty"` // Reason for blocking / unblocking
+	Rule       string `json:",omitempty"` // Original rule text
+	IP         net.IP `json:",omitempty"` // Not nil only in the case of a hosts file syntax
+	FilterID   int64  `json:",omitempty"` // Filter ID the rule belongs to
+
+	// for ReasonRewrite:
+	CanonName string `json:",omitempty"` // CNAME value
+
+	// for RewriteEtcHosts:
+	ReverseHost string `json:",omitempty"`
+
+	// for ReasonRewrite & RewriteEtcHosts:
+	IPList []net.IP `json:",omitempty"` // list of IP addresses
+
+	// for FilteredBlockedService:
+	ServiceName string `json:",omitempty"` // Name of the blocked service
+}
+
+// Matched can be used to see if any match at all was found, no matter filtered or not
+func (r Reason) Matched() bool {
+	return r != NotFilteredNotFound
+}
+
+// CheckHostRules tries to match the host against filtering rules only
+func (d *Dnsfilter) CheckHostRules(host string, qtype uint16, setts *RequestFilteringSettings) (Result, error) {
+	if !setts.FilteringEnabled {
+		return Result{}, nil
+	}
+
+	return d.matchHost(host, qtype, *setts)
+}
+
+// CheckHost tries to match the host against filtering rules,
+// then safebrowsing and parental if they are enabled
+func (d *Dnsfilter) CheckHost(host string, qtype uint16, setts *RequestFilteringSettings) (Result, error) {
+	// sometimes DNS clients will try to resolve ".", which is a request to get root servers
+	if host == "" {
+		return Result{Reason: NotFilteredNotFound}, nil
+	}
+	host = strings.ToLower(host)
+
+	var result Result
+	var err error
+
+	// first - check rewrites, they have the highest priority
+	result = d.processRewrites(host, qtype)
+	if result.Reason == ReasonRewrite {
+		return result, nil
+	}
+
+	// Now check the hosts file -- do we have any rules for it?
+	// just like DNS rewrites, it has higher priority than filtering rules.
+	if d.Config.AutoHosts != nil {
+		ips := d.Config.AutoHosts.Process(host, qtype)
+		if ips != nil {
+			result.Reason = RewriteEtcHosts
+			result.IPList = ips
+			return result, nil
+		}
+
+		revHost := d.Config.AutoHosts.ProcessReverse(host, qtype)
+		if len(revHost) != 0 {
+			result.Reason = RewriteEtcHosts
+			result.ReverseHost = revHost + "."
+			return result, nil
+		}
+	}
+
+	// Then check the filter lists.
+	// if request is blocked -- it should be blocked.
+	// if it is whitelisted -- we should do nothing with it anymore.
+	if setts.FilteringEnabled {
+		result, err = d.matchHost(host, qtype, *setts)
+		if err != nil {
+			return result, err
+		}
+		if result.Reason.Matched() {
+			return result, nil
+		}
+	}
+
+	// are there any blocked services?
+	if len(setts.ServicesRules) != 0 {
+		result = matchBlockedServicesRules(host, setts.ServicesRules)
+		if result.Reason.Matched() {
+			return result, nil
+		}
+	}
+
+	// browsing security web service
+	if setts.SafeBrowsingEnabled {
+		result, err = d.checkSafeBrowsing(host)
+		if err != nil {
+			log.Info("SafeBrowsing: failed: %v", err)
+			return Result{}, nil
+		}
+		if result.Reason.Matched() {
+			return result, nil
+		}
+	}
+
+	// parental control web service
+	if setts.ParentalEnabled {
+		result, err = d.checkParental(host)
+		if err != nil {
+			log.Printf("Parental: failed: %v", err)
+			return Result{}, nil
+		}
+		if result.Reason.Matched() {
+			return result, nil
+		}
+	}
+
+	// apply safe search if needed
+	if setts.SafeSearchEnabled {
+		result, err = d.checkSafeSearch(host)
+		if err != nil {
+			log.Info("SafeSearch: failed: %v", err)
+			return Result{}, nil
+		}
+
+		if result.Reason.Matched() {
+			return result, nil
+		}
+	}
+
+	return Result{}, nil
+}
+
+// Process rewrites table
+// . Find CNAME for a domain name (exact match or by wildcard)
+//  . if found and CNAME equals to domain name - this is an exception;  exit
+//  . if found, set domain name to canonical name
+//  . repeat for the new domain name (Note: we return only the last CNAME)
+// . Find A or AAAA record for a domain name (exact match or by wildcard)
+//  . if found, set IP addresses (IPv4 or IPv6 depending on qtype) in Result.IPList array
+func (d *Dnsfilter) processRewrites(host string, qtype uint16) Result {
+	var res Result
+
+	d.confLock.RLock()
+	defer d.confLock.RUnlock()
+
+	rr := findRewrites(d.Rewrites, host)
+	if len(rr) != 0 {
+		res.Reason = ReasonRewrite
+	}
+
+	cnames := map[string]bool{}
+	origHost := host
+	for len(rr) != 0 && rr[0].Type == dns.TypeCNAME {
+		log.Debug("Rewrite: CNAME for %s is %s", host, rr[0].Answer)
+
+		if host == rr[0].Answer { // "host == CNAME" is an exception
+			res.Reason = 0
+			return res
+		}
+
+		host = rr[0].Answer
+		_, ok := cnames[host]
+		if ok {
+			log.Info("Rewrite: breaking CNAME redirection loop: %s.  Question: %s", host, origHost)
+			return res
+		}
+		cnames[host] = false
+		res.CanonName = rr[0].Answer
+		rr = findRewrites(d.Rewrites, host)
+	}
+
+	for _, r := range rr {
+		if (r.Type == dns.TypeA && qtype == dns.TypeA) ||
+			(r.Type == dns.TypeAAAA && qtype == dns.TypeAAAA) {
+
+			if r.IP == nil { // IP exception
+				res.Reason = 0
+				return res
+			}
+
+			res.IPList = append(res.IPList, r.IP)
+			log.Debug("Rewrite: A/AAAA for %s is %s", host, r.IP)
+		}
+	}
+
+	return res
+}
+
+func matchBlockedServicesRules(host string, svcs []ServiceEntry) Result {
+	req := rules.NewRequestForHostname(host)
+	res := Result{}
+
+	for _, s := range svcs {
+		for _, rule := range s.Rules {
+			if rule.Match(req) {
+				res.Reason = FilteredBlockedService
+				res.IsFiltered = true
+				res.ServiceName = s.Name
+				res.Rule = rule.Text()
+				log.Debug("Blocked Services: matched rule: %s  host: %s  service: %s",
+					res.Rule, host, s.Name)
+				return res
+			}
+		}
+	}
+	return res
+}
+
+//
+// Adding rule and matching against the rules
+//
+
+// Return TRUE if file exists
+func fileExists(fn string) bool {
+	_, err := os.Stat(fn)
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+func createFilteringEngine(filters []Filter) (*filterlist.RuleStorage, *urlfilter.DNSEngine, error) {
+	listArray := []filterlist.RuleList{}
+	for _, f := range filters {
+		var list filterlist.RuleList
+
+		if f.ID == 0 {
+			list = &filterlist.StringRuleList{
+				ID:             0,
+				RulesText:      string(f.Data),
+				IgnoreCosmetic: true,
+			}
+
+		} else if !fileExists(f.FilePath) {
+			list = &filterlist.StringRuleList{
+				ID:             int(f.ID),
+				IgnoreCosmetic: true,
+			}
+
+		} else if runtime.GOOS == "windows" {
+			// On Windows we don't pass a file to urlfilter because
+			//  it's difficult to update this file while it's being used.
+			data, err := ioutil.ReadFile(f.FilePath)
+			if err != nil {
+				return nil, nil, fmt.Errorf("ioutil.ReadFile(): %s: %s", f.FilePath, err)
+			}
+			list = &filterlist.StringRuleList{
+				ID:             int(f.ID),
+				RulesText:      string(data),
+				IgnoreCosmetic: true,
+			}
+
+		} else {
+			var err error
+			list, err = filterlist.NewFileRuleList(int(f.ID), f.FilePath, true)
+			if err != nil {
+				return nil, nil, fmt.Errorf("filterlist.NewFileRuleList(): %s: %s", f.FilePath, err)
+			}
+		}
+		listArray = append(listArray, list)
+	}
+
+	rulesStorage, err := filterlist.NewRuleStorage(listArray)
+	if err != nil {
+		return nil, nil, fmt.Errorf("filterlist.NewRuleStorage(): %s", err)
+	}
+	filteringEngine := urlfilter.NewDNSEngine(rulesStorage)
+	return rulesStorage, filteringEngine, nil
+}
+
+// Initialize urlfilter objects
+func (d *Dnsfilter) initFiltering(allowFilters, blockFilters []Filter) error {
+	rulesStorage, filteringEngine, err := createFilteringEngine(blockFilters)
+	if err != nil {
+		return err
+	}
+	rulesStorageWhite, filteringEngineWhite, err := createFilteringEngine(allowFilters)
+	if err != nil {
+		return err
+	}
+
+	d.engineLock.Lock()
+	d.reset()
+	d.rulesStorage = rulesStorage
+	d.filteringEngine = filteringEngine
+	d.rulesStorageWhite = rulesStorageWhite
+	d.filteringEngineWhite = filteringEngineWhite
+	d.engineLock.Unlock()
+
+	// Make sure that the OS reclaims memory as soon as possible
+	debug.FreeOSMemory()
+	log.Debug("initialized filtering engine")
+
+	return nil
+}
+
+// matchHost is a low-level way to check only if hostname is filtered by rules, skipping expensive safebrowsing and parental lookups
+func (d *Dnsfilter) matchHost(host string, qtype uint16, setts RequestFilteringSettings) (Result, error) {
+	d.engineLock.RLock()
+	// Keep in mind that this lock must be held no just when calling Match()
+	//  but also while using the rules returned by it.
+	defer d.engineLock.RUnlock()
+
+	ureq := urlfilter.DNSRequest{}
+	ureq.Hostname = host
+	ureq.ClientIP = setts.ClientIP
+	ureq.ClientName = setts.ClientName
+	ureq.SortedClientTags = setts.ClientTags
+
+	if d.filteringEngineWhite != nil {
+		rr, ok := d.filteringEngineWhite.MatchRequest(ureq)
+		if ok {
+			var rule rules.Rule
+			if rr.NetworkRule != nil {
+				rule = rr.NetworkRule
+			} else if rr.HostRulesV4 != nil {
+				rule = rr.HostRulesV4[0]
+			} else if rr.HostRulesV6 != nil {
+				rule = rr.HostRulesV6[0]
+			}
+
+			log.Debug("Filtering: found whitelist rule for host '%s': '%s'  list_id: %d",
+				host, rule.Text(), rule.GetFilterListID())
+			res := makeResult(rule, NotFilteredWhiteList)
+			return res, nil
+		}
+	}
+
+	if d.filteringEngine == nil {
+		return Result{}, nil
+	}
+
+	rr, ok := d.filteringEngine.MatchRequest(ureq)
+	if !ok {
+		return Result{}, nil
+	}
+
+	if rr.NetworkRule != nil {
+		log.Debug("Filtering: found rule for host '%s': '%s'  list_id: %d",
+			host, rr.NetworkRule.Text(), rr.NetworkRule.GetFilterListID())
+		reason := FilteredBlackList
+		if rr.NetworkRule.Whitelist {
+			reason = NotFilteredWhiteList
+		}
+		res := makeResult(rr.NetworkRule, reason)
+		return res, nil
+	}
+
+	if qtype == dns.TypeA && rr.HostRulesV4 != nil {
+		rule := rr.HostRulesV4[0] // note that we process only 1 matched rule
+		log.Debug("Filtering: found rule for host '%s': '%s'  list_id: %d",
+			host, rule.Text(), rule.GetFilterListID())
+		res := makeResult(rule, FilteredBlackList)
+		res.IP = rule.IP.To4()
+		return res, nil
+	}
+
+	if qtype == dns.TypeAAAA && rr.HostRulesV6 != nil {
+		rule := rr.HostRulesV6[0] // note that we process only 1 matched rule
+		log.Debug("Filtering: found rule for host '%s': '%s'  list_id: %d",
+			host, rule.Text(), rule.GetFilterListID())
+		res := makeResult(rule, FilteredBlackList)
+		res.IP = rule.IP
+		return res, nil
+	}
+
+	if rr.HostRulesV4 != nil || rr.HostRulesV6 != nil {
+		// Question Type doesn't match the host rules
+		// Return the first matched host rule, but without an IP address
+		var rule rules.Rule
+		if rr.HostRulesV4 != nil {
+			rule = rr.HostRulesV4[0]
+		} else if rr.HostRulesV6 != nil {
+			rule = rr.HostRulesV6[0]
+		}
+		log.Debug("Filtering: found rule for host '%s': '%s'  list_id: %d",
+			host, rule.Text(), rule.GetFilterListID())
+		res := makeResult(rule, FilteredBlackList)
+		res.IP = net.IP{}
+		return res, nil
+	}
+
+	return Result{}, nil
+}
+
+// Construct Result object
+func makeResult(rule rules.Rule, reason Reason) Result {
+	res := Result{}
+	res.FilterID = int64(rule.GetFilterListID())
+	res.Rule = rule.Text()
+	res.Reason = reason
+	if reason == FilteredBlackList {
+		res.IsFiltered = true
+	}
+	return res
+}
+
+// InitModule() - manually initialize blocked services map
+func InitModule() {
+	initBlockedServices()
+}
+
+// New creates properly initialized DNS Filter that is ready to be used
+func New(c *Config, blockFilters []Filter) *Dnsfilter {
+
+	if c != nil {
+		cacheConf := cache.Config{
+			EnableLRU: true,
+		}
+
+		// initialize objects only once
+
+		if gctx.safebrowsingCache == nil {
+			cacheConf.MaxSize = c.SafeBrowsingCacheSize
+			gctx.safebrowsingCache = cache.New(cacheConf)
+		}
+
+		if gctx.safeSearchCache == nil {
+			cacheConf.MaxSize = c.SafeSearchCacheSize
+			gctx.safeSearchCache = cache.New(cacheConf)
+		}
+
+		if gctx.parentalCache == nil {
+			cacheConf.MaxSize = c.ParentalCacheSize
+			gctx.parentalCache = cache.New(cacheConf)
+		}
+	}
+
+	d := new(Dnsfilter)
+
+	err := d.initSecurityServices()
+	if err != nil {
+		log.Error("dnsfilter: initialize services: %s", err)
+		return nil
+	}
+
+	if c != nil {
+		d.Config = *c
+		d.prepareRewrites()
+	}
+
+	bsvcs := []string{}
+	for _, s := range d.BlockedServices {
+		if !BlockedSvcKnown(s) {
+			log.Debug("skipping unknown blocked-service '%s'", s)
+			continue
+		}
+		bsvcs = append(bsvcs, s)
+	}
+	d.BlockedServices = bsvcs
+
+	if blockFilters != nil {
+		err := d.initFiltering(nil, blockFilters)
+		if err != nil {
+			log.Error("Can't initialize filtering subsystem: %s", err)
+			d.Close()
+			return nil
+		}
+	}
+
+	return d
+}
+
+// Start - start the module:
+// . start async filtering initializer goroutine
+// . register web handlers
+func (d *Dnsfilter) Start() {
+	d.filtersInitializerChan = make(chan filtersInitializerParams, 1)
+	go d.filtersInitializer()
+
+	if d.Config.HTTPRegister != nil { // for tests
+		d.registerSecurityHandlers()
+		d.registerRewritesHandlers()
+		d.registerBlockedServicesHandlers()
+	}
+}
+
+//
+// stats
+//
+
+// GetStats return dns filtering stats since startup
+func (d *Dnsfilter) GetStats() Stats {
+	return gctx.stats
+}

internal/dnsfilter/dnsfilter_test.go

@@ -0,0 +1,604 @@
+package dnsfilter
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"path"
+	"runtime"
+	"testing"
+
+	"github.com/AdguardTeam/urlfilter/rules"
+	"github.com/miekg/dns"
+	"github.com/stretchr/testify/assert"
+)
+
+var setts RequestFilteringSettings
+
+// HELPERS
+// SAFE BROWSING
+// SAFE SEARCH
+// PARENTAL
+// FILTERING
+// BENCHMARKS
+
+// HELPERS
+
+func purgeCaches() {
+	if gctx.safebrowsingCache != nil {
+		gctx.safebrowsingCache.Clear()
+	}
+	if gctx.parentalCache != nil {
+		gctx.parentalCache.Clear()
+	}
+	if gctx.safeSearchCache != nil {
+		gctx.safeSearchCache.Clear()
+	}
+}
+
+func _Func() string {
+	pc := make([]uintptr, 10) // at least 1 entry needed
+	runtime.Callers(2, pc)
+	f := runtime.FuncForPC(pc[0])
+	return path.Base(f.Name())
+}
+
+func NewForTest(c *Config, filters []Filter) *Dnsfilter {
+	setts = RequestFilteringSettings{}
+	setts.FilteringEnabled = true
+	if c != nil {
+		c.SafeBrowsingCacheSize = 10000
+		c.ParentalCacheSize = 10000
+		c.SafeSearchCacheSize = 1000
+		c.CacheTime = 30
+		setts.SafeSearchEnabled = c.SafeSearchEnabled
+		setts.SafeBrowsingEnabled = c.SafeBrowsingEnabled
+		setts.ParentalEnabled = c.ParentalEnabled
+	}
+	d := New(c, filters)
+	purgeCaches()
+	return d
+}
+
+func (d *Dnsfilter) checkMatch(t *testing.T, hostname string) {
+	t.Helper()
+	ret, err := d.CheckHost(hostname, dns.TypeA, &setts)
+	if err != nil {
+		t.Errorf("Error while matching host %s: %s", hostname, err)
+	}
+	if !ret.IsFiltered {
+		t.Errorf("Expected hostname %s to match", hostname)
+	}
+}
+
+func (d *Dnsfilter) checkMatchIP(t *testing.T, hostname string, ip string, qtype uint16) {
+	t.Helper()
+	ret, err := d.CheckHost(hostname, qtype, &setts)
+	if err != nil {
+		t.Errorf("Error while matching host %s: %s", hostname, err)
+	}
+	if !ret.IsFiltered {
+		t.Errorf("Expected hostname %s to match", hostname)
+	}
+	if ret.IP == nil || ret.IP.String() != ip {
+		t.Errorf("Expected ip %s to match, actual: %v", ip, ret.IP)
+	}
+}
+
+func (d *Dnsfilter) checkMatchEmpty(t *testing.T, hostname string) {
+	t.Helper()
+	ret, err := d.CheckHost(hostname, dns.TypeA, &setts)
+	if err != nil {
+		t.Errorf("Error while matching host %s: %s", hostname, err)
+	}
+	if ret.IsFiltered {
+		t.Errorf("Expected hostname %s to not match", hostname)
+	}
+}
+
+func TestEtcHostsMatching(t *testing.T) {
+	addr := "216.239.38.120"
+	addr6 := "::1"
+	text := fmt.Sprintf(`  %s  google.com www.google.com   # enforce google's safesearch
+%s  ipv6.com
+0.0.0.0 block.com
+0.0.0.1 host2
+0.0.0.2 host2
+::1 host2
+`,
+		addr, addr6)
+	filters := []Filter{Filter{
+		ID: 0, Data: []byte(text),
+	}}
+	d := NewForTest(nil, filters)
+	defer d.Close()
+
+	d.checkMatchIP(t, "google.com", addr, dns.TypeA)
+	d.checkMatchIP(t, "www.google.com", addr, dns.TypeA)
+	d.checkMatchEmpty(t, "subdomain.google.com")
+	d.checkMatchEmpty(t, "example.org")
+
+	// IPv4
+	d.checkMatchIP(t, "block.com", "0.0.0.0", dns.TypeA)
+
+	// ...but empty IPv6
+	ret, err := d.CheckHost("block.com", dns.TypeAAAA, &setts)
+	assert.True(t, err == nil && ret.IsFiltered && ret.IP != nil && len(ret.IP) == 0)
+	assert.True(t, ret.Rule == "0.0.0.0 block.com")
+
+	// IPv6
+	d.checkMatchIP(t, "ipv6.com", addr6, dns.TypeAAAA)
+
+	// ...but empty IPv4
+	ret, err = d.CheckHost("ipv6.com", dns.TypeA, &setts)
+	assert.True(t, err == nil && ret.IsFiltered && ret.IP != nil && len(ret.IP) == 0)
+
+	// 2 IPv4 (return only the first one)
+	ret, err = d.CheckHost("host2", dns.TypeA, &setts)
+	assert.True(t, err == nil && ret.IsFiltered)
+	assert.True(t, ret.IP != nil && ret.IP.Equal(net.ParseIP("0.0.0.1")))
+
+	// ...and 1 IPv6 address
+	ret, err = d.CheckHost("host2", dns.TypeAAAA, &setts)
+	assert.True(t, err == nil && ret.IsFiltered)
+	assert.True(t, ret.IP != nil && ret.IP.Equal(net.ParseIP("::1")))
+}
+
+// SAFE BROWSING
+
+func TestSafeBrowsing(t *testing.T) {
+	d := NewForTest(&Config{SafeBrowsingEnabled: true}, nil)
+	defer d.Close()
+	gctx.stats.Safebrowsing.Requests = 0
+	d.checkMatch(t, "wmconvirus.narod.ru")
+	d.checkMatch(t, "test.wmconvirus.narod.ru")
+	d.checkMatchEmpty(t, "yandex.ru")
+	d.checkMatchEmpty(t, "pornhub.com")
+
+	// test cached result
+	d.safeBrowsingServer = "127.0.0.1"
+	d.checkMatch(t, "wmconvirus.narod.ru")
+	d.checkMatchEmpty(t, "pornhub.com")
+	d.safeBrowsingServer = defaultSafebrowsingServer
+}
+
+func TestParallelSB(t *testing.T) {
+	d := NewForTest(&Config{SafeBrowsingEnabled: true}, nil)
+	defer d.Close()
+	t.Run("group", func(t *testing.T) {
+		for i := 0; i < 100; i++ {
+			t.Run(fmt.Sprintf("aaa%d", i), func(t *testing.T) {
+				t.Parallel()
+				d.checkMatch(t, "wmconvirus.narod.ru")
+				d.checkMatch(t, "test.wmconvirus.narod.ru")
+				d.checkMatchEmpty(t, "yandex.ru")
+				d.checkMatchEmpty(t, "pornhub.com")
+			})
+		}
+	})
+}
+
+// SAFE SEARCH
+
+func TestSafeSearch(t *testing.T) {
+	d := NewForTest(&Config{SafeSearchEnabled: true}, nil)
+	defer d.Close()
+	val, ok := d.SafeSearchDomain("www.google.com")
+	if !ok {
+		t.Errorf("Expected safesearch to find result for www.google.com")
+	}
+	if val != "forcesafesearch.google.com" {
+		t.Errorf("Expected safesearch for google.com to be forcesafesearch.google.com")
+	}
+}
+
+func TestCheckHostSafeSearchYandex(t *testing.T) {
+	d := NewForTest(&Config{SafeSearchEnabled: true}, nil)
+	defer d.Close()
+
+	// Slice of yandex domains
+	yandex := []string{"yAndeX.ru", "YANdex.COM", "yandex.ua", "yandex.by", "yandex.kz", "www.yandex.com"}
+
+	// Check host for each domain
+	for _, host := range yandex {
+		result, err := d.CheckHost(host, dns.TypeA, &setts)
+		if err != nil {
+			t.Errorf("SafeSearch doesn't work for yandex domain `%s` cause %s", host, err)
+		}
+
+		if result.IP.String() != "213.180.193.56" {
+			t.Errorf("SafeSearch doesn't work for yandex domain `%s`", host)
+		}
+	}
+}
+
+func TestCheckHostSafeSearchGoogle(t *testing.T) {
+	d := NewForTest(&Config{SafeSearchEnabled: true}, nil)
+	defer d.Close()
+
+	// Slice of google domains
+	googleDomains := []string{"www.google.com", "www.google.im", "www.google.co.in", "www.google.iq", "www.google.is", "www.google.it", "www.google.je"}
+
+	// Check host for each domain
+	for _, host := range googleDomains {
+		result, err := d.CheckHost(host, dns.TypeA, &setts)
+		if err != nil {
+			t.Errorf("SafeSearch doesn't work for %s cause %s", host, err)
+		}
+
+		if result.IP == nil {
+			t.Errorf("SafeSearch doesn't work for %s", host)
+		}
+	}
+}
+
+func TestSafeSearchCacheYandex(t *testing.T) {
+	d := NewForTest(nil, nil)
+	defer d.Close()
+	domain := "yandex.ru"
+
+	var result Result
+	var err error
+
+	// Check host with disabled safesearch
+	result, err = d.CheckHost(domain, dns.TypeA, &setts)
+	if err != nil {
+		t.Fatalf("Cannot check host due to %s", err)
+	}
+	if result.IP != nil {
+		t.Fatalf("SafeSearch is not enabled but there is an answer for `%s` !", domain)
+	}
+
+	d = NewForTest(&Config{SafeSearchEnabled: true}, nil)
+	defer d.Close()
+
+	result, err = d.CheckHost(domain, dns.TypeA, &setts)
+	if err != nil {
+		t.Fatalf("CheckHost for safesearh domain %s failed cause %s", domain, err)
+	}
+
+	// Fir yandex we already know valid ip
+	if result.IP.String() != "213.180.193.56" {
+		t.Fatalf("Wrong IP for %s safesearch: %s", domain, result.IP.String())
+	}
+
+	// Check cache
+	cachedValue, isFound := getCachedResult(gctx.safeSearchCache, domain)
+
+	if !isFound {
+		t.Fatalf("Safesearch cache doesn't work for %s!", domain)
+	}
+
+	if cachedValue.IP.String() != "213.180.193.56" {
+		t.Fatalf("Wrong IP in cache for %s safesearch: %s", domain, cachedValue.IP.String())
+	}
+}
+
+func TestSafeSearchCacheGoogle(t *testing.T) {
+	d := NewForTest(nil, nil)
+	defer d.Close()
+	domain := "www.google.ru"
+	result, err := d.CheckHost(domain, dns.TypeA, &setts)
+	if err != nil {
+		t.Fatalf("Cannot check host due to %s", err)
+	}
+	if result.IP != nil {
+		t.Fatalf("SafeSearch is not enabled but there is an answer!")
+	}
+
+	d = NewForTest(&Config{SafeSearchEnabled: true}, nil)
+	defer d.Close()
+
+	// Let's lookup for safesearch domain
+	safeDomain, ok := d.SafeSearchDomain(domain)
+	if !ok {
+		t.Fatalf("Failed to get safesearch domain for %s", domain)
+	}
+
+	ips, err := net.LookupIP(safeDomain)
+	if err != nil {
+		t.Fatalf("Failed to lookup for %s", safeDomain)
+	}
+
+	t.Logf("IP addresses: %v", ips)
+	ip := ips[0]
+	for _, i := range ips {
+		if i.To4() != nil {
+			ip = i
+			break
+		}
+	}
+
+	result, err = d.CheckHost(domain, dns.TypeA, &setts)
+	if err != nil {
+		t.Fatalf("CheckHost for safesearh domain %s failed cause %s", domain, err)
+	}
+
+	if result.IP.String() != ip.String() {
+		t.Fatalf("Wrong IP for %s safesearch: %s.  Should be: %s",
+			domain, result.IP.String(), ip)
+	}
+
+	// Check cache
+	cachedValue, isFound := getCachedResult(gctx.safeSearchCache, domain)
+
+	if !isFound {
+		t.Fatalf("Safesearch cache doesn't work for %s!", domain)
+	}
+
+	if cachedValue.IP.String() != ip.String() {
+		t.Fatalf("Wrong IP in cache for %s safesearch: %s", domain, cachedValue.IP.String())
+	}
+}
+
+// PARENTAL
+
+func TestParentalControl(t *testing.T) {
+	d := NewForTest(&Config{ParentalEnabled: true}, nil)
+	defer d.Close()
+	d.checkMatch(t, "pornhub.com")
+	d.checkMatch(t, "www.pornhub.com")
+	d.checkMatchEmpty(t, "www.yandex.ru")
+	d.checkMatchEmpty(t, "yandex.ru")
+	d.checkMatchEmpty(t, "api.jquery.com")
+
+	// test cached result
+	d.parentalServer = "127.0.0.1"
+	d.checkMatch(t, "pornhub.com")
+	d.checkMatchEmpty(t, "yandex.ru")
+	d.parentalServer = defaultParentalServer
+}
+
+// FILTERING
+
+var blockingRules = "||example.org^\n"
+var whitelistRules = "||example.org^\n@@||test.example.org\n"
+var importantRules = "@@||example.org^\n||test.example.org^$important\n"
+var regexRules = "/example\\.org/\n@@||test.example.org^\n"
+var maskRules = "test*.example.org^\nexam*.com\n"
+
+var tests = []struct {
+	testname   string
+	rules      string
+	hostname   string
+	isFiltered bool
+	reason     Reason
+}{
+	{"sanity", "||doubleclick.net^", "www.doubleclick.net", true, FilteredBlackList},
+	{"sanity", "||doubleclick.net^", "nodoubleclick.net", false, NotFilteredNotFound},
+	{"sanity", "||doubleclick.net^", "doubleclick.net.ru", false, NotFilteredNotFound},
+	{"sanity", "||doubleclick.net^", "wmconvirus.narod.ru", false, NotFilteredNotFound},
+
+	{"blocking", blockingRules, "example.org", true, FilteredBlackList},
+	{"blocking", blockingRules, "test.example.org", true, FilteredBlackList},
+	{"blocking", blockingRules, "test.test.example.org", true, FilteredBlackList},
+	{"blocking", blockingRules, "testexample.org", false, NotFilteredNotFound},
+	{"blocking", blockingRules, "onemoreexample.org", false, NotFilteredNotFound},
+
+	{"whitelist", whitelistRules, "example.org", true, FilteredBlackList},
+	{"whitelist", whitelistRules, "test.example.org", false, NotFilteredWhiteList},
+	{"whitelist", whitelistRules, "test.test.example.org", false, NotFilteredWhiteList},
+	{"whitelist", whitelistRules, "testexample.org", false, NotFilteredNotFound},
+	{"whitelist", whitelistRules, "onemoreexample.org", false, NotFilteredNotFound},
+
+	{"important", importantRules, "example.org", false, NotFilteredWhiteList},
+	{"important", importantRules, "test.example.org", true, FilteredBlackList},
+	{"important", importantRules, "test.test.example.org", true, FilteredBlackList},
+	{"important", importantRules, "testexample.org", false, NotFilteredNotFound},
+	{"important", importantRules, "onemoreexample.org", false, NotFilteredNotFound},
+
+	{"regex", regexRules, "example.org", true, FilteredBlackList},
+	{"regex", regexRules, "test.example.org", false, NotFilteredWhiteList},
+	{"regex", regexRules, "test.test.example.org", false, NotFilteredWhiteList},
+	{"regex", regexRules, "testexample.org", true, FilteredBlackList},
+	{"regex", regexRules, "onemoreexample.org", true, FilteredBlackList},
+
+	{"mask", maskRules, "test.example.org", true, FilteredBlackList},
+	{"mask", maskRules, "test2.example.org", true, FilteredBlackList},
+	{"mask", maskRules, "example.com", true, FilteredBlackList},
+	{"mask", maskRules, "exampleeee.com", true, FilteredBlackList},
+	{"mask", maskRules, "onemoreexamsite.com", true, FilteredBlackList},
+	{"mask", maskRules, "example.org", false, NotFilteredNotFound},
+	{"mask", maskRules, "testexample.org", false, NotFilteredNotFound},
+	{"mask", maskRules, "example.co.uk", false, NotFilteredNotFound},
+}
+
+func TestMatching(t *testing.T) {
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%s-%s", test.testname, test.hostname), func(t *testing.T) {
+			filters := []Filter{Filter{
+				ID: 0, Data: []byte(test.rules),
+			}}
+			d := NewForTest(nil, filters)
+			defer d.Close()
+
+			ret, err := d.CheckHost(test.hostname, dns.TypeA, &setts)
+			if err != nil {
+				t.Errorf("Error while matching host %s: %s", test.hostname, err)
+			}
+			if ret.IsFiltered != test.isFiltered {
+				t.Errorf("Hostname %s has wrong result (%v must be %v)", test.hostname, ret.IsFiltered, test.isFiltered)
+			}
+			if ret.Reason != test.reason {
+				t.Errorf("Hostname %s has wrong reason (%v must be %v)", test.hostname, ret.Reason.String(), test.reason.String())
+			}
+		})
+	}
+}
+
+func TestWhitelist(t *testing.T) {
+	rules := `||host1^
+||host2^
+`
+	filters := []Filter{Filter{
+		ID: 0, Data: []byte(rules),
+	}}
+
+	whiteRules := `||host1^
+||host3^
+`
+	whiteFilters := []Filter{Filter{
+		ID: 0, Data: []byte(whiteRules),
+	}}
+	d := NewForTest(nil, filters)
+	d.SetFilters(filters, whiteFilters, false)
+	defer d.Close()
+
+	// matched by white filter
+	ret, err := d.CheckHost("host1", dns.TypeA, &setts)
+	assert.True(t, err == nil)
+	assert.True(t, !ret.IsFiltered && ret.Reason == NotFilteredWhiteList)
+	assert.True(t, ret.Rule == "||host1^")
+
+	// not matched by white filter, but matched by block filter
+	ret, err = d.CheckHost("host2", dns.TypeA, &setts)
+	assert.True(t, err == nil)
+	assert.True(t, ret.IsFiltered && ret.Reason == FilteredBlackList)
+	assert.True(t, ret.Rule == "||host2^")
+
+}
+
+// CLIENT SETTINGS
+
+func applyClientSettings(setts *RequestFilteringSettings) {
+	setts.FilteringEnabled = false
+	setts.ParentalEnabled = false
+	setts.SafeBrowsingEnabled = true
+
+	rule, _ := rules.NewNetworkRule("||facebook.com^", 0)
+	s := ServiceEntry{}
+	s.Name = "facebook"
+	s.Rules = []*rules.NetworkRule{rule}
+	setts.ServicesRules = append(setts.ServicesRules, s)
+}
+
+// Check behaviour without any per-client settings,
+//  then apply per-client settings and check behaviour once again
+func TestClientSettings(t *testing.T) {
+	var r Result
+	filters := []Filter{Filter{
+		ID: 0, Data: []byte("||example.org^\n"),
+	}}
+	d := NewForTest(&Config{ParentalEnabled: true, SafeBrowsingEnabled: false}, filters)
+	defer d.Close()
+
+	// no client settings:
+
+	// blocked by filters
+	r, _ = d.CheckHost("example.org", dns.TypeA, &setts)
+	if !r.IsFiltered || r.Reason != FilteredBlackList {
+		t.Fatalf("CheckHost FilteredBlackList")
+	}
+
+	// blocked by parental
+	r, _ = d.CheckHost("pornhub.com", dns.TypeA, &setts)
+	if !r.IsFiltered || r.Reason != FilteredParental {
+		t.Fatalf("CheckHost FilteredParental")
+	}
+
+	// safesearch is disabled
+	r, _ = d.CheckHost("wmconvirus.narod.ru", dns.TypeA, &setts)
+	if r.IsFiltered {
+		t.Fatalf("CheckHost safesearch")
+	}
+
+	// not blocked
+	r, _ = d.CheckHost("facebook.com", dns.TypeA, &setts)
+	assert.True(t, !r.IsFiltered)
+
+	// override client settings:
+	applyClientSettings(&setts)
+
+	// override filtering settings
+	r, _ = d.CheckHost("example.org", dns.TypeA, &setts)
+	if r.IsFiltered {
+		t.Fatalf("CheckHost")
+	}
+
+	// override parental settings (force disable parental)
+	r, _ = d.CheckHost("pornhub.com", dns.TypeA, &setts)
+	if r.IsFiltered {
+		t.Fatalf("CheckHost")
+	}
+
+	// override safesearch settings (force enable safesearch)
+	r, _ = d.CheckHost("wmconvirus.narod.ru", dns.TypeA, &setts)
+	if !r.IsFiltered || r.Reason != FilteredSafeBrowsing {
+		t.Fatalf("CheckHost FilteredSafeBrowsing")
+	}
+
+	// blocked by additional rules
+	r, _ = d.CheckHost("facebook.com", dns.TypeA, &setts)
+	assert.True(t, r.IsFiltered && r.Reason == FilteredBlockedService)
+}
+
+func prepareTestDir() string {
+	const dir = "./agh-test"
+	_ = os.RemoveAll(dir)
+	_ = os.MkdirAll(dir, 0755)
+	return dir
+}
+
+// BENCHMARKS
+
+func BenchmarkSafeBrowsing(b *testing.B) {
+	d := NewForTest(&Config{SafeBrowsingEnabled: true}, nil)
+	defer d.Close()
+	for n := 0; n < b.N; n++ {
+		hostname := "wmconvirus.narod.ru"
+		ret, err := d.CheckHost(hostname, dns.TypeA, &setts)
+		if err != nil {
+			b.Errorf("Error while matching host %s: %s", hostname, err)
+		}
+		if !ret.IsFiltered {
+			b.Errorf("Expected hostname %s to match", hostname)
+		}
+	}
+}
+
+func BenchmarkSafeBrowsingParallel(b *testing.B) {
+	d := NewForTest(&Config{SafeBrowsingEnabled: true}, nil)
+	defer d.Close()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			hostname := "wmconvirus.narod.ru"
+			ret, err := d.CheckHost(hostname, dns.TypeA, &setts)
+			if err != nil {
+				b.Errorf("Error while matching host %s: %s", hostname, err)
+			}
+			if !ret.IsFiltered {
+				b.Errorf("Expected hostname %s to match", hostname)
+			}
+		}
+	})
+}
+
+func BenchmarkSafeSearch(b *testing.B) {
+	d := NewForTest(&Config{SafeSearchEnabled: true}, nil)
+	defer d.Close()
+	for n := 0; n < b.N; n++ {
+		val, ok := d.SafeSearchDomain("www.google.com")
+		if !ok {
+			b.Errorf("Expected safesearch to find result for www.google.com")
+		}
+		if val != "forcesafesearch.google.com" {
+			b.Errorf("Expected safesearch for google.com to be forcesafesearch.google.com")
+		}
+	}
+}
+
+func BenchmarkSafeSearchParallel(b *testing.B) {
+	d := NewForTest(&Config{SafeSearchEnabled: true}, nil)
+	defer d.Close()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			val, ok := d.SafeSearchDomain("www.google.com")
+			if !ok {
+				b.Errorf("Expected safesearch to find result for www.google.com")
+			}
+			if val != "forcesafesearch.google.com" {
+				b.Errorf("Expected safesearch for google.com to be forcesafesearch.google.com")
+			}
+		}
+	})
+}

internal/dnsfilter/rewrites.go

@@ -0,0 +1,228 @@
+// DNS Rewrites
+
+package dnsfilter
+
+import (
+	"encoding/json"
+	"net"
+	"net/http"
+	"sort"
+	"strings"
+
+	"github.com/AdguardTeam/golibs/log"
+	"github.com/miekg/dns"
+)
+
+// RewriteEntry is a rewrite array element
+type RewriteEntry struct {
+	Domain string `yaml:"domain"`
+	Answer string `yaml:"answer"` // IP address or canonical name
+	Type   uint16 `yaml:"-"`      // DNS record type: CNAME, A or AAAA
+	IP     net.IP `yaml:"-"`      // Parsed IP address (if Type is A or AAAA)
+}
+
+func (r *RewriteEntry) equals(b RewriteEntry) bool {
+	return r.Domain == b.Domain && r.Answer == b.Answer
+}
+
+func isWildcard(host string) bool {
+	return len(host) >= 2 &&
+		host[0] == '*' && host[1] == '.'
+}
+
+// Return TRUE of host name matches a wildcard pattern
+func matchDomainWildcard(host, wildcard string) bool {
+	return isWildcard(wildcard) &&
+		strings.HasSuffix(host, wildcard[1:])
+}
+
+type rewritesArray []RewriteEntry
+
+func (a rewritesArray) Len() int { return len(a) }
+
+func (a rewritesArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+// Priority:
+//  . CNAME < A/AAAA;
+//  . exact < wildcard;
+//  . higher level wildcard < lower level wildcard
+func (a rewritesArray) Less(i, j int) bool {
+	if a[i].Type == dns.TypeCNAME && a[j].Type != dns.TypeCNAME {
+		return true
+	} else if a[i].Type != dns.TypeCNAME && a[j].Type == dns.TypeCNAME {
+		return false
+	}
+
+	if isWildcard(a[i].Domain) {
+		if !isWildcard(a[j].Domain) {
+			return false
+		}
+	} else {
+		if isWildcard(a[j].Domain) {
+			return true
+		}
+	}
+
+	// both are wildcards
+	return len(a[i].Domain) > len(a[j].Domain)
+}
+
+// Prepare entry for use
+func (r *RewriteEntry) prepare() {
+	if r.Answer == "AAAA" {
+		r.IP = nil
+		r.Type = dns.TypeAAAA
+		return
+	} else if r.Answer == "A" {
+		r.IP = nil
+		r.Type = dns.TypeA
+		return
+	}
+
+	ip := net.ParseIP(r.Answer)
+	if ip == nil {
+		r.Type = dns.TypeCNAME
+		return
+	}
+
+	r.IP = ip
+	r.Type = dns.TypeAAAA
+
+	ip4 := ip.To4()
+	if ip4 != nil {
+		r.IP = ip4
+		r.Type = dns.TypeA
+	}
+}
+
+func (d *Dnsfilter) prepareRewrites() {
+	for i := range d.Rewrites {
+		d.Rewrites[i].prepare()
+	}
+}
+
+// Get the list of matched rewrite entries.
+// Priority: CNAME, A/AAAA;  exact, wildcard.
+// If matched exactly, don't return wildcard entries.
+// If matched by several wildcards, select the more specific one
+func findRewrites(a []RewriteEntry, host string) []RewriteEntry {
+	rr := rewritesArray{}
+	for _, r := range a {
+		if r.Domain != host {
+			if !matchDomainWildcard(host, r.Domain) {
+				continue
+			}
+		}
+		rr = append(rr, r)
+	}
+
+	if len(rr) == 0 {
+		return nil
+	}
+
+	sort.Sort(rr)
+
+	isWC := isWildcard(rr[0].Domain)
+	if !isWC {
+		for i, r := range rr {
+			if isWildcard(r.Domain) {
+				rr = rr[:i]
+				break
+			}
+		}
+	} else {
+		rr = rr[:1]
+	}
+
+	return rr
+}
+
+func rewriteArrayDup(a []RewriteEntry) []RewriteEntry {
+	a2 := make([]RewriteEntry, len(a))
+	copy(a2, a)
+	return a2
+}
+
+type rewriteEntryJSON struct {
+	Domain string `json:"domain"`
+	Answer string `json:"answer"`
+}
+
+func (d *Dnsfilter) handleRewriteList(w http.ResponseWriter, r *http.Request) {
+
+	arr := []*rewriteEntryJSON{}
+
+	d.confLock.Lock()
+	for _, ent := range d.Config.Rewrites {
+		jsent := rewriteEntryJSON{
+			Domain: ent.Domain,
+			Answer: ent.Answer,
+		}
+		arr = append(arr, &jsent)
+	}
+	d.confLock.Unlock()
+
+	w.Header().Set("Content-Type", "application/json")
+	err := json.NewEncoder(w).Encode(arr)
+	if err != nil {
+		httpError(r, w, http.StatusInternalServerError, "json.Encode: %s", err)
+		return
+	}
+}
+
+func (d *Dnsfilter) handleRewriteAdd(w http.ResponseWriter, r *http.Request) {
+
+	jsent := rewriteEntryJSON{}
+	err := json.NewDecoder(r.Body).Decode(&jsent)
+	if err != nil {
+		httpError(r, w, http.StatusBadRequest, "json.Decode: %s", err)
+		return
+	}
+
+	ent := RewriteEntry{
+		Domain: jsent.Domain,
+		Answer: jsent.Answer,
+	}
+	ent.prepare()
+	d.confLock.Lock()
+	d.Config.Rewrites = append(d.Config.Rewrites, ent)
+	d.confLock.Unlock()
+	log.Debug("Rewrites: added element: %s -> %s [%d]",
+		ent.Domain, ent.Answer, len(d.Config.Rewrites))
+
+	d.Config.ConfigModified()
+}
+
+func (d *Dnsfilter) handleRewriteDelete(w http.ResponseWriter, r *http.Request) {
+
+	jsent := rewriteEntryJSON{}
+	err := json.NewDecoder(r.Body).Decode(&jsent)
+	if err != nil {
+		httpError(r, w, http.StatusBadRequest, "json.Decode: %s", err)
+		return
+	}
+
+	entDel := RewriteEntry{
+		Domain: jsent.Domain,
+		Answer: jsent.Answer,
+	}
+	arr := []RewriteEntry{}
+	d.confLock.Lock()
+	for _, ent := range d.Config.Rewrites {
+		if ent.equals(entDel) {
+			log.Debug("Rewrites: removed element: %s -> %s", ent.Domain, ent.Answer)
+			continue
+		}
+		arr = append(arr, ent)
+	}
+	d.Config.Rewrites = arr
+	d.confLock.Unlock()
+
+	d.Config.ConfigModified()
+}
+
+func (d *Dnsfilter) registerRewritesHandlers() {
+	d.Config.HTTPRegister("GET", "/control/rewrite/list", d.handleRewriteList)
+	d.Config.HTTPRegister("POST", "/control/rewrite/add", d.handleRewriteAdd)
+	d.Config.HTTPRegister("POST", "/control/rewrite/delete", d.handleRewriteDelete)
+}

internal/dnsfilter/rewrites_test.go

@@ -0,0 +1,215 @@
+package dnsfilter
+
+import (
+	"net"
+	"testing"
+
+	"github.com/miekg/dns"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRewrites(t *testing.T) {
+	d := Dnsfilter{}
+	// CNAME, A, AAAA
+	d.Rewrites = []RewriteEntry{
+		RewriteEntry{"somecname", "somehost.com", 0, nil},
+		RewriteEntry{"somehost.com", "0.0.0.0", 0, nil},
+
+		RewriteEntry{"host.com", "1.2.3.4", 0, nil},
+		RewriteEntry{"host.com", "1.2.3.5", 0, nil},
+		RewriteEntry{"host.com", "1:2:3::4", 0, nil},
+		RewriteEntry{"www.host.com", "host.com", 0, nil},
+	}
+	d.prepareRewrites()
+	r := d.processRewrites("host2.com", dns.TypeA)
+	assert.Equal(t, NotFilteredNotFound, r.Reason)
+
+	r = d.processRewrites("www.host.com", dns.TypeA)
+	assert.Equal(t, ReasonRewrite, r.Reason)
+	assert.Equal(t, "host.com", r.CanonName)
+	assert.Equal(t, 2, len(r.IPList))
+	assert.True(t, r.IPList[0].Equal(net.ParseIP("1.2.3.4")))
+	assert.True(t, r.IPList[1].Equal(net.ParseIP("1.2.3.5")))
+
+	r = d.processRewrites("www.host.com", dns.TypeAAAA)
+	assert.Equal(t, ReasonRewrite, r.Reason)
+	assert.Equal(t, "host.com", r.CanonName)
+	assert.Equal(t, 1, len(r.IPList))
+	assert.True(t, r.IPList[0].Equal(net.ParseIP("1:2:3::4")))
+
+	// wildcard
+	d.Rewrites = []RewriteEntry{
+		RewriteEntry{"host.com", "1.2.3.4", 0, nil},
+		RewriteEntry{"*.host.com", "1.2.3.5", 0, nil},
+	}
+	d.prepareRewrites()
+	r = d.processRewrites("host.com", dns.TypeA)
+	assert.Equal(t, ReasonRewrite, r.Reason)
+	assert.True(t, r.IPList[0].Equal(net.ParseIP("1.2.3.4")))
+
+	r = d.processRewrites("www.host.com", dns.TypeA)
+	assert.Equal(t, ReasonRewrite, r.Reason)
+	assert.True(t, r.IPList[0].Equal(net.ParseIP("1.2.3.5")))
+
+	r = d.processRewrites("www.host2.com", dns.TypeA)
+	assert.Equal(t, NotFilteredNotFound, r.Reason)
+
+	// override a wildcard
+	d.Rewrites = []RewriteEntry{
+		RewriteEntry{"a.host.com", "1.2.3.4", 0, nil},
+		RewriteEntry{"*.host.com", "1.2.3.5", 0, nil},
+	}
+	d.prepareRewrites()
+	r = d.processRewrites("a.host.com", dns.TypeA)
+	assert.Equal(t, ReasonRewrite, r.Reason)
+	assert.True(t, len(r.IPList) == 1)
+	assert.True(t, r.IPList[0].Equal(net.ParseIP("1.2.3.4")))
+
+	// wildcard + CNAME
+	d.Rewrites = []RewriteEntry{
+		RewriteEntry{"host.com", "1.2.3.4", 0, nil},
+		RewriteEntry{"*.host.com", "host.com", 0, nil},
+	}
+	d.prepareRewrites()
+	r = d.processRewrites("www.host.com", dns.TypeA)
+	assert.Equal(t, ReasonRewrite, r.Reason)
+	assert.Equal(t, "host.com", r.CanonName)
+	assert.True(t, r.IPList[0].Equal(net.ParseIP("1.2.3.4")))
+
+	// 2 CNAMEs
+	d.Rewrites = []RewriteEntry{
+		RewriteEntry{"b.host.com", "a.host.com", 0, nil},
+		RewriteEntry{"a.host.com", "host.com", 0, nil},
+		RewriteEntry{"host.com", "1.2.3.4", 0, nil},
+	}
+	d.prepareRewrites()
+	r = d.processRewrites("b.host.com", dns.TypeA)
+	assert.Equal(t, ReasonRewrite, r.Reason)
+	assert.Equal(t, "host.com", r.CanonName)
+	assert.True(t, len(r.IPList) == 1)
+	assert.True(t, r.IPList[0].Equal(net.ParseIP("1.2.3.4")))
+
+	// 2 CNAMEs + wildcard
+	d.Rewrites = []RewriteEntry{
+		RewriteEntry{"b.host.com", "a.host.com", 0, nil},
+		RewriteEntry{"a.host.com", "x.somehost.com", 0, nil},
+		RewriteEntry{"*.somehost.com", "1.2.3.4", 0, nil},
+	}
+	d.prepareRewrites()
+	r = d.processRewrites("b.host.com", dns.TypeA)
+	assert.Equal(t, ReasonRewrite, r.Reason)
+	assert.Equal(t, "x.somehost.com", r.CanonName)
+	assert.True(t, len(r.IPList) == 1)
+	assert.True(t, r.IPList[0].Equal(net.ParseIP("1.2.3.4")))
+}
+
+func TestRewritesLevels(t *testing.T) {
+	d := Dnsfilter{}
+	// exact host, wildcard L2, wildcard L3
+	d.Rewrites = []RewriteEntry{
+		RewriteEntry{"host.com", "1.1.1.1", 0, nil},
+		RewriteEntry{"*.host.com", "2.2.2.2", 0, nil},
+		RewriteEntry{"*.sub.host.com", "3.3.3.3", 0, nil},
+	}
+	d.prepareRewrites()
+
+	// match exact
+	r := d.processRewrites("host.com", dns.TypeA)
+	assert.Equal(t, ReasonRewrite, r.Reason)
+	assert.Equal(t, 1, len(r.IPList))
+	assert.Equal(t, "1.1.1.1", r.IPList[0].String())
+
+	// match L2
+	r = d.processRewrites("sub.host.com", dns.TypeA)
+	assert.Equal(t, ReasonRewrite, r.Reason)
+	assert.Equal(t, 1, len(r.IPList))
+	assert.Equal(t, "2.2.2.2", r.IPList[0].String())
+
+	// match L3
+	r = d.processRewrites("my.sub.host.com", dns.TypeA)
+	assert.Equal(t, ReasonRewrite, r.Reason)
+	assert.Equal(t, 1, len(r.IPList))
+	assert.Equal(t, "3.3.3.3", r.IPList[0].String())
+}
+
+func TestRewritesExceptionCNAME(t *testing.T) {
+	d := Dnsfilter{}
+	// wildcard; exception for a sub-domain
+	d.Rewrites = []RewriteEntry{
+		RewriteEntry{"*.host.com", "2.2.2.2", 0, nil},
+		RewriteEntry{"sub.host.com", "sub.host.com", 0, nil},
+	}
+	d.prepareRewrites()
+
+	// match sub-domain
+	r := d.processRewrites("my.host.com", dns.TypeA)
+	assert.Equal(t, ReasonRewrite, r.Reason)
+	assert.Equal(t, 1, len(r.IPList))
+	assert.Equal(t, "2.2.2.2", r.IPList[0].String())
+
+	// match sub-domain, but handle exception
+	r = d.processRewrites("sub.host.com", dns.TypeA)
+	assert.Equal(t, NotFilteredNotFound, r.Reason)
+}
+
+func TestRewritesExceptionWC(t *testing.T) {
+	d := Dnsfilter{}
+	// wildcard; exception for a sub-wildcard
+	d.Rewrites = []RewriteEntry{
+		RewriteEntry{"*.host.com", "2.2.2.2", 0, nil},
+		RewriteEntry{"*.sub.host.com", "*.sub.host.com", 0, nil},
+	}
+	d.prepareRewrites()
+
+	// match sub-domain
+	r := d.processRewrites("my.host.com", dns.TypeA)
+	assert.Equal(t, ReasonRewrite, r.Reason)
+	assert.Equal(t, 1, len(r.IPList))
+	assert.Equal(t, "2.2.2.2", r.IPList[0].String())
+
+	// match sub-domain, but handle exception
+	r = d.processRewrites("my.sub.host.com", dns.TypeA)
+	assert.Equal(t, NotFilteredNotFound, r.Reason)
+}
+
+func TestRewritesExceptionIP(t *testing.T) {
+	d := Dnsfilter{}
+	// exception for AAAA record
+	d.Rewrites = []RewriteEntry{
+		RewriteEntry{"host.com", "1.2.3.4", 0, nil},
+		RewriteEntry{"host.com", "AAAA", 0, nil},
+		RewriteEntry{"host2.com", "::1", 0, nil},
+		RewriteEntry{"host2.com", "A", 0, nil},
+		RewriteEntry{"host3.com", "A", 0, nil},
+	}
+	d.prepareRewrites()
+
+	// match domain
+	r := d.processRewrites("host.com", dns.TypeA)
+	assert.Equal(t, ReasonRewrite, r.Reason)
+	assert.Equal(t, 1, len(r.IPList))
+	assert.Equal(t, "1.2.3.4", r.IPList[0].String())
+
+	// match exception
+	r = d.processRewrites("host.com", dns.TypeAAAA)
+	assert.Equal(t, NotFilteredNotFound, r.Reason)
+
+	// match exception
+	r = d.processRewrites("host2.com", dns.TypeA)
+	assert.Equal(t, NotFilteredNotFound, r.Reason)
+
+	// match domain
+	r = d.processRewrites("host2.com", dns.TypeAAAA)
+	assert.Equal(t, ReasonRewrite, r.Reason)
+	assert.Equal(t, 1, len(r.IPList))
+	assert.Equal(t, "::1", r.IPList[0].String())
+
+	// match exception
+	r = d.processRewrites("host3.com", dns.TypeA)
+	assert.Equal(t, NotFilteredNotFound, r.Reason)
+
+	// match domain
+	r = d.processRewrites("host3.com", dns.TypeAAAA)
+	assert.Equal(t, ReasonRewrite, r.Reason)
+	assert.Equal(t, 0, len(r.IPList))
+}

internal/dnsfilter/safe_search.go

@@ -0,0 +1,149 @@
+package dnsfilter
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/gob"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/AdguardTeam/golibs/cache"
+	"github.com/AdguardTeam/golibs/log"
+)
+
+/*
+expire byte[4]
+res Result
+*/
+func (d *Dnsfilter) setCacheResult(cache cache.Cache, host string, res Result) int {
+	var buf bytes.Buffer
+
+	expire := uint(time.Now().Unix()) + d.Config.CacheTime*60
+	var exp []byte
+	exp = make([]byte, 4)
+	binary.BigEndian.PutUint32(exp, uint32(expire))
+	_, _ = buf.Write(exp)
+
+	enc := gob.NewEncoder(&buf)
+	err := enc.Encode(res)
+	if err != nil {
+		log.Error("gob.Encode(): %s", err)
+		return 0
+	}
+	val := buf.Bytes()
+	_ = cache.Set([]byte(host), val)
+	return len(val)
+}
+
+func getCachedResult(cache cache.Cache, host string) (Result, bool) {
+	data := cache.Get([]byte(host))
+	if data == nil {
+		return Result{}, false
+	}
+
+	exp := int(binary.BigEndian.Uint32(data[:4]))
+	if exp <= int(time.Now().Unix()) {
+		cache.Del([]byte(host))
+		return Result{}, false
+	}
+
+	var buf bytes.Buffer
+	buf.Write(data[4:])
+	dec := gob.NewDecoder(&buf)
+	r := Result{}
+	err := dec.Decode(&r)
+	if err != nil {
+		log.Debug("gob.Decode(): %s", err)
+		return Result{}, false
+	}
+
+	return r, true
+}
+
+// SafeSearchDomain returns replacement address for search engine
+func (d *Dnsfilter) SafeSearchDomain(host string) (string, bool) {
+	val, ok := safeSearchDomains[host]
+	return val, ok
+}
+
+func (d *Dnsfilter) checkSafeSearch(host string) (Result, error) {
+	if log.GetLevel() >= log.DEBUG {
+		timer := log.StartTimer()
+		defer timer.LogElapsed("SafeSearch: lookup for %s", host)
+	}
+
+	// Check cache. Return cached result if it was found
+	cachedValue, isFound := getCachedResult(gctx.safeSearchCache, host)
+	if isFound {
+		// atomic.AddUint64(&gctx.stats.Safesearch.CacheHits, 1)
+		log.Tracef("SafeSearch: found in cache: %s", host)
+		return cachedValue, nil
+	}
+
+	safeHost, ok := d.SafeSearchDomain(host)
+	if !ok {
+		return Result{}, nil
+	}
+
+	res := Result{IsFiltered: true, Reason: FilteredSafeSearch}
+	if ip := net.ParseIP(safeHost); ip != nil {
+		res.IP = ip
+		valLen := d.setCacheResult(gctx.safeSearchCache, host, res)
+		log.Debug("SafeSearch: stored in cache: %s (%d bytes)", host, valLen)
+		return res, nil
+	}
+
+	// TODO this address should be resolved with upstream that was configured in dnsforward
+	addrs, err := net.LookupIP(safeHost)
+	if err != nil {
+		log.Tracef("SafeSearchDomain for %s was found but failed to lookup for %s cause %s", host, safeHost, err)
+		return Result{}, err
+	}
+
+	for _, i := range addrs {
+		if ipv4 := i.To4(); ipv4 != nil {
+			res.IP = ipv4
+			break
+		}
+	}
+
+	if len(res.IP) == 0 {
+		return Result{}, fmt.Errorf("no ipv4 addresses in safe search response for %s", safeHost)
+	}
+
+	// Cache result
+	valLen := d.setCacheResult(gctx.safeSearchCache, host, res)
+	log.Debug("SafeSearch: stored in cache: %s (%d bytes)", host, valLen)
+	return res, nil
+}
+
+func (d *Dnsfilter) handleSafeSearchEnable(w http.ResponseWriter, r *http.Request) {
+	d.Config.SafeSearchEnabled = true
+	d.Config.ConfigModified()
+}
+
+func (d *Dnsfilter) handleSafeSearchDisable(w http.ResponseWriter, r *http.Request) {
+	d.Config.SafeSearchEnabled = false
+	d.Config.ConfigModified()
+}
+
+func (d *Dnsfilter) handleSafeSearchStatus(w http.ResponseWriter, r *http.Request) {
+	data := map[string]interface{}{
+		"enabled": d.Config.SafeSearchEnabled,
+	}
+	jsonVal, err := json.Marshal(data)
+	if err != nil {
+		httpError(r, w, http.StatusInternalServerError, "Unable to marshal status json: %s", err)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_, err = w.Write(jsonVal)
+	if err != nil {
+		httpError(r, w, http.StatusInternalServerError, "Unable to write response json: %s", err)
+		return
+	}
+}

internal/dnsfilter/safesearch.go

@@ -0,0 +1,218 @@
+package dnsfilter
+
+var safeSearchDomains = map[string]string{
+	"yandex.com":     "213.180.193.56",
+	"yandex.ru":      "213.180.193.56",
+	"yandex.ua":      "213.180.193.56",
+	"yandex.by":      "213.180.193.56",
+	"yandex.kz":      "213.180.193.56",
+	"www.yandex.com": "213.180.193.56",
+	"www.yandex.ru":  "213.180.193.56",
+	"www.yandex.ua":  "213.180.193.56",
+	"www.yandex.by":  "213.180.193.56",
+	"www.yandex.kz":  "213.180.193.56",
+
+	"www.bing.com": "strict.bing.com",
+
+	"duckduckgo.com":       "safe.duckduckgo.com",
+	"www.duckduckgo.com":   "safe.duckduckgo.com",
+	"start.duckduckgo.com": "safe.duckduckgo.com",
+
+	"www.google.com":    "forcesafesearch.google.com",
+	"www.google.ad":     "forcesafesearch.google.com",
+	"www.google.ae":     "forcesafesearch.google.com",
+	"www.google.com.af": "forcesafesearch.google.com",
+	"www.google.com.ag": "forcesafesearch.google.com",
+	"www.google.com.ai": "forcesafesearch.google.com",
+	"www.google.al":     "forcesafesearch.google.com",
+	"www.google.am":     "forcesafesearch.google.com",
+	"www.google.co.ao":  "forcesafesearch.google.com",
+	"www.google.com.ar": "forcesafesearch.google.com",
+	"www.google.as":     "forcesafesearch.google.com",
+	"www.google.at":     "forcesafesearch.google.com",
+	"www.google.com.au": "forcesafesearch.google.com",
+	"www.google.az":     "forcesafesearch.google.com",
+	"www.google.ba":     "forcesafesearch.google.com",
+	"www.google.com.bd": "forcesafesearch.google.com",
+	"www.google.be":     "forcesafesearch.google.com",
+	"www.google.bf":     "forcesafesearch.google.com",
+	"www.google.bg":     "forcesafesearch.google.com",
+	"www.google.com.bh": "forcesafesearch.google.com",
+	"www.google.bi":     "forcesafesearch.google.com",
+	"www.google.bj":     "forcesafesearch.google.com",
+	"www.google.com.bn": "forcesafesearch.google.com",
+	"www.google.com.bo": "forcesafesearch.google.com",
+	"www.google.com.br": "forcesafesearch.google.com",
+	"www.google.bs":     "forcesafesearch.google.com",
+	"www.google.bt":     "forcesafesearch.google.com",
+	"www.google.co.bw":  "forcesafesearch.google.com",
+	"www.google.by":     "forcesafesearch.google.com",
+	"www.google.com.bz": "forcesafesearch.google.com",
+	"www.google.ca":     "forcesafesearch.google.com",
+	"www.google.cd":     "forcesafesearch.google.com",
+	"www.google.cf":     "forcesafesearch.google.com",
+	"www.google.cg":     "forcesafesearch.google.com",
+	"www.google.ch":     "forcesafesearch.google.com",
+	"www.google.ci":     "forcesafesearch.google.com",
+	"www.google.co.ck":  "forcesafesearch.google.com",
+	"www.google.cl":     "forcesafesearch.google.com",
+	"www.google.cm":     "forcesafesearch.google.com",
+	"www.google.cn":     "forcesafesearch.google.com",
+	"www.google.com.co": "forcesafesearch.google.com",
+	"www.google.co.cr":  "forcesafesearch.google.com",
+	"www.google.com.cu": "forcesafesearch.google.com",
+	"www.google.cv":     "forcesafesearch.google.com",
+	"www.google.com.cy": "forcesafesearch.google.com",
+	"www.google.cz":     "forcesafesearch.google.com",
+	"www.google.de":     "forcesafesearch.google.com",
+	"www.google.dj":     "forcesafesearch.google.com",
+	"www.google.dk":     "forcesafesearch.google.com",
+	"www.google.dm":     "forcesafesearch.google.com",
+	"www.google.com.do": "forcesafesearch.google.com",
+	"www.google.dz":     "forcesafesearch.google.com",
+	"www.google.com.ec": "forcesafesearch.google.com",
+	"www.google.ee":     "forcesafesearch.google.com",
+	"www.google.com.eg": "forcesafesearch.google.com",
+	"www.google.es":     "forcesafesearch.google.com",
+	"www.google.com.et": "forcesafesearch.google.com",
+	"www.google.fi":     "forcesafesearch.google.com",
+	"www.google.com.fj": "forcesafesearch.google.com",
+	"www.google.fm":     "forcesafesearch.google.com",
+	"www.google.fr":     "forcesafesearch.google.com",
+	"www.google.ga":     "forcesafesearch.google.com",
+	"www.google.ge":     "forcesafesearch.google.com",
+	"www.google.gg":     "forcesafesearch.google.com",
+	"www.google.com.gh": "forcesafesearch.google.com",
+	"www.google.com.gi": "forcesafesearch.google.com",
+	"www.google.gl":     "forcesafesearch.google.com",
+	"www.google.gm":     "forcesafesearch.google.com",
+	"www.google.gp":     "forcesafesearch.google.com",
+	"www.google.gr":     "forcesafesearch.google.com",
+	"www.google.com.gt": "forcesafesearch.google.com",
+	"www.google.gy":     "forcesafesearch.google.com",
+	"www.google.com.hk": "forcesafesearch.google.com",
+	"www.google.hn":     "forcesafesearch.google.com",
+	"www.google.hr":     "forcesafesearch.google.com",
+	"www.google.ht":     "forcesafesearch.google.com",
+	"www.google.hu":     "forcesafesearch.google.com",
+	"www.google.co.id":  "forcesafesearch.google.com",
+	"www.google.ie":     "forcesafesearch.google.com",
+	"www.google.co.il":  "forcesafesearch.google.com",
+	"www.google.im":     "forcesafesearch.google.com",
+	"www.google.co.in":  "forcesafesearch.google.com",
+	"www.google.iq":     "forcesafesearch.google.com",
+	"www.google.is":     "forcesafesearch.google.com",
+	"www.google.it":     "forcesafesearch.google.com",
+	"www.google.je":     "forcesafesearch.google.com",
+	"www.google.com.jm": "forcesafesearch.google.com",
+	"www.google.jo":     "forcesafesearch.google.com",
+	"www.google.co.jp":  "forcesafesearch.google.com",
+	"www.google.co.ke":  "forcesafesearch.google.com",
+	"www.google.com.kh": "forcesafesearch.google.com",
+	"www.google.ki":     "forcesafesearch.google.com",
+	"www.google.kg":     "forcesafesearch.google.com",
+	"www.google.co.kr":  "forcesafesearch.google.com",
+	"www.google.com.kw": "forcesafesearch.google.com",
+	"www.google.kz":     "forcesafesearch.google.com",
+	"www.google.la":     "forcesafesearch.google.com",
+	"www.google.com.lb": "forcesafesearch.google.com",
+	"www.google.li":     "forcesafesearch.google.com",
+	"www.google.lk":     "forcesafesearch.google.com",
+	"www.google.co.ls":  "forcesafesearch.google.com",
+	"www.google.lt":     "forcesafesearch.google.com",
+	"www.google.lu":     "forcesafesearch.google.com",
+	"www.google.lv":     "forcesafesearch.google.com",
+	"www.google.com.ly": "forcesafesearch.google.com",
+	"www.google.co.ma":  "forcesafesearch.google.com",
+	"www.google.md":     "forcesafesearch.google.com",
+	"www.google.me":     "forcesafesearch.google.com",
+	"www.google.mg":     "forcesafesearch.google.com",
+	"www.google.mk":     "forcesafesearch.google.com",
+	"www.google.ml":     "forcesafesearch.google.com",
+	"www.google.com.mm": "forcesafesearch.google.com",
+	"www.google.mn":     "forcesafesearch.google.com",
+	"www.google.ms":     "forcesafesearch.google.com",
+	"www.google.com.mt": "forcesafesearch.google.com",
+	"www.google.mu":     "forcesafesearch.google.com",
+	"www.google.mv":     "forcesafesearch.google.com",
+	"www.google.mw":     "forcesafesearch.google.com",
+	"www.google.com.mx": "forcesafesearch.google.com",
+	"www.google.com.my": "forcesafesearch.google.com",
+	"www.google.co.mz":  "forcesafesearch.google.com",
+	"www.google.com.na": "forcesafesearch.google.com",
+	"www.google.com.nf": "forcesafesearch.google.com",
+	"www.google.com.ng": "forcesafesearch.google.com",
+	"www.google.com.ni": "forcesafesearch.google.com",
+	"www.google.ne":     "forcesafesearch.google.com",
+	"www.google.nl":     "forcesafesearch.google.com",
+	"www.google.no":     "forcesafesearch.google.com",
+	"www.google.com.np": "forcesafesearch.google.com",
+	"www.google.nr":     "forcesafesearch.google.com",
+	"www.google.nu":     "forcesafesearch.google.com",
+	"www.google.co.nz":  "forcesafesearch.google.com",
+	"www.google.com.om": "forcesafesearch.google.com",
+	"www.google.com.pa": "forcesafesearch.google.com",
+	"www.google.com.pe": "forcesafesearch.google.com",
+	"www.google.com.pg": "forcesafesearch.google.com",
+	"www.google.com.ph": "forcesafesearch.google.com",
+	"www.google.com.pk": "forcesafesearch.google.com",
+	"www.google.pl":     "forcesafesearch.google.com",
+	"www.google.pn":     "forcesafesearch.google.com",
+	"www.google.com.pr": "forcesafesearch.google.com",
+	"www.google.ps":     "forcesafesearch.google.com",
+	"www.google.pt":     "forcesafesearch.google.com",
+	"www.google.com.py": "forcesafesearch.google.com",
+	"www.google.com.qa": "forcesafesearch.google.com",
+	"www.google.ro":     "forcesafesearch.google.com",
+	"www.google.ru":     "forcesafesearch.google.com",
+	"www.google.rw":     "forcesafesearch.google.com",
+	"www.google.com.sa": "forcesafesearch.google.com",
+	"www.google.com.sb": "forcesafesearch.google.com",
+	"www.google.sc":     "forcesafesearch.google.com",
+	"www.google.se":     "forcesafesearch.google.com",
+	"www.google.com.sg": "forcesafesearch.google.com",
+	"www.google.sh":     "forcesafesearch.google.com",
+	"www.google.si":     "forcesafesearch.google.com",
+	"www.google.sk":     "forcesafesearch.google.com",
+	"www.google.com.sl": "forcesafesearch.google.com",
+	"www.google.sn":     "forcesafesearch.google.com",
+	"www.google.so":     "forcesafesearch.google.com",
+	"www.google.sm":     "forcesafesearch.google.com",
+	"www.google.sr":     "forcesafesearch.google.com",
+	"www.google.st":     "forcesafesearch.google.com",
+	"www.google.com.sv": "forcesafesearch.google.com",
+	"www.google.td":     "forcesafesearch.google.com",
+	"www.google.tg":     "forcesafesearch.google.com",
+	"www.google.co.th":  "forcesafesearch.google.com",
+	"www.google.com.tj": "forcesafesearch.google.com",
+	"www.google.tk":     "forcesafesearch.google.com",
+	"www.google.tl":     "forcesafesearch.google.com",
+	"www.google.tm":     "forcesafesearch.google.com",
+	"www.google.tn":     "forcesafesearch.google.com",
+	"www.google.to":     "forcesafesearch.google.com",
+	"www.google.com.tr": "forcesafesearch.google.com",
+	"www.google.tt":     "forcesafesearch.google.com",
+	"www.google.com.tw": "forcesafesearch.google.com",
+	"www.google.co.tz":  "forcesafesearch.google.com",
+	"www.google.com.ua": "forcesafesearch.google.com",
+	"www.google.co.ug":  "forcesafesearch.google.com",
+	"www.google.co.uk":  "forcesafesearch.google.com",
+	"www.google.com.uy": "forcesafesearch.google.com",
+	"www.google.co.uz":  "forcesafesearch.google.com",
+	"www.google.com.vc": "forcesafesearch.google.com",
+	"www.google.co.ve":  "forcesafesearch.google.com",
+	"www.google.vg":     "forcesafesearch.google.com",
+	"www.google.co.vi":  "forcesafesearch.google.com",
+	"www.google.com.vn": "forcesafesearch.google.com",
+	"www.google.vu":     "forcesafesearch.google.com",
+	"www.google.ws":     "forcesafesearch.google.com",
+	"www.google.rs":     "forcesafesearch.google.com",
+
+	"www.youtube.com":          "restrictmoderate.youtube.com",
+	"m.youtube.com":            "restrictmoderate.youtube.com",
+	"youtubei.googleapis.com":  "restrictmoderate.youtube.com",
+	"youtube.googleapis.com":   "restrictmoderate.youtube.com",
+	"www.youtube-nocookie.com": "restrictmoderate.youtube.com",
+
+	"pixabay.com": "safesearch.pixabay.com",
+}

internal/dnsfilter/sb_pc.go

@@ -0,0 +1,418 @@
+// Safe Browsing, Parental Control
+
+package dnsfilter
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/AdguardTeam/dnsproxy/upstream"
+	"github.com/AdguardTeam/golibs/cache"
+	"github.com/AdguardTeam/golibs/log"
+	"github.com/miekg/dns"
+	"golang.org/x/net/publicsuffix"
+)
+
+const dnsTimeout = 3 * time.Second
+const defaultSafebrowsingServer = "https://dns-family.adguard.com/dns-query"
+const defaultParentalServer = "https://dns-family.adguard.com/dns-query"
+const sbTXTSuffix = "sb.dns.adguard.com."
+const pcTXTSuffix = "pc.dns.adguard.com."
+
+func (d *Dnsfilter) initSecurityServices() error {
+	var err error
+	d.safeBrowsingServer = defaultSafebrowsingServer
+	d.parentalServer = defaultParentalServer
+	opts := upstream.Options{
+		Timeout: dnsTimeout,
+		ServerIPAddrs: []net.IP{
+			net.ParseIP("94.140.14.15"),
+			net.ParseIP("94.140.15.16"),
+			net.ParseIP("2a10:50c0::bad1:ff"),
+			net.ParseIP("2a10:50c0::bad2:ff"),
+		},
+	}
+
+	d.parentalUpstream, err = upstream.AddressToUpstream(d.parentalServer, opts)
+	if err != nil {
+		return err
+	}
+
+	d.safeBrowsingUpstream, err = upstream.AddressToUpstream(d.safeBrowsingServer, opts)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+/*
+expire byte[4]
+hash byte[32]
+...
+*/
+func (c *sbCtx) setCache(prefix []byte, hashes []byte) {
+	d := make([]byte, 4+len(hashes))
+	expire := uint(time.Now().Unix()) + c.cacheTime*60
+	binary.BigEndian.PutUint32(d[:4], uint32(expire))
+	copy(d[4:], hashes)
+	c.cache.Set(prefix, d)
+	log.Debug("%s: stored in cache: %v", c.svc, prefix)
+}
+
+func (c *sbCtx) getCached() int {
+	now := time.Now().Unix()
+	hashesToRequest := map[[32]byte]string{}
+	for k, v := range c.hashToHost {
+		key := k[0:2]
+		val := c.cache.Get(key)
+		if val != nil {
+			expire := binary.BigEndian.Uint32(val)
+			if now >= int64(expire) {
+				val = nil
+			} else {
+				for i := 4; i < len(val); i += 32 {
+					hash := val[i : i+32]
+					var hash32 [32]byte
+					copy(hash32[:], hash[0:32])
+					_, found := c.hashToHost[hash32]
+					if found {
+						log.Debug("%s: found in cache: %s: blocked by %v", c.svc, c.host, hash32)
+						return 1
+					}
+				}
+			}
+		}
+		if val == nil {
+			hashesToRequest[k] = v
+		}
+	}
+
+	if len(hashesToRequest) == 0 {
+		log.Debug("%s: found in cache: %s: not blocked", c.svc, c.host)
+		return -1
+	}
+
+	c.hashToHost = hashesToRequest
+	return 0
+}
+
+type sbCtx struct {
+	host       string
+	svc        string
+	hashToHost map[[32]byte]string
+	cache      cache.Cache
+	cacheTime  uint
+}
+
+func hostnameToHashes(host string) map[[32]byte]string {
+	hashes := map[[32]byte]string{}
+	tld, icann := publicsuffix.PublicSuffix(host)
+	if !icann {
+		// private suffixes like cloudfront.net
+		tld = ""
+	}
+	curhost := host
+
+	nDots := 0
+	for i := len(curhost) - 1; i >= 0; i-- {
+		if curhost[i] == '.' {
+			nDots++
+			if nDots == 4 {
+				curhost = curhost[i+1:] // "xxx.a.b.c.d" -> "a.b.c.d"
+				break
+			}
+		}
+	}
+
+	for {
+		if curhost == "" {
+			// we've reached end of string
+			break
+		}
+		if tld != "" && curhost == tld {
+			// we've reached the TLD, don't hash it
+			break
+		}
+
+		sum := sha256.Sum256([]byte(curhost))
+		hashes[sum] = curhost
+
+		pos := strings.IndexByte(curhost, byte('.'))
+		if pos < 0 {
+			break
+		}
+		curhost = curhost[pos+1:]
+	}
+	return hashes
+}
+
+// convert hash array to string
+func (c *sbCtx) getQuestion() string {
+	q := ""
+	for hash := range c.hashToHost {
+		q += fmt.Sprintf("%s.", hex.EncodeToString(hash[0:2]))
+	}
+	if c.svc == "SafeBrowsing" {
+		q += sbTXTSuffix
+	} else {
+		q += pcTXTSuffix
+	}
+	return q
+}
+
+// Find the target hash in TXT response
+func (c *sbCtx) processTXT(resp *dns.Msg) (bool, [][]byte) {
+	matched := false
+	hashes := [][]byte{}
+	for _, a := range resp.Answer {
+		txt, ok := a.(*dns.TXT)
+		if !ok {
+			continue
+		}
+		log.Debug("%s: received hashes for %s: %v", c.svc, c.host, txt.Txt)
+
+		for _, t := range txt.Txt {
+
+			if len(t) != 32*2 {
+				continue
+			}
+			hash, err := hex.DecodeString(t)
+			if err != nil {
+				continue
+			}
+
+			hashes = append(hashes, hash)
+
+			if !matched {
+				var hash32 [32]byte
+				copy(hash32[:], hash)
+				hashHost, ok := c.hashToHost[hash32]
+				if ok {
+					log.Debug("%s: matched %s by %s/%s", c.svc, c.host, hashHost, t)
+					matched = true
+				}
+			}
+		}
+	}
+
+	return matched, hashes
+}
+
+func (c *sbCtx) storeCache(hashes [][]byte) {
+	sort.Slice(hashes, func(a, b int) bool {
+		return bytes.Compare(hashes[a], hashes[b]) < 0
+	})
+
+	var curData []byte
+	var prevPrefix []byte
+	for i, hash := range hashes {
+		prefix := hash[0:2]
+		if !bytes.Equal(prefix, prevPrefix) {
+			if i != 0 {
+				c.setCache(prevPrefix, curData)
+				curData = nil
+			}
+			prevPrefix = hashes[i][0:2]
+		}
+		curData = append(curData, hash...)
+	}
+
+	if len(prevPrefix) != 0 {
+		c.setCache(prevPrefix, curData)
+	}
+
+	for hash := range c.hashToHost {
+		prefix := hash[0:2]
+		val := c.cache.Get(prefix)
+		if val == nil {
+			c.setCache(prefix, nil)
+		}
+	}
+}
+
+// Disabling "dupl": the algorithm of SB/PC is similar, but it uses different data
+// nolint:dupl
+func (d *Dnsfilter) checkSafeBrowsing(host string) (Result, error) {
+	if log.GetLevel() >= log.DEBUG {
+		timer := log.StartTimer()
+		defer timer.LogElapsed("SafeBrowsing lookup for %s", host)
+	}
+
+	result := Result{}
+	hashes := hostnameToHashes(host)
+
+	c := &sbCtx{
+		host:       host,
+		svc:        "SafeBrowsing",
+		hashToHost: hashes,
+		cache:      gctx.safebrowsingCache,
+		cacheTime:  d.Config.CacheTime,
+	}
+
+	// check cache
+	match := c.getCached()
+	if match < 0 {
+		return result, nil
+	} else if match > 0 {
+		result.IsFiltered = true
+		result.Reason = FilteredSafeBrowsing
+		result.Rule = "adguard-malware-shavar"
+		return result, nil
+	}
+
+	question := c.getQuestion()
+	log.Tracef("SafeBrowsing: checking %s: %s", host, question)
+
+	req := dns.Msg{}
+	req.SetQuestion(question, dns.TypeTXT)
+	resp, err := d.safeBrowsingUpstream.Exchange(&req)
+	if err != nil {
+		return result, err
+	}
+
+	matched, receivedHashes := c.processTXT(resp)
+	if matched {
+		result.IsFiltered = true
+		result.Reason = FilteredSafeBrowsing
+		result.Rule = "adguard-malware-shavar"
+	}
+	c.storeCache(receivedHashes)
+
+	return result, nil
+}
+
+// Disabling "dupl": the algorithm of SB/PC is similar, but it uses different data
+// nolint:dupl
+func (d *Dnsfilter) checkParental(host string) (Result, error) {
+	if log.GetLevel() >= log.DEBUG {
+		timer := log.StartTimer()
+		defer timer.LogElapsed("Parental lookup for %s", host)
+	}
+
+	result := Result{}
+	hashes := hostnameToHashes(host)
+
+	c := &sbCtx{
+		host:       host,
+		svc:        "Parental",
+		hashToHost: hashes,
+		cache:      gctx.parentalCache,
+		cacheTime:  d.Config.CacheTime,
+	}
+
+	// check cache
+	match := c.getCached()
+	if match < 0 {
+		return result, nil
+	} else if match > 0 {
+		result.IsFiltered = true
+		result.Reason = FilteredParental
+		result.Rule = "parental CATEGORY_BLACKLISTED"
+		return result, nil
+	}
+
+	question := c.getQuestion()
+	log.Tracef("Parental: checking %s: %s", host, question)
+
+	req := dns.Msg{}
+	req.SetQuestion(question, dns.TypeTXT)
+	resp, err := d.parentalUpstream.Exchange(&req)
+	if err != nil {
+		return result, err
+	}
+
+	matched, receivedHashes := c.processTXT(resp)
+	if matched {
+		result.IsFiltered = true
+		result.Reason = FilteredParental
+		result.Rule = "parental CATEGORY_BLACKLISTED"
+	}
+	c.storeCache(receivedHashes)
+
+	return result, err
+}
+
+func httpError(r *http.Request, w http.ResponseWriter, code int, format string, args ...interface{}) {
+	text := fmt.Sprintf(format, args...)
+	log.Info("DNSFilter: %s %s: %s", r.Method, r.URL, text)
+	http.Error(w, text, code)
+}
+
+func (d *Dnsfilter) handleSafeBrowsingEnable(w http.ResponseWriter, r *http.Request) {
+	d.Config.SafeBrowsingEnabled = true
+	d.Config.ConfigModified()
+}
+
+func (d *Dnsfilter) handleSafeBrowsingDisable(w http.ResponseWriter, r *http.Request) {
+	d.Config.SafeBrowsingEnabled = false
+	d.Config.ConfigModified()
+}
+
+func (d *Dnsfilter) handleSafeBrowsingStatus(w http.ResponseWriter, r *http.Request) {
+	data := map[string]interface{}{
+		"enabled": d.Config.SafeBrowsingEnabled,
+	}
+	jsonVal, err := json.Marshal(data)
+	if err != nil {
+		httpError(r, w, http.StatusInternalServerError, "Unable to marshal status json: %s", err)
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_, err = w.Write(jsonVal)
+	if err != nil {
+		httpError(r, w, http.StatusInternalServerError, "Unable to write response json: %s", err)
+		return
+	}
+}
+
+func (d *Dnsfilter) handleParentalEnable(w http.ResponseWriter, r *http.Request) {
+	d.Config.ParentalEnabled = true
+	d.Config.ConfigModified()
+}
+
+func (d *Dnsfilter) handleParentalDisable(w http.ResponseWriter, r *http.Request) {
+	d.Config.ParentalEnabled = false
+	d.Config.ConfigModified()
+}
+
+func (d *Dnsfilter) handleParentalStatus(w http.ResponseWriter, r *http.Request) {
+	data := map[string]interface{}{
+		"enabled": d.Config.ParentalEnabled,
+	}
+	jsonVal, err := json.Marshal(data)
+	if err != nil {
+		httpError(r, w, http.StatusInternalServerError, "Unable to marshal status json: %s", err)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_, err = w.Write(jsonVal)
+	if err != nil {
+		httpError(r, w, http.StatusInternalServerError, "Unable to write response json: %s", err)
+		return
+	}
+}
+
+func (d *Dnsfilter) registerSecurityHandlers() {
+	d.Config.HTTPRegister("POST", "/control/safebrowsing/enable", d.handleSafeBrowsingEnable)
+	d.Config.HTTPRegister("POST", "/control/safebrowsing/disable", d.handleSafeBrowsingDisable)
+	d.Config.HTTPRegister("GET", "/control/safebrowsing/status", d.handleSafeBrowsingStatus)
+
+	d.Config.HTTPRegister("POST", "/control/parental/enable", d.handleParentalEnable)
+	d.Config.HTTPRegister("POST", "/control/parental/disable", d.handleParentalDisable)
+	d.Config.HTTPRegister("GET", "/control/parental/status", d.handleParentalStatus)
+
+	d.Config.HTTPRegister("POST", "/control/safesearch/enable", d.handleSafeSearchEnable)
+	d.Config.HTTPRegister("POST", "/control/safesearch/disable", d.handleSafeSearchDisable)
+	d.Config.HTTPRegister("GET", "/control/safesearch/status", d.handleSafeSearchStatus)
+}

internal/dnsfilter/sb_pc_test.go

@@ -0,0 +1,91 @@
+package dnsfilter
+
+import (
+	"crypto/sha256"
+	"strings"
+	"testing"
+
+	"github.com/AdguardTeam/golibs/cache"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSafeBrowsingHash(t *testing.T) {
+	// test hostnameToHashes()
+	hashes := hostnameToHashes("1.2.3.sub.host.com")
+	assert.Equal(t, 3, len(hashes))
+	_, ok := hashes[sha256.Sum256([]byte("3.sub.host.com"))]
+	assert.True(t, ok)
+	_, ok = hashes[sha256.Sum256([]byte("sub.host.com"))]
+	assert.True(t, ok)
+	_, ok = hashes[sha256.Sum256([]byte("host.com"))]
+	assert.True(t, ok)
+	_, ok = hashes[sha256.Sum256([]byte("com"))]
+	assert.False(t, ok)
+
+	c := &sbCtx{
+		svc: "SafeBrowsing",
+	}
+
+	// test getQuestion()
+	c.hashToHost = hashes
+	q := c.getQuestion()
+	assert.True(t, strings.Index(q, "7a1b.") >= 0)
+	assert.True(t, strings.Index(q, "af5a.") >= 0)
+	assert.True(t, strings.Index(q, "eb11.") >= 0)
+	assert.True(t, strings.Index(q, "sb.dns.adguard.com.") > 0)
+}
+
+func TestSafeBrowsingCache(t *testing.T) {
+	c := &sbCtx{
+		svc:       "SafeBrowsing",
+		cacheTime: 100,
+	}
+	conf := cache.Config{}
+	c.cache = cache.New(conf)
+
+	// store in cache hashes for "3.sub.host.com" and "host.com"
+	//  and empty data for hash-prefix for "sub.host.com"
+	hash := sha256.Sum256([]byte("sub.host.com"))
+	c.hashToHost = make(map[[32]byte]string)
+	c.hashToHost[hash] = "sub.host.com"
+	var hashesArray [][]byte
+	hash4 := sha256.Sum256([]byte("3.sub.host.com"))
+	hashesArray = append(hashesArray, hash4[:])
+	hash2 := sha256.Sum256([]byte("host.com"))
+	hashesArray = append(hashesArray, hash2[:])
+	c.storeCache(hashesArray)
+
+	// match "3.sub.host.com" or "host.com" from cache
+	c.hashToHost = make(map[[32]byte]string)
+	hash = sha256.Sum256([]byte("3.sub.host.com"))
+	c.hashToHost[hash] = "3.sub.host.com"
+	hash = sha256.Sum256([]byte("sub.host.com"))
+	c.hashToHost[hash] = "sub.host.com"
+	hash = sha256.Sum256([]byte("host.com"))
+	c.hashToHost[hash] = "host.com"
+	assert.Equal(t, 1, c.getCached())
+
+	// match "sub.host.com" from cache
+	c.hashToHost = make(map[[32]byte]string)
+	hash = sha256.Sum256([]byte("sub.host.com"))
+	c.hashToHost[hash] = "sub.host.com"
+	assert.Equal(t, -1, c.getCached())
+
+	// match "sub.host.com" from cache,
+	//  but another hash for "nonexisting.com" is not in cache
+	//  which means that we must get data from server for it
+	c.hashToHost = make(map[[32]byte]string)
+	hash = sha256.Sum256([]byte("sub.host.com"))
+	c.hashToHost[hash] = "sub.host.com"
+	hash = sha256.Sum256([]byte("nonexisting.com"))
+	c.hashToHost[hash] = "nonexisting.com"
+	assert.Equal(t, 0, c.getCached())
+
+	hash = sha256.Sum256([]byte("sub.host.com"))
+	_, ok := c.hashToHost[hash]
+	assert.False(t, ok)
+
+	hash = sha256.Sum256([]byte("nonexisting.com"))
+	_, ok = c.hashToHost[hash]
+	assert.True(t, ok)
+}