Initial commit

dnsfilter/README.md

@@ -0,0 +1,70 @@
+# AdGuard DNS Go library
+
+Example use:
+```bash
+[ -z "$GOPATH" ] && export GOPATH=$HOME/go
+go get -d github.com/AdguardTeam/AdguardDNS/dnsfilter
+```
+
+Create file filter.go
+```filter.go
+package main
+
+import (
+    "github.com/AdguardTeam/AdguardDNS/dnsfilter"
+    "log"
+)
+
+func main() {
+    filter := dnsfilter.New()
+    filter.AddRule("||dou*ck.net^")
+    host := "www.doubleclick.net"
+    res, err := filter.CheckHost(host)
+    if err != nil {
+        // temporary failure
+        log.Fatalf("Failed to check host '%s': %s", host, err)
+    }
+    if res.IsFiltered {
+        log.Printf("Host %s is filtered, reason - '%s', matched rule: '%s'", host, res.Reason, res.Rule)
+    } else {
+        log.Printf("Host %s is not filtered, reason - '%s'", host, res.Reason)
+    }
+}
+```
+
+And then run it:
+```bash
+go run filter.go
+```
+
+You will get:
+```
+2000/01/01 00:00:00 Host www.doubleclick.net is filtered, reason - 'FilteredBlackList', matched rule: '||dou*ck.net^'
+```
+
+You can also enable checking against AdGuard's SafeBrowsing:
+
+```go
+package main
+
+import (
+    "github.com/AdguardTeam/AdguardDNS/dnsfilter"
+    "log"
+)
+
+func main() {
+    filter := dnsfilter.New()
+    filter.EnableSafeBrowsing()
+    host := "wmconvirus.narod.ru" // hostname for testing safebrowsing
+    res, err := filter.CheckHost(host)
+    if err != nil {
+        // temporary failure
+        log.Fatalf("Failed to check host '%s': %s", host, err)
+    }
+    if res.IsFiltered {
+        log.Printf("Host %s is filtered, reason - '%s', matched rule: '%s'", host, res.Reason, res.Rule)
+    } else {
+        log.Printf("Host %s is not filtered, reason - '%s'", host, res.Reason)
+    }
+}
+```

dnsfilter/dnsfilter.go

@@ -0,0 +1,798 @@
+package dnsfilter
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/sha256"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"regexp"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/bluele/gcache"
+	"golang.org/x/net/publicsuffix"
+)
+
+const defaultCacheSize = 64 * 1024 // in number of elements
+const defaultCacheTime time.Duration = 30 * time.Minute
+
+const defaultHTTPTimeout time.Duration = 5 * time.Minute
+const defaultHTTPMaxIdleConnections = 100
+
+const defaultSafebrowsingServer = "sb.adtidy.org"
+const defaultSafebrowsingURL = "http://%s/safebrowsing-lookup-hash.html?prefixes=%s"
+const defaultParentalURL = "http://pctrl.adguard.com/check-parental-control-hash?prefixes=%s&sensitivity=%d"
+
+var ErrInvalidSyntax = errors.New("dnsfilter: invalid rule syntax")
+var ErrInvalidParental = errors.New("dnsfilter: invalid parental sensitivity, must be either 3, 10, 13 or 17")
+
+const shortcutLength = 6 // used for rule search optimization, 6 hits the sweet spot
+
+const enableFastLookup = true         // flag for debugging, must be true in production for faster performance
+const enableDelayedCompilation = true // flag for debugging, must be true in production for faster performance
+
+type Config struct {
+	safeSearchEnabled   bool
+	safeBrowsingEnabled bool
+	safeBrowsingServer  string
+	parentalEnabled     bool
+	parentalSensitivity int // must be either 3, 10, 13 or 17
+}
+
+type Rule struct {
+	text         string // text without @@ decorators or $ options
+	shortcut     string // for speeding up lookup
+	originalText string // original text for reporting back to applications
+
+	// options
+	options []string // optional options after $
+
+	// parsed options
+	isWhitelist bool
+	isImportant bool
+	apps        []string
+
+	// user-supplied data
+	listID uint32
+
+	// compiled regexp
+	compiled *regexp.Regexp
+
+	sync.RWMutex
+}
+
+type LookupStats struct {
+	Requests   uint64 // number of HTTP requests that were sent
+	CacheHits  uint64 // number of lookups that didn't need HTTP requests
+	Pending    int64  // number of currently pending HTTP requests
+	PendingMax int64  // maximum number of pending HTTP requests
+}
+
+type Stats struct {
+	Safebrowsing LookupStats
+	Parental     LookupStats
+}
+
+// Dnsfilter holds added rules and performs hostname matches against the rules
+type Dnsfilter struct {
+	storage      map[string]*Rule // rule storage, not used for matching, needs to be key->value
+	storageMutex sync.RWMutex
+
+	// rules are checked against these lists in the order defined here
+	important *rulesTable // more important than whitelist and is checked first
+	whiteList *rulesTable // more important than blacklist
+	blackList *rulesTable
+
+	// HTTP lookups for safebrowsing and parental
+	client    http.Client     // handle for http client -- single instance as recommended by docs
+	transport *http.Transport // handle for http transport used by http client
+
+	config Config
+}
+
+//go:generate stringer -type=Reason
+
+// filtered/notfiltered reason
+type Reason int
+
+const (
+	// reasons for not filtering
+	NotFilteredNotFound  Reason = iota // host was not find in any checks, default value for result
+	NotFilteredWhiteList               // the host is explicitly whitelisted
+	NotFilteredError                   // there was a transitive error during check
+
+	// reasons for filtering
+	FilteredBlackList    // the host was matched to be advertising host
+	FilteredSafeBrowsing // the host was matched to be malicious/phishing
+	FilteredParental     // the host was matched to be outside of parental control settings
+	FilteredInvalid      // the request was invalid and was not processed
+	FilteredSafeSearch   // the host was replaced with safesearch variant
+)
+
+// these variables need to survive coredns reload
+var (
+	stats             Stats
+	safebrowsingCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build()
+	parentalCache     = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build()
+)
+
+// search result
+type Result struct {
+	IsFiltered bool
+	Reason     Reason
+	Rule       string
+}
+
+func (r Reason) Matched() bool {
+	return r != NotFilteredNotFound
+}
+
+// CheckHost tries to match host against rules, then safebrowsing and parental if they are enabled
+func (d *Dnsfilter) CheckHost(host string) (Result, error) {
+	// sometimes DNS clients will try to resolve ".", which in turns transforms into "" when it reaches here
+	if host == "" {
+		return Result{Reason: FilteredInvalid}, nil
+	}
+
+	// try filter lists first
+	result, err := d.matchHost(host)
+	if err != nil {
+		return result, err
+	}
+	if result.Reason.Matched() {
+		return result, nil
+	}
+
+	// check safebrowsing if no match
+	if d.config.safeBrowsingEnabled {
+		result, err = d.checkSafeBrowsing(host)
+		if err != nil {
+			// failed to do HTTP lookup -- treat it as if we got empty response, but don't save cache
+			log.Printf("Failed to do safebrowsing HTTP lookup, ignoring check: %v", err)
+			return Result{}, nil
+		}
+		if result.Reason.Matched() {
+			return result, nil
+		}
+	}
+
+	// check parental if no match
+	if d.config.parentalEnabled {
+		result, err = d.checkParental(host)
+		if err != nil {
+			// failed to do HTTP lookup -- treat it as if we got empty response, but don't save cache
+			log.Printf("Failed to do parental HTTP lookup, ignoring check: %v", err)
+			return Result{}, nil
+		}
+		if result.Reason.Matched() {
+			return result, nil
+		}
+	}
+
+	// nothing matched, return nothing
+	return Result{}, nil
+}
+
+//
+// rules table
+//
+
+type rulesTable struct {
+	rulesByShortcut map[string][]*Rule
+	rulesLeftovers  []*Rule
+	sync.RWMutex
+}
+
+func newRulesTable() *rulesTable {
+	return &rulesTable{
+		rulesByShortcut: make(map[string][]*Rule),
+		rulesLeftovers:  make([]*Rule, 0),
+	}
+}
+
+func (r *rulesTable) Add(rule *Rule) {
+	r.Lock()
+	if len(rule.shortcut) == shortcutLength && enableFastLookup {
+		r.rulesByShortcut[rule.shortcut] = append(r.rulesByShortcut[rule.shortcut], rule)
+	} else {
+		r.rulesLeftovers = append(r.rulesLeftovers, rule)
+	}
+	r.Unlock()
+}
+
+func (r *rulesTable) matchByHost(host string) (Result, error) {
+	res, err := r.searchShortcuts(host)
+	if err != nil {
+		return res, err
+	}
+	if res.Reason.Matched() {
+		return res, nil
+	}
+
+	res, err = r.searchLeftovers(host)
+	if err != nil {
+		return res, err
+	}
+	if res.Reason.Matched() {
+		return res, nil
+	}
+
+	return Result{}, nil
+}
+
+func (r *rulesTable) searchShortcuts(host string) (Result, error) {
+	// check in shortcuts first
+	for i := 0; i < len(host); i++ {
+		shortcut := host[i:]
+		if len(shortcut) > shortcutLength {
+			shortcut = shortcut[:shortcutLength]
+		}
+		if len(shortcut) != shortcutLength {
+			continue
+		}
+		rules, ok := r.rulesByShortcut[shortcut]
+		if !ok {
+			continue
+		}
+		for _, rule := range rules {
+			res, err := rule.match(host)
+			// error? stop search
+			if err != nil {
+				return res, err
+			}
+			// matched? stop search
+			if res.Reason.Matched() {
+				return res, err
+			}
+			// continue otherwise
+		}
+	}
+	return Result{}, nil
+}
+
+func (r *rulesTable) searchLeftovers(host string) (Result, error) {
+	for _, rule := range r.rulesLeftovers {
+		res, err := rule.match(host)
+		// error? stop search
+		if err != nil {
+			return res, err
+		}
+		// matched? stop search
+		if res.Reason.Matched() {
+			return res, err
+		}
+		// continue otherwise
+	}
+	return Result{}, nil
+}
+
+func findOptionIndex(text string) int {
+	for i, r := range text {
+		// ignore non-$
+		if r != '$' {
+			continue
+		}
+		// ignore `\$`
+		if i > 0 && text[i-1] == '\\' {
+			continue
+		}
+		// ignore `$/`
+		if i > len(text) && text[i+1] == '/' {
+			continue
+		}
+		return i + 1
+	}
+	return -1
+}
+
+func (rule *Rule) extractOptions() error {
+	optIndex := findOptionIndex(rule.text)
+	if optIndex == 0 { // starts with $
+		return ErrInvalidSyntax
+	}
+	if optIndex == len(rule.text) { // ends with $
+		return ErrInvalidSyntax
+	}
+	if optIndex < 0 {
+		return nil
+	}
+
+	optionsStr := rule.text[optIndex:]
+	rule.text = rule.text[:optIndex-1] // remove options from text
+
+	begin := 0
+	i := 0
+	for i = 0; i < len(optionsStr); i++ {
+		switch optionsStr[i] {
+		case ',':
+			if i > 0 {
+				// it might be escaped, if so, ignore
+				if optionsStr[i-1] == '\\' {
+					break // from switch, not for loop
+				}
+			}
+			rule.options = append(rule.options, optionsStr[begin:i])
+			begin = i + 1
+		}
+	}
+	if begin != i {
+		// there's still an option remaining
+		rule.options = append(rule.options, optionsStr[begin:])
+	}
+
+	return nil
+}
+
+func (rule *Rule) parseOptions() error {
+	err := rule.extractOptions()
+	if err != nil {
+		return err
+	}
+
+	for _, option := range rule.options {
+		switch {
+		case option == "important":
+			rule.isImportant = true
+		case strings.HasPrefix(option, "app="):
+			option = strings.TrimPrefix(option, "app=")
+			rule.apps = strings.Split(option, "|")
+		default:
+			return ErrInvalidSyntax
+		}
+	}
+
+	return nil
+}
+
+func (rule *Rule) extractShortcut() {
+	// regex rules have no shortcuts
+	if rule.text[0] == '/' && rule.text[len(rule.text)-1] == '/' {
+		return
+	}
+
+	fields := strings.FieldsFunc(rule.text, func(r rune) bool {
+		switch r {
+		case '*', '^', '|':
+			return true
+		}
+		return false
+	})
+	longestField := ""
+	for _, field := range fields {
+		if len(field) > len(longestField) {
+			longestField = field
+		}
+	}
+	if len(longestField) > shortcutLength {
+		longestField = longestField[:shortcutLength]
+	}
+	rule.shortcut = strings.ToLower(longestField)
+}
+
+func (rule *Rule) compile() error {
+	rule.RLock()
+	isCompiled := rule.compiled != nil
+	rule.RUnlock()
+	if isCompiled {
+		return nil
+	}
+
+	expr, err := ruleToRegexp(rule.text)
+	if err != nil {
+		return err
+	}
+
+	compiled, err := regexp.Compile(expr)
+	if err != nil {
+		return err
+	}
+
+	rule.Lock()
+	rule.compiled = compiled
+	rule.Unlock()
+
+	return nil
+}
+
+func (rule *Rule) match(host string) (Result, error) {
+	res := Result{}
+	err := rule.compile()
+	if err != nil {
+		return res, err
+	}
+	rule.RLock()
+	matched := rule.compiled.MatchString(host)
+	rule.RUnlock()
+	if matched {
+		res.Reason = FilteredBlackList
+		res.IsFiltered = true
+		if rule.isWhitelist {
+			res.Reason = NotFilteredWhiteList
+			res.IsFiltered = false
+		}
+		res.Rule = rule.text
+	}
+	return res, nil
+}
+
+func getCachedReason(cache gcache.Cache, host string) (result Result, isFound bool, err error) {
+	isFound = false // not found yet
+
+	// get raw value
+	rawValue, err := cache.Get(host)
+	if err == gcache.KeyNotFoundError {
+		// not a real error, just not found
+		err = nil
+		return
+	}
+	if err != nil {
+		// real error
+		return
+	}
+
+	// since it can be something else, validate that it belongs to proper type
+	cachedValue, ok := rawValue.(Result)
+	if ok == false {
+		// this is not our type -- error
+		text := "SHOULD NOT HAPPEN: entry with invalid type was found in lookup cache"
+		log.Println(text)
+		err = errors.New(text)
+		return
+	}
+	isFound = ok
+	return cachedValue, isFound, err
+}
+
+// for each dot, hash it and add it to string
+func hostnameToHashParam(host string, addslash bool) (string, map[string]bool) {
+	var hashparam bytes.Buffer
+	hashes := map[string]bool{}
+	tld, icann := publicsuffix.PublicSuffix(host)
+	if icann == false {
+		// private suffixes like cloudfront.net
+		tld = ""
+	}
+	curhost := host
+	for {
+		if curhost == "" {
+			// we've reached end of string
+			break
+		}
+		if tld != "" && curhost == tld {
+			// we've reached the TLD, don't hash it
+			break
+		}
+		tohash := []byte(curhost)
+		if addslash {
+			tohash = append(tohash, '/')
+		}
+		sum := sha256.Sum256(tohash)
+		hexhash := fmt.Sprintf("%X", sum)
+		hashes[hexhash] = true
+		hashparam.WriteString(fmt.Sprintf("%02X%02X%02X%02X/", sum[0], sum[1], sum[2], sum[3]))
+		pos := strings.IndexByte(curhost, byte('.'))
+		if pos < 0 {
+			break
+		}
+		curhost = curhost[pos+1:]
+	}
+	return hashparam.String(), hashes
+}
+
+func (d *Dnsfilter) checkSafeBrowsing(host string) (Result, error) {
+	format := func(hashparam string) string {
+		url := fmt.Sprintf(defaultSafebrowsingURL, d.config.safeBrowsingServer, hashparam)
+		return url
+	}
+	handleBody := func(body []byte, hashes map[string]bool) (Result, error) {
+		result := Result{}
+		scanner := bufio.NewScanner(strings.NewReader(string(body)))
+		for scanner.Scan() {
+			line := scanner.Text()
+			splitted := strings.Split(line, ":")
+			if len(splitted) < 3 {
+				continue
+			}
+			hash := splitted[2]
+			if _, ok := hashes[hash]; ok {
+				// it's in the hash
+				result.IsFiltered = true
+				result.Reason = FilteredSafeBrowsing
+				result.Rule = splitted[0]
+				break
+			}
+		}
+
+		if err := scanner.Err(); err != nil {
+			// error, don't save cache
+			return Result{}, err
+		}
+		return result, nil
+	}
+	result, err := d.lookupCommon(host, &stats.Safebrowsing, safebrowsingCache, true, format, handleBody)
+	return result, err
+}
+
+func (d *Dnsfilter) checkParental(host string) (Result, error) {
+	format2 := func(hashparam string) string {
+		url := fmt.Sprintf(defaultParentalURL, hashparam, d.config.parentalSensitivity)
+		return url
+	}
+	handleBody2 := func(body []byte, hashes map[string]bool) (Result, error) {
+		// parse json
+		var m []struct {
+			Blocked   bool   `json:"blocked"`
+			ClientTTL int    `json:"clientTtl"`
+			Reason    string `json:"reason"`
+		}
+		err := json.Unmarshal(body, &m)
+		if err != nil {
+			// error, don't save cache
+			log.Printf("Couldn't parse json '%s': %s", body, err)
+			return Result{}, err
+		}
+
+		result := Result{}
+
+		for i := range m {
+			if m[i].Blocked {
+				result.IsFiltered = true
+				result.Reason = FilteredParental
+				result.Rule = fmt.Sprintf("parental %s", m[i].Reason)
+				break
+			}
+		}
+		return result, nil
+	}
+	result, err := d.lookupCommon(host, &stats.Parental, parentalCache, false, format2, handleBody2)
+	return result, err
+}
+
+// real implementation of lookup/check
+func (d *Dnsfilter) lookupCommon(host string, lookupstats *LookupStats, cache gcache.Cache, hashparamNeedSlash bool, format func(hashparam string) string, handleBody func(body []byte, hashes map[string]bool) (Result, error)) (Result, error) {
+	// if host ends with a dot, trim it
+	host = strings.ToLower(strings.Trim(host, "."))
+
+	// check cache
+	cachedValue, isFound, err := getCachedReason(cache, host)
+	if isFound {
+		atomic.AddUint64(&stats.Safebrowsing.CacheHits, 1)
+		return cachedValue, nil
+	}
+	if err != nil {
+		return Result{}, err
+	}
+
+	// convert hostname to hash parameters
+	hashparam, hashes := hostnameToHashParam(host, hashparamNeedSlash)
+
+	// format URL with our hashes
+	url := format(hashparam)
+
+	// do HTTP request
+	atomic.AddUint64(&lookupstats.Requests, 1)
+	atomic.AddInt64(&lookupstats.Pending, 1)
+	updateMax(&lookupstats.Pending, &lookupstats.PendingMax)
+	resp, err := d.client.Get(url)
+	atomic.AddInt64(&lookupstats.Pending, -1)
+	if resp != nil && resp.Body != nil {
+		defer resp.Body.Close()
+	}
+	if err != nil {
+		// error, don't save cache
+		return Result{}, err
+	}
+
+	// get body text
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		// error, don't save cache
+		return Result{}, err
+	}
+
+	// handle status code
+	switch {
+	case resp.StatusCode == 204:
+		// empty result, save cache
+		cache.Set(host, Result{})
+		return Result{}, nil
+	case resp.StatusCode != 200:
+		// error, don't save cache
+		return Result{}, nil
+	}
+
+	result, err := handleBody(body, hashes)
+	if err != nil {
+		// error, don't save cache
+		return Result{}, err
+	}
+
+	cache.Set(host, result)
+	return result, nil
+}
+
+//
+// Adding rule and matching against the rules
+//
+
+// AddRule adds a rule, checking if it is a valid rule first and if it wasn't added already
+func (d *Dnsfilter) AddRule(input string, filterListID uint32) error {
+	input = strings.TrimSpace(input)
+	d.storageMutex.RLock()
+	_, exists := d.storage[input]
+	d.storageMutex.RUnlock()
+	if exists {
+		// already added
+		return ErrInvalidSyntax
+	}
+
+	if !isValidRule(input) {
+		return ErrInvalidSyntax
+	}
+
+	rule := Rule{
+		text:         input, // will be modified
+		originalText: input,
+		listID:       filterListID,
+	}
+
+	// mark rule as whitelist if it starts with @@
+	if strings.HasPrefix(rule.text, "@@") {
+		rule.isWhitelist = true
+		rule.text = rule.text[2:]
+	}
+
+	err := rule.parseOptions()
+	if err != nil {
+		return err
+	}
+
+	rule.extractShortcut()
+
+	if !enableDelayedCompilation {
+		err := rule.compile()
+		if err != nil {
+			return err
+		}
+	}
+
+	destination := d.blackList
+	if rule.isImportant {
+		destination = d.important
+	} else if rule.isWhitelist {
+		destination = d.whiteList
+	}
+
+	d.storageMutex.Lock()
+	d.storage[input] = &rule
+	d.storageMutex.Unlock()
+	destination.Add(&rule)
+	return nil
+}
+
+// matchHost is a low-level way to check only if hostname is filtered by rules, skipping expensive safebrowsing and parental lookups
+func (d *Dnsfilter) matchHost(host string) (Result, error) {
+	lists := []*rulesTable{
+		d.important,
+		d.whiteList,
+		d.blackList,
+	}
+
+	for _, table := range lists {
+		res, err := table.matchByHost(host)
+		if err != nil {
+			return res, err
+		}
+		if res.Reason.Matched() {
+			return res, nil
+		}
+	}
+	return Result{}, nil
+}
+
+//
+// lifecycle helper functions
+//
+
+func New() *Dnsfilter {
+	d := new(Dnsfilter)
+
+	d.storage = make(map[string]*Rule)
+	d.important = newRulesTable()
+	d.whiteList = newRulesTable()
+	d.blackList = newRulesTable()
+
+	// Customize the Transport to have larger connection pool
+	defaultRoundTripper := http.DefaultTransport
+	defaultTransportPointer, ok := defaultRoundTripper.(*http.Transport)
+	if !ok {
+		panic(fmt.Sprintf("defaultRoundTripper not an *http.Transport"))
+	}
+	d.transport = defaultTransportPointer                           // dereference it to get a copy of the struct that the pointer points to
+	d.transport.MaxIdleConns = defaultHTTPMaxIdleConnections        // default 100
+	d.transport.MaxIdleConnsPerHost = defaultHTTPMaxIdleConnections // default 2
+	d.client = http.Client{
+		Transport: d.transport,
+		Timeout:   defaultHTTPTimeout,
+	}
+	d.config.safeBrowsingServer = defaultSafebrowsingServer
+	return d
+}
+
+func (d *Dnsfilter) Destroy() {
+	d.transport.CloseIdleConnections()
+}
+
+//
+// config manipulation helpers
+//
+
+func (d *Dnsfilter) EnableSafeBrowsing() {
+	d.config.safeBrowsingEnabled = true
+}
+
+func (d *Dnsfilter) EnableParental(sensitivity int) error {
+	switch sensitivity {
+	case 3, 10, 13, 17:
+		d.config.parentalSensitivity = sensitivity
+		d.config.parentalEnabled = true
+		return nil
+	default:
+		return ErrInvalidParental
+	}
+}
+
+func (d *Dnsfilter) EnableSafeSearch() {
+	d.config.safeSearchEnabled = true
+}
+
+func (d *Dnsfilter) SetSafeBrowsingServer(host string) {
+	if len(host) == 0 {
+		d.config.safeBrowsingServer = defaultSafebrowsingServer
+	} else {
+		d.config.safeBrowsingServer = host
+	}
+}
+
+func (d *Dnsfilter) SetHTTPTimeout(t time.Duration) {
+	d.client.Timeout = t
+}
+
+func (d *Dnsfilter) ResetHTTPTimeout() {
+	d.client.Timeout = defaultHTTPTimeout
+}
+
+func (d *Dnsfilter) SafeSearchDomain(host string) (string, bool) {
+	if d.config.safeSearchEnabled == false {
+		return "", false
+	}
+	val, ok := safeSearchDomains[host]
+	return val, ok
+}
+
+//
+// stats
+//
+
+func (d *Dnsfilter) GetStats() Stats {
+	return stats
+}
+
+func (d *Dnsfilter) Count() int {
+	return len(d.storage)
+}
+
+//
+// cache control, right now needed only for tests
+//
+func purgeCaches() {
+	safebrowsingCache.Purge()
+	parentalCache.Purge()
+}

dnsfilter/dnsfilter_test.go

@@ -0,0 +1,640 @@
+package dnsfilter
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"bufio"
+	"fmt"
+	"os"
+	"runtime"
+
+	"go.uber.org/goleak"
+)
+
+func TestRuleToRegexp(t *testing.T) {
+	tests := []struct {
+		rule   string
+		result string
+		err    error
+	}{
+		{"/doubleclick/", "doubleclick", nil},
+		{"/", "", ErrInvalidSyntax},
+		{`|double*?.+[]|(){}#$\|`, `^double.*\?\.\+\[\]\|\(\)\{\}\#\$\\$`, nil},
+		{`||doubleclick.net^`, `^([a-z0-9-_.]+\.)?doubleclick\.net([^ a-zA-Z0-9.%]|$)`, nil},
+	}
+	for _, testcase := range tests {
+		converted, err := ruleToRegexp(testcase.rule)
+		if err != testcase.err {
+			t.Error("Errors do not match, got ", err, " expected ", testcase.err)
+		}
+		if converted != testcase.result {
+			t.Error("Results do not match, got ", converted, " expected ", testcase.result)
+		}
+	}
+}
+
+//
+// helper functions
+//
+func (d *Dnsfilter) checkAddRule(t *testing.T, rule string) {
+	t.Helper()
+	err := d.AddRule(rule, 0)
+	if err == nil {
+		// nothing to report
+		return
+	}
+	if err == ErrInvalidSyntax {
+		t.Errorf("This rule has invalid syntax: %s", rule)
+	}
+	if err != nil {
+		t.Errorf("Error while adding rule %s: %s", rule, err)
+	}
+}
+
+func (d *Dnsfilter) checkAddRuleFail(t *testing.T, rule string) {
+	t.Helper()
+	err := d.AddRule(rule, 0)
+	if err == ErrInvalidSyntax {
+		return
+	}
+	if err != nil {
+		t.Errorf("Error while adding rule %s: %s", rule, err)
+	}
+	t.Errorf("Adding this rule should have failed: %s", rule)
+}
+
+func (d *Dnsfilter) checkMatch(t *testing.T, hostname string) {
+	t.Helper()
+	ret, err := d.CheckHost(hostname)
+	if err != nil {
+		t.Errorf("Error while matching host %s: %s", hostname, err)
+	}
+	if !ret.IsFiltered {
+		t.Errorf("Expected hostname %s to match", hostname)
+	}
+}
+
+func (d *Dnsfilter) checkMatchEmpty(t *testing.T, hostname string) {
+	t.Helper()
+	ret, err := d.CheckHost(hostname)
+	if err != nil {
+		t.Errorf("Error while matching host %s: %s", hostname, err)
+	}
+	if ret.IsFiltered {
+		t.Errorf("Expected hostname %s to not match", hostname)
+	}
+}
+
+func loadTestRules(d *Dnsfilter) error {
+	filterFileName := "../tests/dns.txt"
+	file, err := os.Open(filterFileName)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		rule := scanner.Text()
+		err = d.AddRule(rule, 0)
+		if err == ErrInvalidSyntax {
+			continue
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	err = scanner.Err()
+	return err
+}
+
+func NewForTest() *Dnsfilter {
+	d := New()
+	purgeCaches()
+	return d
+}
+
+//
+// tests
+//
+func TestSanityCheck(t *testing.T) {
+	d := NewForTest()
+	defer d.Destroy()
+	d.checkAddRule(t, "||doubleclick.net^")
+	d.checkMatch(t, "www.doubleclick.net")
+	d.checkMatchEmpty(t, "nodoubleclick.net")
+	d.checkMatchEmpty(t, "doubleclick.net.ru")
+	d.checkMatchEmpty(t, "wmconvirus.narod.ru")
+	d.checkAddRuleFail(t, "lkfaojewhoawehfwacoefawr$@#$@3413841384")
+}
+
+func TestCount(t *testing.T) {
+	d := NewForTest()
+	defer d.Destroy()
+	err := loadTestRules(d)
+	if err != nil {
+		t.Fatal(err)
+	}
+	count := d.Count()
+	expected := 12747
+	if count != expected {
+		t.Fatalf("Number of rules parsed should be %d, but it is %d\n", expected, count)
+	}
+}
+
+func TestDnsFilterBlocking(t *testing.T) {
+	d := NewForTest()
+	defer d.Destroy()
+	d.checkAddRule(t, "||example.org^")
+
+	d.checkMatch(t, "example.org")
+	d.checkMatch(t, "test.example.org")
+	d.checkMatch(t, "test.test.example.org")
+	d.checkMatchEmpty(t, "testexample.org")
+	d.checkMatchEmpty(t, "onemoreexample.org")
+}
+
+func TestDnsFilterWhitelist(t *testing.T) {
+	d := NewForTest()
+	defer d.Destroy()
+	d.checkAddRule(t, "||example.org^")
+	d.checkAddRule(t, "@@||test.example.org")
+
+	d.checkMatch(t, "example.org")
+	d.checkMatchEmpty(t, "test.example.org")
+	d.checkMatchEmpty(t, "test.test.example.org")
+}
+
+func TestDnsFilterImportant(t *testing.T) {
+	d := NewForTest()
+	defer d.Destroy()
+	d.checkAddRule(t, "@@||example.org^")
+	d.checkAddRule(t, "||test.example.org^$important")
+
+	d.checkMatchEmpty(t, "example.org")
+	d.checkMatch(t, "test.example.org")
+	d.checkMatch(t, "test.test.example.org")
+	d.checkMatchEmpty(t, "testexample.org")
+	d.checkMatchEmpty(t, "onemoreexample.org")
+}
+
+func TestDnsFilterRegexrule(t *testing.T) {
+	d := NewForTest()
+	defer d.Destroy()
+	d.checkAddRule(t, "/example\\.org/")
+	d.checkAddRule(t, "@@||test.example.org^")
+
+	d.checkMatch(t, "example.org")
+	d.checkMatchEmpty(t, "test.example.org")
+	d.checkMatchEmpty(t, "test.test.example.org")
+	d.checkMatch(t, "testexample.org")
+	d.checkMatch(t, "onemoreexample.org")
+}
+
+func TestDomainMask(t *testing.T) {
+	d := NewForTest()
+	defer d.Destroy()
+	d.checkAddRule(t, "test*.example.org^")
+	d.checkAddRule(t, "exam*.com")
+
+	d.checkMatch(t, "test.example.org")
+	d.checkMatch(t, "test2.example.org")
+	d.checkMatch(t, "example.com")
+	d.checkMatch(t, "exampleeee.com")
+
+	d.checkMatchEmpty(t, "example.org")
+	d.checkMatchEmpty(t, "testexample.org")
+	d.checkMatchEmpty(t, "example.co.uk")
+}
+
+func TestAddRuleFail(t *testing.T) {
+	d := NewForTest()
+	defer d.Destroy()
+	d.checkAddRuleFail(t, "lkfaojewhoawehfwacoefawr$@#$@3413841384")
+}
+
+func printMemStats(r runtime.MemStats) {
+	fmt.Printf("Alloc: %.2f, HeapAlloc: %.2f Mb, Sys: %.2f Mb, HeapSys: %.2f Mb\n",
+		float64(r.Alloc)/1024.0/1024.0, float64(r.HeapAlloc)/1024.0/1024.0,
+		float64(r.Sys)/1024.0/1024.0, float64(r.HeapSys)/1024.0/1024.0)
+}
+
+func TestLotsOfRulesMemoryUsage(t *testing.T) {
+	var start, afterLoad, end runtime.MemStats
+	runtime.GC()
+	runtime.ReadMemStats(&start)
+	fmt.Printf("Memory usage before loading rules - %d kB alloc, %d kB sys\n", start.Alloc/1024, start.Sys/1024)
+
+	d := NewForTest()
+	defer d.Destroy()
+	err := loadTestRules(d)
+	if err != nil {
+		t.Error(err)
+	}
+	runtime.GC()
+	runtime.ReadMemStats(&afterLoad)
+	fmt.Printf("Memory usage after loading rules - %d kB alloc, %d kB sys\n", afterLoad.Alloc/1024, afterLoad.Sys/1024)
+
+	tests := []struct {
+		host  string
+		match bool
+	}{
+		{"asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com", false},
+		{"asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net", true},
+	}
+	for _, testcase := range tests {
+		ret, err := d.CheckHost(testcase.host)
+		if err != nil {
+			t.Errorf("Error while matching host %s: %s", testcase.host, err)
+		}
+		if ret.IsFiltered == false && ret.IsFiltered != testcase.match {
+			t.Errorf("Expected hostname %s to not match", testcase.host)
+		}
+		if ret.IsFiltered == true && ret.IsFiltered != testcase.match {
+			t.Errorf("Expected hostname %s to match", testcase.host)
+		}
+	}
+	runtime.GC()
+	runtime.ReadMemStats(&end)
+	fmt.Printf("Memory usage after matching - %d kB alloc, %d kB sys\n", afterLoad.Alloc/1024, afterLoad.Sys/1024)
+}
+
+func TestSafeBrowsing(t *testing.T) {
+	testCases := []string{
+		"",
+		"sb.adtidy.org",
+	}
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("%s in %s", tc, _Func()), func(t *testing.T) {
+			d := NewForTest()
+			defer d.Destroy()
+			d.EnableSafeBrowsing()
+			stats.Safebrowsing.Requests = 0
+			d.checkMatch(t, "wmconvirus.narod.ru")
+			d.checkMatch(t, "wmconvirus.narod.ru")
+			if stats.Safebrowsing.Requests != 1 {
+				t.Errorf("Safebrowsing lookup positive cache is not working: %v", stats.Safebrowsing.Requests)
+			}
+			d.checkMatch(t, "WMconvirus.narod.ru")
+			if stats.Safebrowsing.Requests != 1 {
+				t.Errorf("Safebrowsing lookup positive cache is not working: %v", stats.Safebrowsing.Requests)
+			}
+			d.checkMatch(t, "wmconvirus.narod.ru.")
+			d.checkMatch(t, "test.wmconvirus.narod.ru")
+			d.checkMatch(t, "test.wmconvirus.narod.ru.")
+			d.checkMatchEmpty(t, "yandex.ru")
+			d.checkMatchEmpty(t, "pornhub.com")
+			l := stats.Safebrowsing.Requests
+			d.checkMatchEmpty(t, "pornhub.com")
+			if stats.Safebrowsing.Requests != l {
+				t.Errorf("Safebrowsing lookup negative cache is not working: %v", stats.Safebrowsing.Requests)
+			}
+		})
+	}
+}
+
+func TestParallelSB(t *testing.T) {
+	d := NewForTest()
+	defer d.Destroy()
+	d.EnableSafeBrowsing()
+	t.Run("group", func(t *testing.T) {
+		for i := 0; i < 100; i++ {
+			t.Run(fmt.Sprintf("aaa%d", i), func(t *testing.T) {
+				t.Parallel()
+				d.checkMatch(t, "wmconvirus.narod.ru")
+				d.checkMatch(t, "wmconvirus.narod.ru.")
+				d.checkMatch(t, "test.wmconvirus.narod.ru")
+				d.checkMatch(t, "test.wmconvirus.narod.ru.")
+				d.checkMatchEmpty(t, "yandex.ru")
+				d.checkMatchEmpty(t, "pornhub.com")
+			})
+		}
+	})
+}
+
+// the only way to verify that custom server option is working is to point it at a server that does serve safebrowsing
+func TestSafeBrowsingCustomServerFail(t *testing.T) {
+	d := NewForTest()
+	defer d.Destroy()
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// w.Write("Hello, client")
+		fmt.Fprintln(w, "Hello, client")
+	}))
+	defer ts.Close()
+	address := ts.Listener.Addr().String()
+
+	d.EnableSafeBrowsing()
+	d.SetHTTPTimeout(time.Second * 5)
+	d.SetSafeBrowsingServer(address) // this will ensure that test fails
+	d.checkMatchEmpty(t, "wmconvirus.narod.ru")
+}
+
+func TestParentalControl(t *testing.T) {
+	d := NewForTest()
+	defer d.Destroy()
+	d.EnableParental(3)
+	d.checkMatch(t, "pornhub.com")
+	d.checkMatch(t, "pornhub.com")
+	if stats.Parental.Requests != 1 {
+		t.Errorf("Parental lookup positive cache is not working")
+	}
+	d.checkMatch(t, "PORNhub.com")
+	if stats.Parental.Requests != 1 {
+		t.Errorf("Parental lookup positive cache is not working")
+	}
+	d.checkMatch(t, "www.pornhub.com")
+	d.checkMatch(t, "pornhub.com.")
+	d.checkMatch(t, "www.pornhub.com.")
+	d.checkMatchEmpty(t, "www.yandex.ru")
+	d.checkMatchEmpty(t, "yandex.ru")
+	l := stats.Parental.Requests
+	d.checkMatchEmpty(t, "yandex.ru")
+	if stats.Parental.Requests != l {
+		t.Errorf("Parental lookup negative cache is not working")
+	}
+}
+
+func TestSafeSearch(t *testing.T) {
+	d := NewForTest()
+	defer d.Destroy()
+	_, ok := d.SafeSearchDomain("www.google.com")
+	if ok {
+		t.Errorf("Expected safesearch to error when disabled")
+	}
+	d.EnableSafeSearch()
+	val, ok := d.SafeSearchDomain("www.google.com")
+	if !ok {
+		t.Errorf("Expected safesearch to find result for www.google.com")
+	}
+	if val != "forcesafesearch.google.com" {
+		t.Errorf("Expected safesearch for google.com to be forcesafesearch.google.com")
+	}
+}
+
+//
+// parametrized testing
+//
+var blockingRules = []string{"||example.org^"}
+var whitelistRules = []string{"||example.org^", "@@||test.example.org"}
+var importantRules = []string{"@@||example.org^", "||test.example.org^$important"}
+var regexRules = []string{"/example\\.org/", "@@||test.example.org^"}
+var maskRules = []string{"test*.example.org^", "exam*.com"}
+
+var tests = []struct {
+	testname string
+	rules    []string
+	hostname string
+	result   bool
+}{
+	{"sanity", []string{"||doubleclick.net^"}, "www.doubleclick.net", true},
+	{"sanity", []string{"||doubleclick.net^"}, "nodoubleclick.net", false},
+	{"sanity", []string{"||doubleclick.net^"}, "doubleclick.net.ru", false},
+	{"sanity", []string{"||doubleclick.net^"}, "wmconvirus.narod.ru", false},
+	{"blocking", blockingRules, "example.org", true},
+	{"blocking", blockingRules, "test.example.org", true},
+	{"blocking", blockingRules, "test.test.example.org", true},
+	{"blocking", blockingRules, "testexample.org", false},
+	{"blocking", blockingRules, "onemoreexample.org", false},
+	{"whitelist", whitelistRules, "example.org", true},
+	{"whitelist", whitelistRules, "test.example.org", false},
+	{"whitelist", whitelistRules, "test.test.example.org", false},
+	{"whitelist", whitelistRules, "testexample.org", false},
+	{"whitelist", whitelistRules, "onemoreexample.org", false},
+	{"important", importantRules, "example.org", false},
+	{"important", importantRules, "test.example.org", true},
+	{"important", importantRules, "test.test.example.org", true},
+	{"important", importantRules, "testexample.org", false},
+	{"important", importantRules, "onemoreexample.org", false},
+	{"regex", regexRules, "example.org", true},
+	{"regex", regexRules, "test.example.org", false},
+	{"regex", regexRules, "test.test.example.org", false},
+	{"regex", regexRules, "testexample.org", true},
+	{"regex", regexRules, "onemoreexample.org", true},
+	{"mask", maskRules, "test.example.org", true},
+	{"mask", maskRules, "test2.example.org", true},
+	{"mask", maskRules, "example.com", true},
+	{"mask", maskRules, "exampleeee.com", true},
+	{"mask", maskRules, "onemoreexamsite.com", true},
+	{"mask", maskRules, "example.org", false},
+	{"mask", maskRules, "testexample.org", false},
+	{"mask", maskRules, "example.co.uk", false},
+}
+
+func TestMatching(t *testing.T) {
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%s-%s", test.testname, test.hostname), func(t *testing.T) {
+			d := NewForTest()
+			defer d.Destroy()
+			for _, rule := range test.rules {
+				err := d.AddRule(rule, 0)
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			ret, err := d.CheckHost(test.hostname)
+			if err != nil {
+				t.Errorf("Error while matching host %s: %s", test.hostname, err)
+			}
+			if ret.IsFiltered != test.result {
+				t.Errorf("Hostname %s has wrong result (%v must be %v)", test.hostname, ret, test.result)
+			}
+		})
+	}
+}
+
+//
+// benchmarks
+//
+func BenchmarkAddRule(b *testing.B) {
+	d := NewForTest()
+	defer d.Destroy()
+	for n := 0; n < b.N; n++ {
+		rule := "||doubleclick.net^"
+		err := d.AddRule(rule, 0)
+		switch err {
+		case nil:
+		case ErrInvalidSyntax: // ignore invalid syntax
+		default:
+			b.Fatalf("Error while adding rule %s: %s", rule, err)
+		}
+	}
+}
+
+func BenchmarkAddRuleParallel(b *testing.B) {
+	d := NewForTest()
+	defer d.Destroy()
+	rule := "||doubleclick.net^"
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		var err error
+		for pb.Next() {
+			err = d.AddRule(rule, 0)
+		}
+		switch err {
+		case nil:
+		case ErrInvalidSyntax: // ignore invalid syntax
+		default:
+			b.Fatalf("Error while adding rule %s: %s", rule, err)
+		}
+	})
+}
+
+func BenchmarkLotsOfRulesNoMatch(b *testing.B) {
+	d := NewForTest()
+	defer d.Destroy()
+	err := loadTestRules(d)
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for n := 0; n < b.N; n++ {
+		hostname := "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com"
+		ret, err := d.CheckHost(hostname)
+		if err != nil {
+			b.Errorf("Error while matching host %s: %s", hostname, err)
+		}
+		if ret.IsFiltered {
+			b.Errorf("Expected hostname %s to not match", hostname)
+		}
+	}
+}
+
+func BenchmarkLotsOfRulesNoMatchParallel(b *testing.B) {
+	d := NewForTest()
+	defer d.Destroy()
+	err := loadTestRules(d)
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	hostname := "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.thisistesthost.com"
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			ret, err := d.CheckHost(hostname)
+			if err != nil {
+				b.Errorf("Error while matching host %s: %s", hostname, err)
+			}
+			if ret.IsFiltered {
+				b.Errorf("Expected hostname %s to not match", hostname)
+			}
+		}
+	})
+}
+
+func BenchmarkLotsOfRulesMatch(b *testing.B) {
+	d := NewForTest()
+	defer d.Destroy()
+	err := loadTestRules(d)
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for n := 0; n < b.N; n++ {
+		const hostname = "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net"
+		ret, err := d.CheckHost(hostname)
+		if err != nil {
+			b.Errorf("Error while matching host %s: %s", hostname, err)
+		}
+		if !ret.IsFiltered {
+			b.Errorf("Expected hostname %s to match", hostname)
+		}
+	}
+}
+
+func BenchmarkLotsOfRulesMatchParallel(b *testing.B) {
+	d := NewForTest()
+	defer d.Destroy()
+	err := loadTestRules(d)
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	const hostname = "asdasdasd_adsajdasda_asdasdjashdkasdasdasdasd_adsajdasda_asdasdjashdkasd.ad.doubleclick.net"
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			ret, err := d.CheckHost(hostname)
+			if err != nil {
+				b.Errorf("Error while matching host %s: %s", hostname, err)
+			}
+			if !ret.IsFiltered {
+				b.Errorf("Expected hostname %s to match", hostname)
+			}
+		}
+	})
+}
+
+func BenchmarkSafeBrowsing(b *testing.B) {
+	d := NewForTest()
+	defer d.Destroy()
+	d.EnableSafeBrowsing()
+	for n := 0; n < b.N; n++ {
+		hostname := "wmconvirus.narod.ru"
+		ret, err := d.CheckHost(hostname)
+		if err != nil {
+			b.Errorf("Error while matching host %s: %s", hostname, err)
+		}
+		if !ret.IsFiltered {
+			b.Errorf("Expected hostname %s to match", hostname)
+		}
+	}
+}
+
+func BenchmarkSafeBrowsingParallel(b *testing.B) {
+	d := NewForTest()
+	defer d.Destroy()
+	d.EnableSafeBrowsing()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			hostname := "wmconvirus.narod.ru"
+			ret, err := d.CheckHost(hostname)
+			if err != nil {
+				b.Errorf("Error while matching host %s: %s", hostname, err)
+			}
+			if !ret.IsFiltered {
+				b.Errorf("Expected hostname %s to match", hostname)
+			}
+		}
+	})
+}
+
+func BenchmarkSafeSearch(b *testing.B) {
+	d := NewForTest()
+	defer d.Destroy()
+	d.EnableSafeSearch()
+	for n := 0; n < b.N; n++ {
+		val, ok := d.SafeSearchDomain("www.google.com")
+		if !ok {
+			b.Errorf("Expected safesearch to find result for www.google.com")
+		}
+		if val != "forcesafesearch.google.com" {
+			b.Errorf("Expected safesearch for google.com to be forcesafesearch.google.com")
+		}
+	}
+}
+
+func BenchmarkSafeSearchParallel(b *testing.B) {
+	d := NewForTest()
+	defer d.Destroy()
+	d.EnableSafeSearch()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			val, ok := d.SafeSearchDomain("www.google.com")
+			if !ok {
+				b.Errorf("Expected safesearch to find result for www.google.com")
+			}
+			if val != "forcesafesearch.google.com" {
+				b.Errorf("Expected safesearch for google.com to be forcesafesearch.google.com")
+			}
+		}
+	})
+}
+
+func TestMain(m *testing.M) {
+	goleak.VerifyTestMain(m)
+}

dnsfilter/helpers.go

@@ -0,0 +1,81 @@
+package dnsfilter
+
+import (
+	"fmt"
+	"path"
+	"runtime"
+	"strings"
+	"sync/atomic"
+)
+
+func isValidRule(rule string) bool {
+	if len(rule) < 4 {
+		return false
+	}
+	if rule[0] == '!' {
+		return false
+	}
+	if rule[0] == '#' {
+		return false
+	}
+	if strings.HasPrefix(rule, "[Adblock") {
+		return false
+	}
+
+	masks := []string{
+		"##",
+		"#@#",
+		"#$#",
+		"#@$#",
+		"$$",
+		"$@$",
+		"#%#",
+		"#@%#",
+	}
+	for _, mask := range masks {
+		if strings.Contains(rule, mask) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func updateMax(valuePtr *int64, maxPtr *int64) {
+	for {
+		current := atomic.LoadInt64(valuePtr)
+		max := atomic.LoadInt64(maxPtr)
+		if current <= max {
+			break
+		}
+		swapped := atomic.CompareAndSwapInt64(maxPtr, max, current)
+		if swapped == true {
+			break
+		}
+		// swapping failed because value has changed after reading, try again
+	}
+}
+
+//
+// helper functions for debugging and testing
+//
+func _Func() string {
+	pc := make([]uintptr, 10) // at least 1 entry needed
+	runtime.Callers(2, pc)
+	f := runtime.FuncForPC(pc[0])
+	return path.Base(f.Name())
+}
+
+func trace(format string, args ...interface{}) {
+	pc := make([]uintptr, 10) // at least 1 entry needed
+	runtime.Callers(2, pc)
+	f := runtime.FuncForPC(pc[0])
+	var buf strings.Builder
+	buf.WriteString(fmt.Sprintf("%s(): ", path.Base(f.Name())))
+	text := fmt.Sprintf(format, args...)
+	buf.WriteString(text)
+	if len(text) == 0 || text[len(text)-1] != '\n' {
+		buf.WriteRune('\n')
+	}
+	fmt.Print(buf.String())
+}

dnsfilter/reason_string.go

@@ -0,0 +1,16 @@
+// Code generated by "stringer -type=Reason"; DO NOT EDIT.
+
+package dnsfilter
+
+import "strconv"
+
+const _Reason_name = "NotFilteredNotFoundNotFilteredWhiteListNotFilteredErrorFilteredBlackListFilteredSafeBrowsingFilteredParentalFilteredInvalidFilteredSafeSearch"
+
+var _Reason_index = [...]uint8{0, 19, 39, 55, 72, 92, 108, 123, 141}
+
+func (i Reason) String() string {
+	if i < 0 || i >= Reason(len(_Reason_index)-1) {
+		return "Reason(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _Reason_name[_Reason_index[i]:_Reason_index[i+1]]
+}

dnsfilter/rule_to_regexp.go

@@ -0,0 +1,51 @@
+package dnsfilter
+
+import (
+	"strings"
+)
+
+func ruleToRegexp(rule string) (string, error) {
+	const hostStart = "^([a-z0-9-_.]+\\.)?"
+	const hostEnd = "([^ a-zA-Z0-9.%]|$)"
+
+	// empty or short rule -- do nothing
+	if !isValidRule(rule) {
+		return "", ErrInvalidSyntax
+	}
+
+	// if starts with / and ends with /, it's already a regexp, just strip the slashes
+	if rule[0] == '/' && rule[len(rule)-1] == '/' {
+		return rule[1 : len(rule)-1], nil
+	}
+
+	var sb strings.Builder
+
+	if rule[0] == '|' && rule[1] == '|' {
+		sb.WriteString(hostStart)
+		rule = rule[2:]
+	}
+
+	for i, r := range rule {
+		switch {
+		case r == '?' || r == '.' || r == '+' || r == '[' || r == ']' || r == '(' || r == ')' || r == '{' || r == '}' || r == '#' || r == '\\' || r == '$':
+			sb.WriteRune('\\')
+			sb.WriteRune(r)
+		case r == '|' && i == 0:
+			// | at start and it's not || at start
+			sb.WriteRune('^')
+		case r == '|' && i == len(rule)-1:
+			// | at end
+			sb.WriteRune('$')
+		case r == '|' && i != 0 && i != len(rule)-1:
+			sb.WriteString(`\|`)
+		case r == '*':
+			sb.WriteString(`.*`)
+		case r == '^':
+			sb.WriteString(hostEnd)
+		default:
+			sb.WriteRune(r)
+		}
+	}
+
+	return sb.String(), nil
+}

dnsfilter/safesearch.go

@@ -0,0 +1,201 @@
+package dnsfilter
+
+var safeSearchDomains = map[string]string{
+	"yandex.com": "213.180.193.56",
+	"yandex.ru":  "213.180.193.56",
+	"yandex.ua":  "213.180.193.56",
+	"yandex.by":  "213.180.193.56",
+	"yandex.kz":  "213.180.193.56",
+
+	"www.bing.com": "strict.bing.com",
+
+	"www.google.com":    "forcesafesearch.google.com",
+	"www.google.ad":     "forcesafesearch.google.com",
+	"www.google.ae":     "forcesafesearch.google.com",
+	"www.google.com.af": "forcesafesearch.google.com",
+	"www.google.com.ag": "forcesafesearch.google.com",
+	"www.google.com.ai": "forcesafesearch.google.com",
+	"www.google.al":     "forcesafesearch.google.com",
+	"www.google.am":     "forcesafesearch.google.com",
+	"www.google.co.ao":  "forcesafesearch.google.com",
+	"www.google.com.ar": "forcesafesearch.google.com",
+	"www.google.as":     "forcesafesearch.google.com",
+	"www.google.at":     "forcesafesearch.google.com",
+	"www.google.com.au": "forcesafesearch.google.com",
+	"www.google.az":     "forcesafesearch.google.com",
+	"www.google.ba":     "forcesafesearch.google.com",
+	"www.google.com.bd": "forcesafesearch.google.com",
+	"www.google.be":     "forcesafesearch.google.com",
+	"www.google.bf":     "forcesafesearch.google.com",
+	"www.google.bg":     "forcesafesearch.google.com",
+	"www.google.com.bh": "forcesafesearch.google.com",
+	"www.google.bi":     "forcesafesearch.google.com",
+	"www.google.bj":     "forcesafesearch.google.com",
+	"www.google.com.bn": "forcesafesearch.google.com",
+	"www.google.com.bo": "forcesafesearch.google.com",
+	"www.google.com.br": "forcesafesearch.google.com",
+	"www.google.bs":     "forcesafesearch.google.com",
+	"www.google.bt":     "forcesafesearch.google.com",
+	"www.google.co.bw":  "forcesafesearch.google.com",
+	"www.google.by":     "forcesafesearch.google.com",
+	"www.google.com.bz": "forcesafesearch.google.com",
+	"www.google.ca":     "forcesafesearch.google.com",
+	"www.google.cd":     "forcesafesearch.google.com",
+	"www.google.cf":     "forcesafesearch.google.com",
+	"www.google.cg":     "forcesafesearch.google.com",
+	"www.google.ch":     "forcesafesearch.google.com",
+	"www.google.ci":     "forcesafesearch.google.com",
+	"www.google.co.ck":  "forcesafesearch.google.com",
+	"www.google.cl":     "forcesafesearch.google.com",
+	"www.google.cm":     "forcesafesearch.google.com",
+	"www.google.cn":     "forcesafesearch.google.com",
+	"www.google.com.co": "forcesafesearch.google.com",
+	"www.google.co.cr":  "forcesafesearch.google.com",
+	"www.google.com.cu": "forcesafesearch.google.com",
+	"www.google.cv":     "forcesafesearch.google.com",
+	"www.google.com.cy": "forcesafesearch.google.com",
+	"www.google.cz":     "forcesafesearch.google.com",
+	"www.google.de":     "forcesafesearch.google.com",
+	"www.google.dj":     "forcesafesearch.google.com",
+	"www.google.dk":     "forcesafesearch.google.com",
+	"www.google.dm":     "forcesafesearch.google.com",
+	"www.google.com.do": "forcesafesearch.google.com",
+	"www.google.dz":     "forcesafesearch.google.com",
+	"www.google.com.ec": "forcesafesearch.google.com",
+	"www.google.ee":     "forcesafesearch.google.com",
+	"www.google.com.eg": "forcesafesearch.google.com",
+	"www.google.es":     "forcesafesearch.google.com",
+	"www.google.com.et": "forcesafesearch.google.com",
+	"www.google.fi":     "forcesafesearch.google.com",
+	"www.google.com.fj": "forcesafesearch.google.com",
+	"www.google.fm":     "forcesafesearch.google.com",
+	"www.google.fr":     "forcesafesearch.google.com",
+	"www.google.ga":     "forcesafesearch.google.com",
+	"www.google.ge":     "forcesafesearch.google.com",
+	"www.google.gg":     "forcesafesearch.google.com",
+	"www.google.com.gh": "forcesafesearch.google.com",
+	"www.google.com.gi": "forcesafesearch.google.com",
+	"www.google.gl":     "forcesafesearch.google.com",
+	"www.google.gm":     "forcesafesearch.google.com",
+	"www.google.gp":     "forcesafesearch.google.com",
+	"www.google.gr":     "forcesafesearch.google.com",
+	"www.google.com.gt": "forcesafesearch.google.com",
+	"www.google.gy":     "forcesafesearch.google.com",
+	"www.google.com.hk": "forcesafesearch.google.com",
+	"www.google.hn":     "forcesafesearch.google.com",
+	"www.google.hr":     "forcesafesearch.google.com",
+	"www.google.ht":     "forcesafesearch.google.com",
+	"www.google.hu":     "forcesafesearch.google.com",
+	"www.google.co.id":  "forcesafesearch.google.com",
+	"www.google.ie":     "forcesafesearch.google.com",
+	"www.google.co.il":  "forcesafesearch.google.com",
+	"www.google.im":     "forcesafesearch.google.com",
+	"www.google.co.in":  "forcesafesearch.google.com",
+	"www.google.iq":     "forcesafesearch.google.com",
+	"www.google.is":     "forcesafesearch.google.com",
+	"www.google.it":     "forcesafesearch.google.com",
+	"www.google.je":     "forcesafesearch.google.com",
+	"www.google.com.jm": "forcesafesearch.google.com",
+	"www.google.jo":     "forcesafesearch.google.com",
+	"www.google.co.jp":  "forcesafesearch.google.com",
+	"www.google.co.ke":  "forcesafesearch.google.com",
+	"www.google.com.kh": "forcesafesearch.google.com",
+	"www.google.ki":     "forcesafesearch.google.com",
+	"www.google.kg":     "forcesafesearch.google.com",
+	"www.google.co.kr":  "forcesafesearch.google.com",
+	"www.google.com.kw": "forcesafesearch.google.com",
+	"www.google.kz":     "forcesafesearch.google.com",
+	"www.google.la":     "forcesafesearch.google.com",
+	"www.google.com.lb": "forcesafesearch.google.com",
+	"www.google.li":     "forcesafesearch.google.com",
+	"www.google.lk":     "forcesafesearch.google.com",
+	"www.google.co.ls":  "forcesafesearch.google.com",
+	"www.google.lt":     "forcesafesearch.google.com",
+	"www.google.lu":     "forcesafesearch.google.com",
+	"www.google.lv":     "forcesafesearch.google.com",
+	"www.google.com.ly": "forcesafesearch.google.com",
+	"www.google.co.ma":  "forcesafesearch.google.com",
+	"www.google.md":     "forcesafesearch.google.com",
+	"www.google.me":     "forcesafesearch.google.com",
+	"www.google.mg":     "forcesafesearch.google.com",
+	"www.google.mk":     "forcesafesearch.google.com",
+	"www.google.ml":     "forcesafesearch.google.com",
+	"www.google.com.mm": "forcesafesearch.google.com",
+	"www.google.mn":     "forcesafesearch.google.com",
+	"www.google.ms":     "forcesafesearch.google.com",
+	"www.google.com.mt": "forcesafesearch.google.com",
+	"www.google.mu":     "forcesafesearch.google.com",
+	"www.google.mv":     "forcesafesearch.google.com",
+	"www.google.mw":     "forcesafesearch.google.com",
+	"www.google.com.mx": "forcesafesearch.google.com",
+	"www.google.com.my": "forcesafesearch.google.com",
+	"www.google.co.mz":  "forcesafesearch.google.com",
+	"www.google.com.na": "forcesafesearch.google.com",
+	"www.google.com.nf": "forcesafesearch.google.com",
+	"www.google.com.ng": "forcesafesearch.google.com",
+	"www.google.com.ni": "forcesafesearch.google.com",
+	"www.google.ne":     "forcesafesearch.google.com",
+	"www.google.nl":     "forcesafesearch.google.com",
+	"www.google.no":     "forcesafesearch.google.com",
+	"www.google.com.np": "forcesafesearch.google.com",
+	"www.google.nr":     "forcesafesearch.google.com",
+	"www.google.nu":     "forcesafesearch.google.com",
+	"www.google.co.nz":  "forcesafesearch.google.com",
+	"www.google.com.om": "forcesafesearch.google.com",
+	"www.google.com.pa": "forcesafesearch.google.com",
+	"www.google.com.pe": "forcesafesearch.google.com",
+	"www.google.com.pg": "forcesafesearch.google.com",
+	"www.google.com.ph": "forcesafesearch.google.com",
+	"www.google.com.pk": "forcesafesearch.google.com",
+	"www.google.pl":     "forcesafesearch.google.com",
+	"www.google.pn":     "forcesafesearch.google.com",
+	"www.google.com.pr": "forcesafesearch.google.com",
+	"www.google.ps":     "forcesafesearch.google.com",
+	"www.google.pt":     "forcesafesearch.google.com",
+	"www.google.com.py": "forcesafesearch.google.com",
+	"www.google.com.qa": "forcesafesearch.google.com",
+	"www.google.ro":     "forcesafesearch.google.com",
+	"www.google.ru":     "forcesafesearch.google.com",
+	"www.google.rw":     "forcesafesearch.google.com",
+	"www.google.com.sa": "forcesafesearch.google.com",
+	"www.google.com.sb": "forcesafesearch.google.com",
+	"www.google.sc":     "forcesafesearch.google.com",
+	"www.google.se":     "forcesafesearch.google.com",
+	"www.google.com.sg": "forcesafesearch.google.com",
+	"www.google.sh":     "forcesafesearch.google.com",
+	"www.google.si":     "forcesafesearch.google.com",
+	"www.google.sk":     "forcesafesearch.google.com",
+	"www.google.com.sl": "forcesafesearch.google.com",
+	"www.google.sn":     "forcesafesearch.google.com",
+	"www.google.so":     "forcesafesearch.google.com",
+	"www.google.sm":     "forcesafesearch.google.com",
+	"www.google.sr":     "forcesafesearch.google.com",
+	"www.google.st":     "forcesafesearch.google.com",
+	"www.google.com.sv": "forcesafesearch.google.com",
+	"www.google.td":     "forcesafesearch.google.com",
+	"www.google.tg":     "forcesafesearch.google.com",
+	"www.google.co.th":  "forcesafesearch.google.com",
+	"www.google.com.tj": "forcesafesearch.google.com",
+	"www.google.tk":     "forcesafesearch.google.com",
+	"www.google.tl":     "forcesafesearch.google.com",
+	"www.google.tm":     "forcesafesearch.google.com",
+	"www.google.tn":     "forcesafesearch.google.com",
+	"www.google.to":     "forcesafesearch.google.com",
+	"www.google.com.tr": "forcesafesearch.google.com",
+	"www.google.tt":     "forcesafesearch.google.com",
+	"www.google.com.tw": "forcesafesearch.google.com",
+	"www.google.co.tz":  "forcesafesearch.google.com",
+	"www.google.com.ua": "forcesafesearch.google.com",
+	"www.google.co.ug":  "forcesafesearch.google.com",
+	"www.google.co.uk":  "forcesafesearch.google.com",
+	"www.google.com.uy": "forcesafesearch.google.com",
+	"www.google.co.uz":  "forcesafesearch.google.com",
+	"www.google.com.vc": "forcesafesearch.google.com",
+	"www.google.co.ve":  "forcesafesearch.google.com",
+	"www.google.vg":     "forcesafesearch.google.com",
+	"www.google.co.vi":  "forcesafesearch.google.com",
+	"www.google.com.vn": "forcesafesearch.google.com",
+	"www.google.vu":     "forcesafesearch.google.com",
+	"www.google.ws":     "forcesafesearch.google.com",
+	"www.google.rs":     "forcesafesearch.google.com",
+}