daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

* dnsfilter: major refactoring

Browse Source 
* dnsfilter is controlled by package home, not dnsforward
* move HTTP handlers to dnsfilter/
* apply filtering settings without DNS server restart
* use only 1 goroutine for filters update
* apply new filters quickly (after they are ready to be used)
This commit is contained in:
Simon Zolin
2019-10-09 19:51:26 +03:00
parent b43c076c4d
commit a59e346d4a
14 changed files with 578 additions and 588 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
home/config.go
View File

@@ -10,6 +10,7 @@ import (
"time"
"github.com/AdguardTeam/AdGuardHome/dhcpd"
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/AdguardTeam/AdGuardHome/dnsforward"
"github.com/AdguardTeam/AdGuardHome/querylog"
"github.com/AdguardTeam/AdGuardHome/stats"
@@ -71,7 +72,6 @@ type configuration struct {
client *http.Client
stats stats.Stats // statistics module
queryLog querylog.QueryLog // query log module
filteringStarted bool // TRUE if filtering module is started
auth *Auth // HTTP authentication module
// cached version.json to avoid hammering github.io for each page reload
@@ -79,6 +79,7 @@ type configuration struct {
versionCheckLastTime time.Time
dnsctx dnsContext
dnsFilter *dnsfilter.Dnsfilter
dnsServer *dnsforward.Server
dhcpServer dhcpd.Server
httpServer *http.Server
@@ -217,10 +218,10 @@ func initConfig() {
}
config.DNS.CacheSize = 4 * 1024 * 1024
config.DNS.SafeBrowsingCacheSize = 1 * 1024 * 1024
config.DNS.SafeSearchCacheSize = 1 * 1024 * 1024
config.DNS.ParentalCacheSize = 1 * 1024 * 1024
config.DNS.CacheTime = 30
config.DNS.DnsfilterConf.SafeBrowsingCacheSize = 1 * 1024 * 1024
config.DNS.DnsfilterConf.SafeSearchCacheSize = 1 * 1024 * 1024
config.DNS.DnsfilterConf.ParentalCacheSize = 1 * 1024 * 1024
config.DNS.DnsfilterConf.CacheTime = 30
config.Filters = defaultFilters()
}
@@ -367,6 +368,12 @@ func (c *configuration) write() error {
config.DNS.QueryLogInterval = dc.Interval
}
if config.dnsFilter != nil {
c := dnsfilter.Config{}
config.dnsFilter.WriteDiskConfig(&c)
config.DNS.DnsfilterConf = c
}
configFile := config.getConfigFilename()
log.Debug("Writing YAML file: %s", configFile)
yamlText, err := yaml.Marshal(&config)

146
home/control.go
View File

@@ -377,142 +377,6 @@ func checkDNS(input string, bootstrap []string) error {
return nil
}
// ------------
// safebrowsing
// ------------
func handleSafeBrowsingEnable(w http.ResponseWriter, r *http.Request) {
config.DNS.SafeBrowsingEnabled = true
httpUpdateConfigReloadDNSReturnOK(w, r)
}
func handleSafeBrowsingDisable(w http.ResponseWriter, r *http.Request) {
config.DNS.SafeBrowsingEnabled = false
httpUpdateConfigReloadDNSReturnOK(w, r)
}
func handleSafeBrowsingStatus(w http.ResponseWriter, r *http.Request) {
data := map[string]interface{}{
"enabled": config.DNS.SafeBrowsingEnabled,
}
jsonVal, err := json.Marshal(data)
if err != nil {
httpError(w, http.StatusInternalServerError, "Unable to marshal status json: %s", err)
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(jsonVal)
if err != nil {
httpError(w, http.StatusInternalServerError, "Unable to write response json: %s", err)
return
}
}
// --------
// parental
// --------
func handleParentalEnable(w http.ResponseWriter, r *http.Request) {
parameters, err := parseParametersFromBody(r.Body)
if err != nil {
httpError(w, http.StatusBadRequest, "failed to parse parameters from body: %s", err)
return
}
sensitivity, ok := parameters["sensitivity"]
if !ok {
http.Error(w, "Sensitivity parameter was not specified", 400)
return
}
switch sensitivity {
case "3":
break
case "EARLY_CHILDHOOD":
sensitivity = "3"
case "10":
break
case "YOUNG":
sensitivity = "10"
case "13":
break
case "TEEN":
sensitivity = "13"
case "17":
break
case "MATURE":
sensitivity = "17"
default:
http.Error(w, "Sensitivity must be set to valid value", 400)
return
}
i, err := strconv.Atoi(sensitivity)
if err != nil {
http.Error(w, "Sensitivity must be set to valid value", 400)
return
}
config.DNS.ParentalSensitivity = i
config.DNS.ParentalEnabled = true
httpUpdateConfigReloadDNSReturnOK(w, r)
}
func handleParentalDisable(w http.ResponseWriter, r *http.Request) {
config.DNS.ParentalEnabled = false
httpUpdateConfigReloadDNSReturnOK(w, r)
}
func handleParentalStatus(w http.ResponseWriter, r *http.Request) {
data := map[string]interface{}{
"enabled": config.DNS.ParentalEnabled,
}
if config.DNS.ParentalEnabled {
data["sensitivity"] = config.DNS.ParentalSensitivity
}
jsonVal, err := json.Marshal(data)
if err != nil {
httpError(w, http.StatusInternalServerError, "Unable to marshal status json: %s", err)
return
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(jsonVal)
if err != nil {
httpError(w, http.StatusInternalServerError, "Unable to write response json: %s", err)
return
}
}
// ------------
// safebrowsing
// ------------
func handleSafeSearchEnable(w http.ResponseWriter, r *http.Request) {
config.DNS.SafeSearchEnabled = true
httpUpdateConfigReloadDNSReturnOK(w, r)
}
func handleSafeSearchDisable(w http.ResponseWriter, r *http.Request) {
config.DNS.SafeSearchEnabled = false
httpUpdateConfigReloadDNSReturnOK(w, r)
}
func handleSafeSearchStatus(w http.ResponseWriter, r *http.Request) {
data := map[string]interface{}{
"enabled": config.DNS.SafeSearchEnabled,
}
jsonVal, err := json.Marshal(data)
if err != nil {
httpError(w, http.StatusInternalServerError, "Unable to marshal status json: %s", err)
return
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(jsonVal)
if err != nil {
httpError(w, http.StatusInternalServerError, "Unable to write response json: %s", err)
return
}
}
// --------------
// DNS-over-HTTPS
// --------------
@@ -543,15 +407,6 @@ func registerControlHandlers() {
httpRegister(http.MethodGet, "/control/i18n/current_language", handleI18nCurrentLanguage)
http.HandleFunc("/control/version.json", postInstall(optionalAuth(handleGetVersionJSON)))
httpRegister(http.MethodPost, "/control/update", handleUpdate)
httpRegister(http.MethodPost, "/control/safebrowsing/enable", handleSafeBrowsingEnable)
httpRegister(http.MethodPost, "/control/safebrowsing/disable", handleSafeBrowsingDisable)
httpRegister(http.MethodGet, "/control/safebrowsing/status", handleSafeBrowsingStatus)
httpRegister(http.MethodPost, "/control/parental/enable", handleParentalEnable)
httpRegister(http.MethodPost, "/control/parental/disable", handleParentalDisable)
httpRegister(http.MethodGet, "/control/parental/status", handleParentalStatus)
httpRegister(http.MethodPost, "/control/safesearch/enable", handleSafeSearchEnable)
httpRegister(http.MethodPost, "/control/safesearch/disable", handleSafeSearchDisable)
httpRegister(http.MethodGet, "/control/safesearch/status", handleSafeSearchStatus)
httpRegister(http.MethodGet, "/control/dhcp/status", handleDHCPStatus)
httpRegister(http.MethodGet, "/control/dhcp/interfaces", handleDHCPInterfaces)
httpRegister(http.MethodPost, "/control/dhcp/set_config", handleDHCPSetConfig)
@@ -565,7 +420,6 @@ func registerControlHandlers() {
RegisterFilteringHandlers()
RegisterTLSHandlers()
RegisterClientsHandlers()
registerRewritesHandlers()
RegisterBlockedServicesHandlers()
RegisterAuthHandlers()

52
home/control_filtering.go
View File

@@ -86,17 +86,8 @@ func handleFilteringAddURL(w http.ResponseWriter, r *http.Request) {
return
}
err = writeAllConfigs()
if err != nil {
httpError(w, http.StatusInternalServerError, "Couldn't write config file: %s", err)
return
}
err = reconfigureDNSServer()
if err != nil {
httpError(w, http.StatusInternalServerError, "Couldn't reconfigure the DNS server: %s", err)
return
}
onConfigModified()
enableFilters(true)
_, err = fmt.Fprintf(w, "OK %d rules\n", f.RulesCount)
if err != nil {
@@ -121,32 +112,28 @@ func handleFilteringRemoveURL(w http.ResponseWriter, r *http.Request) {
return
}
// Stop DNS server:
// we close urlfilter object which in turn closes file descriptors to filter files.
// Otherwise, Windows won't allow us to remove the file which is being currently used.
_ = config.dnsServer.Stop()
// go through each element and delete if url matches
config.Lock()
newFilters := config.Filters[:0]
newFilters := []filter{}
for _, filter := range config.Filters {
if filter.URL != req.URL {
newFilters = append(newFilters, filter)
} else {
// Remove the filter file
err := os.Remove(filter.Path())
if err != nil && !os.IsNotExist(err) {
config.Unlock()
httpError(w, http.StatusInternalServerError, "Couldn't remove the filter file: %s", err)
return
err := os.Rename(filter.Path(), filter.Path()+".old")
if err != nil {
log.Error("os.Rename: %s: %s", filter.Path(), err)
}
log.Debug("os.Remove(%s)", filter.Path())
}
}
// Update the configuration after removing filter files
config.Filters = newFilters
config.Unlock()
httpUpdateConfigReloadDNSReturnOK(w, r)
onConfigModified()
enableFilters(true)
// Note: the old files "filter.txt.old" aren't deleted - it's not really necessary,
// but will require the additional code to run after enableFilters() is finished: i.e. complicated
}
type filterURLJSON struct {
@@ -173,7 +160,8 @@ func handleFilteringSetURL(w http.ResponseWriter, r *http.Request) {
return
}
httpUpdateConfigReloadDNSReturnOK(w, r)
onConfigModified()
enableFilters(true)
}
func handleFilteringSetRules(w http.ResponseWriter, r *http.Request) {
@@ -184,12 +172,13 @@ func handleFilteringSetRules(w http.ResponseWriter, r *http.Request) {
}
config.UserRules = strings.Split(string(body), "\n")
httpUpdateConfigReloadDNSReturnOK(w, r)
_ = writeAllConfigs()
enableFilters(true)
}
func handleFilteringRefresh(w http.ResponseWriter, r *http.Request) {
updated := refreshFiltersIfNecessary(true)
fmt.Fprintf(w, "OK %d filters updated\n", updated)
beginRefreshFilters()
fmt.Fprintf(w, "OK 0 filters updated\n")
}
type filterJSON struct {
@@ -260,9 +249,8 @@ func handleFilteringConfig(w http.ResponseWriter, r *http.Request) {
config.DNS.FilteringEnabled = req.Enabled
config.DNS.FiltersUpdateIntervalHours = req.Interval
httpUpdateConfigReloadDNSReturnOK(w, r)
returnOK(w)
onConfigModified()
enableFilters(true)
}
// RegisterFilteringHandlers - register handlers

47
home/dns.go
View File

@@ -55,7 +55,18 @@ func initDNSServer() {
HTTPRegister: httpRegister,
}
config.queryLog = querylog.New(conf)
config.dnsServer = dnsforward.NewServer(config.stats, config.queryLog)
filterConf := config.DNS.DnsfilterConf
bindhost := config.DNS.BindHost
if config.DNS.BindHost == "0.0.0.0" {
bindhost = "127.0.0.1"
}
filterConf.ResolverAddress = fmt.Sprintf("%s:%d", bindhost, config.DNS.Port)
filterConf.ConfigModified = onConfigModified
filterConf.HTTPRegister = httpRegister
config.dnsFilter = dnsfilter.New(&filterConf, nil)
config.dnsServer = dnsforward.NewServer(config.dnsFilter, config.stats, config.queryLog)
sessFilename := filepath.Join(baseDir, "sessions.db")
config.auth = InitAuth(sessFilename, config.Users)
@@ -159,34 +170,11 @@ func onDNSRequest(d *proxy.DNSContext) {
}
func generateServerConfig() (dnsforward.ServerConfig, error) {
filters := []dnsfilter.Filter{}
userFilter := userFilter()
filters = append(filters, dnsfilter.Filter{
ID: userFilter.ID,
Data: userFilter.Data,
})
for _, filter := range config.Filters {
if !filter.Enabled {
continue
}
filters = append(filters, dnsfilter.Filter{
ID: filter.ID,
FilePath: filter.Path(),
})
}
newconfig := dnsforward.ServerConfig{
UDPListenAddr: &net.UDPAddr{IP: net.ParseIP(config.DNS.BindHost), Port: config.DNS.Port},
TCPListenAddr: &net.TCPAddr{IP: net.ParseIP(config.DNS.BindHost), Port: config.DNS.Port},
FilteringConfig: config.DNS.FilteringConfig,
Filters: filters,
}
newconfig.AsyncStartup = true
bindhost := config.DNS.BindHost
if config.DNS.BindHost == "0.0.0.0" {
bindhost = "127.0.0.1"
}
newconfig.ResolverAddress = fmt.Sprintf("%s:%d", bindhost, config.DNS.Port)
if config.TLS.Enabled {
newconfig.TLSConfig = config.TLS.TLSConfig
@@ -242,20 +230,18 @@ func startDNSServer() error {
return fmt.Errorf("unable to start forwarding DNS server: Already running")
}
enableFilters(false)
newconfig, err := generateServerConfig()
if err != nil {
return errorx.Decorate(err, "Couldn't start forwarding DNS server")
}
err = config.dnsServer.Start(&newconfig)
if err != nil {
return errorx.Decorate(err, "Couldn't start forwarding DNS server")
}
if !config.filteringStarted {
config.filteringStarted = true
startRefreshFilters()
}
return nil
}
@@ -285,6 +271,9 @@ func stopDNSServer() error {
// DNS forward module must be closed BEFORE stats or queryLog because it depends on them
config.dnsServer.Close()
config.dnsFilter.Close()
config.dnsFilter = nil
config.stats.Close()
config.stats = nil

104
home/dns_rewrites.go
View File

@@ -1,104 +0,0 @@
package home
import (
"encoding/json"
"net/http"
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/AdguardTeam/golibs/log"
)
type rewriteEntryJSON struct {
Domain string `json:"domain"`
Answer string `json:"answer"`
}
func handleRewriteList(w http.ResponseWriter, r *http.Request) {
arr := []*rewriteEntryJSON{}
config.RLock()
for _, ent := range config.DNS.Rewrites {
jsent := rewriteEntryJSON{
Domain: ent.Domain,
Answer: ent.Answer,
}
arr = append(arr, &jsent)
}
config.RUnlock()
w.Header().Set("Content-Type", "application/json")
err := json.NewEncoder(w).Encode(arr)
if err != nil {
httpError(w, http.StatusInternalServerError, "json.Encode: %s", err)
return
}
}
func handleRewriteAdd(w http.ResponseWriter, r *http.Request) {
jsent := rewriteEntryJSON{}
err := json.NewDecoder(r.Body).Decode(&jsent)
if err != nil {
httpError(w, http.StatusBadRequest, "json.Decode: %s", err)
return
}
ent := dnsfilter.RewriteEntry{
Domain: jsent.Domain,
Answer: jsent.Answer,
}
config.Lock()
config.DNS.Rewrites = append(config.DNS.Rewrites, ent)
config.Unlock()
log.Debug("Rewrites: added element: %s -> %s [%d]",
ent.Domain, ent.Answer, len(config.DNS.Rewrites))
err = writeAllConfigsAndReloadDNS()
if err != nil {
httpError(w, http.StatusBadRequest, "%s", err)
return
}
returnOK(w)
}
func handleRewriteDelete(w http.ResponseWriter, r *http.Request) {
jsent := rewriteEntryJSON{}
err := json.NewDecoder(r.Body).Decode(&jsent)
if err != nil {
httpError(w, http.StatusBadRequest, "json.Decode: %s", err)
return
}
entDel := dnsfilter.RewriteEntry{
Domain: jsent.Domain,
Answer: jsent.Answer,
}
arr := []dnsfilter.RewriteEntry{}
config.Lock()
for _, ent := range config.DNS.Rewrites {
if ent == entDel {
log.Debug("Rewrites: removed element: %s -> %s", ent.Domain, ent.Answer)
continue
}
arr = append(arr, ent)
}
config.DNS.Rewrites = arr
config.Unlock()
err = writeAllConfigsAndReloadDNS()
if err != nil {
httpError(w, http.StatusBadRequest, "%s", err)
return
}
returnOK(w)
}
func registerRewritesHandlers() {
httpRegister(http.MethodGet, "/control/rewrite/list", handleRewriteList)
httpRegister(http.MethodPost, "/control/rewrite/add", handleRewriteAdd)
httpRegister(http.MethodPost, "/control/rewrite/delete", handleRewriteDelete)
}

91
home/filter.go
View File

@@ -19,18 +19,13 @@ import (
var (
nextFilterID = time.Now().Unix() // semi-stable way to generate an unique ID
filterTitleRegexp = regexp.MustCompile(`^! Title: +(.*)$`)
forceRefresh bool
)
func initFiltering() {
loadFilters()
deduplicateFilters()
updateUniqueFilterID(config.Filters)
}
func startRefreshFilters() {
go func() {
_ = refreshFiltersIfNecessary(false)
}()
go periodicallyRefreshFilters()
}
@@ -180,32 +175,43 @@ func assignUniqueFilterID() int64 {
// Sets up a timer that will be checking for filters updates periodically
func periodicallyRefreshFilters() {
nextRefresh := int64(0)
for {
time.Sleep(1 * time.Hour)
if config.DNS.FiltersUpdateIntervalHours == 0 {
continue
if forceRefresh {
_ = refreshFiltersIfNecessary(true)
forceRefresh = false
}
refreshFiltersIfNecessary(false)
if config.DNS.FiltersUpdateIntervalHours != 0 && nextRefresh <= time.Now().Unix() {
_ = refreshFiltersIfNecessary(false)
nextRefresh = time.Now().Add(1 * time.Hour).Unix()
}
time.Sleep(1 * time.Second)
}
}
// Schedule the procedure to refresh filters
func beginRefreshFilters() {
forceRefresh = true
log.Debug("Filters: schedule update")
}
// Checks filters updates if necessary
// If force is true, it ignores the filter.LastUpdated field value
//
// Algorithm:
// . Get the list of filters to be updated
// . For each filter run the download and checksum check operation
// . Stop server
// . For each filter:
// . If filter data hasn't changed, just set new update time on file
// . If filter data has changed, save it on disk
// . Apply changes to the current configuration
// . Start server
// . If filter data has changed: rename the old file, store the new data on disk
// . Pass new filters to dnsfilter object
func refreshFiltersIfNecessary(force bool) int {
var updateFilters []filter
var updateFlags []bool // 'true' if filter data has changed
log.Debug("Filters: updating...")
now := time.Now()
config.RLock()
for i := range config.Filters {
@@ -229,7 +235,6 @@ func refreshFiltersIfNecessary(force bool) int {
}
config.RUnlock()
updateCount := 0
for i := range updateFilters {
uf := &updateFilters[i]
updated, err := uf.update()
@@ -239,24 +244,14 @@ func refreshFiltersIfNecessary(force bool) int {
continue
}
uf.LastUpdated = now
if updated {
updateCount++
}
}
stopped := false
if updateCount != 0 {
_ = config.dnsServer.Stop()
stopped = true
}
updateCount = 0
updateCount := 0
for i := range updateFilters {
uf := &updateFilters[i]
updated := updateFlags[i]
if updated {
// Saving it to the filters dir now
err := uf.save()
err := uf.saveAndBackupOld()
if err != nil {
log.Printf("Failed to save the updated filter %d: %s", uf.ID, err)
continue
@@ -290,12 +285,20 @@ func refreshFiltersIfNecessary(force bool) int {
config.Unlock()
}
if stopped {
err := reconfigureDNSServer()
if err != nil {
log.Error("cannot reconfigure DNS server with the new filters: %s", err)
if updateCount != 0 {
enableFilters(false)
for i := range updateFilters {
uf := &updateFilters[i]
updated := updateFlags[i]
if !updated {
continue
}
_ = os.Remove(uf.Path() + ".old")
}
}
log.Debug("Filters: update finished")
return updateCount
}
@@ -413,6 +416,12 @@ func (filter *filter) save() error {
return err
}
func (filter *filter) saveAndBackupOld() error {
filterFilePath := filter.Path()
_ = os.Rename(filterFilePath, filterFilePath+".old")
return filter.save()
}
// loads filter contents from the file in dataDir
func (filter *filter) load() error {
filterFilePath := filter.Path()
@@ -467,3 +476,23 @@ func (filter *filter) LastTimeUpdated() time.Time {
// filter file modified time
return s.ModTime()
}
func enableFilters(async bool) {
var filters map[int]string
if config.DNS.FilteringConfig.FilteringEnabled {
// convert array of filters
filters = make(map[int]string)
userFilter := userFilter()
filters[int(userFilter.ID)] = string(userFilter.Data)
for _, filter := range config.Filters {
if !filter.Enabled {
continue
}
filters[int(filter.ID)] = filter.Path()
}
}
_ = config.dnsFilter.SetFilters(filters, async)
}

25
home/helpers.go
View File

@@ -1,12 +1,10 @@
package home
import (
"bufio"
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
@@ -155,29 +153,6 @@ func postInstallHandler(handler http.Handler) http.Handler {
return &postInstallHandlerStruct{handler}
}
// -------------------------------------------------
// helper functions for parsing parameters from body
// -------------------------------------------------
func parseParametersFromBody(r io.Reader) (map[string]string, error) {
parameters := map[string]string{}
scanner := bufio.NewScanner(r)
for scanner.Scan() {
line := scanner.Text()
if len(line) == 0 {
// skip empty lines
continue
}
parts := strings.SplitN(line, "=", 2)
if len(parts) != 2 {
return parameters, errors.New("Got invalid request body")
}
parameters[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
}
return parameters, nil
}
// ------------------
// network interfaces
// ------------------

11
home/home.go
View File

@@ -143,11 +143,12 @@ func run(args options) {
}
initDNSServer()
err = startDNSServer()
if err != nil {
log.Fatal(err)
}
go func() {
err = startDNSServer()
if err != nil {
log.Fatal(err)
}
}()
err = startDHCPServer()
if err != nil {