Merge: + DNS: use rules from /etc/hosts

- fix filtering logic: don't do DNS response check for Rewrite rules

Close #1478

Squashed commit of the following:

commit 1206b94881289ff664b7c8998ea97c1455da1ff8
Merge: c462577a 5fe98474
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Fri Mar 20 15:00:25 2020 +0300

    Merge remote-tracking branch 'origin/master' into 1478-auto-records

commit c462577ad84754f5b3ea4cd58339838af817fe36
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Fri Mar 20 14:33:17 2020 +0300

    minor

commit 7e824ba5f432648a976bc4b8076a645ba875ef70
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Fri Mar 20 14:29:54 2020 +0300

    more tests

commit a22b62136c5cfd84cd0450897aef9e7d2e20585a
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Fri Mar 20 14:09:52 2020 +0300

    rename, move

commit 9e5ed49ad3c27c57d540edf18b78d29e56afb067
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Mar 19 15:33:27 2020 +0300

    fix logic - don't do DNS response check for Rewrite rules

commit 6cfabc0348a41883b8bba834626a7e8760b76bf2
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Mar 19 11:35:07 2020 +0300

    minor

commit 4540aed9327566078e5087d43c30f4e8bffab7b9
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Mar 19 11:03:24 2020 +0300

    fix

commit 9ddddf7bded812da48613cc07084e360c15ddd0e
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Mar 19 10:49:13 2020 +0300

    fix

commit c5f8ef745b6f2a768be8a2ab23ad80b01b0aa54f
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Mar 19 10:37:26 2020 +0300

    fix

commit f4be00947bf0528c9a7cd4f09c4090db444c4694
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Mon Mar 16 20:13:00 2020 +0300

    + auto DNS records from /etc/hosts

home/clients.go

@@ -3,9 +3,7 @@ package home
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
 	"net"
-	"os"
 	"os/exec"
 	"runtime"
 	"sort"
@@ -16,6 +14,7 @@ import (
 	"github.com/AdguardTeam/AdGuardHome/dhcpd"
 	"github.com/AdguardTeam/AdGuardHome/dnsfilter"
 	"github.com/AdguardTeam/AdGuardHome/dnsforward"
+	"github.com/AdguardTeam/AdGuardHome/util"
 	"github.com/AdguardTeam/dnsproxy/upstream"
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/AdguardTeam/golibs/utils"
@@ -79,12 +78,14 @@ type clientsContainer struct {
 	// dhcpServer is used for looking up clients IP addresses by MAC addresses
 	dhcpServer *dhcpd.Server
 
+	autoHosts *util.AutoHosts // get entries from system hosts-files
+
 	testing bool // if TRUE, this object is used for internal tests
 }
 
 // Init initializes clients container
 // Note: this function must be called only once
-func (clients *clientsContainer) Init(objects []clientObject, dhcpServer *dhcpd.Server) {
+func (clients *clientsContainer) Init(objects []clientObject, dhcpServer *dhcpd.Server, autoHosts *util.AutoHosts) {
 	if clients.list != nil {
 		log.Fatal("clients.list != nil")
 	}
@@ -98,11 +99,13 @@ func (clients *clientsContainer) Init(objects []clientObject, dhcpServer *dhcpd.
 	}
 
 	clients.dhcpServer = dhcpServer
+	clients.autoHosts = autoHosts
 	clients.addFromConfig(objects)
 
 	if !clients.testing {
 		clients.addFromDHCP()
 		clients.dhcpServer.SetOnLeaseChanged(clients.onDHCPLeaseChanged)
+		clients.autoHosts.SetOnChanged(clients.onHostsChanged)
 	}
 }
 
@@ -120,7 +123,6 @@ func (clients *clientsContainer) Start() {
 
 // Reload - reload auto-clients
 func (clients *clientsContainer) Reload() {
-	clients.addFromHostsFile()
 	clients.addFromSystemARP()
 }
 
@@ -225,6 +227,10 @@ func (clients *clientsContainer) onDHCPLeaseChanged(flags int) {
 	}
 }
 
+func (clients *clientsContainer) onHostsChanged() {
+	clients.addFromHostsFile()
+}
+
 // Exists checks if client with this IP already exists
 func (clients *clientsContainer) Exists(ip string, source clientSource) bool {
 	clients.lock.Lock()
@@ -605,46 +611,28 @@ func (clients *clientsContainer) rmHosts(source clientSource) int {
 	return n
 }
 
-// Parse system 'hosts' file and fill clients array
+// Fill clients array from system hosts-file
 func (clients *clientsContainer) addFromHostsFile() {
-	hostsFn := "/etc/hosts"
-	if runtime.GOOS == "windows" {
-		hostsFn = os.ExpandEnv("$SystemRoot\\system32\\drivers\\etc\\hosts")
-	}
-
-	d, e := ioutil.ReadFile(hostsFn)
-	if e != nil {
-		log.Info("Can't read file %s: %v", hostsFn, e)
-		return
-	}
+	hosts := clients.autoHosts.List()
 
 	clients.lock.Lock()
 	defer clients.lock.Unlock()
 	_ = clients.rmHosts(ClientSourceHostsFile)
 
-	lines := strings.Split(string(d), "\n")
 	n := 0
-	for _, ln := range lines {
-		ln = strings.TrimSpace(ln)
-		if len(ln) == 0 || ln[0] == '#' {
-			continue
-		}
-
-		fields := strings.Fields(ln)
-		if len(fields) < 2 {
-			continue
-		}
-
-		ok, e := clients.addHost(fields[0], fields[1], ClientSourceHostsFile)
-		if e != nil {
-			log.Tracef("%s", e)
-		}
-		if ok {
-			n++
+	for ip, names := range hosts {
+		for _, name := range names {
+			ok, err := clients.addHost(ip, name.String(), ClientSourceHostsFile)
+			if err != nil {
+				log.Debug("Clients: %s", err)
+			}
+			if ok {
+				n++
+			}
 		}
 	}
 
-	log.Debug("Clients: added %d client aliases from %s", n, hostsFn)
+	log.Debug("Clients: added %d client aliases from system hosts-file", n)
 }
 
 // Add IP -> Host pairs from the system's `arp -a` command output

home/clients_test.go

@@ -18,7 +18,7 @@ func TestClients(t *testing.T) {
 	clients := clientsContainer{}
 	clients.testing = true
 
-	clients.Init(nil, nil)
+	clients.Init(nil, nil, nil)
 
 	// add
 	c = Client{
@@ -156,7 +156,7 @@ func TestClientsWhois(t *testing.T) {
 	var c Client
 	clients := clientsContainer{}
 	clients.testing = true
-	clients.Init(nil, nil)
+	clients.Init(nil, nil, nil)
 
 	whois := [][]string{{"orgname", "orgname-val"}, {"country", "country-val"}}
 	// set whois info on new client
@@ -183,7 +183,7 @@ func TestClientsAddExisting(t *testing.T) {
 	var c Client
 	clients := clientsContainer{}
 	clients.testing = true
-	clients.Init(nil, nil)
+	clients.Init(nil, nil, nil)
 
 	// some test variables
 	mac, _ := net.ParseMAC("aa:aa:aa:aa:aa:aa")

home/dns.go

@@ -56,6 +56,7 @@ func initDNSServer() error {
 		bindhost = "127.0.0.1"
 	}
 	filterConf.ResolverAddress = fmt.Sprintf("%s:%d", bindhost, config.DNS.Port)
+	filterConf.AutoHosts = &Context.autoHosts
 	filterConf.ConfigModified = onConfigModified
 	filterConf.HTTPRegister = httpRegister
 	Context.dnsFilter = dnsfilter.New(&filterConf, nil)

home/home.go

@@ -68,6 +68,7 @@ type homeContext struct {
 	filters    Filtering            // DNS filtering module
 	web        *Web                 // Web (HTTP, HTTPS) module
 	tls        *TLSMod              // TLS module
+	autoHosts  util.AutoHosts       // IP-hostname pairs taken from system configuration (e.g. /etc/hosts) files
 
 	// Runtime properties
 	// --
@@ -210,7 +211,8 @@ func run(args options) {
 	if Context.dhcpServer == nil {
 		os.Exit(1)
 	}
-	Context.clients.Init(config.Clients, Context.dhcpServer)
+	Context.autoHosts.Init("")
+	Context.clients.Init(config.Clients, Context.dhcpServer, &Context.autoHosts)
 	config.Clients = nil
 
 	if (runtime.GOOS == "linux" || runtime.GOOS == "darwin") &&
@@ -270,6 +272,7 @@ func run(args options) {
 			log.Fatalf("%s", err)
 		}
 		Context.tls.Start()
+		Context.autoHosts.Start()
 
 		go func() {
 			err := startDNSServer()
@@ -434,6 +437,8 @@ func cleanup() {
 		log.Error("Couldn't stop DHCP server: %s", err)
 	}
 
+	Context.autoHosts.Close()
+
 	if Context.tls != nil {
 		Context.tls.Close()
 		Context.tls = nil

home/service.go

@@ -196,7 +196,7 @@ func handleServiceInstallCommand(s service.Service) {
 		log.Fatal(err)
 	}
 
-	if isOpenWrt() {
+	if util.IsOpenWrt() {
 		// On OpenWrt it is important to run enable after the service installation
 		// Otherwise, the service won't start on the system startup
 		_, err := runInitdCommand("enable")
@@ -223,7 +223,7 @@ Click on the link below and follow the Installation Wizard steps to finish setup
 
 // handleServiceStatusCommand handles service "uninstall" command
 func handleServiceUninstallCommand(s service.Service) {
-	if isOpenWrt() {
+	if util.IsOpenWrt() {
 		// On OpenWrt it is important to run disable command first
 		// as it will remove the symlink
 		_, err := runInitdCommand("disable")
@@ -270,7 +270,7 @@ func configureService(c *service.Config) {
 	c.Option["SysvScript"] = sysvScript
 
 	// On OpenWrt we're using a different type of sysvScript
-	if isOpenWrt() {
+	if util.IsOpenWrt() {
 		c.Option["SysvScript"] = openWrtScript
 	}
 }
@@ -283,20 +283,6 @@ func runInitdCommand(action string) (int, error) {
 	return code, err
 }
 
-// isOpenWrt checks if OS is OpenWRT
-func isOpenWrt() bool {
-	if runtime.GOOS != "linux" {
-		return false
-	}
-
-	body, err := ioutil.ReadFile("/etc/os-release")
-	if err != nil {
-		return false
-	}
-
-	return strings.Contains(string(body), "OpenWrt")
-}
-
 // Basically the same template as the one defined in github.com/kardianos/service
 // but with two additional keys - StandardOutPath and StandardErrorPath
 var launchdConfig = `<?xml version='1.0' encoding='UTF-8'?>