Pull request: 3846 hosts querylog

Merge in DNS/adguard-home from 3846-hosts-querylog to master Updates #3846. Squashed commit of the following: commit 722e96628b1ccca1a5b5a716b8bcb1da2aefcc3b Merge: a20ad71e ed868fa4 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Tue Nov 23 17:52:08 2021 +0300 Merge branch 'master' into 3846-hosts-querylog commit a20ad71e723dbfa3483c3bdf9e4c8fd15c8b0e3c Author: Ildar Kamalov <ik@adguard.com> Date: Tue Nov 23 17:28:12 2021 +0300 client: fix variable name commit 7013bff05d6cff75c6c25a38d614db8b4b2f0b87 Author: Ildar Kamalov <ik@adguard.com> Date: Tue Nov 23 17:03:26 2021 +0300 client: fix missing import commit 8e4a0fb047b4d39ab44a285f59420573d7ba5eec Author: Ildar Kamalov <ik@adguard.com> Date: Tue Nov 23 16:56:50 2021 +0300 client: handle system host filter id commit abbbf662d2f3ea3f5d3569a9c45418e356adbf3c Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Nov 22 13:54:52 2021 +0300 all: imp code commit c2df63e46e75f84f70a610d18deccbeee672ebda Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Nov 22 12:50:51 2021 +0300 querylog: rm unused test data commit 8a1d47d266254fd4aedd4c61c7ea9e48168ea375 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Mon Nov 22 02:52:50 2021 +0300 aghnet: final imps commit ade3acb4bebc8bdd755e56f314cdf19bc9375557 Author: Eugene Burkov <E.Burkov@AdGuard.COM> Date: Fri Nov 19 15:48:40 2021 +0300 all: add hosts container rule list support
2021-11-23 18:01:48 +03:00
parent ed868fa46a
commit 51f11d2f8e
11 changed files with 475 additions and 287 deletions
--- a/internal/aghnet/hostscontainer.go
+++ b/internal/aghnet/hostscontainer.go
@@ -28,6 +28,69 @@ func DefaultHostsPaths() (paths []string) {
 	return defaultHostsPaths()
 }

+// requestMatcher combines the logic for matching requests and translating the
+// appropriate rules.
+type requestMatcher struct {
+	// stateLock protects all the fields of requestMatcher.
+	stateLock *sync.RWMutex
+
+	// rulesStrg stores the rules obtained from the hosts' file.
+	rulesStrg *filterlist.RuleStorage
+	// engine serves rulesStrg.
+	engine *urlfilter.DNSEngine
+
+	// translator maps generated $dnsrewrite rules into hosts-syntax rules.
+	//
+	// TODO(e.burkov):  Store the filename from which the rule was parsed.
+	translator map[string]string
+}
+
+// MatchRequest processes the request rewriting hostnames and addresses read
+// from the operating system's hosts files.
+//
+// res is nil for any request having not an A/AAAA or PTR type.  Results
+// containing CNAME information may be queried again with the same question type
+// and the returned CNAME for Host field of request.  Results are guaranteed to
+// be direct, i.e. any returned CNAME resolves into actual address like an alias
+// in hosts does, see man hosts (5).
+//
+// It's safe for concurrent use.
+func (rm *requestMatcher) MatchRequest(
+	req urlfilter.DNSRequest,
+) (res *urlfilter.DNSResult, ok bool) {
+	switch req.DNSType {
+	case dns.TypeA, dns.TypeAAAA, dns.TypePTR:
+		log.Debug("%s: handling the request", hostsContainerPref)
+	default:
+		return nil, false
+	}
+
+	rm.stateLock.RLock()
+	defer rm.stateLock.RUnlock()
+
+	return rm.engine.MatchRequest(req)
+}
+
+// Translate returns the source hosts-syntax rule for the generated dnsrewrite
+// rule or an empty string if the last doesn't exist.
+func (rm *requestMatcher) Translate(rule string) (hostRule string) {
+	rm.stateLock.RLock()
+	defer rm.stateLock.RUnlock()
+
+	return rm.translator[rule]
+}
+
+// resetEng updates container's engine and the translation map.
+func (rm *requestMatcher) resetEng(rulesStrg *filterlist.RuleStorage, tr map[string]string) {
+	rm.stateLock.Lock()
+	defer rm.stateLock.Unlock()
+
+	rm.rulesStrg = rulesStrg
+	rm.engine = urlfilter.NewDNSEngine(rm.rulesStrg)
+
+	rm.translator = tr
+}
+
 // hostsContainerPref is a prefix for logging and wrapping errors in
 // HostsContainer's methods.
 const hostsContainerPref = "hosts container"
@@ -35,13 +98,9 @@ const hostsContainerPref = "hosts container"
 // HostsContainer stores the relevant hosts database provided by the OS and
 // processes both A/AAAA and PTR DNS requests for those.
 type HostsContainer struct {
-	// engLock protects rulesStrg and engine.
-	engLock *sync.RWMutex
-
-	// rulesStrg stores the rules obtained from the hosts' file.
-	rulesStrg *filterlist.RuleStorage
-	// engine serves rulesStrg.
-	engine *urlfilter.DNSEngine
+	// requestMatcher matches the requests and translates the rules.  It's
+	// embedded to implement MatchRequest and Translate for *HostsContainer.
+	requestMatcher

 	// done is the channel to sign closing the container.
 	done chan struct{}
@@ -87,7 +146,9 @@ func NewHostsContainer(
 	}

 	hc = &HostsContainer{
-		engLock:  &sync.RWMutex{},
+		requestMatcher: requestMatcher{
+			stateLock: &sync.RWMutex{},
+		},
 		done:     make(chan struct{}, 1),
 		updates:  make(chan *netutil.IPMap, 1),
 		fsys:     fsys,
@@ -117,25 +178,6 @@ func NewHostsContainer(
 	return hc, nil
 }

-// MatchRequest is the request processing method to resolve hostnames and
-// addresses from the operating system's hosts files.  res is nil for any
-// request having not an A/AAAA or PTR type.  It's safe for concurrent use.
-func (hc *HostsContainer) MatchRequest(
-	req urlfilter.DNSRequest,
-) (res *urlfilter.DNSResult, ok bool) {
-	switch req.DNSType {
-	case dns.TypeA, dns.TypeAAAA, dns.TypePTR:
-		log.Debug("%s: handling the request", hostsContainerPref)
-	default:
-		return nil, false
-	}
-
-	hc.engLock.RLock()
-	defer hc.engLock.RUnlock()
-
-	return hc.engine.MatchRequest(req)
-}
-
 // Close implements the io.Closer interface for *HostsContainer.  Close must
 // only be called once.  The returned err is always nil.
 func (hc *HostsContainer) Close() (err error) {
@@ -203,10 +245,17 @@ func (hc *HostsContainer) handleEvents() {
 }

 // hostsParser is a helper type to parse rules from the operating system's hosts
-// file.
+// file.  It exists for only a single refreshing session.
 type hostsParser struct {
-	// rules builds the resulting rules list content.
-	rules *strings.Builder
+	// rulesBuilder builds the resulting rulesBuilder list content.
+	rulesBuilder *strings.Builder
+
+	// translations maps generated $dnsrewrite rules to the hosts-translations
+	// rules.
+	translations map[string]string
+
+	// cnameSet prevents duplicating cname rules.
+	cnameSet *stringutil.Set

 	// table stores only the unique IP-hostname pairs.  It's also sent to the
 	// updates channel afterwards.
@@ -215,8 +264,11 @@ type hostsParser struct {

 func (hc *HostsContainer) newHostsParser() (hp *hostsParser) {
 	return &hostsParser{
-		rules: &strings.Builder{},
-		table: netutil.NewIPMap(hc.last.Len()),
+		rulesBuilder: &strings.Builder{},
+		// For A/AAAA and PTRs.
+		translations: make(map[string]string, hc.last.Len()*2),
+		cnameSet:     stringutil.NewSet(),
+		table:        netutil.NewIPMap(hc.last.Len()),
 	}
 }

@@ -234,9 +286,7 @@ func (hp *hostsParser) parseFile(
 			continue
 		}

-		for _, host := range hosts {
-			hp.addPair(ip, host)
-		}
+		hp.addPairs(ip, hosts)
 	}

 	return nil, true, s.Err()
@@ -244,7 +294,6 @@ func (hp *hostsParser) parseFile(

 // parseLine parses the line having the hosts syntax ignoring invalid ones.
 func (hp *hostsParser) parseLine(line string) (ip net.IP, hosts []string) {
-	line = strings.TrimSpace(line)
 	fields := strings.Fields(line)
 	if len(fields) < 2 {
 		return nil, nil
@@ -274,74 +323,142 @@ loop:
 	return ip, hosts
 }

-// add returns true if the pair of ip and host wasn't added to the hp before.
-func (hp *hostsParser) add(ip net.IP, host string) (added bool) {
+// Simple types of hosts in hosts database.  Zero value isn't used to be able
+// quizzaciously emulate nil with 0.
+const (
+	_ = iota
+	hostAlias
+	hostMain
+)
+
+// add tries to add the ip-host pair.  It returns:
+//
+//   hostAlias if the host is not the first one added for the ip.
+//   hostMain  if the host is the first one added for the ip.
+//   0         if the ip-host pair has already been added.
+//
+func (hp *hostsParser) add(ip net.IP, host string) (hostType int) {
 	v, ok := hp.table.Get(ip)
-	hosts, _ := v.(*stringutil.Set)
-	switch {
+	switch hosts, _ := v.(*stringutil.Set); {
 	case ok && hosts.Has(host):
-		return false
+		return 0
 	case hosts == nil:
 		hosts = stringutil.NewSet(host)
 		hp.table.Set(ip, hosts)
+
+		return hostMain
 	default:
 		hosts.Add(host)
-	}

-	return true
+		return hostAlias
+	}
 }

-// addPair puts the pair of ip and host to the rules builder if needed.
-func (hp *hostsParser) addPair(ip net.IP, host string) {
+// addPair puts the pair of ip and host to the rules builder if needed.  For
+// each ip the first member of hosts will become the main one.
+func (hp *hostsParser) addPairs(ip net.IP, hosts []string) {
+	// Put the rule in a preproccesed format like:
+	//
+	//   ip host1 host2 ...
+	//
+	hostsLine := strings.Join(append([]string{ip.String()}, hosts...), " ")
+	var mainHost string
+	for _, host := range hosts {
+		switch hp.add(ip, host) {
+		case 0:
+			continue
+		case hostMain:
+			mainHost = host
+			added, addedPtr := hp.writeMainHostRule(host, ip)
+			hp.translations[added], hp.translations[addedPtr] = hostsLine, hostsLine
+		case hostAlias:
+			pair := fmt.Sprint(host, " ", mainHost)
+			if hp.cnameSet.Has(pair) {
+				continue
+			}
+			// Since the hostAlias couldn't be returned from add before the
+			// hostMain the mainHost shouldn't appear empty.
+			hp.writeAliasHostRule(host, mainHost)
+			hp.cnameSet.Add(pair)
+		}
+
+		log.Debug("%s: added ip-host pair %q-%q", hostsContainerPref, ip, host)
+	}
+}
+
+// writeAliasHostRule writes the CNAME rule for the alias-host pair into
+// internal builders.
+func (hp *hostsParser) writeAliasHostRule(alias, host string) {
+	const (
+		nl = "\n"
+		sc = ";"
+
+		rwSuccess = rules.MaskSeparator + "$dnsrewrite=NOERROR" + sc + "CNAME" + sc
+		constLen  = len(rules.MaskStartURL) + len(rwSuccess) + len(nl)
+	)
+
+	hp.rulesBuilder.Grow(constLen + len(host) + len(alias))
+	stringutil.WriteToBuilder(hp.rulesBuilder, rules.MaskStartURL, alias, rwSuccess, host, nl)
+}
+
+// writeMainHostRule writes the actual rule for the qtype and the PTR for the
+// host-ip pair into internal builders.
+func (hp *hostsParser) writeMainHostRule(host string, ip net.IP) (added, addedPtr string) {
 	arpa, err := netutil.IPToReversedAddr(ip)
 	if err != nil {
 		return
 	}

-	if !hp.add(ip, host) {
-		return
-	}
-
-	qtype := "AAAA"
-	if ip.To4() != nil {
-		// Assume the validation of the IP address is performed already.
-		qtype = "A"
-	}
-
 	const (
 		nl = "\n"
-		sc = ";"

-		rewriteSuccess    = "$dnsrewrite=NOERROR" + sc
-		rewriteSuccessPTR = rewriteSuccess + "PTR" + sc
+		rwSuccess    = "^$dnsrewrite=NOERROR;"
+		rwSuccessPTR = "^$dnsrewrite=NOERROR;PTR;"
+
+		modLen    = len("||") + len(rwSuccess)
+		modLenPTR = len("||") + len(rwSuccessPTR)
 	)

+	var qtype string
+	// The validation of the IP address has been performed earlier so it is
+	// guaranteed to be either an IPv4 or an IPv6.
+	if ip.To4() != nil {
+		qtype = "A"
+	} else {
+		qtype = "AAAA"
+	}
+
 	ipStr := ip.String()
 	fqdn := dns.Fqdn(host)

-	for _, ruleData := range [...][]string{{
-		// A/AAAA.
-		rules.MaskStartURL,
+	ruleBuilder := &strings.Builder{}
+	ruleBuilder.Grow(modLen + len(host) + len(qtype) + len(ipStr))
+	stringutil.WriteToBuilder(
+		ruleBuilder,
+		"||",
 		host,
-		rules.MaskSeparator,
-		rewriteSuccess,
+		rwSuccess,
 		qtype,
-		sc,
+		";",
 		ipStr,
-		nl,
-	}, {
-		// PTR.
-		rules.MaskStartURL,
-		arpa,
-		rules.MaskSeparator,
-		rewriteSuccessPTR,
-		fqdn,
-		nl,
-	}} {
-		stringutil.WriteToBuilder(hp.rules, ruleData...)
-	}
+	)
+	added = ruleBuilder.String()

-	log.Debug("%s: added ip-host pair %q/%q", hostsContainerPref, ip, host)
+	ruleBuilder.Reset()
+	ruleBuilder.Grow(modLenPTR + len(arpa) + len(fqdn))
+	stringutil.WriteToBuilder(
+		ruleBuilder,
+		"||",
+		arpa,
+		rwSuccessPTR,
+		fqdn,
+	)
+	addedPtr = ruleBuilder.String()
+
+	hp.rulesBuilder.Grow(len(added) + len(addedPtr) + 2*len(nl))
+	stringutil.WriteToBuilder(hp.rulesBuilder, added, nl, addedPtr, nl)
+
+	return added, addedPtr
 }

 // equalSet returns true if the internal hosts table just parsed equals target.
@@ -385,15 +502,16 @@ func (hp *hostsParser) sendUpd(ch chan *netutil.IPMap) {
 	case ch <- upd:
 		// The previous update was just read and the next one pushed.  Go on.
 	default:
-		log.Debug("%s: the channel is broken", hostsContainerPref)
+		log.Error("%s: the updates channel is broken", hostsContainerPref)
 	}
 }

 // newStrg creates a new rules storage from parsed data.
 func (hp *hostsParser) newStrg() (s *filterlist.RuleStorage, err error) {
 	return filterlist.NewRuleStorage([]filterlist.RuleList{&filterlist.StringRuleList{
+		// TODO(e.burkov):  Make configurable.
 		ID:             -1,
-		RulesText:      hp.rules.String(),
+		RulesText:      hp.rulesBuilder.String(),
 		IgnoreCosmetic: true,
 	}})
 }
@@ -424,15 +542,7 @@ func (hc *HostsContainer) refresh() (err error) {
 		return fmt.Errorf("initializing rules storage: %w", err)
 	}

-	hc.resetEng(rulesStrg)
+	hc.resetEng(rulesStrg, hp.translations)

 	return nil
 }
-
-func (hc *HostsContainer) resetEng(rulesStrg *filterlist.RuleStorage) {
-	hc.engLock.Lock()
-	defer hc.engLock.Unlock()
-
-	hc.rulesStrg = rulesStrg
-	hc.engine = urlfilter.NewDNSEngine(hc.rulesStrg)
-}