daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pull request 1964: AG-23599 use hostsfile

Browse Source 
Merge in DNS/adguard-home from AG-23599-use-hostsfile to master

Squashed commit of the following:

commit 4766e67a9d5faa4bc89a2a935d187ce4829f7214
Merge: 38369360b 762e5be97
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Tue Aug 22 16:33:54 2023 +0300

    Merge branch 'master' into AG-23599-use-hostsfile

commit 38369360b7d0e5c9ec373c5a06bac8792ca9cd69
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Mon Aug 21 18:09:15 2023 +0300

    filtering: imp tests

commit 1c4d4a9f9639f048173e1c949f39f9ecb6ed0347
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Mon Aug 21 14:00:10 2023 +0300

    filtering: imp cognit, cyclo

commit c50c33d7240c2812a715759fabf140e02184b729
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Mon Aug 21 12:57:31 2023 +0300

    filtering: imp code

commit 92203b16719a717a2946c0401e166b1b38ddb7bc
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Fri Aug 18 17:39:11 2023 +0300

    all: imp code, docs

commit 523e8cd50f9136feede657385b7274fa6ba64131
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Aug 17 15:14:02 2023 +0300

    all: fix ipv6

commit 6ce4537132615cbdc34a0b1f326fedd2b63c355d
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Aug 17 14:17:27 2023 +0300

    all: rm urlfilter from hosts

commit d6666e851680c7e586325ea5970e0356ab919074
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Aug 16 15:09:52 2023 +0300

    WIP

commit 4a2732960558bef6636d3c428bad4c7c830016ca
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Aug 16 14:47:13 2023 +0300

    all: use hostsfile
This commit is contained in:
Eugene Burkov
2023-08-22 16:45:11 +03:00
parent 762e5be97a
commit 4b4036fa6a
14 changed files with 639 additions and 753 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
internal/filtering/dnsrewrite.go
View File

@@ -1,6 +1,10 @@
package filtering
import (
"net"
"github.com/AdguardTeam/golibs/hostsfile"
"github.com/AdguardTeam/urlfilter"
"github.com/AdguardTeam/urlfilter/rules"
"github.com/miekg/dns"
)
@@ -73,3 +77,59 @@ func (d *DNSFilter) processDNSRewrites(dnsr []*rules.NetworkRule) (res Result) {
Reason: RewrittenRule,
}
}
// processDNSResultRewrites returns an empty Result if there are no dnsrewrite
// rules in dnsres. Otherwise, it returns the processed Result.
func (d *DNSFilter) processDNSResultRewrites(
dnsres *urlfilter.DNSResult,
host string,
) (dnsRWRes Result) {
dnsr := dnsres.DNSRewrites()
if len(dnsr) == 0 {
return Result{}
}
res := d.processDNSRewrites(dnsr)
if res.Reason == RewrittenRule && res.CanonName == host {
// A rewrite of a host to itself. Go on and try matching other things.
return Result{}
}
return res
}
// appendRewriteResultFromHost appends the rewrite result from rec to vals and
// resRules.
func appendRewriteResultFromHost(
vals []rules.RRValue,
resRules []*ResultRule,
rec *hostsfile.Record,
qtype uint16,
) (updatedVals []rules.RRValue, updatedRules []*ResultRule) {
switch qtype {
case dns.TypeA:
if !rec.Addr.Is4() {
return vals, resRules
}
vals = append(vals, net.IP(rec.Addr.AsSlice()))
case dns.TypeAAAA:
if !rec.Addr.Is6() {
return vals, resRules
}
vals = append(vals, net.IP(rec.Addr.AsSlice()))
case dns.TypePTR:
for _, name := range rec.Names {
vals = append(vals, name)
}
}
recText, _ := rec.MarshalText()
resRules = append(resRules, &ResultRule{
FilterListID: SysHostsListID,
Text: string(recText),
})
return vals, resRules
}

158
internal/filtering/dnsrewrite_test.go
View File

@@ -1,10 +1,17 @@
package filtering
import (
"fmt"
"net"
"net/netip"
"path"
"testing"
"testing/fstest"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/golibs/testutil"
"github.com/AdguardTeam/urlfilter/rules"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -202,3 +209,154 @@ func TestDNSFilter_CheckHostRules_dnsrewrite(t *testing.T) {
assert.Equal(t, "new-ptr-with-dot.", ptr)
})
}
func TestDNSFilter_CheckHost_hostsContainer(t *testing.T) {
addrv4 := netip.MustParseAddr("1.2.3.4")
addrv6 := netip.MustParseAddr("::1")
addrMapped := netip.MustParseAddr("::ffff:1.2.3.4")
data := fmt.Sprintf(
""+
"%s v4.host.example\n"+
"%s v6.host.example\n"+
"%s mapped.host.example\n",
addrv4,
addrv6,
addrMapped,
)
files := fstest.MapFS{
"hosts": &fstest.MapFile{
Data: []byte(data),
},
}
watcher := &aghtest.FSWatcher{
OnEvents: func() (e <-chan struct{}) { return nil },
OnAdd: func(name string) (err error) { return nil },
OnClose: func() (err error) { return nil },
}
hc, err := aghnet.NewHostsContainer(files, watcher, "hosts")
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, hc.Close)
f, _ := newForTest(t, &Config{EtcHosts: hc}, nil)
setts := &Settings{
FilteringEnabled: true,
}
testCases := []struct {
name string
host string
wantRules []*ResultRule
wantResps []rules.RRValue
dtyp uint16
}{{
name: "v4",
host: "v4.host.example",
dtyp: dns.TypeA,
wantRules: []*ResultRule{{
Text: "1.2.3.4 v4.host.example",
FilterListID: SysHostsListID,
}},
wantResps: []rules.RRValue{net.IP(addrv4.AsSlice())},
}, {
name: "v6",
host: "v6.host.example",
dtyp: dns.TypeAAAA,
wantRules: []*ResultRule{{
Text: "::1 v6.host.example",
FilterListID: SysHostsListID,
}},
wantResps: []rules.RRValue{net.IP(addrv6.AsSlice())},
}, {
name: "mapped",
host: "mapped.host.example",
dtyp: dns.TypeAAAA,
wantRules: []*ResultRule{{
Text: "::ffff:1.2.3.4 mapped.host.example",
FilterListID: SysHostsListID,
}},
wantResps: []rules.RRValue{net.IP(addrMapped.AsSlice())},
}, {
name: "ptr",
host: "4.3.2.1.in-addr.arpa",
dtyp: dns.TypePTR,
wantRules: []*ResultRule{{
Text: "1.2.3.4 v4.host.example",
FilterListID: SysHostsListID,
}},
wantResps: []rules.RRValue{"v4.host.example"},
}, {
name: "ptr-mapped",
host: "4.0.3.0.2.0.1.0.f.f.f.f.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa",
dtyp: dns.TypePTR,
wantRules: []*ResultRule{{
Text: "::ffff:1.2.3.4 mapped.host.example",
FilterListID: SysHostsListID,
}},
wantResps: []rules.RRValue{"mapped.host.example"},
}, {
name: "not_found_v4",
host: "non.existent.example",
dtyp: dns.TypeA,
wantRules: nil,
wantResps: nil,
}, {
name: "not_found_v6",
host: "non.existent.example",
dtyp: dns.TypeAAAA,
wantRules: nil,
wantResps: nil,
}, {
name: "not_found_ptr",
host: "4.3.2.2.in-addr.arpa",
dtyp: dns.TypePTR,
wantRules: nil,
wantResps: nil,
}, {
name: "v4_mismatch",
host: "v4.host.example",
dtyp: dns.TypeAAAA,
wantRules: nil,
wantResps: nil,
}, {
name: "v6_mismatch",
host: "v6.host.example",
dtyp: dns.TypeA,
wantRules: nil,
wantResps: nil,
}, {
name: "wrong_ptr",
host: "4.3.2.1.ip6.arpa",
dtyp: dns.TypePTR,
wantRules: nil,
wantResps: nil,
}, {
name: "unsupported_type",
host: "v4.host.example",
dtyp: dns.TypeCNAME,
wantRules: nil,
wantResps: nil,
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
var res Result
res, err = f.CheckHost(tc.host, tc.dtyp, setts)
require.NoError(t, err)
if len(tc.wantRules) == 0 {
assert.Empty(t, res.Rules)
assert.Nil(t, res.DNSRewriteResult)
return
}
require.NotNil(t, res.DNSRewriteResult)
require.Contains(t, res.DNSRewriteResult.Response, tc.dtyp)
assert.Equal(t, tc.wantResps, res.DNSRewriteResult.Response[tc.dtyp])
assert.Equal(t, tc.wantRules, res.Rules)
})
}
}

106
internal/filtering/filtering.go
View File

@@ -7,6 +7,7 @@ import (
"io/fs"
"net"
"net/http"
"net/netip"
"os"
"path/filepath"
"runtime"
@@ -19,8 +20,10 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/hostsfile"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/mathutil"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/AdguardTeam/urlfilter"
"github.com/AdguardTeam/urlfilter/filterlist"
@@ -326,16 +329,6 @@ func (d *DNSFilter) WriteDiskConfig(c *Config) {
c.UserRules = slices.Clone(d.UserRules)
}
// cloneRewrites returns a deep copy of entries.
func cloneRewrites(entries []*LegacyRewrite) (clone []*LegacyRewrite) {
clone = make([]*LegacyRewrite, len(entries))
for i, rw := range entries {
clone[i] = rw.clone()
}
return clone
}
// setFilters sets new filters, synchronously or asynchronously. When filters
// are set asynchronously, the old filters continue working until the new
// filters are ready.
@@ -503,31 +496,50 @@ func (d *DNSFilter) matchSysHosts(
qtype uint16,
setts *Settings,
) (res Result, err error) {
// TODO(e.burkov): Where else is this checked?
if !setts.FilteringEnabled || d.EtcHosts == nil {
return res, nil
}
dnsres, _ := d.EtcHosts.MatchRequest(&urlfilter.DNSRequest{
Hostname: host,
SortedClientTags: setts.ClientTags,
// TODO(e.burkov): Wait for urlfilter update to pass netip.Addr.
ClientIP: setts.ClientIP.String(),
ClientName: setts.ClientName,
DNSType: qtype,
})
if dnsres == nil {
return res, nil
var recs []*hostsfile.Record
switch qtype {
case dns.TypeA, dns.TypeAAAA:
recs = d.EtcHosts.MatchName(host)
case dns.TypePTR:
var ip net.IP
ip, err = netutil.IPFromReversedAddr(host)
if err != nil {
log.Debug("filtering: failed to parse PTR record %q: %s", host, err)
return res, nil
}
addr, _ := netip.AddrFromSlice(ip)
recs = d.EtcHosts.MatchAddr(addr)
default:
log.Debug("filtering: unsupported query type %s", dns.Type(qtype))
}
dnsr := dnsres.DNSRewrites()
if len(dnsr) == 0 {
return res, nil
var vals []rules.RRValue
var resRules []*ResultRule
resRulesLen := 0
for _, rec := range recs {
vals, resRules = appendRewriteResultFromHost(vals, resRules, rec, qtype)
if len(resRules) > resRulesLen {
resRulesLen = len(resRules)
log.Debug("filtering: matched %s in %q", host, rec.Source)
}
}
res = d.processDNSRewrites(dnsr)
res.Reason = RewrittenAutoHosts
for _, r := range res.Rules {
r.Text = stringutil.Coalesce(d.EtcHosts.Translate(r.Text), r.Text)
if len(vals) > 0 {
res.DNSRewriteResult = &DNSRewriteResult{
Response: DNSRewriteResultResponse{
qtype: vals,
},
RCode: dns.RcodeSuccess,
}
res.Rules = resRules
res.Reason = RewrittenRule
}
return res, nil
@@ -594,25 +606,6 @@ func (d *DNSFilter) processRewrites(host string, qtype uint16) (res Result) {
return res
}
// setRewriteResult sets the Reason or IPList of res if necessary. res must not
// be nil.
func setRewriteResult(res *Result, host string, rewrites []*LegacyRewrite, qtype uint16) {
for _, rw := range rewrites {
if rw.Type == qtype && (qtype == dns.TypeA || qtype == dns.TypeAAAA) {
if rw.IP == nil {
// "A"/"AAAA" exception: allow getting from upstream.
res.Reason = NotFilteredNotFound
return
}
res.IPList = append(res.IPList, rw.IP)
log.Debug("rewrite: a/aaaa for %s is %s", host, rw.IP)
}
}
}
// matchBlockedServicesRules checks the host against the blocked services rules
// in settings, if any. The err is always nil, it is only there to make this
// a valid hostChecker function.
@@ -895,26 +888,6 @@ func (d *DNSFilter) matchHost(
return res, nil
}
// processDNSResultRewrites returns an empty Result if there are no dnsrewrite
// rules in dnsres. Otherwise, it returns the processed Result.
func (d *DNSFilter) processDNSResultRewrites(
dnsres *urlfilter.DNSResult,
host string,
) (dnsRWRes Result) {
dnsr := dnsres.DNSRewrites()
if len(dnsr) == 0 {
return Result{}
}
res := d.processDNSRewrites(dnsr)
if res.Reason == RewrittenRule && res.CanonName == host {
// A rewrite of a host to itself. Go on and try matching other things.
return Result{}
}
return res
}
// makeResult returns a properly constructed Result.
func makeResult(matchedRules []rules.Rule, reason Reason) (res Result) {
resRules := make([]*ResultRule, len(matchedRules))
@@ -987,7 +960,6 @@ func New(c *Config, blockFilters []Filter) (d *DNSFilter, err error) {
if d.BlockedServices != nil {
err = d.BlockedServices.Validate()
if err != nil {
return nil, fmt.Errorf("filtering: %w", err)
}

2
internal/filtering/filtering_test.go
View File

@@ -91,7 +91,7 @@ func (d *DNSFilter) checkMatchEmpty(t *testing.T, hostname string, setts *Settin
assert.Falsef(t, res.IsFiltered, "host %q", hostname)
}
func TestEtcHostsMatching(t *testing.T) {
func TestDNSFilter_CheckHost_hostRules(t *testing.T) {
addr := "216.239.38.120"
addr6 := "::1"
text := fmt.Sprintf(` %s google.com www.google.com # enforce google's safesearch

30
internal/filtering/rewrites.go
View File

@@ -6,6 +6,7 @@ import (
"strings"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/mathutil"
"github.com/miekg/dns"
"golang.org/x/exp/slices"
@@ -200,3 +201,32 @@ func findRewrites(
return rewrites, matched
}
// setRewriteResult sets the Reason or IPList of res if necessary. res must not
// be nil.
func setRewriteResult(res *Result, host string, rewrites []*LegacyRewrite, qtype uint16) {
for _, rw := range rewrites {
if rw.Type == qtype && (qtype == dns.TypeA || qtype == dns.TypeAAAA) {
if rw.IP == nil {
// "A"/"AAAA" exception: allow getting from upstream.
res.Reason = NotFilteredNotFound
return
}
res.IPList = append(res.IPList, rw.IP)
log.Debug("rewrite: a/aaaa for %s is %s", host, rw.IP)
}
}
}
// cloneRewrites returns a deep copy of entries.
func cloneRewrites(entries []*LegacyRewrite) (clone []*LegacyRewrite) {
clone = make([]*LegacyRewrite, len(entries))
for i, rw := range entries {
clone[i] = rw.clone()
}
return clone
}