all: sync with master; upd chlog

This commit is contained in:
Ainar Garipov
2023-09-07 17:13:48 +03:00
parent 3be7676970
commit 7b93f5d7cf
306 changed files with 19770 additions and 4916 deletions

View File

@@ -1,212 +0,0 @@
package aghnet
import (
"bufio"
"bytes"
"fmt"
"net"
"net/netip"
"sync"
"github.com/AdguardTeam/golibs/errors"
"golang.org/x/exp/slices"
)
// ARPDB: The Network Neighborhood Database
// ARPDB stores and refreshes the network neighborhood reported by ARP (Address
// Resolution Protocol).
type ARPDB interface {
// Refresh updates the stored data. It must be safe for concurrent use.
Refresh() (err error)
// Neighbors returnes the last set of data reported by ARP. Both the method
// and it's result must be safe for concurrent use.
Neighbors() (ns []Neighbor)
}
// NewARPDB returns the ARPDB properly initialized for the OS.
func NewARPDB() (arp ARPDB) {
return newARPDB()
}
// Empty ARPDB implementation
// EmptyARPDB is the ARPDB implementation that does nothing.
type EmptyARPDB struct{}
// type check
var _ ARPDB = EmptyARPDB{}
// Refresh implements the ARPDB interface for EmptyARPContainer. It does
// nothing and always returns nil error.
func (EmptyARPDB) Refresh() (err error) { return nil }
// Neighbors implements the ARPDB interface for EmptyARPContainer. It always
// returns nil.
func (EmptyARPDB) Neighbors() (ns []Neighbor) { return nil }
// ARPDB Helper Types
// Neighbor is the pair of IP address and MAC address reported by ARP.
type Neighbor struct {
// Name is the hostname of the neighbor. Empty name is valid since not each
// implementation of ARP is able to retrieve that.
Name string
// IP contains either IPv4 or IPv6.
IP netip.Addr
// MAC contains the hardware address.
MAC net.HardwareAddr
}
// Clone returns the deep copy of n.
func (n Neighbor) Clone() (clone Neighbor) {
return Neighbor{
Name: n.Name,
IP: n.IP,
MAC: slices.Clone(n.MAC),
}
}
// neighs is the helper type that stores neighbors to avoid copying its methods
// among all the ARPDB implementations.
type neighs struct {
mu *sync.RWMutex
ns []Neighbor
}
// len returns the length of the neighbors slice. It's safe for concurrent use.
func (ns *neighs) len() (l int) {
ns.mu.RLock()
defer ns.mu.RUnlock()
return len(ns.ns)
}
// clone returns a deep copy of the underlying neighbors slice. It's safe for
// concurrent use.
func (ns *neighs) clone() (cloned []Neighbor) {
ns.mu.RLock()
defer ns.mu.RUnlock()
cloned = make([]Neighbor, len(ns.ns))
for i, n := range ns.ns {
cloned[i] = n.Clone()
}
return cloned
}
// reset replaces the underlying slice with the new one. It's safe for
// concurrent use.
func (ns *neighs) reset(with []Neighbor) {
ns.mu.Lock()
defer ns.mu.Unlock()
ns.ns = with
}
// Command ARPDB
// parseNeighsFunc parses the text from sc as if it'd be an output of some
// ARP-related command. lenHint is a hint for the size of the allocated slice
// of Neighbors.
type parseNeighsFunc func(sc *bufio.Scanner, lenHint int) (ns []Neighbor)
// cmdARPDB is the implementation of the ARPDB that uses command line to
// retrieve data.
type cmdARPDB struct {
parse parseNeighsFunc
ns *neighs
cmd string
args []string
}
// type check
var _ ARPDB = (*cmdARPDB)(nil)
// Refresh implements the ARPDB interface for *cmdARPDB.
func (arp *cmdARPDB) Refresh() (err error) {
defer func() { err = errors.Annotate(err, "cmd arpdb: %w") }()
code, out, err := aghosRunCommand(arp.cmd, arp.args...)
if err != nil {
return fmt.Errorf("running command: %w", err)
} else if code != 0 {
return fmt.Errorf("running command: unexpected exit code %d", code)
}
sc := bufio.NewScanner(bytes.NewReader(out))
ns := arp.parse(sc, arp.ns.len())
if err = sc.Err(); err != nil {
// TODO(e.burkov): This error seems unreachable. Investigate.
return fmt.Errorf("scanning the output: %w", err)
}
arp.ns.reset(ns)
return nil
}
// Neighbors implements the ARPDB interface for *cmdARPDB.
func (arp *cmdARPDB) Neighbors() (ns []Neighbor) {
return arp.ns.clone()
}
// Composite ARPDB
// arpdbs is the ARPDB that combines several ARPDB implementations and
// consequently switches between those.
type arpdbs struct {
// arps is the set of ARPDB implementations to range through.
arps []ARPDB
neighs
}
// newARPDBs returns a properly initialized *arpdbs. It begins refreshing from
// the first of arps.
func newARPDBs(arps ...ARPDB) (arp *arpdbs) {
return &arpdbs{
arps: arps,
neighs: neighs{
mu: &sync.RWMutex{},
ns: make([]Neighbor, 0),
},
}
}
// type check
var _ ARPDB = (*arpdbs)(nil)
// Refresh implements the ARPDB interface for *arpdbs.
func (arp *arpdbs) Refresh() (err error) {
var errs []error
for _, a := range arp.arps {
err = a.Refresh()
if err != nil {
errs = append(errs, err)
continue
}
arp.reset(a.Neighbors())
return nil
}
if len(errs) > 0 {
err = errors.List("each arpdb failed", errs...)
}
return err
}
// Neighbors implements the ARPDB interface for *arpdbs.
//
// TODO(e.burkov): Think of a way to avoid cloning the slice twice.
func (arp *arpdbs) Neighbors() (ns []Neighbor) {
return arp.clone()
}

View File

@@ -1,81 +0,0 @@
//go:build darwin || freebsd
package aghnet
import (
"bufio"
"net"
"net/netip"
"strings"
"sync"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/netutil"
)
func newARPDB() (arp *cmdARPDB) {
return &cmdARPDB{
parse: parseArpA,
ns: &neighs{
mu: &sync.RWMutex{},
ns: make([]Neighbor, 0),
},
cmd: "arp",
// Use -n flag to avoid resolving the hostnames of the neighbors. By
// default ARP attempts to resolve the hostnames via DNS. See man 8
// arp.
//
// See also https://github.com/AdguardTeam/AdGuardHome/issues/3157.
args: []string{"-a", "-n"},
}
}
// parseArpA parses the output of the "arp -a -n" command on macOS and FreeBSD.
// The expected input format:
//
// host.name (192.168.0.1) at ff:ff:ff:ff:ff:ff on en0 ifscope [ethernet]
func parseArpA(sc *bufio.Scanner, lenHint int) (ns []Neighbor) {
ns = make([]Neighbor, 0, lenHint)
for sc.Scan() {
ln := sc.Text()
fields := strings.Fields(ln)
if len(fields) < 4 {
continue
}
n := Neighbor{}
if ipStr := fields[1]; len(ipStr) < 2 {
continue
} else if ip, err := netip.ParseAddr(ipStr[1 : len(ipStr)-1]); err != nil {
log.Debug("arpdb: parsing arp output: ip: %s", err)
continue
} else {
n.IP = ip
}
hwStr := fields[3]
mac, err := net.ParseMAC(hwStr)
if err != nil {
log.Debug("arpdb: parsing arp output: mac: %s", err)
continue
} else {
n.MAC = mac
}
host := fields[0]
err = netutil.ValidateHostname(host)
if err != nil {
log.Debug("arpdb: parsing arp output: host: %s", err)
} else {
n.Name = host
}
ns = append(ns, n)
}
return ns
}

View File

@@ -1,31 +0,0 @@
//go:build darwin || freebsd
package aghnet
import (
"net"
"net/netip"
)
const arpAOutput = `
invalid.mac (1.2.3.4) at 12:34:56:78:910 on el0 ifscope [ethernet]
invalid.ip (1.2.3.4.5) at ab:cd:ef:ab:cd:12 on ek0 ifscope [ethernet]
invalid.fmt 1 at 12:cd:ef:ab:cd:ef on er0 ifscope [ethernet]
hostname.one (192.168.1.2) at ab:cd:ef:ab:cd:ef on en0 ifscope [ethernet]
hostname.two (::ffff:ffff) at ef:cd:ab:ef:cd:ab on em0 expires in 1198 seconds [ethernet]
? (::1234) at aa:bb:cc:dd:ee:ff on ej0 expires in 1918 seconds [ethernet]
`
var wantNeighs = []Neighbor{{
Name: "hostname.one",
IP: netip.MustParseAddr("192.168.1.2"),
MAC: net.HardwareAddr{0xAB, 0xCD, 0xEF, 0xAB, 0xCD, 0xEF},
}, {
Name: "hostname.two",
IP: netip.MustParseAddr("::ffff:ffff"),
MAC: net.HardwareAddr{0xEF, 0xCD, 0xAB, 0xEF, 0xCD, 0xAB},
}, {
Name: "",
IP: netip.MustParseAddr("::1234"),
MAC: net.HardwareAddr{0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF},
}}

View File

@@ -1,251 +0,0 @@
//go:build linux
package aghnet
import (
"bufio"
"fmt"
"io/fs"
"net"
"net/netip"
"strings"
"sync"
"github.com/AdguardTeam/AdGuardHome/internal/aghos"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/stringutil"
)
func newARPDB() (arp *arpdbs) {
// Use the common storage among the implementations.
ns := &neighs{
mu: &sync.RWMutex{},
ns: make([]Neighbor, 0),
}
var parseF parseNeighsFunc
if aghos.IsOpenWrt() {
parseF = parseArpAWrt
} else {
parseF = parseArpA
}
return newARPDBs(
// Try /proc/net/arp first.
&fsysARPDB{
ns: ns,
fsys: rootDirFS,
filename: "proc/net/arp",
},
// Then, try "arp -a -n".
&cmdARPDB{
parse: parseF,
ns: ns,
cmd: "arp",
// Use -n flag to avoid resolving the hostnames of the neighbors.
// By default ARP attempts to resolve the hostnames via DNS. See
// man 8 arp.
//
// See also https://github.com/AdguardTeam/AdGuardHome/issues/3157.
args: []string{"-a", "-n"},
},
// Finally, try "ip neigh".
&cmdARPDB{
parse: parseIPNeigh,
ns: ns,
cmd: "ip",
args: []string{"neigh"},
},
)
}
// fsysARPDB accesses the ARP cache file to update the database.
type fsysARPDB struct {
ns *neighs
fsys fs.FS
filename string
}
// type check
var _ ARPDB = (*fsysARPDB)(nil)
// Refresh implements the ARPDB interface for *fsysARPDB.
func (arp *fsysARPDB) Refresh() (err error) {
var f fs.File
f, err = arp.fsys.Open(arp.filename)
if err != nil {
return fmt.Errorf("opening %q: %w", arp.filename, err)
}
sc := bufio.NewScanner(f)
// Skip the header.
if !sc.Scan() {
return nil
} else if err = sc.Err(); err != nil {
return err
}
ns := make([]Neighbor, 0, arp.ns.len())
for sc.Scan() {
ln := sc.Text()
fields := stringutil.SplitTrimmed(ln, " ")
if len(fields) != 6 {
continue
}
n := Neighbor{}
n.IP, err = netip.ParseAddr(fields[0])
if err != nil || n.IP.IsUnspecified() {
continue
} else if n.MAC, err = net.ParseMAC(fields[3]); err != nil {
continue
}
ns = append(ns, n)
}
arp.ns.reset(ns)
return nil
}
// Neighbors implements the ARPDB interface for *fsysARPDB.
func (arp *fsysARPDB) Neighbors() (ns []Neighbor) {
return arp.ns.clone()
}
// parseArpAWrt parses the output of the "arp -a -n" command on OpenWrt. The
// expected input format:
//
// IP address HW type Flags HW address Mask Device
// 192.168.11.98 0x1 0x2 5a:92:df:a9:7e:28 * wan
func parseArpAWrt(sc *bufio.Scanner, lenHint int) (ns []Neighbor) {
if !sc.Scan() {
// Skip the header.
return
}
ns = make([]Neighbor, 0, lenHint)
for sc.Scan() {
ln := sc.Text()
fields := strings.Fields(ln)
if len(fields) < 4 {
continue
}
n := Neighbor{}
ip, err := netip.ParseAddr(fields[0])
if err != nil || n.IP.IsUnspecified() {
log.Debug("arpdb: parsing arp output: ip: %s", err)
continue
} else {
n.IP = ip
}
hwStr := fields[3]
mac, err := net.ParseMAC(hwStr)
if err != nil {
log.Debug("arpdb: parsing arp output: mac: %s", err)
continue
} else {
n.MAC = mac
}
ns = append(ns, n)
}
return ns
}
// parseArpA parses the output of the "arp -a -n" command on Linux. The
// expected input format:
//
// hostname (192.168.1.1) at ab:cd:ef:ab:cd:ef [ether] on enp0s3
func parseArpA(sc *bufio.Scanner, lenHint int) (ns []Neighbor) {
ns = make([]Neighbor, 0, lenHint)
for sc.Scan() {
ln := sc.Text()
fields := strings.Fields(ln)
if len(fields) < 4 {
continue
}
n := Neighbor{}
if ipStr := fields[1]; len(ipStr) < 2 {
continue
} else if ip, err := netip.ParseAddr(ipStr[1 : len(ipStr)-1]); err != nil {
log.Debug("arpdb: parsing arp output: ip: %s", err)
continue
} else {
n.IP = ip
}
hwStr := fields[3]
if mac, err := net.ParseMAC(hwStr); err != nil {
log.Debug("arpdb: parsing arp output: mac: %s", err)
continue
} else {
n.MAC = mac
}
host := fields[0]
if verr := netutil.ValidateHostname(host); verr != nil {
log.Debug("arpdb: parsing arp output: host: %s", verr)
} else {
n.Name = host
}
ns = append(ns, n)
}
return ns
}
// parseIPNeigh parses the output of the "ip neigh" command on Linux. The
// expected input format:
//
// 192.168.1.1 dev enp0s3 lladdr ab:cd:ef:ab:cd:ef REACHABLE
func parseIPNeigh(sc *bufio.Scanner, lenHint int) (ns []Neighbor) {
ns = make([]Neighbor, 0, lenHint)
for sc.Scan() {
ln := sc.Text()
fields := strings.Fields(ln)
if len(fields) < 5 {
continue
}
n := Neighbor{}
ip, err := netip.ParseAddr(fields[0])
if err != nil {
log.Debug("arpdb: parsing arp output: ip: %s", err)
continue
} else {
n.IP = ip
}
mac, err := net.ParseMAC(fields[4])
if err != nil {
log.Debug("arpdb: parsing arp output: mac: %s", err)
continue
} else {
n.MAC = mac
}
ns = append(ns, n)
}
return ns
}

View File

@@ -1,102 +0,0 @@
//go:build linux
package aghnet
import (
"net"
"net/netip"
"sync"
"testing"
"testing/fstest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const arpAOutputWrt = `
IP address HW type Flags HW address Mask Device
1.2.3.4.5 0x1 0x2 aa:bb:cc:dd:ee:ff * wan
1.2.3.4 0x1 0x2 12:34:56:78:910 * wan
192.168.1.2 0x1 0x2 ab:cd:ef:ab:cd:ef * wan
::ffff:ffff 0x1 0x2 ef:cd:ab:ef:cd:ab * wan`
const arpAOutput = `
invalid.mac (1.2.3.4) at 12:34:56:78:910 on el0 ifscope [ethernet]
invalid.ip (1.2.3.4.5) at ab:cd:ef:ab:cd:12 on ek0 ifscope [ethernet]
invalid.fmt 1 at 12:cd:ef:ab:cd:ef on er0 ifscope [ethernet]
? (192.168.1.2) at ab:cd:ef:ab:cd:ef on en0 ifscope [ethernet]
? (::ffff:ffff) at ef:cd:ab:ef:cd:ab on em0 expires in 100 seconds [ethernet]`
const ipNeighOutput = `
1.2.3.4.5 dev enp0s3 lladdr aa:bb:cc:dd:ee:ff DELAY
1.2.3.4 dev enp0s3 lladdr 12:34:56:78:910 DELAY
192.168.1.2 dev enp0s3 lladdr ab:cd:ef:ab:cd:ef DELAY
::ffff:ffff dev enp0s3 lladdr ef:cd:ab:ef:cd:ab router STALE`
var wantNeighs = []Neighbor{{
IP: netip.MustParseAddr("192.168.1.2"),
MAC: net.HardwareAddr{0xAB, 0xCD, 0xEF, 0xAB, 0xCD, 0xEF},
}, {
IP: netip.MustParseAddr("::ffff:ffff"),
MAC: net.HardwareAddr{0xEF, 0xCD, 0xAB, 0xEF, 0xCD, 0xAB},
}}
func TestFSysARPDB(t *testing.T) {
require.NoError(t, fstest.TestFS(testdata, "proc_net_arp"))
a := &fsysARPDB{
ns: &neighs{
mu: &sync.RWMutex{},
ns: make([]Neighbor, 0),
},
fsys: testdata,
filename: "proc_net_arp",
}
err := a.Refresh()
require.NoError(t, err)
ns := a.Neighbors()
assert.Equal(t, wantNeighs, ns)
}
func TestCmdARPDB_linux(t *testing.T) {
sh := mapShell{
"arp -a": {err: nil, out: arpAOutputWrt, code: 0},
"ip neigh": {err: nil, out: ipNeighOutput, code: 0},
}
substShell(t, sh.RunCmd)
t.Run("wrt", func(t *testing.T) {
a := &cmdARPDB{
parse: parseArpAWrt,
cmd: "arp",
args: []string{"-a"},
ns: &neighs{
mu: &sync.RWMutex{},
ns: make([]Neighbor, 0),
},
}
err := a.Refresh()
require.NoError(t, err)
assert.Equal(t, wantNeighs, a.Neighbors())
})
t.Run("ip_neigh", func(t *testing.T) {
a := &cmdARPDB{
parse: parseIPNeigh,
cmd: "ip",
args: []string{"neigh"},
ns: &neighs{
mu: &sync.RWMutex{},
ns: make([]Neighbor, 0),
},
}
err := a.Refresh()
require.NoError(t, err)
assert.Equal(t, wantNeighs, a.Neighbors())
})
}

View File

@@ -1,76 +0,0 @@
//go:build openbsd
package aghnet
import (
"bufio"
"net"
"net/netip"
"strings"
"sync"
"github.com/AdguardTeam/golibs/log"
)
func newARPDB() (arp *cmdARPDB) {
return &cmdARPDB{
parse: parseArpA,
ns: &neighs{
mu: &sync.RWMutex{},
ns: make([]Neighbor, 0),
},
cmd: "arp",
// Use -n flag to avoid resolving the hostnames of the neighbors. By
// default ARP attempts to resolve the hostnames via DNS. See man 8
// arp.
//
// See also https://github.com/AdguardTeam/AdGuardHome/issues/3157.
args: []string{"-a", "-n"},
}
}
// parseArpA parses the output of the "arp -a -n" command on OpenBSD. The
// expected input format:
//
// Host Ethernet Address Netif Expire Flags
// 192.168.1.1 ab:cd:ef:ab:cd:ef em0 19m59s
func parseArpA(sc *bufio.Scanner, lenHint int) (ns []Neighbor) {
// Skip the header.
if !sc.Scan() {
return nil
}
ns = make([]Neighbor, 0, lenHint)
for sc.Scan() {
ln := sc.Text()
fields := strings.Fields(ln)
if len(fields) < 2 {
continue
}
n := Neighbor{}
ip, err := netip.ParseAddr(fields[0])
if err != nil {
log.Debug("arpdb: parsing arp output: ip: %s", err)
continue
} else {
n.IP = ip
}
mac, err := net.ParseMAC(fields[1])
if err != nil {
log.Debug("arpdb: parsing arp output: mac: %s", err)
continue
} else {
n.MAC = mac
}
ns = append(ns, n)
}
return ns
}

View File

@@ -1,24 +0,0 @@
//go:build openbsd
package aghnet
import (
"net"
"net/netip"
)
const arpAOutput = `
Host Ethernet Address Netif Expire Flags
1.2.3.4.5 aa:bb:cc:dd:ee:ff em0 permanent
1.2.3.4 12:34:56:78:910 em0 permanent
192.168.1.2 ab:cd:ef:ab:cd:ef em0 19m56s
::ffff:ffff ef:cd:ab:ef:cd:ab em0 permanent l
`
var wantNeighs = []Neighbor{{
IP: netip.MustParseAddr("192.168.1.2"),
MAC: net.HardwareAddr{0xAB, 0xCD, 0xEF, 0xAB, 0xCD, 0xEF},
}, {
IP: netip.MustParseAddr("::ffff:ffff"),
MAC: net.HardwareAddr{0xEF, 0xCD, 0xAB, 0xEF, 0xCD, 0xAB},
}}

View File

@@ -1,217 +0,0 @@
package aghnet
import (
"net"
"net/netip"
"sync"
"testing"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/testutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNewARPDB(t *testing.T) {
var a ARPDB
require.NotPanics(t, func() { a = NewARPDB() })
assert.NotNil(t, a)
}
// TestARPDB is the mock implementation of ARPDB to use in tests.
type TestARPDB struct {
OnRefresh func() (err error)
OnNeighbors func() (ns []Neighbor)
}
// Refresh implements the ARPDB interface for *TestARPDB.
func (arp *TestARPDB) Refresh() (err error) {
return arp.OnRefresh()
}
// Neighbors implements the ARPDB interface for *TestARPDB.
func (arp *TestARPDB) Neighbors() (ns []Neighbor) {
return arp.OnNeighbors()
}
func TestARPDBS(t *testing.T) {
knownIP := netip.MustParseAddr("1.2.3.4")
knownMAC := net.HardwareAddr{0xAB, 0xCD, 0xEF, 0xAB, 0xCD, 0xEF}
succRefrCount, failRefrCount := 0, 0
clnp := func() {
succRefrCount, failRefrCount = 0, 0
}
succDB := &TestARPDB{
OnRefresh: func() (err error) { succRefrCount++; return nil },
OnNeighbors: func() (ns []Neighbor) {
return []Neighbor{{Name: "abc", IP: knownIP, MAC: knownMAC}}
},
}
failDB := &TestARPDB{
OnRefresh: func() (err error) { failRefrCount++; return errors.Error("refresh failed") },
OnNeighbors: func() (ns []Neighbor) { return nil },
}
t.Run("begin_with_success", func(t *testing.T) {
t.Cleanup(clnp)
a := newARPDBs(succDB, failDB)
err := a.Refresh()
require.NoError(t, err)
assert.Equal(t, 1, succRefrCount)
assert.Zero(t, failRefrCount)
assert.NotEmpty(t, a.Neighbors())
})
t.Run("begin_with_fail", func(t *testing.T) {
t.Cleanup(clnp)
a := newARPDBs(failDB, succDB)
err := a.Refresh()
require.NoError(t, err)
assert.Equal(t, 1, succRefrCount)
assert.Equal(t, 1, failRefrCount)
assert.NotEmpty(t, a.Neighbors())
})
t.Run("fail_only", func(t *testing.T) {
t.Cleanup(clnp)
wantMsg := `each arpdb failed: 2 errors: "refresh failed", "refresh failed"`
a := newARPDBs(failDB, failDB)
err := a.Refresh()
require.Error(t, err)
testutil.AssertErrorMsg(t, wantMsg, err)
assert.Equal(t, 2, failRefrCount)
assert.Empty(t, a.Neighbors())
})
t.Run("fail_after_success", func(t *testing.T) {
t.Cleanup(clnp)
shouldFail := false
unstableDB := &TestARPDB{
OnRefresh: func() (err error) {
if shouldFail {
err = errors.Error("unstable failed")
}
shouldFail = !shouldFail
return err
},
OnNeighbors: func() (ns []Neighbor) {
if !shouldFail {
return failDB.OnNeighbors()
}
return succDB.OnNeighbors()
},
}
a := newARPDBs(unstableDB, succDB)
// Unstable ARPDB should refresh successfully.
err := a.Refresh()
require.NoError(t, err)
assert.Zero(t, succRefrCount)
assert.NotEmpty(t, a.Neighbors())
// Unstable ARPDB should fail and the succDB should be used.
err = a.Refresh()
require.NoError(t, err)
assert.Equal(t, 1, succRefrCount)
assert.NotEmpty(t, a.Neighbors())
// Unstable ARPDB should refresh successfully again.
err = a.Refresh()
require.NoError(t, err)
assert.Equal(t, 1, succRefrCount)
assert.NotEmpty(t, a.Neighbors())
})
t.Run("empty", func(t *testing.T) {
a := newARPDBs()
require.NoError(t, a.Refresh())
assert.Empty(t, a.Neighbors())
})
}
func TestCmdARPDB_arpa(t *testing.T) {
a := &cmdARPDB{
cmd: "cmd",
parse: parseArpA,
ns: &neighs{
mu: &sync.RWMutex{},
ns: make([]Neighbor, 0),
},
}
t.Run("arp_a", func(t *testing.T) {
sh := theOnlyCmd("cmd", 0, arpAOutput, nil)
substShell(t, sh.RunCmd)
err := a.Refresh()
require.NoError(t, err)
assert.Equal(t, wantNeighs, a.Neighbors())
})
t.Run("runcmd_error", func(t *testing.T) {
sh := theOnlyCmd("cmd", 0, "", errors.Error("can't run"))
substShell(t, sh.RunCmd)
err := a.Refresh()
testutil.AssertErrorMsg(t, "cmd arpdb: running command: can't run", err)
})
t.Run("bad_code", func(t *testing.T) {
sh := theOnlyCmd("cmd", 1, "", nil)
substShell(t, sh.RunCmd)
err := a.Refresh()
testutil.AssertErrorMsg(t, "cmd arpdb: running command: unexpected exit code 1", err)
})
t.Run("empty", func(t *testing.T) {
sh := theOnlyCmd("cmd", 0, "", nil)
substShell(t, sh.RunCmd)
err := a.Refresh()
require.NoError(t, err)
assert.Empty(t, a.Neighbors())
})
}
func TestEmptyARPDB(t *testing.T) {
a := EmptyARPDB{}
t.Run("refresh", func(t *testing.T) {
var err error
require.NotPanics(t, func() {
err = a.Refresh()
})
assert.NoError(t, err)
})
t.Run("neighbors", func(t *testing.T) {
var ns []Neighbor
require.NotPanics(t, func() {
ns = a.Neighbors()
})
assert.Empty(t, ns)
})
}

View File

@@ -1,65 +0,0 @@
//go:build windows
package aghnet
import (
"bufio"
"net"
"net/netip"
"strings"
"sync"
)
func newARPDB() (arp *cmdARPDB) {
return &cmdARPDB{
parse: parseArpA,
ns: &neighs{
mu: &sync.RWMutex{},
ns: make([]Neighbor, 0),
},
cmd: "arp",
args: []string{"/a"},
}
}
// parseArpA parses the output of the "arp /a" command on Windows. The expected
// input format (the first line is empty):
//
// Interface: 192.168.56.16 --- 0x7
// Internet Address Physical Address Type
// 192.168.56.1 0a-00-27-00-00-00 dynamic
// 192.168.56.255 ff-ff-ff-ff-ff-ff static
func parseArpA(sc *bufio.Scanner, lenHint int) (ns []Neighbor) {
ns = make([]Neighbor, 0, lenHint)
for sc.Scan() {
ln := sc.Text()
if ln == "" {
continue
}
fields := strings.Fields(ln)
if len(fields) != 3 {
continue
}
n := Neighbor{}
ip, err := netip.ParseAddr(fields[0])
if err != nil {
continue
} else {
n.IP = ip
}
mac, err := net.ParseMAC(fields[1])
if err != nil {
continue
} else {
n.MAC = mac
}
ns = append(ns, n)
}
return ns
}

View File

@@ -1,23 +0,0 @@
//go:build windows
package aghnet
import (
"net"
"net/netip"
)
const arpAOutput = `
Interface: 192.168.1.1 --- 0x7
Internet Address Physical Address Type
192.168.1.2 ab-cd-ef-ab-cd-ef dynamic
::ffff:ffff ef-cd-ab-ef-cd-ab static`
var wantNeighs = []Neighbor{{
IP: netip.MustParseAddr("192.168.1.2"),
MAC: net.HardwareAddr{0xAB, 0xCD, 0xEF, 0xAB, 0xCD, 0xEF},
}, {
IP: netip.MustParseAddr("::ffff:ffff"),
MAC: net.HardwareAddr{0xEF, 0xCD, 0xAB, 0xEF, 0xCD, 0xAB},
}}

View File

@@ -60,7 +60,7 @@ func ifaceIPv4Subnet(iface *net.Interface) (subnet netip.Prefix, err error) {
}
if ip = ip.To4(); ip != nil {
return netip.PrefixFrom(netip.AddrFrom4(*(*[4]byte)(ip)), maskLen), nil
return netip.PrefixFrom(netip.AddrFrom4([4]byte(ip)), maskLen), nil
}
}

View File

@@ -1,25 +1,19 @@
package aghnet
import (
"bufio"
"fmt"
"io"
"io/fs"
"net/netip"
"path"
"strings"
"sync"
"sync/atomic"
"github.com/AdguardTeam/AdGuardHome/internal/aghos"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/hostsfile"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/AdguardTeam/urlfilter"
"github.com/AdguardTeam/urlfilter/filterlist"
"github.com/AdguardTeam/urlfilter/rules"
"github.com/miekg/dns"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)
// DefaultHostsPaths returns the slice of paths default for the operating system
@@ -29,95 +23,51 @@ func DefaultHostsPaths() (paths []string) {
return defaultHostsPaths()
}
// requestMatcher combines the logic for matching requests and translating the
// appropriate rules.
type requestMatcher struct {
// stateLock protects all the fields of requestMatcher.
stateLock *sync.RWMutex
// rulesStrg stores the rules obtained from the hosts' file.
rulesStrg *filterlist.RuleStorage
// engine serves rulesStrg.
engine *urlfilter.DNSEngine
// translator maps generated $dnsrewrite rules into hosts-syntax rules.
//
// TODO(e.burkov): Store the filename from which the rule was parsed.
translator map[string]string
}
// MatchRequest processes the request rewriting hostnames and addresses read
// from the operating system's hosts files. res is nil for any request having
// not an A/AAAA or PTR type, see man 5 hosts.
//
// It's safe for concurrent use.
func (rm *requestMatcher) MatchRequest(
req *urlfilter.DNSRequest,
) (res *urlfilter.DNSResult, ok bool) {
switch req.DNSType {
case dns.TypeA, dns.TypeAAAA, dns.TypePTR:
log.Debug(
"%s: handling %s request for %s",
hostsContainerPrefix,
dns.Type(req.DNSType),
req.Hostname,
)
rm.stateLock.RLock()
defer rm.stateLock.RUnlock()
return rm.engine.MatchRequest(req)
default:
return nil, false
// MatchAddr returns the records for the IP address.
func (hc *HostsContainer) MatchAddr(ip netip.Addr) (recs []*hostsfile.Record) {
cur := hc.current.Load()
if cur == nil {
return nil
}
return cur.addrs[ip]
}
// Translate returns the source hosts-syntax rule for the generated dnsrewrite
// rule or an empty string if the last doesn't exist. The returned rules are in
// a processed format like:
//
// ip host1 host2 ...
func (rm *requestMatcher) Translate(rule string) (hostRule string) {
rm.stateLock.RLock()
defer rm.stateLock.RUnlock()
// MatchName returns the records for the hostname.
func (hc *HostsContainer) MatchName(name string) (recs []*hostsfile.Record) {
cur := hc.current.Load()
if cur != nil {
recs = cur.names[name]
}
return rm.translator[rule]
}
// resetEng updates container's engine and the translation map.
func (rm *requestMatcher) resetEng(rulesStrg *filterlist.RuleStorage, tr map[string]string) {
rm.stateLock.Lock()
defer rm.stateLock.Unlock()
rm.rulesStrg = rulesStrg
rm.engine = urlfilter.NewDNSEngine(rm.rulesStrg)
rm.translator = tr
return recs
}
// hostsContainerPrefix is a prefix for logging and wrapping errors in
// HostsContainer's methods.
const hostsContainerPrefix = "hosts container"
// Hosts is a map of IP addresses to the records, as it primarily stored in the
// [HostsContainer]. It should not be accessed for writing since it may be read
// concurrently, users should clone it before modifying.
//
// The order of records for each address is preserved from original files, but
// the order of the addresses, being a map key, is not.
//
// TODO(e.burkov): Probably, this should be a sorted slice of records.
type Hosts map[netip.Addr][]*hostsfile.Record
// HostsContainer stores the relevant hosts database provided by the OS and
// processes both A/AAAA and PTR DNS requests for those.
//
// TODO(e.burkov): Improve API and move to golibs.
type HostsContainer struct {
// requestMatcher matches the requests and translates the rules. It's
// embedded to implement MatchRequest and Translate for *HostsContainer.
//
// TODO(a.garipov, e.burkov): Consider fully merging into HostsContainer.
requestMatcher
// done is the channel to sign closing the container.
done chan struct{}
// updates is the channel for receiving updated hosts.
updates chan HostsRecords
updates chan Hosts
// last is the set of hosts that was cached within last detected change.
last HostsRecords
// current is the last set of hosts parsed.
current atomic.Pointer[hostsIndex]
// fsys is the working file system to read hosts files from.
fsys fs.FS
@@ -127,30 +77,6 @@ type HostsContainer struct {
// patterns stores specified paths in the fs.Glob-compatible form.
patterns []string
// listID is the identifier for the list of generated rules.
listID int
}
// HostsRecords is a mapping of an IP address to its hosts data.
type HostsRecords map[netip.Addr]*HostsRecord
// HostsRecord represents a single hosts file record.
type HostsRecord struct {
Aliases *stringutil.Set
Canonical string
}
// Equal returns true if all fields of rec are equal to field in other or they
// both are nil.
func (rec *HostsRecord) Equal(other *HostsRecord) (ok bool) {
if rec == nil {
return other == nil
} else if other == nil {
return false
}
return rec.Canonical == other.Canonical && rec.Aliases.Equal(other.Aliases)
}
// ErrNoHostsPaths is returned when there are no valid paths to watch passed to
@@ -162,7 +88,6 @@ const ErrNoHostsPaths errors.Error = "no valid paths to hosts files provided"
// shouldn't be empty and each of paths should locate either a file or a
// directory in fsys. fsys and w must be non-nil.
func NewHostsContainer(
listID int,
fsys fs.FS,
w aghos.FSWatcher,
paths ...string,
@@ -182,12 +107,8 @@ func NewHostsContainer(
}
hc = &HostsContainer{
requestMatcher: requestMatcher{
stateLock: &sync.RWMutex{},
},
listID: listID,
done: make(chan struct{}, 1),
updates: make(chan HostsRecords, 1),
updates: make(chan Hosts, 1),
fsys: fsys,
watcher: w,
patterns: patterns,
@@ -233,7 +154,7 @@ func (hc *HostsContainer) Close() (err error) {
}
// Upd returns the channel into which the updates are sent.
func (hc *HostsContainer) Upd() (updates <-chan HostsRecords) {
func (hc *HostsContainer) Upd() (updates <-chan Hosts) {
return hc.updates
}
@@ -280,7 +201,7 @@ func (hc *HostsContainer) handleEvents() {
}
if err := hc.refresh(); err != nil {
log.Error("%s: %s", hostsContainerPrefix, err)
log.Error("%s: warning: refreshing: %s", hostsContainerPrefix, err)
}
case _, ok = <-hc.done:
// Go on.
@@ -288,198 +209,83 @@ func (hc *HostsContainer) handleEvents() {
}
}
// hostsParser is a helper type to parse rules from the operating system's hosts
// file. It exists for only a single refreshing session.
type hostsParser struct {
// rulesBuilder builds the resulting rules list content.
rulesBuilder *strings.Builder
// translations maps generated rules into actual hosts file lines.
translations map[string]string
// table stores only the unique IP-hostname pairs. It's also sent to the
// updates channel afterwards.
table HostsRecords
}
// newHostsParser creates a new *hostsParser with buffers of size taken from the
// previous parse.
func (hc *HostsContainer) newHostsParser() (hp *hostsParser) {
return &hostsParser{
rulesBuilder: &strings.Builder{},
translations: map[string]string{},
table: make(HostsRecords, len(hc.last)),
}
}
// parseFile is a aghos.FileWalker for parsing the files with hosts syntax. It
// never signs to stop walking and never returns any additional patterns.
//
// See man hosts(5).
func (hp *hostsParser) parseFile(r io.Reader) (patterns []string, cont bool, err error) {
s := bufio.NewScanner(r)
for s.Scan() {
ip, hosts := hp.parseLine(s.Text())
if ip == (netip.Addr{}) || len(hosts) == 0 {
continue
}
hp.addRecord(ip, hosts)
}
return nil, true, s.Err()
}
// parseLine parses the line having the hosts syntax ignoring invalid ones.
func (hp *hostsParser) parseLine(line string) (ip netip.Addr, hosts []string) {
fields := strings.Fields(line)
if len(fields) < 2 {
return netip.Addr{}, nil
}
ip, err := netip.ParseAddr(fields[0])
if err != nil {
return netip.Addr{}, nil
}
for _, f := range fields[1:] {
hashIdx := strings.IndexByte(f, '#')
if hashIdx == 0 {
// The rest of the fields are a part of the comment so return.
break
} else if hashIdx > 0 {
// Only a part of the field is a comment.
f = f[:hashIdx]
}
// Make sure that invalid hosts aren't turned into rules.
//
// See https://github.com/AdguardTeam/AdGuardHome/issues/3946.
//
// TODO(e.burkov): Investigate if hosts may contain DNS-SD domains.
err = netutil.ValidateHostname(f)
if err != nil {
log.Error("%s: host %q is invalid, ignoring", hostsContainerPrefix, f)
continue
}
hosts = append(hosts, f)
}
return ip, hosts
}
// addRecord puts the record for the IP address to the rules builder if needed.
// The first host is considered to be the canonical name for the IP address.
// hosts must have at least one name.
func (hp *hostsParser) addRecord(ip netip.Addr, hosts []string) {
line := strings.Join(append([]string{ip.String()}, hosts...), " ")
rec, ok := hp.table[ip]
if !ok {
rec = &HostsRecord{
Aliases: stringutil.NewSet(),
}
rec.Canonical, hosts = hosts[0], hosts[1:]
hp.addRules(ip, rec.Canonical, line)
hp.table[ip] = rec
}
for _, host := range hosts {
if rec.Canonical == host || rec.Aliases.Has(host) {
continue
}
rec.Aliases.Add(host)
hp.addRules(ip, host, line)
}
}
// addRules adds rules and rule translations for the line.
func (hp *hostsParser) addRules(ip netip.Addr, host, line string) {
rule, rulePtr := hp.writeRules(host, ip)
hp.translations[rule], hp.translations[rulePtr] = line, line
log.Debug("%s: added ip-host pair %q-%q", hostsContainerPrefix, ip, host)
}
// writeRules writes the actual rule for the qtype and the PTR for the host-ip
// pair into internal builders.
func (hp *hostsParser) writeRules(host string, ip netip.Addr) (rule, rulePtr string) {
// TODO(a.garipov): Add a netip.Addr version to netutil.
arpa, err := netutil.IPToReversedAddr(ip.AsSlice())
if err != nil {
return "", ""
}
const (
nl = "\n"
rwSuccess = "^$dnsrewrite=NOERROR;"
rwSuccessPTR = "^$dnsrewrite=NOERROR;PTR;"
modLen = len(rules.MaskPipe) + len(rwSuccess) + len(";")
modLenPTR = len(rules.MaskPipe) + len(rwSuccessPTR)
)
var qtype string
// The validation of the IP address has been performed earlier so it is
// guaranteed to be either an IPv4 or an IPv6.
if ip.Is4() {
qtype = "A"
} else {
qtype = "AAAA"
}
ipStr := ip.String()
fqdn := dns.Fqdn(host)
ruleBuilder := &strings.Builder{}
ruleBuilder.Grow(modLen + len(host) + len(qtype) + len(ipStr))
stringutil.WriteToBuilder(ruleBuilder, rules.MaskPipe, host, rwSuccess, qtype, ";", ipStr)
rule = ruleBuilder.String()
ruleBuilder.Reset()
ruleBuilder.Grow(modLenPTR + len(arpa) + len(fqdn))
stringutil.WriteToBuilder(ruleBuilder, rules.MaskPipe, arpa, rwSuccessPTR, fqdn)
rulePtr = ruleBuilder.String()
hp.rulesBuilder.Grow(len(rule) + len(rulePtr) + 2*len(nl))
stringutil.WriteToBuilder(hp.rulesBuilder, rule, nl, rulePtr, nl)
return rule, rulePtr
}
// sendUpd tries to send the parsed data to the ch.
func (hp *hostsParser) sendUpd(ch chan HostsRecords) {
func (hc *HostsContainer) sendUpd(recs Hosts) {
log.Debug("%s: sending upd", hostsContainerPrefix)
upd := hp.table
ch := hc.updates
select {
case ch <- upd:
case ch <- recs:
// Updates are delivered. Go on.
case <-ch:
ch <- upd
ch <- recs
log.Debug("%s: replaced the last update", hostsContainerPrefix)
case ch <- upd:
case ch <- recs:
// The previous update was just read and the next one pushed. Go on.
default:
log.Error("%s: the updates channel is broken", hostsContainerPrefix)
}
}
// newStrg creates a new rules storage from parsed data.
func (hp *hostsParser) newStrg(id int) (s *filterlist.RuleStorage, err error) {
return filterlist.NewRuleStorage([]filterlist.RuleList{&filterlist.StringRuleList{
ID: id,
RulesText: hp.rulesBuilder.String(),
IgnoreCosmetic: true,
}})
// hostsIndex is a [hostsfile.Set] to enumerate all the records.
type hostsIndex struct {
// addrs maps IP addresses to the records.
addrs Hosts
// names maps hostnames to the records.
names map[string][]*hostsfile.Record
}
// walk is a file walking function for hostsIndex.
func (idx *hostsIndex) walk(r io.Reader) (patterns []string, cont bool, err error) {
return nil, true, hostsfile.Parse(idx, r, nil)
}
// type check
var _ hostsfile.Set = (*hostsIndex)(nil)
// Add implements the [hostsfile.Set] interface for *hostsIndex.
func (idx *hostsIndex) Add(rec *hostsfile.Record) {
idx.addrs[rec.Addr] = append(idx.addrs[rec.Addr], rec)
for _, name := range rec.Names {
idx.names[name] = append(idx.names[name], rec)
}
}
// type check
var _ hostsfile.HandleSet = (*hostsIndex)(nil)
// HandleInvalid implements the [hostsfile.HandleSet] interface for *hostsIndex.
func (idx *hostsIndex) HandleInvalid(src string, _ []byte, err error) {
lineErr := &hostsfile.LineError{}
if !errors.As(err, &lineErr) {
// Must not happen if idx passed to [hostsfile.Parse].
return
} else if errors.Is(lineErr, hostsfile.ErrEmptyLine) {
// Ignore empty lines.
return
}
log.Info("%s: warning: parsing %q: %s", hostsContainerPrefix, src, lineErr)
}
// equalRecs is an equality function for [*hostsfile.Record].
func equalRecs(a, b *hostsfile.Record) (ok bool) {
return a.Addr == b.Addr && a.Source == b.Source && slices.Equal(a.Names, b.Names)
}
// equalRecSlices is an equality function for slices of [*hostsfile.Record].
func equalRecSlices(a, b []*hostsfile.Record) (ok bool) { return slices.EqualFunc(a, b, equalRecs) }
// Equal returns true if indexes are equal.
func (idx *hostsIndex) Equal(other *hostsIndex) (ok bool) {
if idx == nil {
return other == nil
} else if other == nil {
return false
}
return maps.EqualFunc(idx.addrs, other.addrs, equalRecSlices)
}
// refresh gets the data from specified files and propagates the updates if
@@ -489,27 +295,27 @@ func (hp *hostsParser) newStrg(id int) (s *filterlist.RuleStorage, err error) {
func (hc *HostsContainer) refresh() (err error) {
log.Debug("%s: refreshing", hostsContainerPrefix)
hp := hc.newHostsParser()
if _, err = aghos.FileWalker(hp.parseFile).Walk(hc.fsys, hc.patterns...); err != nil {
return fmt.Errorf("refreshing : %w", err)
var addrLen, nameLen int
last := hc.current.Load()
if last != nil {
addrLen, nameLen = len(last.addrs), len(last.names)
}
idx := &hostsIndex{
addrs: make(Hosts, addrLen),
names: make(map[string][]*hostsfile.Record, nameLen),
}
// hc.last is nil on the first refresh, so let that one through.
if hc.last != nil && maps.EqualFunc(hp.table, hc.last, (*HostsRecord).Equal) {
log.Debug("%s: no changes detected", hostsContainerPrefix)
return nil
}
defer hp.sendUpd(hc.updates)
hc.last = maps.Clone(hp.table)
var rulesStrg *filterlist.RuleStorage
if rulesStrg, err = hp.newStrg(hc.listID); err != nil {
return fmt.Errorf("initializing rules storage: %w", err)
_, err = aghos.FileWalker(idx.walk).Walk(hc.fsys, hc.patterns...)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return err
}
hc.resetEng(rulesStrg, hp.translations)
// TODO(e.burkov): Serialize updates using time.
if !last.Equal(idx) {
hc.current.Store(idx)
hc.sendUpd(idx.addrs)
}
return nil
}

View File

@@ -2,13 +2,11 @@ package aghnet
import (
"io/fs"
"net/netip"
"path"
"testing"
"testing/fstest"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/testutil/fakefs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -76,69 +74,3 @@ func TestHostsContainer_PathsToPatterns(t *testing.T) {
assert.ErrorIs(t, err, errStat)
})
}
func TestUniqueRules_ParseLine(t *testing.T) {
ip := netutil.IPv4Localhost()
ipStr := ip.String()
testCases := []struct {
name string
line string
wantIP netip.Addr
wantHosts []string
}{{
name: "simple",
line: ipStr + ` hostname`,
wantIP: ip,
wantHosts: []string{"hostname"},
}, {
name: "aliases",
line: ipStr + ` hostname alias`,
wantIP: ip,
wantHosts: []string{"hostname", "alias"},
}, {
name: "invalid_line",
line: ipStr,
wantIP: netip.Addr{},
wantHosts: nil,
}, {
name: "invalid_line_hostname",
line: ipStr + ` # hostname`,
wantIP: ip,
wantHosts: nil,
}, {
name: "commented_aliases",
line: ipStr + ` hostname # alias`,
wantIP: ip,
wantHosts: []string{"hostname"},
}, {
name: "whole_comment",
line: `# ` + ipStr + ` hostname`,
wantIP: netip.Addr{},
wantHosts: nil,
}, {
name: "partial_comment",
line: ipStr + ` host#name`,
wantIP: ip,
wantHosts: []string{"host"},
}, {
name: "empty",
line: ``,
wantIP: netip.Addr{},
wantHosts: nil,
}, {
name: "bad_hosts",
line: ipStr + ` bad..host bad._tld empty.tld. ok.host`,
wantIP: ip,
wantHosts: []string{"ok.host"},
}}
for _, tc := range testCases {
hp := hostsParser{}
t.Run(tc.name, func(t *testing.T) {
got, hosts := hp.parseLine(tc.line)
assert.Equal(t, tc.wantIP, got)
assert.Equal(t, tc.wantHosts, hosts)
})
}
}

View File

@@ -1,9 +1,9 @@
package aghnet_test
import (
"net"
"net/netip"
"path"
"strings"
"path/filepath"
"sync/atomic"
"testing"
"testing/fstest"
@@ -13,18 +13,146 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/hostsfile"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/AdguardTeam/golibs/testutil"
"github.com/AdguardTeam/urlfilter"
"github.com/AdguardTeam/urlfilter/rules"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// nl is a newline character.
const nl = "\n"
// Variables mirroring the etc_hosts file from testdata.
var (
addr1000 = netip.MustParseAddr("1.0.0.0")
addr1001 = netip.MustParseAddr("1.0.0.1")
addr1002 = netip.MustParseAddr("1.0.0.2")
addr1003 = netip.MustParseAddr("1.0.0.3")
addr1004 = netip.MustParseAddr("1.0.0.4")
addr1357 = netip.MustParseAddr("1.3.5.7")
addr4216 = netip.MustParseAddr("4.2.1.6")
addr7531 = netip.MustParseAddr("7.5.3.1")
addr0 = netip.MustParseAddr("::")
addr1 = netip.MustParseAddr("::1")
addr2 = netip.MustParseAddr("::2")
addr3 = netip.MustParseAddr("::3")
addr4 = netip.MustParseAddr("::4")
addr42 = netip.MustParseAddr("::42")
addr13 = netip.MustParseAddr("::13")
addr31 = netip.MustParseAddr("::31")
hostsSrc = "./" + filepath.Join("./testdata", "etc_hosts")
testHosts = map[netip.Addr][]*hostsfile.Record{
addr1000: {{
Addr: addr1000,
Source: hostsSrc,
Names: []string{"hello", "hello.world"},
}, {
Addr: addr1000,
Source: hostsSrc,
Names: []string{"hello.world.again"},
}, {
Addr: addr1000,
Source: hostsSrc,
Names: []string{"hello.world"},
}},
addr1001: {{
Addr: addr1001,
Source: hostsSrc,
Names: []string{"simplehost"},
}, {
Addr: addr1001,
Source: hostsSrc,
Names: []string{"simplehost"},
}},
addr1002: {{
Addr: addr1002,
Source: hostsSrc,
Names: []string{"a.whole", "lot.of", "aliases", "for.testing"},
}},
addr1003: {{
Addr: addr1003,
Source: hostsSrc,
Names: []string{"*"},
}},
addr1004: {{
Addr: addr1004,
Source: hostsSrc,
Names: []string{"*.com"},
}},
addr1357: {{
Addr: addr1357,
Source: hostsSrc,
Names: []string{"domain4", "domain4.alias"},
}},
addr7531: {{
Addr: addr7531,
Source: hostsSrc,
Names: []string{"domain4.alias", "domain4"},
}},
addr4216: {{
Addr: addr4216,
Source: hostsSrc,
Names: []string{"domain", "domain.alias"},
}},
addr0: {{
Addr: addr0,
Source: hostsSrc,
Names: []string{"hello", "hello.world"},
}, {
Addr: addr0,
Source: hostsSrc,
Names: []string{"hello.world.again"},
}, {
Addr: addr0,
Source: hostsSrc,
Names: []string{"hello.world"},
}},
addr1: {{
Addr: addr1,
Source: hostsSrc,
Names: []string{"simplehost"},
}, {
Addr: addr1,
Source: hostsSrc,
Names: []string{"simplehost"},
}},
addr2: {{
Addr: addr2,
Source: hostsSrc,
Names: []string{"a.whole", "lot.of", "aliases", "for.testing"},
}},
addr3: {{
Addr: addr3,
Source: hostsSrc,
Names: []string{"*"},
}},
addr4: {{
Addr: addr4,
Source: hostsSrc,
Names: []string{"*.com"},
}},
addr42: {{
Addr: addr42,
Source: hostsSrc,
Names: []string{"domain.alias", "domain"},
}},
addr13: {{
Addr: addr13,
Source: hostsSrc,
Names: []string{"domain6", "domain6.alias"},
}},
addr31: {{
Addr: addr31,
Source: hostsSrc,
Names: []string{"domain6.alias", "domain6"},
}},
}
)
func TestNewHostsContainer(t *testing.T) {
const dirname = "dir"
const filename = "file1"
@@ -73,7 +201,7 @@ func TestNewHostsContainer(t *testing.T) {
return eventsCh
}
hc, err := aghnet.NewHostsContainer(0, testFS, &aghtest.FSWatcher{
hc, err := aghnet.NewHostsContainer(testFS, &aghtest.FSWatcher{
OnEvents: onEvents,
OnAdd: onAdd,
OnClose: func() (err error) { return nil },
@@ -99,7 +227,7 @@ func TestNewHostsContainer(t *testing.T) {
t.Run("nil_fs", func(t *testing.T) {
require.Panics(t, func() {
_, _ = aghnet.NewHostsContainer(0, nil, &aghtest.FSWatcher{
_, _ = aghnet.NewHostsContainer(nil, &aghtest.FSWatcher{
// Those shouldn't panic.
OnEvents: func() (e <-chan struct{}) { return nil },
OnAdd: func(name string) (err error) { return nil },
@@ -110,7 +238,7 @@ func TestNewHostsContainer(t *testing.T) {
t.Run("nil_watcher", func(t *testing.T) {
require.Panics(t, func() {
_, _ = aghnet.NewHostsContainer(0, testFS, nil, p)
_, _ = aghnet.NewHostsContainer(testFS, nil, p)
})
})
@@ -123,7 +251,7 @@ func TestNewHostsContainer(t *testing.T) {
OnClose: func() (err error) { return nil },
}
hc, err := aghnet.NewHostsContainer(0, testFS, errWatcher, p)
hc, err := aghnet.NewHostsContainer(testFS, errWatcher, p)
require.ErrorIs(t, err, errOnAdd)
assert.Nil(t, hc)
@@ -136,6 +264,9 @@ func TestHostsContainer_refresh(t *testing.T) {
ip := netutil.IPv4Localhost()
ipStr := ip.String()
anotherIPStr := "1.2.3.4"
anotherIP := netip.MustParseAddr(anotherIPStr)
testFS := fstest.MapFS{"dir/file1": &fstest.MapFile{Data: []byte(ipStr + ` hostname` + nl)}}
// event is a convenient alias for an empty struct{} to emit test events.
@@ -154,40 +285,44 @@ func TestHostsContainer_refresh(t *testing.T) {
OnClose: func() (err error) { return nil },
}
hc, err := aghnet.NewHostsContainer(0, testFS, w, "dir")
hc, err := aghnet.NewHostsContainer(testFS, w, "dir")
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, hc.Close)
checkRefresh := func(t *testing.T, want *aghnet.HostsRecord) {
checkRefresh := func(t *testing.T, want aghnet.Hosts) {
t.Helper()
upd, ok := aghchan.MustReceive(hc.Upd(), 1*time.Second)
require.True(t, ok)
require.NotNil(t, upd)
assert.Len(t, upd, 1)
rec, ok := upd[ip]
require.True(t, ok)
require.NotNil(t, rec)
assert.Truef(t, rec.Equal(want), "%+v != %+v", rec, want)
assert.Equal(t, want, upd)
}
t.Run("initial_refresh", func(t *testing.T) {
checkRefresh(t, &aghnet.HostsRecord{
Aliases: stringutil.NewSet(),
Canonical: "hostname",
checkRefresh(t, aghnet.Hosts{
ip: {{
Addr: ip,
Source: "file1",
Names: []string{"hostname"},
}},
})
})
t.Run("second_refresh", func(t *testing.T) {
testFS["dir/file2"] = &fstest.MapFile{Data: []byte(ipStr + ` alias` + nl)}
testFS["dir/file2"] = &fstest.MapFile{Data: []byte(anotherIPStr + ` alias` + nl)}
eventsCh <- event{}
checkRefresh(t, &aghnet.HostsRecord{
Aliases: stringutil.NewSet("alias"),
Canonical: "hostname",
checkRefresh(t, aghnet.Hosts{
ip: {{
Addr: ip,
Source: "file1",
Names: []string{"hostname"},
}},
anotherIP: {{
Addr: anotherIP,
Source: "file2",
Names: []string{"alias"},
}},
})
})
@@ -198,12 +333,9 @@ func TestHostsContainer_refresh(t *testing.T) {
// Require the changes are written.
require.Eventually(t, func() bool {
res, ok := hc.MatchRequest(&urlfilter.DNSRequest{
Hostname: "hostname",
DNSType: dns.TypeA,
})
ips := hc.MatchName("hostname")
return !ok && res.DNSRewrites() == nil
return len(ips) == 0
}, 5*time.Second, time.Second/2)
// Make a change again.
@@ -212,285 +344,117 @@ func TestHostsContainer_refresh(t *testing.T) {
// Require the changes are written.
require.Eventually(t, func() bool {
res, ok := hc.MatchRequest(&urlfilter.DNSRequest{
Hostname: "hostname",
DNSType: dns.TypeA,
})
ips := hc.MatchName("hostname")
return !ok && res.DNSRewrites() != nil
return len(ips) > 0
}, 5*time.Second, time.Second/2)
assert.Len(t, hc.Upd(), 1)
})
}
func TestHostsContainer_Translate(t *testing.T) {
func TestHostsContainer_MatchName(t *testing.T) {
require.NoError(t, fstest.TestFS(testdata, "etc_hosts"))
stubWatcher := aghtest.FSWatcher{
OnEvents: func() (e <-chan struct{}) { return nil },
OnAdd: func(name string) (err error) { return nil },
OnClose: func() (err error) { return nil },
}
require.NoError(t, fstest.TestFS(testdata, "etc_hosts"))
hc, err := aghnet.NewHostsContainer(0, testdata, &stubWatcher, "etc_hosts")
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, hc.Close)
testCases := []struct {
name string
rule string
wantTrans []string
}{{
name: "simplehost",
rule: "|simplehost^$dnsrewrite=NOERROR;A;1.0.0.1",
wantTrans: []string{"1.0.0.1", "simplehost"},
}, {
name: "hello",
rule: "|hello^$dnsrewrite=NOERROR;A;1.0.0.0",
wantTrans: []string{"1.0.0.0", "hello", "hello.world"},
}, {
name: "hello-alias",
rule: "|hello.world.again^$dnsrewrite=NOERROR;A;1.0.0.0",
wantTrans: []string{"1.0.0.0", "hello.world.again"},
}, {
name: "simplehost_v6",
rule: "|simplehost^$dnsrewrite=NOERROR;AAAA;::1",
wantTrans: []string{"::1", "simplehost"},
}, {
name: "hello_v6",
rule: "|hello^$dnsrewrite=NOERROR;AAAA;::",
wantTrans: []string{"::", "hello", "hello.world"},
}, {
name: "hello_v6-alias",
rule: "|hello.world.again^$dnsrewrite=NOERROR;AAAA;::",
wantTrans: []string{"::", "hello.world.again"},
}, {
name: "simplehost_ptr",
rule: "|1.0.0.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;simplehost.",
wantTrans: []string{"1.0.0.1", "simplehost"},
}, {
name: "hello_ptr",
rule: "|0.0.0.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;hello.",
wantTrans: []string{"1.0.0.0", "hello", "hello.world"},
}, {
name: "hello_ptr-alias",
rule: "|0.0.0.1.in-addr.arpa^$dnsrewrite=NOERROR;PTR;hello.world.again.",
wantTrans: []string{"1.0.0.0", "hello.world.again"},
}, {
name: "simplehost_ptr_v6",
rule: "|1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa" +
"^$dnsrewrite=NOERROR;PTR;simplehost.",
wantTrans: []string{"::1", "simplehost"},
}, {
name: "hello_ptr_v6",
rule: "|0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa" +
"^$dnsrewrite=NOERROR;PTR;hello.",
wantTrans: []string{"::", "hello", "hello.world"},
}, {
name: "hello_ptr_v6-alias",
rule: "|0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa" +
"^$dnsrewrite=NOERROR;PTR;hello.world.again.",
wantTrans: []string{"::", "hello.world.again"},
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
got := stringutil.NewSet(strings.Fields(hc.Translate(tc.rule))...)
assert.True(t, stringutil.NewSet(tc.wantTrans...).Equal(got))
})
}
}
func TestHostsContainer(t *testing.T) {
const listID = 1234
require.NoError(t, fstest.TestFS(testdata, "etc_hosts"))
testCases := []struct {
req *urlfilter.DNSRequest
req string
name string
want []*rules.DNSRewrite
want []*hostsfile.Record
}{{
req: &urlfilter.DNSRequest{
Hostname: "simplehost",
DNSType: dns.TypeA,
},
req: "simplehost",
name: "simple",
want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess,
Value: net.IPv4(1, 0, 0, 1),
RRType: dns.TypeA,
}, {
RCode: dns.RcodeSuccess,
Value: net.ParseIP("::1"),
RRType: dns.TypeAAAA,
}},
want: append(testHosts[addr1001], testHosts[addr1]...),
}, {
req: &urlfilter.DNSRequest{
Hostname: "hello.world",
DNSType: dns.TypeA,
},
req: "hello.world",
name: "hello_alias",
want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess,
Value: net.IPv4(1, 0, 0, 0),
RRType: dns.TypeA,
}, {
RCode: dns.RcodeSuccess,
Value: net.ParseIP("::"),
RRType: dns.TypeAAAA,
}},
}, {
req: &urlfilter.DNSRequest{
Hostname: "hello.world.again",
DNSType: dns.TypeA,
want: []*hostsfile.Record{
testHosts[addr1000][0],
testHosts[addr1000][2],
testHosts[addr0][0],
testHosts[addr0][2],
},
}, {
req: "hello.world.again",
name: "other_line_alias",
want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess,
Value: net.IPv4(1, 0, 0, 0),
RRType: dns.TypeA,
}, {
RCode: dns.RcodeSuccess,
Value: net.ParseIP("::"),
RRType: dns.TypeAAAA,
}},
}, {
req: &urlfilter.DNSRequest{
Hostname: "say.hello",
DNSType: dns.TypeA,
want: []*hostsfile.Record{
testHosts[addr1000][1],
testHosts[addr0][1],
},
}, {
req: "say.hello",
name: "hello_subdomain",
want: []*rules.DNSRewrite{},
}, {
req: &urlfilter.DNSRequest{
Hostname: "say.hello.world",
DNSType: dns.TypeA,
},
name: "hello_alias_subdomain",
want: []*rules.DNSRewrite{},
}, {
req: &urlfilter.DNSRequest{
Hostname: "for.testing",
DNSType: dns.TypeA,
},
name: "lots_of_aliases",
want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess,
RRType: dns.TypeA,
Value: net.IPv4(1, 0, 0, 2),
}, {
RCode: dns.RcodeSuccess,
RRType: dns.TypeAAAA,
Value: net.ParseIP("::2"),
}},
}, {
req: &urlfilter.DNSRequest{
Hostname: "1.0.0.1.in-addr.arpa",
DNSType: dns.TypePTR,
},
name: "reverse",
want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess,
RRType: dns.TypePTR,
Value: "simplehost.",
}},
}, {
req: &urlfilter.DNSRequest{
Hostname: "nonexistent.example",
DNSType: dns.TypeA,
},
name: "non-existing",
want: []*rules.DNSRewrite{},
}, {
req: &urlfilter.DNSRequest{
Hostname: "1.0.0.1.in-addr.arpa",
DNSType: dns.TypeSRV,
},
name: "bad_type",
want: nil,
}, {
req: &urlfilter.DNSRequest{
Hostname: "domain",
DNSType: dns.TypeA,
},
req: "say.hello.world",
name: "hello_alias_subdomain",
want: nil,
}, {
req: "for.testing",
name: "lots_of_aliases",
want: append(testHosts[addr1002], testHosts[addr2]...),
}, {
req: "nonexistent.example",
name: "non-existing",
want: nil,
}, {
req: "domain",
name: "issue_4216_4_6",
want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess,
RRType: dns.TypeA,
Value: net.IPv4(4, 2, 1, 6),
}, {
RCode: dns.RcodeSuccess,
RRType: dns.TypeAAAA,
Value: net.ParseIP("::42"),
}},
want: append(testHosts[addr4216], testHosts[addr42]...),
}, {
req: &urlfilter.DNSRequest{
Hostname: "domain4",
DNSType: dns.TypeA,
},
req: "domain4",
name: "issue_4216_4",
want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess,
RRType: dns.TypeA,
Value: net.IPv4(7, 5, 3, 1),
}, {
RCode: dns.RcodeSuccess,
RRType: dns.TypeA,
Value: net.IPv4(1, 3, 5, 7),
}},
want: append(testHosts[addr1357], testHosts[addr7531]...),
}, {
req: &urlfilter.DNSRequest{
Hostname: "domain6",
DNSType: dns.TypeAAAA,
},
req: "domain6",
name: "issue_4216_6",
want: []*rules.DNSRewrite{{
RCode: dns.RcodeSuccess,
RRType: dns.TypeAAAA,
Value: net.ParseIP("::13"),
}, {
RCode: dns.RcodeSuccess,
RRType: dns.TypeAAAA,
Value: net.ParseIP("::31"),
}},
want: append(testHosts[addr13], testHosts[addr31]...),
}}
hc, err := aghnet.NewHostsContainer(testdata, &stubWatcher, "etc_hosts")
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, hc.Close)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
recs := hc.MatchName(tc.req)
assert.Equal(t, tc.want, recs)
})
}
}
func TestHostsContainer_MatchAddr(t *testing.T) {
require.NoError(t, fstest.TestFS(testdata, "etc_hosts"))
stubWatcher := aghtest.FSWatcher{
OnEvents: func() (e <-chan struct{}) { return nil },
OnAdd: func(name string) (err error) { return nil },
OnClose: func() (err error) { return nil },
}
hc, err := aghnet.NewHostsContainer(listID, testdata, &stubWatcher, "etc_hosts")
hc, err := aghnet.NewHostsContainer(testdata, &stubWatcher, "etc_hosts")
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, hc.Close)
testCases := []struct {
req netip.Addr
name string
want []*hostsfile.Record
}{{
req: netip.AddrFrom4([4]byte{1, 0, 0, 1}),
name: "reverse",
want: testHosts[addr1001],
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
res, ok := hc.MatchRequest(tc.req)
require.False(t, ok)
if tc.want == nil {
assert.Nil(t, res)
return
}
require.NotNil(t, res)
rewrites := res.DNSRewrites()
require.Len(t, rewrites, len(tc.want))
for i, rewrite := range rewrites {
require.Equal(t, listID, rewrite.FilterListID)
rw := rewrite.DNSRewrite
require.NotNil(t, rw)
assert.Equal(t, tc.want[i], rw)
}
recs := hc.MatchAddr(tc.req)
assert.Equal(t, tc.want, recs)
})
}
}

56
internal/aghnet/ignore.go Normal file
View File

@@ -0,0 +1,56 @@
package aghnet
import (
"strings"
"github.com/AdguardTeam/urlfilter"
"github.com/AdguardTeam/urlfilter/filterlist"
"golang.org/x/exp/slices"
)
// IgnoreEngine contains the list of rules for ignoring hostnames and matches
// them.
//
// TODO(s.chzhen): Move all urlfilter stuff to aghfilter.
type IgnoreEngine struct {
// engine is the filtering engine that can match rules for ignoring
// hostnames.
engine *urlfilter.DNSEngine
// ignored is the list of rules for ignoring hostnames.
ignored []string
}
// NewIgnoreEngine creates a new instance of the IgnoreEngine and stores the
// list of rules for ignoring hostnames.
func NewIgnoreEngine(ignored []string) (e *IgnoreEngine, err error) {
ruleList := &filterlist.StringRuleList{
RulesText: strings.ToLower(strings.Join(ignored, "\n")),
IgnoreCosmetic: true,
}
ruleStorage, err := filterlist.NewRuleStorage([]filterlist.RuleList{ruleList})
if err != nil {
return nil, err
}
return &IgnoreEngine{
engine: urlfilter.NewDNSEngine(ruleStorage),
ignored: ignored,
}, nil
}
// Has returns true if IgnoreEngine matches the host.
func (e *IgnoreEngine) Has(host string) (ignore bool) {
if e == nil {
return false
}
_, ignore = e.engine.Match(host)
return ignore
}
// Values returns a copy of list of rules for ignoring hostnames.
func (e *IgnoreEngine) Values() (ignored []string) {
return slices.Clone(e.ignored)
}

View File

@@ -0,0 +1,46 @@
package aghnet_test
import (
"testing"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/stretchr/testify/require"
)
func TestIgnoreEngine_Has(t *testing.T) {
hostnames := []string{
"*.example.com",
"example.com",
"|.^",
}
engine, err := aghnet.NewIgnoreEngine(hostnames)
require.NotNil(t, engine)
require.NoError(t, err)
testCases := []struct {
name string
host string
ignore bool
}{{
name: "basic",
host: "example.com",
ignore: true,
}, {
name: "root",
host: ".",
ignore: true,
}, {
name: "wildcard",
host: "www.example.com",
ignore: true,
}, {
name: "not_ignored",
host: "something.com",
ignore: false,
}}
for _, tc := range testCases {
require.Equal(t, tc.ignore, engine.Has(tc.host))
}
}

View File

@@ -4,7 +4,6 @@ package aghnet
import (
"context"
"fmt"
"net"
"os"
"syscall"
@@ -24,20 +23,9 @@ func reuseAddrCtrl(_, _ string, c syscall.RawConn) (err error) {
}
})
const (
errMsg = "setting control options"
errMsgFmt = errMsg + ": %w"
)
err = errors.Join(err, cerr)
if err != nil && cerr != nil {
err = errors.List(errMsg, err, cerr)
} else if err != nil {
err = fmt.Errorf(errMsgFmt, err)
} else if cerr != nil {
err = fmt.Errorf(errMsgFmt, cerr)
}
return err
return errors.Annotate(err, "setting control options: %w")
}
// listenPacketReusable announces on the local network address additionally

View File

@@ -390,9 +390,5 @@ func (m *ipsetMgr) Close() (err error) {
errs = append(errs, err)
}
if len(errs) != 0 {
return errors.List("closing ipsets", errs...)
}
return nil
return errors.Annotate(errors.Join(errs...), "closing ipsets: %w")
}

View File

@@ -7,7 +7,6 @@ import (
"io/fs"
"net"
"net/netip"
"os"
"strings"
"testing"
@@ -18,9 +17,6 @@ import (
"github.com/stretchr/testify/require"
)
// testdata is the filesystem containing data for testing the package.
var testdata fs.FS = os.DirFS("./testdata")
// substRootDirFS replaces the aghos.RootDirFS function used throughout the
// package with fsys for tests ran under t.
func substRootDirFS(t testing.TB, fsys fs.FS) {

View File

@@ -1,6 +0,0 @@
IP address HW type Flags HW address Mask Device
192.168.1.2 0x1 0x2 ab:cd:ef:ab:cd:ef * wan
::ffff:ffff 0x1 0x0 ef:cd:ab:ef:cd:ab * br-lan
0.0.0.0 0x0 0x0 00:00:00:00:00:00 * unspec
1.2.3.4.5 0x1 0x2 aa:bb:cc:dd:ee:ff * wan
1.2.3.4 0x1 0x2 12:34:56:78:910 * wan