daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

all: sync with master; upd chlog

Browse Source
This commit is contained in:
Ainar Garipov
2023-06-07 20:04:01 +03:00
parent 7030c7c24c
commit c65700923a
76 changed files with 2998 additions and 1909 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/aghhttp/aghhttp.go
View File

@@ -72,8 +72,8 @@ func WriteJSONResponse(w http.ResponseWriter, r *http.Request, resp any) (err er
// WriteJSONResponseCode is like [WriteJSONResponse] but adds the ability to
// redefine the status code.
func WriteJSONResponseCode(w http.ResponseWriter, r *http.Request, code int, resp any) (err error) {
w.WriteHeader(code)
w.Header().Set(httphdr.ContentType, HdrValApplicationJSON)
w.WriteHeader(code)
err = json.NewEncoder(w).Encode(resp)
if err != nil {
Error(r, w, http.StatusInternalServerError, "encoding resp: %s", err)

2
internal/aghnet/dhcp_unix.go
View File

@@ -304,7 +304,7 @@ func tryConn6(req *dhcpv6.Message, c net.PacketConn) (ok, next bool, err error)
if !(response.Type() == dhcpv6.MessageTypeAdvertise &&
msg.TransactionID == req.TransactionID &&
rcid != nil &&
cid.Equal(*rcid)) {
cid.Equal(rcid)) {
log.Debug("dhcpv6: received message from server doesn't match our request")

6
internal/aghos/service.go Normal file
View File

@@ -0,0 +1,6 @@
package aghos
// PreCheckActionStart performs the service start action pre-check.
func PreCheckActionStart() (err error) {
return preCheckActionStart()
}

32
internal/aghos/service_darwin.go Normal file
View File

@@ -0,0 +1,32 @@
//go:build darwin
package aghos
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/AdguardTeam/golibs/log"
)
// preCheckActionStart performs the service start action pre-check. It warns
// user that the service should be installed into Applications directory.
func preCheckActionStart() (err error) {
exe, err := os.Executable()
if err != nil {
return fmt.Errorf("getting executable path: %v", err)
}
exe, err = filepath.EvalSymlinks(exe)
if err != nil {
return fmt.Errorf("evaluating executable symlinks: %v", err)
}
if !strings.HasPrefix(exe, "/Applications/") {
log.Info("warning: service must be started from within the /Applications directory")
}
return err
}

8
internal/aghos/service_others.go Normal file
View File

@@ -0,0 +1,8 @@
//go:build !darwin
package aghos
// preCheckActionStart performs the service start action pre-check.
func preCheckActionStart() (err error) {
return nil
}

86
internal/dhcpd/README.md
View File

@@ -1,46 +1,60 @@
# DHCP server
# Testing DHCP Server
Contents:
* [Test setup with Virtual Box](#vbox)
* [Test setup with Virtual Box](#vbox)
* [Quick test with DHCPTest](#dhcptest)
<a id="vbox"></a>
## Test setup with Virtual Box
## <a href="#vbox" id="vbox" name="vbox">Test setup with Virtual Box</a>
To set up a test environment for DHCP server you need:
### Prerequisites
* Linux host machine
* Virtual Box
* Virtual machine (guest OS doesn't matter)
To set up a test environment for DHCP server you will need:
### Configure client
* Linux AG Home host machine (Virtual).
* Virtual Box.
* Virtual machine (guest OS doesn't matter).
1. Install Virtual Box and run the following command to create a Host-Only network:
### Configure Virtual Box
$ VBoxManage hostonlyif create
1. Install Virtual Box and run the following command to create a Host-Only
network:
You can check its status by `ip a` command.
```sh
$ VBoxManage hostonlyif create
```
You can check its status by `ip a` command.
You can also set up Host-Only network using Virtual Box menu:
You can also set up Host-Only network using Virtual Box menu:
```
File -> Host Network Manager...
```
File -> Host Network Manager...
2. Create your virtual machine and set up its network:
2. Create your virtual machine and set up its network:
```
VM Settings -> Network -> Host-only Adapter
```
VM Settings -> Network -> Host-only Adapter
3. Start your VM, install an OS. Configure your network interface to use
DHCP and the OS should ask for a IP address from our DHCP server.
3. Start your VM, install an OS. Configure your network interface to use DHCP and the OS should ask for a IP address from our DHCP server.
4. To see the current IP addresses on client OS you can use `ip a` command on
Linux or `ipconfig` on Windows.
4. To see the current IP address on client OS you can use `ip a` command on Linux or `ipconfig` on Windows.
5. To force the client OS to request an IP from DHCP server again, you can
use `dhclient` on Linux or `ipconfig /release` on Windows.
5. To force the client OS to request an IP from DHCP server again, you can use `dhclient` on Linux or `ipconfig /release` on Windows.
### Configure server
### Configure server
1. Edit server configuration file `AdGuardHome.yaml`, for example:
1. Edit server configuration file 'AdGuardHome.yaml', for example:
dhcp:
```yaml
dhcp:
enabled: true
interface_name: vboxnet0
local_domain_name: lan
dhcpv4:
gateway_ip: 192.168.56.1
subnet_mask: 255.255.255.0
@@ -54,11 +68,29 @@ To set up a test environment for DHCP server you need:
lease_duration: 86400
ra_slaac_only: false
ra_allow_slaac: false
```
2. Start the server
2. Start the server
./AdGuardHome
```sh
./AdGuardHome -v
```
There should be a message in log which shows that DHCP server is ready:
There should be a message in log which shows that DHCP server is ready:
[info] DHCP: listening on 0.0.0.0:67
```
[info] DHCP: listening on 0.0.0.0:67
```
## <a href="#dhcptest" id="dhcptest" name="dhcptest">Quick test with DHCPTest utility</a>
### Prerequisites
* [DHCP test utility][dhcptest-gh].
### Quick test
The DHCP server could be tested for DISCOVER-OFFER packets with in
interactive mode.
[dhcptest-gh]: https://github.com/CyberShadow/dhcptest

2
internal/dhcpd/conn_darwin.go → internal/dhcpd/conn_bsd.go
View File

@@ -1,4 +1,4 @@
//go:build darwin
//go:build darwin || freebsd || openbsd
package dhcpd

2
internal/dhcpd/conn_darwin_internal_test.go → internal/dhcpd/conn_bsd_internal_test.go
View File

@@ -1,4 +1,4 @@
//go:build darwin
//go:build darwin || freebsd || openbsd
package dhcpd

2
internal/dhcpd/conn_unix.go → internal/dhcpd/conn_linux.go
View File

@@ -1,4 +1,4 @@
//go:build freebsd || linux || openbsd
//go:build linux
package dhcpd

2
internal/dhcpd/conn_unix_internal_test.go → internal/dhcpd/conn_linux_internal_test.go
View File

@@ -1,4 +1,4 @@
//go:build freebsd || linux || openbsd
//go:build linux
package dhcpd

61
internal/dhcpd/dhcpd.go
View File

@@ -239,36 +239,16 @@ func Create(conf *ServerConfig) (s *server, err error) {
// [aghhttp.RegisterFunc].
s.registerHandlers()
v4conf := conf.Conf4
v4conf.InterfaceName = s.conf.InterfaceName
v4conf.notify = s.onNotify
v4conf.Enabled = s.conf.Enabled && v4conf.RangeStart.IsValid()
s.srv4, err = v4Create(&v4conf)
v4Enabled, v6Enabled, err := s.setServers(conf)
if err != nil {
if v4conf.Enabled {
return nil, fmt.Errorf("creating dhcpv4 srv: %w", err)
}
log.Debug("dhcpd: warning: creating dhcpv4 srv: %s", err)
}
v6conf := conf.Conf6
v6conf.Enabled = s.conf.Enabled
if len(v6conf.RangeStart) == 0 {
v6conf.Enabled = false
}
v6conf.InterfaceName = s.conf.InterfaceName
v6conf.notify = s.onNotify
s.srv6, err = v6Create(v6conf)
if err != nil {
return nil, fmt.Errorf("creating dhcpv6 srv: %w", err)
// Don't wrap the error, because it's informative enough as is.
return nil, err
}
s.conf.Conf4 = conf.Conf4
s.conf.Conf6 = conf.Conf6
if s.conf.Enabled && !v4conf.Enabled && !v6conf.Enabled {
if s.conf.Enabled && !v4Enabled && !v6Enabled {
return nil, fmt.Errorf("neither dhcpv4 nor dhcpv6 srv is configured")
}
@@ -289,6 +269,39 @@ func Create(conf *ServerConfig) (s *server, err error) {
return s, nil
}
// setServers updates DHCPv4 and DHCPv6 servers created from the provided
// configuration conf.
func (s *server) setServers(conf *ServerConfig) (v4Enabled, v6Enabled bool, err error) {
v4conf := conf.Conf4
v4conf.InterfaceName = s.conf.InterfaceName
v4conf.notify = s.onNotify
v4conf.Enabled = s.conf.Enabled && v4conf.RangeStart.IsValid()
s.srv4, err = v4Create(&v4conf)
if err != nil {
if v4conf.Enabled {
return true, false, fmt.Errorf("creating dhcpv4 srv: %w", err)
}
log.Debug("dhcpd: warning: creating dhcpv4 srv: %s", err)
}
v6conf := conf.Conf6
v6conf.InterfaceName = s.conf.InterfaceName
v6conf.notify = s.onNotify
v6conf.Enabled = s.conf.Enabled
if len(v6conf.RangeStart) == 0 {
v6conf.Enabled = false
}
s.srv6, err = v6Create(v6conf)
if err != nil {
return v4conf.Enabled, v6conf.Enabled, fmt.Errorf("creating dhcpv6 srv: %w", err)
}
return v4conf.Enabled, v6conf.Enabled, nil
}
// Enabled returns true when the server is enabled.
func (s *server) Enabled() (ok bool) {
return s.conf.Enabled

172
internal/dhcpd/http_unix.go
View File

@@ -16,6 +16,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/netutil"
)
type v4ServerConfJSON struct {
@@ -263,6 +264,28 @@ func (s *server) handleDHCPSetConfigV6(
return srv6, enabled, err
}
// createServers returns DHCPv4 and DHCPv6 servers created from the provided
// configuration conf.
func (s *server) createServers(conf *dhcpServerConfigJSON) (srv4, srv6 DHCPServer, err error) {
srv4, v4Enabled, err := s.handleDHCPSetConfigV4(conf)
if err != nil {
return nil, nil, fmt.Errorf("bad dhcpv4 configuration: %s", err)
}
srv6, v6Enabled, err := s.handleDHCPSetConfigV6(conf)
if err != nil {
return nil, nil, fmt.Errorf("bad dhcpv6 configuration: %s", err)
}
if conf.Enabled == aghalg.NBTrue && !v4Enabled && !v6Enabled {
return nil, nil, fmt.Errorf("dhcpv4 or dhcpv6 configuration must be complete")
}
return srv4, srv6, nil
}
// handleDHCPSetConfig is the handler for the POST /control/dhcp/set_config
// HTTP API.
func (s *server) handleDHCPSetConfig(w http.ResponseWriter, r *http.Request) {
conf := &dhcpServerConfigJSON{}
conf.Enabled = aghalg.BoolToNullBool(s.conf.Enabled)
@@ -275,22 +298,9 @@ func (s *server) handleDHCPSetConfig(w http.ResponseWriter, r *http.Request) {
return
}
srv4, v4Enabled, err := s.handleDHCPSetConfigV4(conf)
srv4, srv6, err := s.createServers(conf)
if err != nil {
aghhttp.Error(r, w, http.StatusBadRequest, "bad dhcpv4 configuration: %s", err)
return
}
srv6, v6Enabled, err := s.handleDHCPSetConfigV6(conf)
if err != nil {
aghhttp.Error(r, w, http.StatusBadRequest, "bad dhcpv6 configuration: %s", err)
return
}
if conf.Enabled == aghalg.NBTrue && !v4Enabled && !v6Enabled {
aghhttp.Error(r, w, http.StatusBadRequest, "dhcpv4 or dhcpv6 configuration must be complete")
aghhttp.Error(r, w, http.StatusBadRequest, "%s", err)
return
}
@@ -350,8 +360,10 @@ type netInterfaceJSON struct {
Addrs6 []netip.Addr `json:"ipv6_addresses"`
}
// handleDHCPInterfaces is the handler for the GET /control/dhcp/interfaces
// HTTP API.
func (s *server) handleDHCPInterfaces(w http.ResponseWriter, r *http.Request) {
response := map[string]netInterfaceJSON{}
resp := map[string]*netInterfaceJSON{}
ifaces, err := net.Interfaces()
if err != nil {
@@ -362,82 +374,86 @@ func (s *server) handleDHCPInterfaces(w http.ResponseWriter, r *http.Request) {
for _, iface := range ifaces {
if iface.Flags&net.FlagLoopback != 0 {
// it's a loopback, skip it
continue
}
if iface.Flags&net.FlagBroadcast == 0 {
// this interface doesn't support broadcast, skip it
// It's a loopback, skip it.
continue
}
var addrs []net.Addr
addrs, err = iface.Addrs()
if err != nil {
aghhttp.Error(
r,
w,
http.StatusInternalServerError,
"Failed to get addresses for interface %s: %s",
iface.Name,
err,
)
if iface.Flags&net.FlagBroadcast == 0 {
// This interface doesn't support broadcast, skip it.
continue
}
jsonIface, iErr := newNetInterfaceJSON(iface)
if iErr != nil {
aghhttp.Error(r, w, http.StatusInternalServerError, "%s", iErr)
return
}
jsonIface := netInterfaceJSON{
Name: iface.Name,
HardwareAddr: iface.HardwareAddr.String(),
}
if iface.Flags != 0 {
jsonIface.Flags = iface.Flags.String()
}
// we don't want link-local addresses in json, so skip them
for _, addr := range addrs {
ipnet, ok := addr.(*net.IPNet)
if !ok {
// not an IPNet, should not happen
aghhttp.Error(
r,
w,
http.StatusInternalServerError,
"got iface.Addrs() element %[1]s that is not net.IPNet, it is %[1]T",
addr)
return
}
// ignore link-local
//
// TODO(e.burkov): Try to listen DHCP on LLA as well.
if ipnet.IP.IsLinkLocalUnicast() {
continue
}
if ip4 := ipnet.IP.To4(); ip4 != nil {
addr := netip.AddrFrom4(*(*[4]byte)(ip4))
jsonIface.Addrs4 = append(jsonIface.Addrs4, addr)
} else {
addr := netip.AddrFrom16(*(*[16]byte)(ipnet.IP))
jsonIface.Addrs6 = append(jsonIface.Addrs6, addr)
}
}
if len(jsonIface.Addrs4)+len(jsonIface.Addrs6) != 0 {
jsonIface.GatewayIP = aghnet.GatewayIP(iface.Name)
response[iface.Name] = jsonIface
if jsonIface != nil {
resp[iface.Name] = jsonIface
}
}
err = json.NewEncoder(w).Encode(response)
_ = aghhttp.WriteJSONResponse(w, r, resp)
}
// newNetInterfaceJSON creates a JSON object from a [net.Interface] iface.
func newNetInterfaceJSON(iface net.Interface) (out *netInterfaceJSON, err error) {
addrs, err := iface.Addrs()
if err != nil {
aghhttp.Error(
r,
w,
http.StatusInternalServerError,
"Failed to marshal json with available interfaces: %s",
return nil, fmt.Errorf(
"failed to get addresses for interface %s: %s",
iface.Name,
err,
)
}
out = &netInterfaceJSON{
Name: iface.Name,
HardwareAddr: iface.HardwareAddr.String(),
}
if iface.Flags != 0 {
out.Flags = iface.Flags.String()
}
// We don't want link-local addresses in JSON, so skip them.
for _, addr := range addrs {
ipNet, ok := addr.(*net.IPNet)
if !ok {
// Not an IPNet, should not happen.
return nil, fmt.Errorf("got iface.Addrs() element %[1]s that is not"+
" net.IPNet, it is %[1]T", addr)
}
// Ignore link-local.
//
// TODO(e.burkov): Try to listen DHCP on LLA as well.
if ipNet.IP.IsLinkLocalUnicast() {
continue
}
vAddr, iErr := netutil.IPToAddrNoMapped(ipNet.IP)
if iErr != nil {
// Not an IPNet, should not happen.
return nil, fmt.Errorf("failed to convert IP address %[1]s: %w", addr, iErr)
}
if vAddr.Is4() {
out.Addrs4 = append(out.Addrs4, vAddr)
} else {
out.Addrs6 = append(out.Addrs6, vAddr)
}
}
if len(out.Addrs4)+len(out.Addrs6) == 0 {
return nil, nil
}
out.GatewayIP = aghnet.GatewayIP(iface.Name)
return out, nil
}
// dhcpSearchOtherResult contains information about other DHCP server for

16
internal/dhcpd/routeradv.go
View File

@@ -7,6 +7,7 @@ import (
"sync/atomic"
"time"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/netutil"
"golang.org/x/net/icmp"
@@ -195,7 +196,7 @@ func createICMPv6RAPacket(params icmpv6RA) (data []byte, err error) {
return data, nil
}
// Init - initialize RA module
// Init initializes RA module.
func (ra *raCtx) Init() (err error) {
ra.stop.Store(0)
ra.conn = nil
@@ -203,8 +204,7 @@ func (ra *raCtx) Init() (err error) {
return nil
}
log.Debug("dhcpv6 ra: source IP address: %s DNS IP address: %s",
ra.ipAddr, ra.dnsIPAddr)
log.Debug("dhcpv6 ra: source IP address: %s DNS IP address: %s", ra.ipAddr, ra.dnsIPAddr)
params := icmpv6RA{
managedAddressConfiguration: !ra.raSLAACOnly,
@@ -223,18 +223,15 @@ func (ra *raCtx) Init() (err error) {
return fmt.Errorf("creating packet: %w", err)
}
success := false
ipAndScope := ra.ipAddr.String() + "%" + ra.ifaceName
ra.conn, err = icmp.ListenPacket("ip6:ipv6-icmp", ipAndScope)
if err != nil {
return fmt.Errorf("dhcpv6 ra: icmp.ListenPacket: %w", err)
}
defer func() {
if !success {
derr := ra.Close()
if derr != nil {
log.Error("closing context: %s", derr)
}
if err != nil {
err = errors.WithDeferred(err, ra.Close())
}
}()
@@ -269,7 +266,6 @@ func (ra *raCtx) Init() (err error) {
log.Debug("dhcpv6 ra: loop exit")
}()
success = true
return nil
}

271
internal/dhcpd/v4_unix.go
View File

@@ -342,8 +342,8 @@ func (s *v4Server) rmLease(lease *Lease) (err error) {
// server to be configured and it's not.
const ErrUnconfigured errors.Error = "server is unconfigured"
// AddStaticLease implements the DHCPServer interface for *v4Server. It is safe
// for concurrent use.
// AddStaticLease implements the DHCPServer interface for *v4Server. It is
// safe for concurrent use.
func (s *v4Server) AddStaticLease(l *Lease) (err error) {
defer func() { err = errors.Annotate(err, "dhcpv4: adding static lease: %w") }()
@@ -354,21 +354,23 @@ func (s *v4Server) AddStaticLease(l *Lease) (err error) {
l.IP = l.IP.Unmap()
if !l.IP.Is4() {
return fmt.Errorf("invalid ip %q, only ipv4 is supported", l.IP)
return fmt.Errorf("invalid IP %q: only IPv4 is supported", l.IP)
} else if gwIP := s.conf.GatewayIP; gwIP == l.IP {
return fmt.Errorf("can't assign the gateway IP %s to the lease", gwIP)
return fmt.Errorf("can't assign the gateway IP %q to the lease", gwIP)
}
l.IsStatic = true
err = netutil.ValidateMAC(l.HWAddr)
if err != nil {
// Don't wrap the error, because it's informative enough as is.
return err
}
if hostname := l.Hostname; hostname != "" {
hostname, err = normalizeHostname(hostname)
if err != nil {
// Don't wrap the error, because it's informative enough as is.
return err
}
@@ -386,32 +388,9 @@ func (s *v4Server) AddStaticLease(l *Lease) (err error) {
l.Hostname = hostname
}
// Perform the following actions in an anonymous function to make sure
// that the lock gets unlocked before the notification step.
func() {
s.leasesLock.Lock()
defer s.leasesLock.Unlock()
err = s.rmDynamicLease(l)
if err != nil {
err = fmt.Errorf(
"removing dynamic leases for %s (%s): %w",
l.IP,
l.HWAddr,
err,
)
return
}
err = s.addLease(l)
if err != nil {
err = fmt.Errorf("adding static lease for %s (%s): %w", l.IP, l.HWAddr, err)
return
}
}()
err = s.updateStaticLease(l)
if err != nil {
// Don't wrap the error, because it's informative enough as is.
return err
}
@@ -421,6 +400,25 @@ func (s *v4Server) AddStaticLease(l *Lease) (err error) {
return nil
}
// updateStaticLease safe removes dynamic lease with the same properties and
// then adds a static lease l.
func (s *v4Server) updateStaticLease(l *Lease) (err error) {
s.leasesLock.Lock()
defer s.leasesLock.Unlock()
err = s.rmDynamicLease(l)
if err != nil {
return fmt.Errorf("removing dynamic leases for %s (%s): %w", l.IP, l.HWAddr, err)
}
err = s.addLease(l)
if err != nil {
return fmt.Errorf("adding static lease for %s (%s): %w", l.IP, l.HWAddr, err)
}
return nil
}
// RemoveStaticLease removes a static lease. It is safe for concurrent use.
func (s *v4Server) RemoveStaticLease(l *Lease) (err error) {
defer func() { err = errors.Annotate(err, "dhcpv4: %w") }()
@@ -894,24 +892,9 @@ func (s *v4Server) handleDecline(req, resp *dhcpv4.DHCPv4) (err error) {
reqIP = req.ClientIPAddr
}
netIP, ok := netip.AddrFromSlice(reqIP)
if !ok {
log.Info("dhcpv4: invalid IP: %s", reqIP)
return nil
}
var oldLease *Lease
for _, l := range s.leases {
if bytes.Equal(l.HWAddr, mac) && l.IP == netIP {
oldLease = l
break
}
}
oldLease := s.findLeaseForIP(reqIP, mac)
if oldLease == nil {
log.Info("dhcpv4: lease with ip %s for %s not found", reqIP, mac)
log.Info("dhcpv4: lease with IP %s for %s not found", reqIP, mac)
return nil
}
@@ -925,7 +908,7 @@ func (s *v4Server) handleDecline(req, resp *dhcpv4.DHCPv4) (err error) {
if err != nil {
return fmt.Errorf("allocating new lease for %s: %w", mac, err)
} else if newLease == nil {
log.Info("dhcpv4: allocating new lease for %s: no more ip addresses", mac)
log.Info("dhcpv4: allocating new lease for %s: no more IP addresses", mac)
resp.YourIPAddr = make([]byte, 4)
resp.UpdateOption(dhcpv4.OptMessageType(dhcpv4.MessageTypeAck))
@@ -941,15 +924,32 @@ func (s *v4Server) handleDecline(req, resp *dhcpv4.DHCPv4) (err error) {
return fmt.Errorf("adding new lease for %s: %w", mac, err)
}
log.Info("dhcpv4: changed ip from %s to %s for %s", reqIP, newLease.IP, mac)
resp.YourIPAddr = net.IP(newLease.IP.AsSlice())
log.Info("dhcpv4: changed IP from %s to %s for %s", reqIP, newLease.IP, mac)
resp.YourIPAddr = newLease.IP.AsSlice()
resp.UpdateOption(dhcpv4.OptMessageType(dhcpv4.MessageTypeAck))
return nil
}
// findLeaseForIP returns a lease for provided ip and mac.
func (s *v4Server) findLeaseForIP(ip net.IP, mac net.HardwareAddr) (l *Lease) {
netIP, ok := netip.AddrFromSlice(ip)
if !ok {
log.Info("dhcpv4: invalid IP: %s", ip)
return nil
}
for _, il := range s.leases {
if bytes.Equal(il.HWAddr, mac) && il.IP == netIP {
return il
}
}
return nil
}
// handleRelease is the handler for the DHCP Release request.
func (s *v4Server) handleRelease(req, resp *dhcpv4.DHCPv4) (err error) {
mac := req.ClientHWAddr
@@ -995,11 +995,80 @@ func (s *v4Server) handleRelease(req, resp *dhcpv4.DHCPv4) (err error) {
return nil
}
// Find a lease associated with MAC and prepare response
// Return 1: OK
// Return 0: error; reply with Nak
// Return -1: error; don't reply
func (s *v4Server) handle(req, resp *dhcpv4.DHCPv4) int {
// messageHandler describes a DHCPv4 message handler function.
type messageHandler func(s *v4Server, req, resp *dhcpv4.DHCPv4) (rCode int, l *Lease, err error)
// messageHandlers is a map of handlers for various messages with message types
// keys.
var messageHandlers = map[dhcpv4.MessageType]messageHandler{
dhcpv4.MessageTypeDiscover: func(
s *v4Server,
req *dhcpv4.DHCPv4,
resp *dhcpv4.DHCPv4,
) (rCode int, l *Lease, err error) {
l, err = s.handleDiscover(req, resp)
if err != nil {
return 0, nil, fmt.Errorf("handling discover: %s", err)
}
if l == nil {
return 0, nil, nil
}
return 1, l, nil
},
dhcpv4.MessageTypeRequest: func(
s *v4Server,
req *dhcpv4.DHCPv4,
resp *dhcpv4.DHCPv4,
) (rCode int, l *Lease, err error) {
var toReply bool
l, toReply = s.handleRequest(req, resp)
if l == nil {
if toReply {
return 0, nil, nil
}
// Drop the packet.
return -1, nil, nil
}
return 1, l, nil
},
dhcpv4.MessageTypeDecline: func(
s *v4Server,
req *dhcpv4.DHCPv4,
resp *dhcpv4.DHCPv4,
) (rCode int, l *Lease, err error) {
err = s.handleDecline(req, resp)
if err != nil {
return 0, nil, fmt.Errorf("handling decline: %s", err)
}
return 1, nil, nil
},
dhcpv4.MessageTypeRelease: func(
s *v4Server,
req *dhcpv4.DHCPv4,
resp *dhcpv4.DHCPv4,
) (rCode int, l *Lease, err error) {
err = s.handleRelease(req, resp)
if err != nil {
return 0, nil, fmt.Errorf("handling release: %s", err)
}
return 1, nil, nil
},
}
// handle processes request, it finds a lease associated with MAC address and
// prepares response.
//
// Possible return values are:
// - "1": OK,
// - "0": error, reply with Nak,
// - "-1": error, don't reply.
func (s *v4Server) handle(req, resp *dhcpv4.DHCPv4) (rCode int) {
var err error
// Include server's identifier option since any reply should contain it.
@@ -1007,47 +1076,26 @@ func (s *v4Server) handle(req, resp *dhcpv4.DHCPv4) int {
// See https://datatracker.ietf.org/doc/html/rfc2131#page-29.
resp.UpdateOption(dhcpv4.OptServerIdentifier(s.conf.dnsIPAddrs[0].AsSlice()))
// TODO(a.garipov): Refactor this into handlers.
var l *Lease
switch mt := req.MessageType(); mt {
case dhcpv4.MessageTypeDiscover:
l, err = s.handleDiscover(req, resp)
if err != nil {
log.Error("dhcpv4: handling discover: %s", err)
handler := messageHandlers[req.MessageType()]
if handler == nil {
s.updateOptions(req, resp)
return 0
}
return 1
}
if l == nil {
return 0
}
case dhcpv4.MessageTypeRequest:
var toReply bool
l, toReply = s.handleRequest(req, resp)
if l == nil {
if toReply {
return 0
}
return -1 // drop packet
}
case dhcpv4.MessageTypeDecline:
err = s.handleDecline(req, resp)
if err != nil {
log.Error("dhcpv4: handling decline: %s", err)
rCode, l, err := handler(s, req, resp)
if err != nil {
log.Error("dhcpv4: %s", err)
return 0
}
case dhcpv4.MessageTypeRelease:
err = s.handleRelease(req, resp)
if err != nil {
log.Error("dhcpv4: handling release: %s", err)
return 0
}
return 0
}
if rCode != 1 {
return rCode
}
if l != nil {
resp.YourIPAddr = net.IP(l.IP.AsSlice())
resp.YourIPAddr = l.IP.AsSlice()
}
s.updateOptions(req, resp)
@@ -1162,23 +1210,8 @@ func (s *v4Server) Start() (err error) {
// No available IP addresses which may appear later.
return nil
}
// Update the value of Domain Name Server option separately from others if
// not assigned yet since its value is available only at server's start.
//
// TODO(e.burkov): Initialize as implicit option with the rest of default
// options when it will be possible to do before the call to Start.
if !s.explicitOpts.Has(dhcpv4.OptionDomainNameServer) {
s.implicitOpts.Update(dhcpv4.OptDNS(dnsIPAddrs...))
}
for _, ip := range dnsIPAddrs {
ip = ip.To4()
if ip == nil {
continue
}
s.conf.dnsIPAddrs = append(s.conf.dnsIPAddrs, netip.AddrFrom4(*(*[4]byte)(ip)))
}
s.configureDNSIPAddrs(dnsIPAddrs)
var c net.PacketConn
if c, err = s.newDHCPConn(iface); err != nil {
@@ -1199,10 +1232,10 @@ func (s *v4Server) Start() (err error) {
log.Info("dhcpv4: listening")
go func() {
if serr := s.srv.Serve(); errors.Is(serr, net.ErrClosed) {
if sErr := s.srv.Serve(); errors.Is(sErr, net.ErrClosed) {
log.Info("dhcpv4: server is closed")
} else if serr != nil {
log.Error("dhcpv4: srv.Serve: %s", serr)
} else if sErr != nil {
log.Error("dhcpv4: srv.Serve: %s", sErr)
}
}()
@@ -1213,6 +1246,28 @@ func (s *v4Server) Start() (err error) {
return nil
}
// configureDNSIPAddrs updates v4Server configuration with provided slice of
// dns IP addresses.
func (s *v4Server) configureDNSIPAddrs(dnsIPAddrs []net.IP) {
// Update the value of Domain Name Server option separately from others if
// not assigned yet since its value is available only at server's start.
//
// TODO(e.burkov): Initialize as implicit option with the rest of default
// options when it will be possible to do before the call to Start.
if !s.explicitOpts.Has(dhcpv4.OptionDomainNameServer) {
s.implicitOpts.Update(dhcpv4.OptDNS(dnsIPAddrs...))
}
for _, ip := range dnsIPAddrs {
vAddr, err := netutil.IPToAddr(ip, netutil.AddrFamilyIPv4)
if err != nil {
continue
}
s.conf.dnsIPAddrs = append(s.conf.dnsIPAddrs, vAddr)
}
}
// Stop - stop server
func (s *v4Server) Stop() (err error) {
if s.srv == nil {

4
internal/dhcpd/v4_unix_test.go
View File

@@ -227,7 +227,7 @@ func TestV4Server_AddRemove_static(t *testing.T) {
},
name: "with_gateway_ip",
wantErrMsg: "dhcpv4: adding static lease: " +
"can't assign the gateway IP 192.168.10.1 to the lease",
`can't assign the gateway IP "192.168.10.1" to the lease`,
}, {
lease: &Lease{
Hostname: "ip6.local",
@@ -236,7 +236,7 @@ func TestV4Server_AddRemove_static(t *testing.T) {
},
name: "ipv6",
wantErrMsg: `dhcpv4: adding static lease: ` +
`invalid ip "ffff::1", only ipv4 is supported`,
`invalid IP "ffff::1": only IPv4 is supported`,
}, {
lease: &Lease{
Hostname: "bad-mac.local",

73
internal/dhcpd/v6_unix.go
View File

@@ -30,7 +30,7 @@ type v6Server struct {
leasesLock sync.Mutex
leases []*Lease
ipAddrs [256]byte
sid dhcpv6.Duid
sid dhcpv6.DUID
ra raCtx // RA module
@@ -586,9 +586,31 @@ func (s *v6Server) packetHandler(conn net.PacketConn, peer net.Addr, req dhcpv6.
}
}
// initialize RA module
func (s *v6Server) initRA(iface *net.Interface) error {
// choose the source IP address - should be link-local-unicast
// configureDNSIPAddrs updates v6Server configuration with the slice of DNS IP
// addresses of provided interface iface. Initializes RA module.
func (s *v6Server) configureDNSIPAddrs(iface *net.Interface) (ok bool, err error) {
dnsIPAddrs, err := aghnet.IfaceDNSIPAddrs(
iface,
aghnet.IPVersion6,
defaultMaxAttempts,
defaultBackoff,
)
if err != nil {
return false, fmt.Errorf("interface %s: %w", iface.Name, err)
}
if len(dnsIPAddrs) == 0 {
return false, nil
}
s.conf.dnsIPAddrs = dnsIPAddrs
return true, s.initRA(iface)
}
// initRA initializes RA module.
func (s *v6Server) initRA(iface *net.Interface) (err error) {
// Choose the source IP address - should be link-local-unicast.
s.ra.ipAddr = s.conf.dnsIPAddrs[0]
for _, ip := range s.conf.dnsIPAddrs {
if ip.IsLinkLocalUnicast() {
@@ -604,6 +626,7 @@ func (s *v6Server) initRA(iface *net.Interface) error {
s.ra.ifaceName = s.conf.InterfaceName
s.ra.iface = iface
s.ra.packetSendPeriod = 1 * time.Second
return s.ra.Init()
}
@@ -623,63 +646,47 @@ func (s *v6Server) Start() (err error) {
log.Debug("dhcpv6: starting...")
dnsIPAddrs, err := aghnet.IfaceDNSIPAddrs(
iface,
aghnet.IPVersion6,
defaultMaxAttempts,
defaultBackoff,
)
ok, err := s.configureDNSIPAddrs(iface)
if err != nil {
return fmt.Errorf("interface %s: %w", ifaceName, err)
// Don't wrap the error, because it's informative enough as is.
return err
}
if len(dnsIPAddrs) == 0 {
if !ok {
// No available IP addresses which may appear later.
return nil
}
s.conf.dnsIPAddrs = dnsIPAddrs
err = s.initRA(iface)
if err != nil {
return err
}
// don't initialize DHCPv6 server if we must force the clients to use SLAAC
// Don't initialize DHCPv6 server if we must force the clients to use SLAAC.
if s.conf.RASLAACOnly {
log.Debug("not starting dhcpv6 server due to ra_slaac_only=true")
return nil
}
log.Debug("dhcpv6: listening...")
err = netutil.ValidateMAC(iface.HardwareAddr)
if err != nil {
return fmt.Errorf("validating interface %s: %w", iface.Name, err)
}
s.sid = dhcpv6.Duid{
Type: dhcpv6.DUID_LLT,
HwType: iana.HWTypeEthernet,
s.sid = &dhcpv6.DUIDLLT{
HWType: iana.HWTypeEthernet,
LinkLayerAddr: iface.HardwareAddr,
Time: dhcpv6.GetTime(),
}
laddr := &net.UDPAddr{
IP: net.ParseIP("::"),
Port: dhcpv6.DefaultServerPort,
}
s.srv, err = server6.NewServer(iface.Name, laddr, s.packetHandler, server6.WithDebugLogger())
s.srv, err = server6.NewServer(iface.Name, nil, s.packetHandler, server6.WithDebugLogger())
if err != nil {
return err
}
log.Debug("dhcpv6: listening...")
go func() {
if serr := s.srv.Serve(); errors.Is(serr, net.ErrClosed) {
if sErr := s.srv.Serve(); errors.Is(sErr, net.ErrClosed) {
log.Info("dhcpv6: server is closed")
} else if serr != nil {
log.Error("dhcpv6: srv.Serve: %s", serr)
} else if sErr != nil {
log.Error("dhcpv6: srv.Serve: %s", sErr)
}
}()

10
internal/dhcpd/v6_unix_test.go
View File

@@ -121,9 +121,8 @@ func TestV6GetLease(t *testing.T) {
dnsAddr := net.ParseIP("2000::1")
s.conf.dnsIPAddrs = []net.IP{dnsAddr}
s.sid = dhcpv6.Duid{
Type: dhcpv6.DUID_LLT,
HwType: iana.HWTypeEthernet,
s.sid = &dhcpv6.DUIDLL{
HWType: iana.HWTypeEthernet,
LinkLayerAddr: net.HardwareAddr{0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA},
}
@@ -216,9 +215,8 @@ func TestV6GetDynamicLease(t *testing.T) {
dnsAddr := net.ParseIP("2000::1")
s.conf.dnsIPAddrs = []net.IP{dnsAddr}
s.sid = dhcpv6.Duid{
Type: dhcpv6.DUID_LLT,
HwType: iana.HWTypeEthernet,
s.sid = &dhcpv6.DUIDLL{
HWType: iana.HWTypeEthernet,
LinkLayerAddr: net.HardwareAddr{0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA},
}

16
internal/dnsforward/dnsforward_test.go
View File

@@ -23,6 +23,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/AdGuardHome/internal/dhcpd"
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/AdGuardHome/internal/filtering/hashprefix"
"github.com/AdguardTeam/AdGuardHome/internal/filtering/safesearch"
"github.com/AdguardTeam/dnsproxy/proxy"
"github.com/AdguardTeam/dnsproxy/upstream"
@@ -915,13 +916,23 @@ func TestBlockedByHosts(t *testing.T) {
}
func TestBlockedBySafeBrowsing(t *testing.T) {
const hostname = "wmconvirus.narod.ru"
const (
hostname = "wmconvirus.narod.ru"
cacheTime = 10 * time.Minute
cacheSize = 10000
)
sbChecker := hashprefix.New(&hashprefix.Config{
CacheTime: cacheTime,
CacheSize: cacheSize,
Upstream: aghtest.NewBlockUpstream(hostname, true),
})
sbUps := aghtest.NewBlockUpstream(hostname, true)
ans4, _ := (&aghtest.TestResolver{}).HostToIPs(hostname)
filterConf := &filtering.Config{
SafeBrowsingEnabled: true,
SafeBrowsingChecker: sbChecker,
}
forwardConf := ServerConfig{
UDPListenAddrs: []*net.UDPAddr{{}},
@@ -935,7 +946,6 @@ func TestBlockedBySafeBrowsing(t *testing.T) {
},
}
s := createTestServer(t, filterConf, forwardConf, nil)
s.dnsFilter.SetSafeBrowsingUpstream(sbUps)
startDeferStop(t, s)
addr := s.dnsProxy.Addr(proxy.ProtoUDP)

10
internal/dnsforward/http_test.go
View File

@@ -205,8 +205,8 @@ func TestDNSForwardHTTP_handleSetConfig(t *testing.T) {
wantSet: `validating upstream servers: validating upstream "!!!": not an ip:port`,
}, {
name: "bootstraps_bad",
wantSet: `checking bootstrap a: invalid address: ` +
`Resolver a is not eligible to be a bootstrap DNS server`,
wantSet: `checking bootstrap a: invalid address: bootstrap a:53: ` +
`ParseAddr("a"): unable to parse IP`,
}, {
name: "cache_bad_ttl",
wantSet: `cache_ttl_min must be less or equal than cache_ttl_max`,
@@ -487,7 +487,8 @@ func TestServer_handleTestUpstreaDNS(t *testing.T) {
},
wantResp: map[string]any{
badUps: `upstream "` + badUps + `" fails to exchange: ` +
`couldn't communicate with upstream: dns: id mismatch`,
`couldn't communicate with upstream: exchanging with ` +
badUps + ` over tcp: dns: id mismatch`,
},
name: "broken",
}, {
@@ -497,7 +498,8 @@ func TestServer_handleTestUpstreaDNS(t *testing.T) {
wantResp: map[string]any{
goodUps: "OK",
badUps: `upstream "` + badUps + `" fails to exchange: ` +
`couldn't communicate with upstream: dns: id mismatch`,
`couldn't communicate with upstream: exchanging with ` +
badUps + ` over tcp: dns: id mismatch`,
},
name: "both",
}}

117
internal/filtering/filtering.go
View File

@@ -18,8 +18,6 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/cache"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/mathutil"
@@ -75,6 +73,12 @@ type Resolver interface {
// Config allows you to configure DNS filtering with New() or just change variables directly.
type Config struct {
// SafeBrowsingChecker is the safe browsing hash-prefix checker.
SafeBrowsingChecker Checker `yaml:"-"`
// ParentControl is the parental control hash-prefix checker.
ParentalControlChecker Checker `yaml:"-"`
// enabled is used to be returned within Settings.
//
// It is of type uint32 to be accessed by atomic.
@@ -158,8 +162,22 @@ type hostChecker struct {
name string
}
// Checker is used for safe browsing or parental control hash-prefix filtering.
type Checker interface {
// Check returns true if request for the host should be blocked.
Check(host string) (block bool, err error)
}
// DNSFilter matches hostnames and DNS requests against filtering rules.
type DNSFilter struct {
safeSearch SafeSearch
// safeBrowsingChecker is the safe browsing hash-prefix checker.
safeBrowsingChecker Checker
// parentalControl is the parental control hash-prefix checker.
parentalControlChecker Checker
rulesStorage *filterlist.RuleStorage
filteringEngine *urlfilter.DNSEngine
@@ -168,14 +186,6 @@ type DNSFilter struct {
engineLock sync.RWMutex
parentalServer string // access via methods
safeBrowsingServer string // access via methods
parentalUpstream upstream.Upstream
safeBrowsingUpstream upstream.Upstream
safebrowsingCache cache.Cache
parentalCache cache.Cache
Config // for direct access by library users, even a = assignment
// confLock protects Config.
confLock sync.RWMutex
@@ -192,7 +202,6 @@ type DNSFilter struct {
// TODO(e.burkov): Don't use regexp for such a simple text processing task.
filterTitleRegexp *regexp.Regexp
safeSearch SafeSearch
hostCheckers []hostChecker
}
@@ -940,19 +949,12 @@ func InitModule() {
// be non-nil.
func New(c *Config, blockFilters []Filter) (d *DNSFilter, err error) {
d = &DNSFilter{
refreshLock: &sync.Mutex{},
filterTitleRegexp: regexp.MustCompile(`^! Title: +(.*)$`),
refreshLock: &sync.Mutex{},
filterTitleRegexp: regexp.MustCompile(`^! Title: +(.*)$`),
safeBrowsingChecker: c.SafeBrowsingChecker,
parentalControlChecker: c.ParentalControlChecker,
}
d.safebrowsingCache = cache.New(cache.Config{
EnableLRU: true,
MaxSize: c.SafeBrowsingCacheSize,
})
d.parentalCache = cache.New(cache.Config{
EnableLRU: true,
MaxSize: c.ParentalCacheSize,
})
d.safeSearch = c.SafeSearch
d.hostCheckers = []hostChecker{{
@@ -977,11 +979,6 @@ func New(c *Config, blockFilters []Filter) (d *DNSFilter, err error) {
defer func() { err = errors.Annotate(err, "filtering: %w") }()
err = d.initSecurityServices()
if err != nil {
return nil, fmt.Errorf("initializing services: %s", err)
}
d.Config = *c
d.filtersMu = &sync.RWMutex{}
@@ -1038,3 +1035,69 @@ func (d *DNSFilter) Start() {
// So for now we just start this periodic task from here.
go d.periodicallyRefreshFilters()
}
// Safe browsing and parental control methods.
// TODO(a.garipov): Unify with checkParental.
func (d *DNSFilter) checkSafeBrowsing(
host string,
_ uint16,
setts *Settings,
) (res Result, err error) {
if !setts.ProtectionEnabled || !setts.SafeBrowsingEnabled {
return Result{}, nil
}
if log.GetLevel() >= log.DEBUG {
timer := log.StartTimer()
defer timer.LogElapsed("safebrowsing lookup for %q", host)
}
res = Result{
Rules: []*ResultRule{{
Text: "adguard-malware-shavar",
FilterListID: SafeBrowsingListID,
}},
Reason: FilteredSafeBrowsing,
IsFiltered: true,
}
block, err := d.safeBrowsingChecker.Check(host)
if !block || err != nil {
return Result{}, err
}
return res, nil
}
// TODO(a.garipov): Unify with checkSafeBrowsing.
func (d *DNSFilter) checkParental(
host string,
_ uint16,
setts *Settings,
) (res Result, err error) {
if !setts.ProtectionEnabled || !setts.ParentalEnabled {
return Result{}, nil
}
if log.GetLevel() >= log.DEBUG {
timer := log.StartTimer()
defer timer.LogElapsed("parental lookup for %q", host)
}
res = Result{
Rules: []*ResultRule{{
Text: "parental CATEGORY_BLACKLISTED",
FilterListID: ParentalListID,
}},
Reason: FilteredParental,
IsFiltered: true,
}
block, err := d.parentalControlChecker.Check(host)
if !block || err != nil {
return Result{}, err
}
return res, nil
}

70
internal/filtering/filtering_test.go
View File

@@ -7,7 +7,7 @@ import (
"testing"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/golibs/cache"
"github.com/AdguardTeam/AdGuardHome/internal/filtering/hashprefix"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/testutil"
"github.com/AdguardTeam/urlfilter/rules"
@@ -27,17 +27,6 @@ const (
// Helpers.
func purgeCaches(d *DNSFilter) {
for _, c := range []cache.Cache{
d.safebrowsingCache,
d.parentalCache,
} {
if c != nil {
c.Clear()
}
}
}
func newForTest(t testing.TB, c *Config, filters []Filter) (f *DNSFilter, setts *Settings) {
setts = &Settings{
ProtectionEnabled: true,
@@ -58,11 +47,17 @@ func newForTest(t testing.TB, c *Config, filters []Filter) (f *DNSFilter, setts
f, err := New(c, filters)
require.NoError(t, err)
purgeCaches(f)
return f, setts
}
func newChecker(host string) Checker {
return hashprefix.New(&hashprefix.Config{
CacheTime: 10,
CacheSize: 100000,
Upstream: aghtest.NewBlockUpstream(host, true),
})
}
func (d *DNSFilter) checkMatch(t *testing.T, hostname string, setts *Settings) {
t.Helper()
@@ -175,10 +170,14 @@ func TestSafeBrowsing(t *testing.T) {
aghtest.ReplaceLogWriter(t, logOutput)
aghtest.ReplaceLogLevel(t, log.DEBUG)
d, setts := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil)
sbChecker := newChecker(sbBlocked)
d, setts := newForTest(t, &Config{
SafeBrowsingEnabled: true,
SafeBrowsingChecker: sbChecker,
}, nil)
t.Cleanup(d.Close)
d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
d.checkMatch(t, sbBlocked, setts)
require.Contains(t, logOutput.String(), fmt.Sprintf("safebrowsing lookup for %q", sbBlocked))
@@ -188,18 +187,17 @@ func TestSafeBrowsing(t *testing.T) {
d.checkMatchEmpty(t, pcBlocked, setts)
// Cached result.
d.safeBrowsingServer = "127.0.0.1"
d.checkMatch(t, sbBlocked, setts)
d.checkMatchEmpty(t, pcBlocked, setts)
d.safeBrowsingServer = defaultSafebrowsingServer
}
func TestParallelSB(t *testing.T) {
d, setts := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil)
d, setts := newForTest(t, &Config{
SafeBrowsingEnabled: true,
SafeBrowsingChecker: newChecker(sbBlocked),
}, nil)
t.Cleanup(d.Close)
d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
t.Run("group", func(t *testing.T) {
for i := 0; i < 100; i++ {
t.Run(fmt.Sprintf("aaa%d", i), func(t *testing.T) {
@@ -220,10 +218,12 @@ func TestParentalControl(t *testing.T) {
aghtest.ReplaceLogWriter(t, logOutput)
aghtest.ReplaceLogLevel(t, log.DEBUG)
d, setts := newForTest(t, &Config{ParentalEnabled: true}, nil)
d, setts := newForTest(t, &Config{
ParentalEnabled: true,
ParentalControlChecker: newChecker(pcBlocked),
}, nil)
t.Cleanup(d.Close)
d.SetParentalUpstream(aghtest.NewBlockUpstream(pcBlocked, true))
d.checkMatch(t, pcBlocked, setts)
require.Contains(t, logOutput.String(), fmt.Sprintf("parental lookup for %q", pcBlocked))
@@ -233,7 +233,6 @@ func TestParentalControl(t *testing.T) {
d.checkMatchEmpty(t, "api.jquery.com", setts)
// Test cached result.
d.parentalServer = "127.0.0.1"
d.checkMatch(t, pcBlocked, setts)
d.checkMatchEmpty(t, "yandex.ru", setts)
}
@@ -593,8 +592,10 @@ func applyClientSettings(setts *Settings) {
func TestClientSettings(t *testing.T) {
d, setts := newForTest(t,
&Config{
ParentalEnabled: true,
SafeBrowsingEnabled: false,
ParentalEnabled: true,
SafeBrowsingEnabled: false,
SafeBrowsingChecker: newChecker(sbBlocked),
ParentalControlChecker: newChecker(pcBlocked),
},
[]Filter{{
ID: 0, Data: []byte("||example.org^\n"),
@@ -602,9 +603,6 @@ func TestClientSettings(t *testing.T) {
)
t.Cleanup(d.Close)
d.SetParentalUpstream(aghtest.NewBlockUpstream(pcBlocked, true))
d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
type testCase struct {
name string
host string
@@ -665,11 +663,12 @@ func TestClientSettings(t *testing.T) {
// Benchmarks.
func BenchmarkSafeBrowsing(b *testing.B) {
d, setts := newForTest(b, &Config{SafeBrowsingEnabled: true}, nil)
d, setts := newForTest(b, &Config{
SafeBrowsingEnabled: true,
SafeBrowsingChecker: newChecker(sbBlocked),
}, nil)
b.Cleanup(d.Close)
d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
for n := 0; n < b.N; n++ {
res, err := d.CheckHost(sbBlocked, dns.TypeA, setts)
require.NoError(b, err)
@@ -679,11 +678,12 @@ func BenchmarkSafeBrowsing(b *testing.B) {
}
func BenchmarkSafeBrowsingParallel(b *testing.B) {
d, setts := newForTest(b, &Config{SafeBrowsingEnabled: true}, nil)
d, setts := newForTest(b, &Config{
SafeBrowsingEnabled: true,
SafeBrowsingChecker: newChecker(sbBlocked),
}, nil)
b.Cleanup(d.Close)
d.SetSafeBrowsingUpstream(aghtest.NewBlockUpstream(sbBlocked, true))
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
res, err := d.CheckHost(sbBlocked, dns.TypeA, setts)

130
internal/filtering/hashprefix/cache.go Normal file
View File

@@ -0,0 +1,130 @@
package hashprefix
import (
"encoding/binary"
"time"
"github.com/AdguardTeam/golibs/log"
)
// expirySize is the size of expiry in cacheItem.
const expirySize = 8
// cacheItem represents an item that we will store in the cache.
type cacheItem struct {
// expiry is the time when cacheItem will expire.
expiry time.Time
// hashes is the hashed hostnames.
hashes []hostnameHash
}
// toCacheItem decodes cacheItem from data. data must be at least equal to
// expiry size.
func toCacheItem(data []byte) *cacheItem {
t := time.Unix(int64(binary.BigEndian.Uint64(data)), 0)
data = data[expirySize:]
hashes := make([]hostnameHash, len(data)/hashSize)
for i := 0; i < len(data); i += hashSize {
var hash hostnameHash
copy(hash[:], data[i:i+hashSize])
hashes = append(hashes, hash)
}
return &cacheItem{
expiry: t,
hashes: hashes,
}
}
// fromCacheItem encodes cacheItem into data.
func fromCacheItem(item *cacheItem) (data []byte) {
data = make([]byte, len(item.hashes)*hashSize+expirySize)
expiry := item.expiry.Unix()
binary.BigEndian.PutUint64(data[:expirySize], uint64(expiry))
for _, v := range item.hashes {
// nolint:looppointer // The subsilce is used for a copy.
data = append(data, v[:]...)
}
return data
}
// findInCache finds hashes in the cache. If nothing found returns list of
// hashes, prefixes of which will be sent to upstream.
func (c *Checker) findInCache(
hashes []hostnameHash,
) (found, blocked bool, hashesToRequest []hostnameHash) {
now := time.Now()
i := 0
for _, hash := range hashes {
// nolint:looppointer // The subsilce is used for a safe cache lookup.
data := c.cache.Get(hash[:prefixLen])
if data == nil {
hashes[i] = hash
i++
continue
}
item := toCacheItem(data)
if now.After(item.expiry) {
hashes[i] = hash
i++
continue
}
if ok := findMatch(hashes, item.hashes); ok {
return true, true, nil
}
}
if i == 0 {
return true, false, nil
}
return false, false, hashes[:i]
}
// storeInCache caches hashes.
func (c *Checker) storeInCache(hashesToRequest, respHashes []hostnameHash) {
hashToStore := make(map[prefix][]hostnameHash)
for _, hash := range respHashes {
var pref prefix
// nolint:looppointer // The subsilce is used for a copy.
copy(pref[:], hash[:])
hashToStore[pref] = append(hashToStore[pref], hash)
}
for pref, hash := range hashToStore {
// nolint:looppointer // The subsilce is used for a safe cache lookup.
c.setCache(pref[:], hash)
}
for _, hash := range hashesToRequest {
// nolint:looppointer // The subsilce is used for a safe cache lookup.
pref := hash[:prefixLen]
val := c.cache.Get(pref)
if val == nil {
c.setCache(pref, nil)
}
}
}
// setCache stores hash in cache.
func (c *Checker) setCache(pref []byte, hashes []hostnameHash) {
item := &cacheItem{
expiry: time.Now().Add(c.cacheTime),
hashes: hashes,
}
c.cache.Set(pref, fromCacheItem(item))
log.Debug("%s: stored in cache: %v", c.svc, pref)
}

245
internal/filtering/hashprefix/hashprefix.go Normal file
View File

@@ -0,0 +1,245 @@
// Package hashprefix used for safe browsing and parent control.
package hashprefix
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"strings"
"time"
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/cache"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/miekg/dns"
"golang.org/x/exp/slices"
"golang.org/x/net/publicsuffix"
)
const (
// prefixLen is the length of the hash prefix of the filtered hostname.
prefixLen = 2
// hashSize is the size of hashed hostname.
hashSize = sha256.Size
// hexSize is the size of hexadecimal representation of hashed hostname.
hexSize = hashSize * 2
)
// prefix is the type of the SHA256 hash prefix used to match against the
// domain-name database.
type prefix [prefixLen]byte
// hostnameHash is the hashed hostname.
//
// TODO(s.chzhen): Split into prefix and suffix.
type hostnameHash [hashSize]byte
// findMatch returns true if one of the a hostnames matches one of the b.
func findMatch(a, b []hostnameHash) (matched bool) {
for _, hash := range a {
if slices.Contains(b, hash) {
return true
}
}
return false
}
// Config is the configuration structure for safe browsing and parental
// control.
type Config struct {
// Upstream is the upstream DNS server.
Upstream upstream.Upstream
// ServiceName is the name of the service.
ServiceName string
// TXTSuffix is the TXT suffix for DNS request.
TXTSuffix string
// CacheTime is the time period to store hash.
CacheTime time.Duration
// CacheSize is the maximum size of the cache. If it's zero, cache size is
// unlimited.
CacheSize uint
}
type Checker struct {
// upstream is the upstream DNS server.
upstream upstream.Upstream
// cache stores hostname hashes.
cache cache.Cache
// svc is the name of the service.
svc string
// txtSuffix is the TXT suffix for DNS request.
txtSuffix string
// cacheTime is the time period to store hash.
cacheTime time.Duration
}
// New returns Checker.
func New(conf *Config) (c *Checker) {
return &Checker{
upstream: conf.Upstream,
cache: cache.New(cache.Config{
EnableLRU: true,
MaxSize: conf.CacheSize,
}),
svc: conf.ServiceName,
txtSuffix: conf.TXTSuffix,
cacheTime: conf.CacheTime,
}
}
// Check returns true if request for the host should be blocked.
func (c *Checker) Check(host string) (ok bool, err error) {
hashes := hostnameToHashes(host)
found, blocked, hashesToRequest := c.findInCache(hashes)
if found {
log.Debug("%s: found %q in cache, blocked: %t", c.svc, host, blocked)
return blocked, nil
}
question := c.getQuestion(hashesToRequest)
log.Debug("%s: checking %s: %s", c.svc, host, question)
req := (&dns.Msg{}).SetQuestion(question, dns.TypeTXT)
resp, err := c.upstream.Exchange(req)
if err != nil {
return false, fmt.Errorf("getting hashes: %w", err)
}
matched, receivedHashes := c.processAnswer(hashesToRequest, resp, host)
c.storeInCache(hashesToRequest, receivedHashes)
return matched, nil
}
// hostnameToHashes returns hashes that should be checked by the hash prefix
// filter.
func hostnameToHashes(host string) (hashes []hostnameHash) {
// subDomainNum defines how many labels should be hashed to match against a
// hash prefix filter.
const subDomainNum = 4
pubSuf, icann := publicsuffix.PublicSuffix(host)
if !icann {
// Check the full private domain space.
pubSuf = ""
}
nDots := 0
i := strings.LastIndexFunc(host, func(r rune) (ok bool) {
if r == '.' {
nDots++
}
return nDots == subDomainNum
})
if i != -1 {
host = host[i+1:]
}
sub := netutil.Subdomains(host)
for _, s := range sub {
if s == pubSuf {
break
}
sum := sha256.Sum256([]byte(s))
hashes = append(hashes, sum)
}
return hashes
}
// getQuestion combines hexadecimal encoded prefixes of hashed hostnames into
// string.
func (c *Checker) getQuestion(hashes []hostnameHash) (q string) {
b := &strings.Builder{}
for _, hash := range hashes {
// nolint:looppointer // The subsilce is used for safe hex encoding.
stringutil.WriteToBuilder(b, hex.EncodeToString(hash[:prefixLen]), ".")
}
stringutil.WriteToBuilder(b, c.txtSuffix)
return b.String()
}
// processAnswer returns true if DNS response matches the hash, and received
// hashed hostnames from the upstream.
func (c *Checker) processAnswer(
hashesToRequest []hostnameHash,
resp *dns.Msg,
host string,
) (matched bool, receivedHashes []hostnameHash) {
txtCount := 0
for _, a := range resp.Answer {
txt, ok := a.(*dns.TXT)
if !ok {
continue
}
txtCount++
receivedHashes = c.appendHashesFromTXT(receivedHashes, txt, host)
}
log.Debug("%s: received answer for %s with %d TXT count", c.svc, host, txtCount)
matched = findMatch(hashesToRequest, receivedHashes)
if matched {
log.Debug("%s: matched %s", c.svc, host)
return true, receivedHashes
}
return false, receivedHashes
}
// appendHashesFromTXT appends received hashed hostnames.
func (c *Checker) appendHashesFromTXT(
hashes []hostnameHash,
txt *dns.TXT,
host string,
) (receivedHashes []hostnameHash) {
log.Debug("%s: received hashes for %s: %v", c.svc, host, txt.Txt)
for _, t := range txt.Txt {
if len(t) != hexSize {
log.Debug("%s: wrong hex size %d for %s %s", c.svc, len(t), host, t)
continue
}
buf, err := hex.DecodeString(t)
if err != nil {
log.Debug("%s: decoding hex string %s: %s", c.svc, t, err)
continue
}
var hash hostnameHash
copy(hash[:], buf)
hashes = append(hashes, hash)
}
return hashes
}

248
internal/filtering/hashprefix/hashprefix_internal_test.go Normal file
View File

@@ -0,0 +1,248 @@
package hashprefix
import (
"crypto/sha256"
"encoding/hex"
"strings"
"testing"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/golibs/cache"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/exp/slices"
)
const (
cacheTime = 10 * time.Minute
cacheSize = 10000
)
func TestChcker_getQuestion(t *testing.T) {
const suf = "sb.dns.adguard.com."
// test hostnameToHashes()
hashes := hostnameToHashes("1.2.3.sub.host.com")
assert.Len(t, hashes, 3)
hash := sha256.Sum256([]byte("3.sub.host.com"))
hexPref1 := hex.EncodeToString(hash[:prefixLen])
assert.True(t, slices.Contains(hashes, hash))
hash = sha256.Sum256([]byte("sub.host.com"))
hexPref2 := hex.EncodeToString(hash[:prefixLen])
assert.True(t, slices.Contains(hashes, hash))
hash = sha256.Sum256([]byte("host.com"))
hexPref3 := hex.EncodeToString(hash[:prefixLen])
assert.True(t, slices.Contains(hashes, hash))
hash = sha256.Sum256([]byte("com"))
assert.False(t, slices.Contains(hashes, hash))
c := &Checker{
svc: "SafeBrowsing",
txtSuffix: suf,
}
q := c.getQuestion(hashes)
assert.Contains(t, q, hexPref1)
assert.Contains(t, q, hexPref2)
assert.Contains(t, q, hexPref3)
assert.True(t, strings.HasSuffix(q, suf))
}
func TestHostnameToHashes(t *testing.T) {
testCases := []struct {
name string
host string
wantLen int
}{{
name: "basic",
host: "example.com",
wantLen: 1,
}, {
name: "sub_basic",
host: "www.example.com",
wantLen: 2,
}, {
name: "private_domain",
host: "foo.co.uk",
wantLen: 1,
}, {
name: "sub_private_domain",
host: "bar.foo.co.uk",
wantLen: 2,
}, {
name: "private_domain_v2",
host: "foo.blogspot.co.uk",
wantLen: 4,
}, {
name: "sub_private_domain_v2",
host: "bar.foo.blogspot.co.uk",
wantLen: 4,
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
hashes := hostnameToHashes(tc.host)
assert.Len(t, hashes, tc.wantLen)
})
}
}
func TestChecker_storeInCache(t *testing.T) {
c := &Checker{
svc: "SafeBrowsing",
cacheTime: cacheTime,
}
conf := cache.Config{}
c.cache = cache.New(conf)
// store in cache hashes for "3.sub.host.com" and "host.com"
// and empty data for hash-prefix for "sub.host.com"
hashes := []hostnameHash{}
hash := sha256.Sum256([]byte("sub.host.com"))
hashes = append(hashes, hash)
var hashesArray []hostnameHash
hash4 := sha256.Sum256([]byte("3.sub.host.com"))
hashesArray = append(hashesArray, hash4)
hash2 := sha256.Sum256([]byte("host.com"))
hashesArray = append(hashesArray, hash2)
c.storeInCache(hashes, hashesArray)
// match "3.sub.host.com" or "host.com" from cache
hashes = []hostnameHash{}
hash = sha256.Sum256([]byte("3.sub.host.com"))
hashes = append(hashes, hash)
hash = sha256.Sum256([]byte("sub.host.com"))
hashes = append(hashes, hash)
hash = sha256.Sum256([]byte("host.com"))
hashes = append(hashes, hash)
found, blocked, _ := c.findInCache(hashes)
assert.True(t, found)
assert.True(t, blocked)
// match "sub.host.com" from cache
hashes = []hostnameHash{}
hash = sha256.Sum256([]byte("sub.host.com"))
hashes = append(hashes, hash)
found, blocked, _ = c.findInCache(hashes)
assert.True(t, found)
assert.False(t, blocked)
// Match "sub.host.com" from cache. Another hash for "host.example" is not
// in the cache, so get data for it from the server.
hashes = []hostnameHash{}
hash = sha256.Sum256([]byte("sub.host.com"))
hashes = append(hashes, hash)
hash = sha256.Sum256([]byte("host.example"))
hashes = append(hashes, hash)
found, _, hashesToRequest := c.findInCache(hashes)
assert.False(t, found)
hash = sha256.Sum256([]byte("sub.host.com"))
ok := slices.Contains(hashesToRequest, hash)
assert.False(t, ok)
hash = sha256.Sum256([]byte("host.example"))
ok = slices.Contains(hashesToRequest, hash)
assert.True(t, ok)
c = &Checker{
svc: "SafeBrowsing",
cacheTime: cacheTime,
}
c.cache = cache.New(cache.Config{})
hashes = []hostnameHash{}
hash = sha256.Sum256([]byte("sub.host.com"))
hashes = append(hashes, hash)
c.cache.Set(hash[:prefixLen], make([]byte, expirySize+hashSize))
found, _, _ = c.findInCache(hashes)
assert.False(t, found)
}
func TestChecker_Check(t *testing.T) {
const hostname = "example.org"
testCases := []struct {
name string
wantBlock bool
}{{
name: "sb_no_block",
wantBlock: false,
}, {
name: "sb_block",
wantBlock: true,
}, {
name: "pc_no_block",
wantBlock: false,
}, {
name: "pc_block",
wantBlock: true,
}}
for _, tc := range testCases {
c := New(&Config{
CacheTime: cacheTime,
CacheSize: cacheSize,
})
// Prepare the upstream.
ups := aghtest.NewBlockUpstream(hostname, tc.wantBlock)
var numReq int
onExchange := ups.OnExchange
ups.OnExchange = func(req *dns.Msg) (resp *dns.Msg, err error) {
numReq++
return onExchange(req)
}
c.upstream = ups
t.Run(tc.name, func(t *testing.T) {
// Firstly, check the request blocking.
hits := 0
res := false
res, err := c.Check(hostname)
require.NoError(t, err)
if tc.wantBlock {
assert.True(t, res)
hits++
} else {
require.False(t, res)
}
// Check the cache state, check the response is now cached.
assert.Equal(t, 1, c.cache.Stats().Count)
assert.Equal(t, hits, c.cache.Stats().Hit)
// There was one request to an upstream.
assert.Equal(t, 1, numReq)
// Now make the same request to check the cache was used.
res, err = c.Check(hostname)
require.NoError(t, err)
if tc.wantBlock {
assert.True(t, res)
} else {
require.False(t, res)
}
// Check the cache state, it should've been used.
assert.Equal(t, 1, c.cache.Stats().Count)
assert.Equal(t, hits+1, c.cache.Stats().Hit)
// Check that there were no additional requests.
assert.Equal(t, 1, numReq)
})
}
}

75
internal/filtering/http.go
View File

@@ -8,6 +8,7 @@ import (
"net/url"
"os"
"path/filepath"
"sync"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
@@ -458,6 +459,80 @@ func (d *DNSFilter) handleCheckHost(w http.ResponseWriter, r *http.Request) {
_ = aghhttp.WriteJSONResponse(w, r, resp)
}
// setProtectedBool sets the value of a boolean pointer under a lock. l must
// protect the value under ptr.
//
// TODO(e.burkov): Make it generic?
func setProtectedBool(mu *sync.RWMutex, ptr *bool, val bool) {
mu.Lock()
defer mu.Unlock()
*ptr = val
}
// protectedBool gets the value of a boolean pointer under a read lock. l must
// protect the value under ptr.
//
// TODO(e.burkov): Make it generic?
func protectedBool(mu *sync.RWMutex, ptr *bool) (val bool) {
mu.RLock()
defer mu.RUnlock()
return *ptr
}
// handleSafeBrowsingEnable is the handler for the POST
// /control/safebrowsing/enable HTTP API.
func (d *DNSFilter) handleSafeBrowsingEnable(w http.ResponseWriter, r *http.Request) {
setProtectedBool(&d.confLock, &d.Config.SafeBrowsingEnabled, true)
d.Config.ConfigModified()
}
// handleSafeBrowsingDisable is the handler for the POST
// /control/safebrowsing/disable HTTP API.
func (d *DNSFilter) handleSafeBrowsingDisable(w http.ResponseWriter, r *http.Request) {
setProtectedBool(&d.confLock, &d.Config.SafeBrowsingEnabled, false)
d.Config.ConfigModified()
}
// handleSafeBrowsingStatus is the handler for the GET
// /control/safebrowsing/status HTTP API.
func (d *DNSFilter) handleSafeBrowsingStatus(w http.ResponseWriter, r *http.Request) {
resp := &struct {
Enabled bool `json:"enabled"`
}{
Enabled: protectedBool(&d.confLock, &d.Config.SafeBrowsingEnabled),
}
_ = aghhttp.WriteJSONResponse(w, r, resp)
}
// handleParentalEnable is the handler for the POST /control/parental/enable
// HTTP API.
func (d *DNSFilter) handleParentalEnable(w http.ResponseWriter, r *http.Request) {
setProtectedBool(&d.confLock, &d.Config.ParentalEnabled, true)
d.Config.ConfigModified()
}
// handleParentalDisable is the handler for the POST /control/parental/disable
// HTTP API.
func (d *DNSFilter) handleParentalDisable(w http.ResponseWriter, r *http.Request) {
setProtectedBool(&d.confLock, &d.Config.ParentalEnabled, false)
d.Config.ConfigModified()
}
// handleParentalStatus is the handler for the GET /control/parental/status
// HTTP API.
func (d *DNSFilter) handleParentalStatus(w http.ResponseWriter, r *http.Request) {
resp := &struct {
Enabled bool `json:"enabled"`
}{
Enabled: protectedBool(&d.confLock, &d.Config.ParentalEnabled),
}
_ = aghhttp.WriteJSONResponse(w, r, resp)
}
// RegisterFilteringHandlers - register handlers
func (d *DNSFilter) RegisterFilteringHandlers() {
registerHTTP := d.HTTPRegister

169
internal/filtering/http_test.go
View File

@@ -8,6 +8,7 @@ import (
"testing"
"time"
"github.com/AdguardTeam/golibs/testutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -136,3 +137,171 @@ func TestDNSFilter_handleFilteringSetURL(t *testing.T) {
})
}
}
func TestDNSFilter_handleSafeBrowsingStatus(t *testing.T) {
const (
testTimeout = time.Second
statusURL = "/control/safebrowsing/status"
)
confModCh := make(chan struct{})
filtersDir := t.TempDir()
testCases := []struct {
name string
url string
enabled bool
wantStatus assert.BoolAssertionFunc
}{{
name: "enable_off",
url: "/control/safebrowsing/enable",
enabled: false,
wantStatus: assert.True,
}, {
name: "enable_on",
url: "/control/safebrowsing/enable",
enabled: true,
wantStatus: assert.True,
}, {
name: "disable_on",
url: "/control/safebrowsing/disable",
enabled: true,
wantStatus: assert.False,
}, {
name: "disable_off",
url: "/control/safebrowsing/disable",
enabled: false,
wantStatus: assert.False,
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
handlers := make(map[string]http.Handler)
d, err := New(&Config{
ConfigModified: func() {
testutil.RequireSend(testutil.PanicT{}, confModCh, struct{}{}, testTimeout)
},
DataDir: filtersDir,
HTTPRegister: func(_, url string, handler http.HandlerFunc) {
handlers[url] = handler
},
SafeBrowsingEnabled: tc.enabled,
}, nil)
require.NoError(t, err)
t.Cleanup(d.Close)
d.RegisterFilteringHandlers()
require.NotEmpty(t, handlers)
require.Contains(t, handlers, statusURL)
r := httptest.NewRequest(http.MethodPost, tc.url, nil)
w := httptest.NewRecorder()
go handlers[tc.url].ServeHTTP(w, r)
testutil.RequireReceive(t, confModCh, testTimeout)
r = httptest.NewRequest(http.MethodGet, statusURL, nil)
w = httptest.NewRecorder()
handlers[statusURL].ServeHTTP(w, r)
require.Equal(t, http.StatusOK, w.Code)
status := struct {
Enabled bool `json:"enabled"`
}{
Enabled: false,
}
err = json.NewDecoder(w.Body).Decode(&status)
require.NoError(t, err)
tc.wantStatus(t, status.Enabled)
})
}
}
func TestDNSFilter_handleParentalStatus(t *testing.T) {
const (
testTimeout = time.Second
statusURL = "/control/parental/status"
)
confModCh := make(chan struct{})
filtersDir := t.TempDir()
testCases := []struct {
name string
url string
enabled bool
wantStatus assert.BoolAssertionFunc
}{{
name: "enable_off",
url: "/control/parental/enable",
enabled: false,
wantStatus: assert.True,
}, {
name: "enable_on",
url: "/control/parental/enable",
enabled: true,
wantStatus: assert.True,
}, {
name: "disable_on",
url: "/control/parental/disable",
enabled: true,
wantStatus: assert.False,
}, {
name: "disable_off",
url: "/control/parental/disable",
enabled: false,
wantStatus: assert.False,
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
handlers := make(map[string]http.Handler)
d, err := New(&Config{
ConfigModified: func() {
testutil.RequireSend(testutil.PanicT{}, confModCh, struct{}{}, testTimeout)
},
DataDir: filtersDir,
HTTPRegister: func(_, url string, handler http.HandlerFunc) {
handlers[url] = handler
},
ParentalEnabled: tc.enabled,
}, nil)
require.NoError(t, err)
t.Cleanup(d.Close)
d.RegisterFilteringHandlers()
require.NotEmpty(t, handlers)
require.Contains(t, handlers, statusURL)
r := httptest.NewRequest(http.MethodPost, tc.url, nil)
w := httptest.NewRecorder()
go handlers[tc.url].ServeHTTP(w, r)
testutil.RequireReceive(t, confModCh, testTimeout)
r = httptest.NewRequest(http.MethodGet, statusURL, nil)
w = httptest.NewRecorder()
handlers[statusURL].ServeHTTP(w, r)
require.Equal(t, http.StatusOK, w.Code)
status := struct {
Enabled bool `json:"enabled"`
}{
Enabled: false,
}
err = json.NewDecoder(w.Body).Decode(&status)
require.NoError(t, err)
tc.wantStatus(t, status.Enabled)
})
}
}

433
internal/filtering/safebrowsing.go
View File

@@ -1,433 +0,0 @@
package filtering
import (
"bytes"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"fmt"
"net"
"net/http"
"strings"
"sync"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/cache"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/miekg/dns"
"golang.org/x/exp/slices"
"golang.org/x/net/publicsuffix"
)
// Safe browsing and parental control methods.
// TODO(a.garipov): Make configurable.
const (
dnsTimeout = 3 * time.Second
defaultSafebrowsingServer = `https://family.adguard-dns.com/dns-query`
defaultParentalServer = `https://family.adguard-dns.com/dns-query`
sbTXTSuffix = `sb.dns.adguard.com.`
pcTXTSuffix = `pc.dns.adguard.com.`
)
// SetParentalUpstream sets the parental upstream for *DNSFilter.
//
// TODO(e.burkov): Remove this in v1 API to forbid the direct access.
func (d *DNSFilter) SetParentalUpstream(u upstream.Upstream) {
d.parentalUpstream = u
}
// SetSafeBrowsingUpstream sets the safe browsing upstream for *DNSFilter.
//
// TODO(e.burkov): Remove this in v1 API to forbid the direct access.
func (d *DNSFilter) SetSafeBrowsingUpstream(u upstream.Upstream) {
d.safeBrowsingUpstream = u
}
func (d *DNSFilter) initSecurityServices() error {
var err error
d.safeBrowsingServer = defaultSafebrowsingServer
d.parentalServer = defaultParentalServer
opts := &upstream.Options{
Timeout: dnsTimeout,
ServerIPAddrs: []net.IP{
{94, 140, 14, 15},
{94, 140, 15, 16},
net.ParseIP("2a10:50c0::bad1:ff"),
net.ParseIP("2a10:50c0::bad2:ff"),
},
}
parUps, err := upstream.AddressToUpstream(d.parentalServer, opts)
if err != nil {
return fmt.Errorf("converting parental server: %w", err)
}
d.SetParentalUpstream(parUps)
sbUps, err := upstream.AddressToUpstream(d.safeBrowsingServer, opts)
if err != nil {
return fmt.Errorf("converting safe browsing server: %w", err)
}
d.SetSafeBrowsingUpstream(sbUps)
return nil
}
/*
expire byte[4]
hash byte[32]
...
*/
func (c *sbCtx) setCache(prefix, hashes []byte) {
d := make([]byte, 4+len(hashes))
expire := uint(time.Now().Unix()) + c.cacheTime*60
binary.BigEndian.PutUint32(d[:4], uint32(expire))
copy(d[4:], hashes)
c.cache.Set(prefix, d)
log.Debug("%s: stored in cache: %v", c.svc, prefix)
}
// findInHash returns 32-byte hash if it's found in hashToHost.
func (c *sbCtx) findInHash(val []byte) (hash32 [32]byte, found bool) {
for i := 4; i < len(val); i += 32 {
hash := val[i : i+32]
copy(hash32[:], hash[0:32])
_, found = c.hashToHost[hash32]
if found {
return hash32, found
}
}
return [32]byte{}, false
}
func (c *sbCtx) getCached() int {
now := time.Now().Unix()
hashesToRequest := map[[32]byte]string{}
for k, v := range c.hashToHost {
// nolint:looppointer // The subsilce is used for a safe cache lookup.
val := c.cache.Get(k[0:2])
if val == nil || now >= int64(binary.BigEndian.Uint32(val)) {
hashesToRequest[k] = v
continue
}
if hash32, found := c.findInHash(val); found {
log.Debug("%s: found in cache: %s: blocked by %v", c.svc, c.host, hash32)
return 1
}
}
if len(hashesToRequest) == 0 {
log.Debug("%s: found in cache: %s: not blocked", c.svc, c.host)
return -1
}
c.hashToHost = hashesToRequest
return 0
}
type sbCtx struct {
host string
svc string
hashToHost map[[32]byte]string
cache cache.Cache
cacheTime uint
}
func hostnameToHashes(host string) map[[32]byte]string {
hashes := map[[32]byte]string{}
tld, icann := publicsuffix.PublicSuffix(host)
if !icann {
// private suffixes like cloudfront.net
tld = ""
}
curhost := host
nDots := 0
for i := len(curhost) - 1; i >= 0; i-- {
if curhost[i] == '.' {
nDots++
if nDots == 4 {
curhost = curhost[i+1:] // "xxx.a.b.c.d" -> "a.b.c.d"
break
}
}
}
for {
if curhost == "" {
// we've reached end of string
break
}
if tld != "" && curhost == tld {
// we've reached the TLD, don't hash it
break
}
sum := sha256.Sum256([]byte(curhost))
hashes[sum] = curhost
pos := strings.IndexByte(curhost, byte('.'))
if pos < 0 {
break
}
curhost = curhost[pos+1:]
}
return hashes
}
// convert hash array to string
func (c *sbCtx) getQuestion() string {
b := &strings.Builder{}
for hash := range c.hashToHost {
// nolint:looppointer // The subsilce is used for safe hex encoding.
stringutil.WriteToBuilder(b, hex.EncodeToString(hash[0:2]), ".")
}
if c.svc == "SafeBrowsing" {
stringutil.WriteToBuilder(b, sbTXTSuffix)
return b.String()
}
stringutil.WriteToBuilder(b, pcTXTSuffix)
return b.String()
}
// Find the target hash in TXT response
func (c *sbCtx) processTXT(resp *dns.Msg) (bool, [][]byte) {
matched := false
hashes := [][]byte{}
for _, a := range resp.Answer {
txt, ok := a.(*dns.TXT)
if !ok {
continue
}
log.Debug("%s: received hashes for %s: %v", c.svc, c.host, txt.Txt)
for _, t := range txt.Txt {
if len(t) != 32*2 {
continue
}
hash, err := hex.DecodeString(t)
if err != nil {
continue
}
hashes = append(hashes, hash)
if !matched {
var hash32 [32]byte
copy(hash32[:], hash)
var hashHost string
hashHost, ok = c.hashToHost[hash32]
if ok {
log.Debug("%s: matched %s by %s/%s", c.svc, c.host, hashHost, t)
matched = true
}
}
}
}
return matched, hashes
}
func (c *sbCtx) storeCache(hashes [][]byte) {
slices.SortFunc(hashes, func(a, b []byte) (sortsBefore bool) {
return bytes.Compare(a, b) == -1
})
var curData []byte
var prevPrefix []byte
for i, hash := range hashes {
// nolint:looppointer // The subsilce is used for a safe comparison.
if !bytes.Equal(hash[0:2], prevPrefix) {
if i != 0 {
c.setCache(prevPrefix, curData)
curData = nil
}
prevPrefix = hashes[i][0:2]
}
curData = append(curData, hash...)
}
if len(prevPrefix) != 0 {
c.setCache(prevPrefix, curData)
}
for hash := range c.hashToHost {
// nolint:looppointer // The subsilce is used for a safe cache lookup.
prefix := hash[0:2]
val := c.cache.Get(prefix)
if val == nil {
c.setCache(prefix, nil)
}
}
}
func check(c *sbCtx, r Result, u upstream.Upstream) (Result, error) {
c.hashToHost = hostnameToHashes(c.host)
switch c.getCached() {
case -1:
return Result{}, nil
case 1:
return r, nil
}
question := c.getQuestion()
log.Tracef("%s: checking %s: %s", c.svc, c.host, question)
req := (&dns.Msg{}).SetQuestion(question, dns.TypeTXT)
resp, err := u.Exchange(req)
if err != nil {
return Result{}, err
}
matched, receivedHashes := c.processTXT(resp)
c.storeCache(receivedHashes)
if matched {
return r, nil
}
return Result{}, nil
}
// TODO(a.garipov): Unify with checkParental.
func (d *DNSFilter) checkSafeBrowsing(
host string,
_ uint16,
setts *Settings,
) (res Result, err error) {
if !setts.ProtectionEnabled || !setts.SafeBrowsingEnabled {
return Result{}, nil
}
if log.GetLevel() >= log.DEBUG {
timer := log.StartTimer()
defer timer.LogElapsed("safebrowsing lookup for %q", host)
}
sctx := &sbCtx{
host: host,
svc: "SafeBrowsing",
cache: d.safebrowsingCache,
cacheTime: d.Config.CacheTime,
}
res = Result{
Rules: []*ResultRule{{
Text: "adguard-malware-shavar",
FilterListID: SafeBrowsingListID,
}},
Reason: FilteredSafeBrowsing,
IsFiltered: true,
}
return check(sctx, res, d.safeBrowsingUpstream)
}
// TODO(a.garipov): Unify with checkSafeBrowsing.
func (d *DNSFilter) checkParental(
host string,
_ uint16,
setts *Settings,
) (res Result, err error) {
if !setts.ProtectionEnabled || !setts.ParentalEnabled {
return Result{}, nil
}
if log.GetLevel() >= log.DEBUG {
timer := log.StartTimer()
defer timer.LogElapsed("parental lookup for %q", host)
}
sctx := &sbCtx{
host: host,
svc: "Parental",
cache: d.parentalCache,
cacheTime: d.Config.CacheTime,
}
res = Result{
Rules: []*ResultRule{{
Text: "parental CATEGORY_BLACKLISTED",
FilterListID: ParentalListID,
}},
Reason: FilteredParental,
IsFiltered: true,
}
return check(sctx, res, d.parentalUpstream)
}
// setProtectedBool sets the value of a boolean pointer under a lock. l must
// protect the value under ptr.
//
// TODO(e.burkov): Make it generic?
func setProtectedBool(mu *sync.RWMutex, ptr *bool, val bool) {
mu.Lock()
defer mu.Unlock()
*ptr = val
}
// protectedBool gets the value of a boolean pointer under a read lock. l must
// protect the value under ptr.
//
// TODO(e.burkov): Make it generic?
func protectedBool(mu *sync.RWMutex, ptr *bool) (val bool) {
mu.RLock()
defer mu.RUnlock()
return *ptr
}
func (d *DNSFilter) handleSafeBrowsingEnable(w http.ResponseWriter, r *http.Request) {
setProtectedBool(&d.confLock, &d.Config.SafeBrowsingEnabled, true)
d.Config.ConfigModified()
}
func (d *DNSFilter) handleSafeBrowsingDisable(w http.ResponseWriter, r *http.Request) {
setProtectedBool(&d.confLock, &d.Config.SafeBrowsingEnabled, false)
d.Config.ConfigModified()
}
func (d *DNSFilter) handleSafeBrowsingStatus(w http.ResponseWriter, r *http.Request) {
resp := &struct {
Enabled bool `json:"enabled"`
}{
Enabled: protectedBool(&d.confLock, &d.Config.SafeBrowsingEnabled),
}
_ = aghhttp.WriteJSONResponse(w, r, resp)
}
func (d *DNSFilter) handleParentalEnable(w http.ResponseWriter, r *http.Request) {
setProtectedBool(&d.confLock, &d.Config.ParentalEnabled, true)
d.Config.ConfigModified()
}
func (d *DNSFilter) handleParentalDisable(w http.ResponseWriter, r *http.Request) {
setProtectedBool(&d.confLock, &d.Config.ParentalEnabled, false)
d.Config.ConfigModified()
}
func (d *DNSFilter) handleParentalStatus(w http.ResponseWriter, r *http.Request) {
resp := &struct {
Enabled bool `json:"enabled"`
}{
Enabled: protectedBool(&d.confLock, &d.Config.ParentalEnabled),
}
_ = aghhttp.WriteJSONResponse(w, r, resp)
}

226
internal/filtering/safebrowsing_test.go
View File

@@ -1,226 +0,0 @@
package filtering
import (
"crypto/sha256"
"strings"
"testing"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/golibs/cache"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSafeBrowsingHash(t *testing.T) {
// test hostnameToHashes()
hashes := hostnameToHashes("1.2.3.sub.host.com")
assert.Len(t, hashes, 3)
_, ok := hashes[sha256.Sum256([]byte("3.sub.host.com"))]
assert.True(t, ok)
_, ok = hashes[sha256.Sum256([]byte("sub.host.com"))]
assert.True(t, ok)
_, ok = hashes[sha256.Sum256([]byte("host.com"))]
assert.True(t, ok)
_, ok = hashes[sha256.Sum256([]byte("com"))]
assert.False(t, ok)
c := &sbCtx{
svc: "SafeBrowsing",
hashToHost: hashes,
}
q := c.getQuestion()
assert.Contains(t, q, "7a1b.")
assert.Contains(t, q, "af5a.")
assert.Contains(t, q, "eb11.")
assert.True(t, strings.HasSuffix(q, "sb.dns.adguard.com."))
}
func TestSafeBrowsingCache(t *testing.T) {
c := &sbCtx{
svc: "SafeBrowsing",
cacheTime: 100,
}
conf := cache.Config{}
c.cache = cache.New(conf)
// store in cache hashes for "3.sub.host.com" and "host.com"
// and empty data for hash-prefix for "sub.host.com"
hash := sha256.Sum256([]byte("sub.host.com"))
c.hashToHost = make(map[[32]byte]string)
c.hashToHost[hash] = "sub.host.com"
var hashesArray [][]byte
hash4 := sha256.Sum256([]byte("3.sub.host.com"))
hashesArray = append(hashesArray, hash4[:])
hash2 := sha256.Sum256([]byte("host.com"))
hashesArray = append(hashesArray, hash2[:])
c.storeCache(hashesArray)
// match "3.sub.host.com" or "host.com" from cache
c.hashToHost = make(map[[32]byte]string)
hash = sha256.Sum256([]byte("3.sub.host.com"))
c.hashToHost[hash] = "3.sub.host.com"
hash = sha256.Sum256([]byte("sub.host.com"))
c.hashToHost[hash] = "sub.host.com"
hash = sha256.Sum256([]byte("host.com"))
c.hashToHost[hash] = "host.com"
assert.Equal(t, 1, c.getCached())
// match "sub.host.com" from cache
c.hashToHost = make(map[[32]byte]string)
hash = sha256.Sum256([]byte("sub.host.com"))
c.hashToHost[hash] = "sub.host.com"
assert.Equal(t, -1, c.getCached())
// Match "sub.host.com" from cache. Another hash for "host.example" is not
// in the cache, so get data for it from the server.
c.hashToHost = make(map[[32]byte]string)
hash = sha256.Sum256([]byte("sub.host.com"))
c.hashToHost[hash] = "sub.host.com"
hash = sha256.Sum256([]byte("host.example"))
c.hashToHost[hash] = "host.example"
assert.Empty(t, c.getCached())
hash = sha256.Sum256([]byte("sub.host.com"))
_, ok := c.hashToHost[hash]
assert.False(t, ok)
hash = sha256.Sum256([]byte("host.example"))
_, ok = c.hashToHost[hash]
assert.True(t, ok)
c = &sbCtx{
svc: "SafeBrowsing",
cacheTime: 100,
}
conf = cache.Config{}
c.cache = cache.New(conf)
hash = sha256.Sum256([]byte("sub.host.com"))
c.hashToHost = make(map[[32]byte]string)
c.hashToHost[hash] = "sub.host.com"
c.cache.Set(hash[0:2], make([]byte, 32))
assert.Empty(t, c.getCached())
}
func TestSBPC_checkErrorUpstream(t *testing.T) {
d, _ := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil)
t.Cleanup(d.Close)
ups := aghtest.NewErrorUpstream()
d.SetSafeBrowsingUpstream(ups)
d.SetParentalUpstream(ups)
setts := &Settings{
ProtectionEnabled: true,
SafeBrowsingEnabled: true,
ParentalEnabled: true,
}
_, err := d.checkSafeBrowsing("smthng.com", dns.TypeA, setts)
assert.Error(t, err)
_, err = d.checkParental("smthng.com", dns.TypeA, setts)
assert.Error(t, err)
}
func TestSBPC(t *testing.T) {
d, _ := newForTest(t, &Config{SafeBrowsingEnabled: true}, nil)
t.Cleanup(d.Close)
const hostname = "example.org"
setts := &Settings{
ProtectionEnabled: true,
SafeBrowsingEnabled: true,
ParentalEnabled: true,
}
testCases := []struct {
testCache cache.Cache
testFunc func(host string, _ uint16, _ *Settings) (res Result, err error)
name string
block bool
}{{
testCache: d.safebrowsingCache,
testFunc: d.checkSafeBrowsing,
name: "sb_no_block",
block: false,
}, {
testCache: d.safebrowsingCache,
testFunc: d.checkSafeBrowsing,
name: "sb_block",
block: true,
}, {
testCache: d.parentalCache,
testFunc: d.checkParental,
name: "pc_no_block",
block: false,
}, {
testCache: d.parentalCache,
testFunc: d.checkParental,
name: "pc_block",
block: true,
}}
for _, tc := range testCases {
// Prepare the upstream.
ups := aghtest.NewBlockUpstream(hostname, tc.block)
var numReq int
onExchange := ups.OnExchange
ups.OnExchange = func(req *dns.Msg) (resp *dns.Msg, err error) {
numReq++
return onExchange(req)
}
d.SetSafeBrowsingUpstream(ups)
d.SetParentalUpstream(ups)
t.Run(tc.name, func(t *testing.T) {
// Firstly, check the request blocking.
hits := 0
res, err := tc.testFunc(hostname, dns.TypeA, setts)
require.NoError(t, err)
if tc.block {
assert.True(t, res.IsFiltered)
require.Len(t, res.Rules, 1)
hits++
} else {
require.False(t, res.IsFiltered)
}
// Check the cache state, check the response is now cached.
assert.Equal(t, 1, tc.testCache.Stats().Count)
assert.Equal(t, hits, tc.testCache.Stats().Hit)
// There was one request to an upstream.
assert.Equal(t, 1, numReq)
// Now make the same request to check the cache was used.
res, err = tc.testFunc(hostname, dns.TypeA, setts)
require.NoError(t, err)
if tc.block {
assert.True(t, res.IsFiltered)
require.Len(t, res.Rules, 1)
} else {
require.False(t, res.IsFiltered)
}
// Check the cache state, it should've been used.
assert.Equal(t, 1, tc.testCache.Stats().Count)
assert.Equal(t, hits+1, tc.testCache.Stats().Hit)
// Check that there were no additional requests.
assert.Equal(t, 1, numReq)
})
purgeCaches(d)
}
}

38
internal/filtering/servicelist.go
View File

@@ -12,6 +12,14 @@ type blockedService struct {
// blockedServices contains raw blocked service data.
var blockedServices = []blockedService{{
ID: "500px",
Name: "500px",
IconSVG: []byte("<svg xmlns=\"http://www.w3.org/2000/svg\" fill=\"currentColor\" viewBox=\"0 0 50 50\"><path d=\"M 5 14 L 2.5 26 L 6.800781 26 C 6.800781 26 7.699219 24.300781 10.199219 24.300781 C 12.699219 24.300781 14 26.199219 14 28.300781 C 14 30.402344 12.5 32.800781 10.199219 32.800781 C 7.898438 32.800781 6.5 30.398438 6.5 29 L 2 29 C 2 30.199219 3 36 10.199219 36 C 15.15625 36 17.417969 33.121094 18.015625 31.898438 C 19.386719 34.34375 21.992188 36 24.984375 36 C 27.253906 36 29.777344 34.808594 32.5 32.453125 C 35.222656 34.808594 37.746094 36 40.015625 36 C 44.417969 36 48 32.410156 48 28 C 48 23.589844 44.417969 20 40.015625 20 C 37.746094 20 35.222656 21.191406 32.5 23.546875 C 29.777344 21.191406 27.253906 20 24.984375 20 C 21.832031 20 19.105469 21.847656 17.8125 24.511719 C 17.113281 23.382813 15.414063 21 11.902344 21 C 8.101563 21 7.300781 22.597656 7.300781 22.597656 C 7.300781 22.597656 7.699219 21.300781 8.300781 18 L 17 18 L 17 14 Z M 24.984375 25 C 25.453125 25 26.800781 25.226563 29.230469 27.328125 L 30.011719 28 L 29.230469 28.671875 C 26.800781 30.773438 25.453125 31 24.984375 31 C 23.339844 31 22 29.652344 22 28 C 22 26.347656 23.339844 25 24.984375 25 Z M 40.015625 25 C 41.660156 25 43 26.347656 43 28 C 43 29.652344 41.660156 31 40.015625 31 C 39.546875 31 38.199219 30.773438 35.769531 28.671875 L 34.988281 28 L 35.769531 27.328125 C 38.199219 25.226563 39.546875 25 40.015625 25 Z\"/></svg>"),
Rules: []string{
"||500px.com^",
"||500px.org^",
},
}, {
ID: "9gag",
Name: "9GAG",
IconSVG: []byte("<svg xmlns=\"http://www.w3.org/2000/svg\" fill=\"currentColor\" viewBox=\"0 0 50 50\"><path d=\"M 44 14 C 44 13.644531 43.8125 13.316406 43.507813 13.136719 C 40.453125 11.347656 28.46875 4.847656 25.535156 3.136719 C 25.222656 2.957031 24.839844 2.957031 24.527344 3.136719 C 21.128906 5.117188 10.089844 11.621094 7.496094 13.136719 C 7.1875 13.316406 7 13.644531 7 14 L 7 20 C 7 20.378906 7.214844 20.722656 7.550781 20.894531 C 7.660156 20.949219 18.597656 26.453125 24.5 29.867188 C 24.8125 30.046875 25.195313 30.046875 25.507813 29.863281 C 27.269531 28.828125 29.117188 27.859375 30.902344 26.921875 C 32.253906 26.214844 33.636719 25.488281 35.003906 24.722656 C 35.007813 26.820313 35.003906 29.296875 35 30.40625 L 25 35.859375 L 14.480469 30.121094 C 14.144531 29.9375 13.730469 29.964844 13.417969 30.1875 L 6.417969 35.1875 C 6.140625 35.386719 5.980469 35.714844 6.003906 36.054688 C 6.023438 36.398438 6.214844 36.707031 6.515625 36.871094 L 24.542969 46.871094 C 24.695313 46.957031 24.859375 47 25.027344 47 C 25.195313 47 25.363281 46.957031 25.515625 46.875 L 43.484375 36.875 C 43.804688 36.695313 44 36.363281 44 36 C 44 36 43.992188 21.011719 44 14 Z M 25 20 L 18 16 L 25 12 L 32 16 Z\" /></svg>"),
@@ -1180,6 +1188,18 @@ var blockedServices = []blockedService{{
"||zuckerberg.com^",
"||zuckerberg.net^",
},
}, {
ID: "flickr",
Name: "Flickr",
IconSVG: []byte("<svg xmlns=\"http://www.w3.org/2000/svg\" fill=\"currentColor\" viewBox=\"0 0 50 50\"><path d=\"M 9 4 C 6.2504839 4 4 6.2504839 4 9 L 4 41 C 4 43.749516 6.2504839 46 9 46 L 41 46 C 43.749516 46 46 43.749516 46 41 L 46 9 C 46 6.2504839 43.749516 4 41 4 L 9 4 z M 9 6 L 41 6 C 42.668484 6 44 7.3315161 44 9 L 44 41 C 44 42.668484 42.668484 44 41 44 L 9 44 C 7.3315161 44 6 42.668484 6 41 L 6 9 C 6 7.3315161 7.3315161 6 9 6 z M 16 17 C 11.59 17 8 20.59 8 25 C 8 29.41 11.59 33 16 33 C 20.41 33 24 29.41 24 25 C 24 20.59 20.41 17 16 17 z M 34 17 C 29.59 17 26 20.59 26 25 C 26 29.41 29.59 33 34 33 C 38.41 33 42 29.41 42 25 C 42 20.59 38.41 17 34 17 z\"/></svg>"),
Rules: []string{
"||flic.kr^",
"||flickr.com^",
"||flickr.net^",
"||flickrprints.com^",
"||flickrpro.com^",
"||staticflickr.com^",
},
}, {
ID: "gog",
Name: "GOG",
@@ -1325,6 +1345,13 @@ var blockedServices = []blockedService{{
"||kakao.com^",
"||kgslb.com^",
},
}, {
ID: "kik",
Name: "Kik",
IconSVG: []byte("<svg xmlns=\"http://www.w3.org/2000/svg\" fill=\"currentColor\" viewBox=\"0 0 50 50\"><path d=\"M 3.5039062 12 C 1.9347705 11.994817 0.87857579 12.97636 0.4453125 13.849609 C 0.01204921 14.722858 0 15.564453 0 15.564453 A 1.0001 1.0001 0 0 0 0 15.59375 L 0 35 A 1.0001 1.0001 0 0 0 0.00390625 35.078125 C 0.00390625 35.078125 0.05696144 35.828363 0.5390625 36.554688 C 1.0211636 37.281011 2.0459252 38.004441 3.5019531 38.001953 C 4.8916439 38.000053 5.8837351 37.273604 6.3769531 36.578125 C 6.8701712 35.882646 6.9863281 35.166016 6.9863281 35.166016 A 1.0001 1.0001 0 0 0 7 35 L 7 31.802734 L 10.167969 36.554688 L 10.130859 36.494141 C 10.511831 37.164615 11.143097 37.525465 11.742188 37.730469 C 12.341278 37.935473 12.950104 38.001953 13.5 38.001953 C 15.411725 38.001953 17 36.431487 17 34.5 C 17 34.056649 16.90825 34.03442 16.851562 33.912109 C 16.794882 33.789799 16.730864 33.671331 16.654297 33.537109 C 16.501163 33.268666 16.298339 32.944015 16.058594 32.572266 C 15.579103 31.828767 14.950355 30.90254 14.322266 29.992188 C 13.310206 28.525308 12.655222 27.610988 12.300781 27.113281 L 14.707031 24.707031 A 1.0001 1.0001 0 0 0 14.738281 24.673828 C 14.738281 24.673828 15.354706 24.012223 15.748047 23.042969 C 16.141388 22.073714 16.298687 20.56089 15.259766 19.349609 C 14.281705 18.208994 12.842689 18.141009 11.925781 18.416016 C 11.008874 18.691022 10.371094 19.222656 10.371094 19.222656 A 1.0001 1.0001 0 0 0 10.292969 19.292969 L 6.9980469 22.587891 L 6.9921875 15.646484 A 1.0001 1.0001 0 0 0 6.9902344 15.580078 C 6.9902344 15.580078 6.9441634 14.743069 6.5058594 13.875 C 6.0675579 13.006938 5.0412971 12.005313 3.5039062 12 z M 30.503906 12 C 28.93477 11.9948 27.878577 12.97636 27.445312 13.849609 C 27.012049 14.722858 27 15.564453 27 15.564453 A 1.0001 1.0001 0 0 0 27 15.59375 L 27 35 A 1.0001 1.0001 0 0 0 27.003906 35.078125 C 27.003906 35.078125 27.056966 35.828363 27.539062 36.554688 C 28.021165 37.281011 29.045925 38.004441 30.501953 38.001953 C 31.891644 38.000053 32.883735 37.273604 33.376953 36.578125 C 33.870171 35.882646 33.986328 35.166016 33.986328 35.166016 A 1.0001 1.0001 0 0 0 34 35 L 34 31.802734 L 37.167969 36.554688 L 37.130859 36.494141 C 37.511831 37.164615 38.143096 37.525465 38.742188 37.730469 C 39.341277 37.935473 39.950104 38.001953 40.5 38.001953 C 42.411725 38.001953 44 36.431487 44 34.5 C 44 34.056649 43.908251 34.03442 43.851562 33.912109 C 43.794882 33.789799 43.730864 33.671331 43.654297 33.537109 C 43.501163 33.268666 43.298339 32.944015 43.058594 32.572266 C 42.579103 31.828767 41.950355 30.90254 41.322266 29.992188 C 40.310206 28.525308 39.655222 27.610988 39.300781 27.113281 L 41.707031 24.707031 A 1.0001 1.0001 0 0 0 41.738281 24.673828 C 41.738281 24.673828 42.354706 24.012223 42.748047 23.042969 C 43.141388 22.073714 43.298687 20.56089 42.259766 19.349609 C 41.281705 18.208994 39.842689 18.141009 38.925781 18.416016 C 38.008874 18.691022 37.371094 19.222656 37.371094 19.222656 A 1.0001 1.0001 0 0 0 37.292969 19.292969 L 33.998047 22.587891 L 33.992188 15.646484 A 1.0001 1.0001 0 0 0 33.990234 15.580078 C 33.990234 15.580078 33.944164 14.743069 33.505859 13.875 C 33.067647 13.006938 32.041297 12.005313 30.503906 12 z M 21.507812 18 C 19.85324 17.98686 18.785557 19.124468 18.382812 20.09375 C 18.181441 20.578391 18.090615 21.031738 18.044922 21.375 C 18.022072 21.546631 18.011459 21.69063 18.005859 21.796875 C 18.000252 21.90312 18 22.065333 18 21.984375 L 17.982422 34.998047 A 1.0001 1.0001 0 0 0 17.990234 35.134766 C 17.990234 35.134766 18.085674 35.862804 18.576172 36.568359 C 19.06667 37.273915 20.071581 37.997467 21.486328 38 C 22.885358 38.0026 23.885897 37.278643 24.380859 36.580078 C 24.875822 35.881513 24.986328 35.160156 24.986328 35.160156 A 1.0001 1.0001 0 0 0 25 35 L 25 21.996094 C 25 21.996094 25.02572 21.084043 24.625 20.117188 C 24.224283 19.150332 23.164841 18.013078 21.507812 18 z M 46.5 24 C 44.578848 24 43 25.578848 43 27.5 C 43 29.421152 44.578848 31 46.5 31 C 48.421152 31 50 29.421152 50 27.5 C 50 25.578848 48.421152 24 46.5 24 z M 46.5 26 C 47.340272 26 48 26.659728 48 27.5 C 48 28.340272 47.340272 29 46.5 29 C 45.659728 29 45 28.340272 45 27.5 C 45 26.659728 45.659728 26 46.5 26 z\"/></svg>"),
Rules: []string{
"||kik.com^",
},
}, {
ID: "lazada",
Name: "Lazada",
@@ -1385,6 +1412,7 @@ var blockedServices = []blockedService{{
Rules: []string{
"||aus.social^",
"||awscommunity.social^",
"||climatejustice.social^",
"||cyberplace.social^",
"||defcon.social^",
"||det.social^",
@@ -1442,13 +1470,13 @@ var blockedServices = []blockedService{{
"||mstdn.plus^",
"||mstdn.social^",
"||muenchen.social^",
"||muenster.im^",
"||newsie.social^",
"||noc.social^",
"||norden.social^",
"||nrw.social^",
"||o3o.ca^",
"||ohai.social^",
"||pewtix.com^",
"||piaille.fr^",
"||pol.social^",
"||ravenation.club^",
@@ -1480,7 +1508,6 @@ var blockedServices = []blockedService{{
"||union.place^",
"||universeodon.com^",
"||urbanists.social^",
"||wien.rocks^",
"||wxw.moe^",
},
}, {
@@ -1827,6 +1854,13 @@ var blockedServices = []blockedService{{
"||tx.me^",
"||usercontent.dev^",
},
}, {
ID: "tidal",
Name: "Tidal",
IconSVG: []byte("<svg xmlns=\"http://www.w3.org/2000/svg\" fill=\"currentColor\" viewBox=\"0 0 50 50\"><path d=\"M 9 12 C 8.7615 12 8.5237969 12.091437 8.3417969 12.273438 L 1.2734375 19.341797 C 0.9094375 19.705797 0.9094375 20.294203 1.2734375 20.658203 L 8.3417969 27.726562 C 8.7057969 28.090563 9.2942031 28.090563 9.6582031 27.726562 L 16.726562 20.658203 C 16.908563 20.476203 17 20.2385 17 20 C 17 19.7615 16.908563 19.523797 16.726562 19.341797 L 9.6582031 12.273438 C 9.4762031 12.091437 9.2385 12 9 12 z M 17 20 C 17 20.2385 17.091438 20.476203 17.273438 20.658203 L 24.341797 27.726562 C 24.523797 27.908563 24.7615 28 25 28 C 25.2385 28 25.476203 27.908563 25.658203 27.726562 L 32.726562 20.658203 C 32.908563 20.476203 33 20.2385 33 20 C 33 19.7615 32.908563 19.523797 32.726562 19.341797 L 25.658203 12.273438 C 25.294203 11.909437 24.705797 11.909437 24.341797 12.273438 L 17.273438 19.341797 C 17.091437 19.523797 17 19.7615 17 20 z M 33 20 C 33 20.2385 33.091437 20.476203 33.273438 20.658203 L 40.341797 27.726562 C 40.705797 28.090563 41.294203 28.090563 41.658203 27.726562 L 48.726562 20.658203 C 49.090563 20.294203 49.090563 19.705797 48.726562 19.341797 L 41.658203 12.273438 C 41.294203 11.909437 40.705797 11.909437 40.341797 12.273438 L 33.273438 19.341797 C 33.091437 19.523797 33 19.7615 33 20 z M 25 28 C 24.7615 28 24.523797 28.091437 24.341797 28.273438 L 17.273438 35.341797 C 16.909437 35.705797 16.909437 36.294203 17.273438 36.658203 L 24.341797 43.726562 C 24.705797 44.090562 25.294203 44.090562 25.658203 43.726562 L 32.726562 36.658203 C 33.090563 36.294203 33.090563 35.705797 32.726562 35.341797 L 25.658203 28.273438 C 25.476203 28.091437 25.2385 28 25 28 z\"/></svg>"),
Rules: []string{
"||tidal.com^",
},
}, {
ID: "tiktok",
Name: "TikTok",

14
internal/home/client.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/AdGuardHome/internal/filtering/safesearch"
"github.com/AdguardTeam/dnsproxy/proxy"
"github.com/AdguardTeam/golibs/stringutil"
)
// Client contains information about persistent clients.
@@ -37,6 +38,19 @@ type Client struct {
IgnoreStatistics bool
}
// ShallowClone returns a deep copy of the client, except upstreamConfig,
// safeSearchConf, SafeSearch fields, because it's difficult to copy them.
func (c *Client) ShallowClone() (sh *Client) {
clone := *c
clone.IDs = stringutil.CloneSlice(c.IDs)
clone.Tags = stringutil.CloneSlice(c.Tags)
clone.BlockedServices = stringutil.CloneSlice(c.BlockedServices)
clone.Upstreams = stringutil.CloneSlice(c.Upstreams)
return &clone
}
// closeUpstreams closes the client-specific upstream config of c if any.
func (c *Client) closeUpstreams() (err error) {
if c.upstreamConfig != nil {

101
internal/home/clients.go
View File

@@ -378,6 +378,7 @@ func (clients *clientsContainer) clientOrArtificial(
}, true
}
// Find returns a shallow copy of the client if there is one found.
func (clients *clientsContainer) Find(id string) (c *Client, ok bool) {
clients.lock.Lock()
defer clients.lock.Unlock()
@@ -387,20 +388,18 @@ func (clients *clientsContainer) Find(id string) (c *Client, ok bool) {
return nil, false
}
c.IDs = stringutil.CloneSlice(c.IDs)
c.Tags = stringutil.CloneSlice(c.Tags)
c.BlockedServices = stringutil.CloneSlice(c.BlockedServices)
c.Upstreams = stringutil.CloneSlice(c.Upstreams)
return c, true
return c.ShallowClone(), true
}
// shouldCountClient is a wrapper around Find to make it a valid client
// information finder for the statistics. If no information about the client
// is found, it returns true.
func (clients *clientsContainer) shouldCountClient(ids []string) (y bool) {
clients.lock.Lock()
defer clients.lock.Unlock()
for _, id := range ids {
client, ok := clients.Find(id)
client, ok := clients.findLocked(id)
if ok {
return !client.IgnoreStatistics
}
@@ -617,6 +616,15 @@ func (clients *clientsContainer) Add(c *Client) (ok bool, err error) {
}
}
clients.add(c)
log.Debug("clients: added %q: ID:%q [%d]", c.Name, c.IDs, len(clients.list))
return true, nil
}
// add c to the indexes. clients.lock is expected to be locked.
func (clients *clientsContainer) add(c *Client) {
// update Name index
clients.list[c.Name] = c
@@ -624,10 +632,6 @@ func (clients *clientsContainer) Add(c *Client) (ok bool, err error) {
for _, id := range c.IDs {
clients.idIndex[id] = c
}
log.Debug("clients: added %q: ID:%q [%d]", c.Name, c.IDs, len(clients.list))
return true, nil
}
// Del removes a client. ok is false if there is no such client.
@@ -645,86 +649,53 @@ func (clients *clientsContainer) Del(name string) (ok bool) {
log.Error("client container: removing client %s: %s", name, err)
}
clients.del(c)
return true
}
// del removes c from the indexes. clients.lock is expected to be locked.
func (clients *clientsContainer) del(c *Client) {
// update Name index
delete(clients.list, name)
delete(clients.list, c.Name)
// update ID index
for _, id := range c.IDs {
delete(clients.idIndex, id)
}
return true
}
// Update updates a client by its name.
func (clients *clientsContainer) Update(name string, c *Client) (err error) {
func (clients *clientsContainer) Update(prev, c *Client) (err error) {
err = clients.check(c)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return err
}
clients.lock.Lock()
defer clients.lock.Unlock()
prev, ok := clients.list[name]
if !ok {
return errors.Error("client not found")
}
// First, check the name index.
// Check the name index.
if prev.Name != c.Name {
_, ok = clients.list[c.Name]
_, ok := clients.list[c.Name]
if ok {
return errors.Error("client already exists")
}
}
// Second, update the ID index.
err = clients.updateIDIndex(prev, c.IDs)
if err != nil {
// Don't wrap the error, because it's informative enough as is.
return err
}
// Update name index.
if prev.Name != c.Name {
delete(clients.list, prev.Name)
clients.list[c.Name] = prev
}
// Update upstreams cache.
err = c.closeUpstreams()
if err != nil {
return err
}
*prev = *c
return nil
}
// updateIDIndex updates the ID index data for cli using the information from
// newIDs.
func (clients *clientsContainer) updateIDIndex(cli *Client, newIDs []string) (err error) {
if slices.Equal(cli.IDs, newIDs) {
return nil
}
for _, id := range newIDs {
existing, ok := clients.idIndex[id]
if ok && existing != cli {
return fmt.Errorf("id %q is used by client with name %q", id, existing.Name)
// Check the ID index.
if !slices.Equal(prev.IDs, c.IDs) {
for _, id := range c.IDs {
existing, ok := clients.idIndex[id]
if ok && existing != prev {
return fmt.Errorf("id %q is used by client with name %q", id, existing.Name)
}
}
}
// Update the IDs in the index.
for _, id := range cli.IDs {
delete(clients.idIndex, id)
}
for _, id := range newIDs {
clients.idIndex[id] = cli
}
clients.del(prev)
clients.add(c)
return nil
}

26
internal/home/clients_test.go
View File

@@ -98,22 +98,8 @@ func TestClients(t *testing.T) {
assert.False(t, ok)
})
t.Run("update_fail_name", func(t *testing.T) {
err := clients.Update("client3", &Client{
IDs: []string{"1.2.3.0"},
Name: "client3",
})
require.Error(t, err)
err = clients.Update("client3", &Client{
IDs: []string{"1.2.3.0"},
Name: "client2",
})
assert.Error(t, err)
})
t.Run("update_fail_ip", func(t *testing.T) {
err := clients.Update("client1", &Client{
err := clients.Update(&Client{Name: "client1"}, &Client{
IDs: []string{"2.2.2.2"},
Name: "client1",
})
@@ -129,7 +115,10 @@ func TestClients(t *testing.T) {
cliNewIP = netip.MustParseAddr(cliNew)
)
err := clients.Update("client1", &Client{
prev, ok := clients.list["client1"]
require.True(t, ok)
err := clients.Update(prev, &Client{
IDs: []string{cliNew},
Name: "client1",
})
@@ -138,7 +127,10 @@ func TestClients(t *testing.T) {
assert.Equal(t, clients.clientSource(cliOldIP), ClientSourceNone)
assert.Equal(t, clients.clientSource(cliNewIP), ClientSourcePersistent)
err = clients.Update("client1", &Client{
prev, ok = clients.list["client1"]
require.True(t, ok)
err = clients.Update(prev, &Client{
IDs: []string{cliNew},
Name: "client1-renamed",
UseOwnSettings: true,

2
internal/home/clientshttp.go
View File

@@ -289,7 +289,7 @@ func (clients *clientsContainer) handleUpdateClient(w http.ResponseWriter, r *ht
return
}
err = clients.Update(dj.Name, c)
err = clients.Update(prev, c)
if err != nil {
aghhttp.Error(r, w, http.StatusBadRequest, "%s", err)

41
internal/home/config.go
View File

@@ -399,19 +399,39 @@ func (c *configuration) getConfigFilename() string {
return configFile
}
// getLogSettings reads logging settings from the config file.
// we do it in a separate method in order to configure logger before the actual configuration is parsed and applied.
func getLogSettings() logSettings {
l := logSettings{}
// readLogSettings reads logging settings from the config file. We do it in a
// separate method in order to configure logger before the actual configuration
// is parsed and applied.
func readLogSettings() (ls *logSettings) {
ls = &logSettings{}
yamlFile, err := readConfigFile()
if err != nil {
return l
return ls
}
err = yaml.Unmarshal(yamlFile, &l)
err = yaml.Unmarshal(yamlFile, ls)
if err != nil {
log.Error("Couldn't get logging settings from the configuration: %s", err)
}
return l
return ls
}
// validateBindHosts returns error if any of binding hosts from configuration is
// not a valid IP address.
func validateBindHosts(conf *configuration) (err error) {
if !conf.BindHost.IsValid() {
return errors.Error("bind_host is not a valid ip address")
}
for i, addr := range conf.DNS.BindHosts {
if !addr.IsValid() {
return fmt.Errorf("dns.bind_hosts at index %d is not a valid ip address", i)
}
}
return nil
}
// parseConfig loads configuration from the YAML file
@@ -425,6 +445,13 @@ func parseConfig() (err error) {
config.fileData = nil
err = yaml.Unmarshal(fileData, &config)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return err
}
err = validateBindHosts(config)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return err
}

2
internal/home/control.go
View File

@@ -180,7 +180,7 @@ func registerControlHandlers() {
httpRegister(http.MethodGet, "/control/status", handleStatus)
httpRegister(http.MethodPost, "/control/i18n/change_language", handleI18nChangeLanguage)
httpRegister(http.MethodGet, "/control/i18n/current_language", handleI18nCurrentLanguage)
Context.mux.HandleFunc("/control/version.json", postInstall(optionalAuth(handleGetVersionJSON)))
Context.mux.HandleFunc("/control/version.json", postInstall(optionalAuth(handleVersionJSON)))
httpRegister(http.MethodPost, "/control/update", handleUpdate)
httpRegister(http.MethodGet, "/control/profile", handleGetProfile)
httpRegister(http.MethodPut, "/control/profile/update", handlePutProfile)

11
internal/home/controlupdate.go
View File

@@ -26,15 +26,14 @@ type temporaryError interface {
Temporary() (ok bool)
}
// Get the latest available version from the Internet
func handleGetVersionJSON(w http.ResponseWriter, r *http.Request) {
// handleVersionJSON is the handler for the POST /control/version.json HTTP API.
//
// TODO(a.garipov): Find out if this API used with a GET method by anyone.
func handleVersionJSON(w http.ResponseWriter, r *http.Request) {
resp := &versionResponse{}
if Context.disableUpdate {
resp.Disabled = true
err := json.NewEncoder(w).Encode(resp)
if err != nil {
aghhttp.Error(r, w, http.StatusInternalServerError, "writing body: %s", err)
}
_ = aghhttp.WriteJSONResponse(w, r, resp)
return
}

368
internal/home/home.go
View File

@@ -27,14 +27,17 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/dhcpd"
"github.com/AdguardTeam/AdGuardHome/internal/dnsforward"
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/AdGuardHome/internal/filtering/hashprefix"
"github.com/AdguardTeam/AdGuardHome/internal/filtering/safesearch"
"github.com/AdguardTeam/AdGuardHome/internal/querylog"
"github.com/AdguardTeam/AdGuardHome/internal/stats"
"github.com/AdguardTeam/AdGuardHome/internal/updater"
"github.com/AdguardTeam/AdGuardHome/internal/version"
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/stringutil"
"golang.org/x/exp/slices"
"gopkg.in/natefinch/lumberjack.v2"
)
@@ -143,7 +146,9 @@ func Main(clientBuildFS fs.FS) {
run(opts, clientBuildFS)
}
func setupContext(opts options) {
// setupContext initializes [Context] fields. It also reads and upgrades
// config file if necessary.
func setupContext(opts options) (err error) {
setupContextFlags(opts)
Context.tlsRoots = aghtls.SystemRootCAs()
@@ -160,10 +165,15 @@ func setupContext(opts options) {
},
}
Context.mux = http.NewServeMux()
if !Context.firstRun {
// Do the upgrade if necessary.
err := upgradeConfig()
fatalOnError(err)
err = upgradeConfig()
if err != nil {
// Don't wrap the error, because it's informative enough as is.
return err
}
if err = parseConfig(); err != nil {
log.Error("parsing configuration file: %s", err)
@@ -179,11 +189,14 @@ func setupContext(opts options) {
if !opts.noEtcHosts && config.Clients.Sources.HostsFile {
err = setupHostsContainer()
fatalOnError(err)
if err != nil {
// Don't wrap the error, because it's informative enough as is.
return err
}
}
}
Context.mux = http.NewServeMux()
return nil
}
// setupContextFlags sets global flags and prints their status to the log.
@@ -285,25 +298,27 @@ func setupHostsContainer() (err error) {
return nil
}
func setupConfig(opts options) (err error) {
config.DNS.DnsfilterConf.EtcHosts = Context.etcHosts
config.DNS.DnsfilterConf.ConfigModified = onConfigModified
config.DNS.DnsfilterConf.HTTPRegister = httpRegister
config.DNS.DnsfilterConf.DataDir = Context.getDataDir()
config.DNS.DnsfilterConf.Filters = slices.Clone(config.Filters)
config.DNS.DnsfilterConf.WhitelistFilters = slices.Clone(config.WhitelistFilters)
config.DNS.DnsfilterConf.UserRules = slices.Clone(config.UserRules)
config.DNS.DnsfilterConf.HTTPClient = Context.client
config.DNS.DnsfilterConf.SafeSearchConf.CustomResolver = safeSearchResolver{}
config.DNS.DnsfilterConf.SafeSearch, err = safesearch.NewDefault(
config.DNS.DnsfilterConf.SafeSearchConf,
"default",
config.DNS.DnsfilterConf.SafeSearchCacheSize,
time.Minute*time.Duration(config.DNS.DnsfilterConf.CacheTime),
)
// setupOpts sets up command-line options.
func setupOpts(opts options) (err error) {
err = setupBindOpts(opts)
if err != nil {
return fmt.Errorf("initializing safesearch: %w", err)
// Don't wrap the error, because it's informative enough as is.
return err
}
if len(opts.pidFile) != 0 && writePIDFile(opts.pidFile) {
Context.pidFileName = opts.pidFile
}
return nil
}
// initContextClients initializes Context clients and related fields.
func initContextClients() (err error) {
err = setupDNSFilteringConf(config.DNS.DnsfilterConf)
if err != nil {
// Don't wrap the error, because it's informative enough as is.
return err
}
//lint:ignore SA1019 Migration is not over.
@@ -338,8 +353,19 @@ func setupConfig(opts options) (err error) {
arpdb = aghnet.NewARPDB()
}
Context.clients.Init(config.Clients.Persistent, Context.dhcpServer, Context.etcHosts, arpdb, config.DNS.DnsfilterConf)
Context.clients.Init(
config.Clients.Persistent,
Context.dhcpServer,
Context.etcHosts,
arpdb,
config.DNS.DnsfilterConf,
)
return nil
}
// setupBindOpts overrides bind host/port from the opts.
func setupBindOpts(opts options) (err error) {
if opts.bindPort != 0 {
config.BindPort = opts.bindPort
@@ -350,12 +376,83 @@ func setupConfig(opts options) (err error) {
}
}
// override bind host/port from the console
if opts.bindHost.IsValid() {
config.BindHost = opts.bindHost
}
if len(opts.pidFile) != 0 && writePIDFile(opts.pidFile) {
Context.pidFileName = opts.pidFile
return nil
}
// setupDNSFilteringConf sets up DNS filtering configuration settings.
func setupDNSFilteringConf(conf *filtering.Config) (err error) {
const (
dnsTimeout = 3 * time.Second
sbService = "safe browsing"
defaultSafeBrowsingServer = `https://family.adguard-dns.com/dns-query`
sbTXTSuffix = `sb.dns.adguard.com.`
pcService = "parental control"
defaultParentalServer = `https://family.adguard-dns.com/dns-query`
pcTXTSuffix = `pc.dns.adguard.com.`
)
conf.EtcHosts = Context.etcHosts
conf.ConfigModified = onConfigModified
conf.HTTPRegister = httpRegister
conf.DataDir = Context.getDataDir()
conf.Filters = slices.Clone(config.Filters)
conf.WhitelistFilters = slices.Clone(config.WhitelistFilters)
conf.UserRules = slices.Clone(config.UserRules)
conf.HTTPClient = Context.client
cacheTime := time.Duration(conf.CacheTime) * time.Minute
upsOpts := &upstream.Options{
Timeout: dnsTimeout,
ServerIPAddrs: []net.IP{
{94, 140, 14, 15},
{94, 140, 15, 16},
net.ParseIP("2a10:50c0::bad1:ff"),
net.ParseIP("2a10:50c0::bad2:ff"),
},
}
sbUps, err := upstream.AddressToUpstream(defaultSafeBrowsingServer, upsOpts)
if err != nil {
return fmt.Errorf("converting safe browsing server: %w", err)
}
conf.SafeBrowsingChecker = hashprefix.New(&hashprefix.Config{
Upstream: sbUps,
ServiceName: sbService,
TXTSuffix: sbTXTSuffix,
CacheTime: cacheTime,
CacheSize: conf.SafeBrowsingCacheSize,
})
parUps, err := upstream.AddressToUpstream(defaultParentalServer, upsOpts)
if err != nil {
return fmt.Errorf("converting parental server: %w", err)
}
conf.ParentalControlChecker = hashprefix.New(&hashprefix.Config{
Upstream: parUps,
ServiceName: pcService,
TXTSuffix: pcTXTSuffix,
CacheTime: cacheTime,
CacheSize: conf.SafeBrowsingCacheSize,
})
conf.SafeSearchConf.CustomResolver = safeSearchResolver{}
conf.SafeSearch, err = safesearch.NewDefault(
conf.SafeSearchConf,
"default",
conf.SafeSearchCacheSize,
cacheTime,
)
if err != nil {
return fmt.Errorf("initializing safesearch: %w", err)
}
return nil
@@ -432,14 +529,16 @@ func fatalOnError(err error) {
// run configures and starts AdGuard Home.
func run(opts options, clientBuildFS fs.FS) {
// configure config filename
// Configure config filename.
initConfigFilename(opts)
// configure working dir and config path
initWorkingDir(opts)
// Configure working dir and config path.
err := initWorkingDir(opts)
fatalOnError(err)
// configure log level and output
configureLogger(opts)
// Configure log level and output.
err = configureLogger(opts)
fatalOnError(err)
// Print the first message after logger is configured.
log.Info(version.Full())
@@ -448,25 +547,29 @@ func run(opts options, clientBuildFS fs.FS) {
log.Info("AdGuard Home is running as a service")
}
setupContext(opts)
err := configureOS(config)
err = setupContext(opts)
fatalOnError(err)
// clients package uses filtering package's static data (filtering.BlockedSvcKnown()),
// so we have to initialize filtering's static data first,
// but also avoid relying on automatic Go init() function
err = configureOS(config)
fatalOnError(err)
// Clients package uses filtering package's static data
// (filtering.BlockedSvcKnown()), so we have to initialize filtering static
// data first, but also to avoid relying on automatic Go init() function.
filtering.InitModule()
err = setupConfig(opts)
err = initContextClients()
fatalOnError(err)
// TODO(e.burkov): This could be made earlier, probably as the option's
err = setupOpts(opts)
fatalOnError(err)
// TODO(e.burkov): This could be made earlier, probably as the option's
// effect.
cmdlineUpdate(opts)
if !Context.firstRun {
// Save the updated config
// Save the updated config.
err = config.write()
fatalOnError(err)
@@ -476,33 +579,15 @@ func run(opts options, clientBuildFS fs.FS) {
}
}
err = os.MkdirAll(Context.getDataDir(), 0o755)
if err != nil {
log.Fatalf("Cannot create DNS data dir at %s: %s", Context.getDataDir(), err)
}
dir := Context.getDataDir()
err = os.MkdirAll(dir, 0o755)
fatalOnError(errors.Annotate(err, "creating DNS data dir at %s: %w", dir))
sessFilename := filepath.Join(Context.getDataDir(), "sessions.db")
GLMode = opts.glinetMode
var rateLimiter *authRateLimiter
if config.AuthAttempts > 0 && config.AuthBlockMin > 0 {
rateLimiter = newAuthRateLimiter(
time.Duration(config.AuthBlockMin)*time.Minute,
config.AuthAttempts,
)
} else {
log.Info("authratelimiter is disabled")
}
Context.auth = InitAuth(
sessFilename,
config.Users,
config.WebSessionTTLHours*60*60,
rateLimiter,
)
if Context.auth == nil {
log.Fatalf("Couldn't initialize Auth module")
}
config.Users = nil
// Init auth module.
Context.auth, err = initUsers()
fatalOnError(err)
Context.tls, err = newTLSManager(config.TLS)
if err != nil {
@@ -520,10 +605,10 @@ func run(opts options, clientBuildFS fs.FS) {
Context.tls.start()
go func() {
serr := startDNSServer()
if serr != nil {
sErr := startDNSServer()
if sErr != nil {
closeDNSServer()
fatalOnError(serr)
fatalOnError(sErr)
}
}()
@@ -537,10 +622,33 @@ func run(opts options, clientBuildFS fs.FS) {
Context.web.start()
// wait indefinitely for other go-routines to complete their job
// Wait indefinitely for other goroutines to complete their job.
select {}
}
// initUsers initializes context auth module. Clears config users field.
func initUsers() (auth *Auth, err error) {
sessFilename := filepath.Join(Context.getDataDir(), "sessions.db")
var rateLimiter *authRateLimiter
if config.AuthAttempts > 0 && config.AuthBlockMin > 0 {
blockDur := time.Duration(config.AuthBlockMin) * time.Minute
rateLimiter = newAuthRateLimiter(blockDur, config.AuthAttempts)
} else {
log.Info("authratelimiter is disabled")
}
sessionTTL := config.WebSessionTTLHours * 60 * 60
auth = InitAuth(sessFilename, config.Users, sessionTTL, rateLimiter)
if auth == nil {
return nil, errors.Error("initializing auth module failed")
}
config.Users = nil
return auth, nil
}
func (c *configuration) anonymizer() (ipmut *aghnet.IPMut) {
var anonFunc aghnet.IPMutFunc
if c.DNS.AnonymizeClientIP {
@@ -613,22 +721,19 @@ func writePIDFile(fn string) bool {
return true
}
// initConfigFilename sets up context config file path. This file path can be
// overridden by command-line arguments, or is set to default.
func initConfigFilename(opts options) {
// config file path can be overridden by command-line arguments:
if opts.confFilename != "" {
Context.configFilename = opts.confFilename
} else {
// Default config file name
Context.configFilename = "AdGuardHome.yaml"
}
Context.configFilename = stringutil.Coalesce(opts.confFilename, "AdGuardHome.yaml")
}
// initWorkingDir initializes the workDir
// if no command-line arguments specified, we use the directory where our binary file is located
func initWorkingDir(opts options) {
// initWorkingDir initializes the workDir. If no command-line arguments are
// specified, the directory with the binary file is used.
func initWorkingDir(opts options) (err error) {
execPath, err := os.Executable()
if err != nil {
panic(err)
// Don't wrap the error, because it's informative enough as is.
return err
}
if opts.workDir != "" {
@@ -640,34 +745,20 @@ func initWorkingDir(opts options) {
workDir, err := filepath.EvalSymlinks(Context.workDir)
if err != nil {
panic(err)
// Don't wrap the error, because it's informative enough as is.
return err
}
Context.workDir = workDir
return nil
}
// configureLogger configures logger level and output
func configureLogger(opts options) {
ls := getLogSettings()
// configureLogger configures logger level and output.
func configureLogger(opts options) (err error) {
ls := getLogSettings(opts)
// command-line arguments can override config settings
if opts.verbose || config.Verbose {
ls.Verbose = true
}
if opts.logFile != "" {
ls.File = opts.logFile
} else if config.File != "" {
ls.File = config.File
}
// Handle default log settings overrides
ls.Compress = config.Compress
ls.LocalTime = config.LocalTime
ls.MaxBackups = config.MaxBackups
ls.MaxSize = config.MaxSize
ls.MaxAge = config.MaxAge
// log.SetLevel(log.INFO) - default
// Configure logger level.
if ls.Verbose {
log.SetLevel(log.DEBUG)
}
@@ -676,38 +767,63 @@ func configureLogger(opts options) {
// happen pretty quickly.
log.SetFlags(log.LstdFlags | log.Lmicroseconds)
if opts.runningAsService && ls.File == "" && runtime.GOOS == "windows" {
// When running as a Windows service, use eventlog by default if nothing
// else is configured. Otherwise, we'll simply lose the log output.
ls.File = configSyslog
}
// logs are written to stdout (default)
// Write logs to stdout by default.
if ls.File == "" {
return
return nil
}
if ls.File == configSyslog {
// Use syslog where it is possible and eventlog on Windows
err := aghos.ConfigureSyslog(serviceName)
// Use syslog where it is possible and eventlog on Windows.
err = aghos.ConfigureSyslog(serviceName)
if err != nil {
log.Fatalf("cannot initialize syslog: %s", err)
}
} else {
logFilePath := ls.File
if !filepath.IsAbs(logFilePath) {
logFilePath = filepath.Join(Context.workDir, logFilePath)
return fmt.Errorf("cannot initialize syslog: %w", err)
}
log.SetOutput(&lumberjack.Logger{
Filename: logFilePath,
Compress: ls.Compress, // disabled by default
LocalTime: ls.LocalTime,
MaxBackups: ls.MaxBackups,
MaxSize: ls.MaxSize, // megabytes
MaxAge: ls.MaxAge, // days
})
return nil
}
logFilePath := ls.File
if !filepath.IsAbs(logFilePath) {
logFilePath = filepath.Join(Context.workDir, logFilePath)
}
log.SetOutput(&lumberjack.Logger{
Filename: logFilePath,
Compress: ls.Compress,
LocalTime: ls.LocalTime,
MaxBackups: ls.MaxBackups,
MaxSize: ls.MaxSize,
MaxAge: ls.MaxAge,
})
return nil
}
// getLogSettings returns a log settings object properly initialized from opts.
func getLogSettings(opts options) (ls *logSettings) {
ls = readLogSettings()
// Command-line arguments can override config settings.
if opts.verbose || config.Verbose {
ls.Verbose = true
}
ls.File = stringutil.Coalesce(opts.logFile, config.File, ls.File)
// Handle default log settings overrides.
ls.Compress = config.Compress
ls.LocalTime = config.LocalTime
ls.MaxBackups = config.MaxBackups
ls.MaxSize = config.MaxSize
ls.MaxAge = config.MaxAge
if opts.runningAsService && ls.File == "" && runtime.GOOS == "windows" {
// When running as a Windows service, use eventlog by default if
// nothing else is configured. Otherwise, we'll lose the log output.
ls.File = configSyslog
}
return ls
}
// cleanup stops and resets all the modules.

48
internal/home/service.go
View File

@@ -4,7 +4,6 @@ import (
"fmt"
"io/fs"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
@@ -84,14 +83,9 @@ func svcStatus(s service.Service) (status service.Status, err error) {
// On OpenWrt, the service utility may not exist. We use our service script
// directly in this case.
func svcAction(s service.Service, action string) (err error) {
if runtime.GOOS == "darwin" && action == "start" {
var exe string
if exe, err = os.Executable(); err != nil {
log.Error("starting service: getting executable path: %s", err)
} else if exe, err = filepath.EvalSymlinks(exe); err != nil {
log.Error("starting service: evaluating executable symlinks: %s", err)
} else if !strings.HasPrefix(exe, "/Applications/") {
log.Info("warning: service must be started from within the /Applications directory")
if action == "start" {
if err = aghos.PreCheckActionStart(); err != nil {
log.Error("starting service: %s", err)
}
}
@@ -99,8 +93,6 @@ func svcAction(s service.Service, action string) (err error) {
if err != nil && service.Platform() == "unix-systemv" &&
(action == "start" || action == "stop" || action == "restart") {
_, err = runInitdCommand(action)
return err
}
return err
@@ -224,6 +216,7 @@ func handleServiceControlAction(opts options, clientBuildFS fs.FS) {
runOpts := opts
runOpts.serviceControlAction = "run"
svcConfig := &service.Config{
Name: serviceName,
DisplayName: serviceDisplayName,
@@ -233,35 +226,48 @@ func handleServiceControlAction(opts options, clientBuildFS fs.FS) {
}
configureService(svcConfig)
prg := &program{
clientBuildFS: clientBuildFS,
opts: runOpts,
}
var s service.Service
if s, err = service.New(prg, svcConfig); err != nil {
s, err := service.New(&program{clientBuildFS: clientBuildFS, opts: runOpts}, svcConfig)
if err != nil {
log.Fatalf("service: initializing service: %s", err)
}
err = handleServiceCommand(s, action, opts)
if err != nil {
log.Fatalf("service: %s", err)
}
log.Printf(
"service: action %s has been done successfully on %s",
action,
service.ChosenSystem(),
)
}
// handleServiceCommand handles service command.
func handleServiceCommand(s service.Service, action string, opts options) (err error) {
switch action {
case "status":
handleServiceStatusCommand(s)
case "run":
if err = s.Run(); err != nil {
log.Fatalf("service: failed to run service: %s", err)
return fmt.Errorf("failed to run service: %w", err)
}
case "install":
initConfigFilename(opts)
initWorkingDir(opts)
if err = initWorkingDir(opts); err != nil {
return fmt.Errorf("failed to init working dir: %w", err)
}
handleServiceInstallCommand(s)
case "uninstall":
handleServiceUninstallCommand(s)
default:
if err = svcAction(s, action); err != nil {
log.Fatalf("service: executing action %q: %s", action, err)
return fmt.Errorf("executing action %q: %w", action, err)
}
}
log.Printf("service: action %s has been done successfully on %s", action, service.ChosenSystem())
return nil
}
// handleServiceStatusCommand handles service "status" command.

44
internal/home/tls.go
View File

@@ -172,9 +172,32 @@ func loadTLSConf(tlsConf *tlsConfigSettings, status *tlsConfigStatus) (err error
}
}()
tlsConf.CertificateChainData = []byte(tlsConf.CertificateChain)
tlsConf.PrivateKeyData = []byte(tlsConf.PrivateKey)
err = loadCertificateChainData(tlsConf, status)
if err != nil {
// Don't wrap the error, because it's informative enough as is.
return err
}
err = loadPrivateKeyData(tlsConf, status)
if err != nil {
// Don't wrap the error, because it's informative enough as is.
return err
}
err = validateCertificates(
status,
tlsConf.CertificateChainData,
tlsConf.PrivateKeyData,
tlsConf.ServerName,
)
return errors.Annotate(err, "validating certificate pair: %w")
}
// loadCertificateChainData loads PEM-encoded certificates chain data to the
// TLS configuration.
func loadCertificateChainData(tlsConf *tlsConfigSettings, status *tlsConfigStatus) (err error) {
tlsConf.CertificateChainData = []byte(tlsConf.CertificateChain)
if tlsConf.CertificatePath != "" {
if tlsConf.CertificateChain != "" {
return errors.Error("certificate data and file can't be set together")
@@ -190,6 +213,13 @@ func loadTLSConf(tlsConf *tlsConfigSettings, status *tlsConfigStatus) (err error
status.ValidCert = true
}
return nil
}
// loadPrivateKeyData loads PEM-encoded private key data to the TLS
// configuration.
func loadPrivateKeyData(tlsConf *tlsConfigSettings, status *tlsConfigStatus) (err error) {
tlsConf.PrivateKeyData = []byte(tlsConf.PrivateKey)
if tlsConf.PrivateKeyPath != "" {
if tlsConf.PrivateKey != "" {
return errors.Error("private key data and file can't be set together")
@@ -203,16 +233,6 @@ func loadTLSConf(tlsConf *tlsConfigSettings, status *tlsConfigStatus) (err error
status.ValidKey = true
}
err = validateCertificates(
status,
tlsConf.CertificateChainData,
tlsConf.PrivateKeyData,
tlsConf.ServerName,
)
if err != nil {
return fmt.Errorf("validating certificate pair: %w", err)
}
return nil
}

97
internal/home/upgrade.go
View File

@@ -41,7 +41,8 @@ func upgradeConfig() error {
err = yaml.Unmarshal(body, &diskConf)
if err != nil {
log.Printf("Couldn't parse config file: %s", err)
log.Printf("parsing config file for upgrade: %s", err)
return err
}
@@ -293,71 +294,61 @@ func upgradeSchema4to5(diskConf yobj) error {
return nil
}
// clients:
// ...
// upgradeSchema5to6 performs the following changes:
//
// ip: 127.0.0.1
// mac: ...
// # BEFORE:
// 'clients':
// ...
// 'ip': 127.0.0.1
// 'mac': ...
//
// ->
//
// clients:
// ...
//
// ids:
// - 127.0.0.1
// - ...
// # AFTER:
// 'clients':
// ...
// 'ids':
// - 127.0.0.1
// - ...
func upgradeSchema5to6(diskConf yobj) error {
log.Printf("%s(): called", funcName())
log.Printf("Upgrade yaml: 5 to 6")
diskConf["schema_version"] = 6
clients, ok := diskConf["clients"]
clientsVal, ok := diskConf["clients"]
if !ok {
return nil
}
switch arr := clients.(type) {
case []any:
for i := range arr {
switch c := arr[i].(type) {
case map[any]any:
var ipVal any
ipVal, ok = c["ip"]
ids := []string{}
if ok {
var ip string
ip, ok = ipVal.(string)
if !ok {
log.Fatalf("client.ip is not a string: %v", ipVal)
return nil
}
if len(ip) != 0 {
ids = append(ids, ip)
}
}
clients, ok := clientsVal.([]yobj)
if !ok {
return fmt.Errorf("unexpected type of clients: %T", clientsVal)
}
var macVal any
macVal, ok = c["mac"]
if ok {
var mac string
mac, ok = macVal.(string)
if !ok {
log.Fatalf("client.mac is not a string: %v", macVal)
return nil
}
if len(mac) != 0 {
ids = append(ids, mac)
}
}
for i := range clients {
c := clients[i]
var ids []string
c["ids"] = ids
default:
continue
if ipVal, hasIP := c["ip"]; hasIP {
var ip string
if ip, ok = ipVal.(string); !ok {
return fmt.Errorf("client.ip is not a string: %v", ipVal)
}
if ip != "" {
ids = append(ids, ip)
}
}
default:
return nil
if macVal, hasMac := c["mac"]; hasMac {
var mac string
if mac, ok = macVal.(string); !ok {
return fmt.Errorf("client.mac is not a string: %v", macVal)
}
if mac != "" {
ids = append(ids, mac)
}
}
c["ids"] = ids
}
return nil

89
internal/home/upgrade_test.go
View File

@@ -68,6 +68,95 @@ func TestUpgradeSchema2to3(t *testing.T) {
assertEqualExcept(t, oldDiskConf, diskConf, excludedEntries, excludedEntries)
}
func TestUpgradeSchema5to6(t *testing.T) {
const newSchemaVer = 6
testCases := []struct {
in yobj
want yobj
wantErr string
name string
}{{
in: yobj{
"clients": []yobj{},
},
want: yobj{
"clients": []yobj{},
"schema_version": newSchemaVer,
},
wantErr: "",
name: "no_clients",
}, {
in: yobj{
"clients": []yobj{{"ip": "127.0.0.1"}},
},
want: yobj{
"clients": []yobj{{
"ids": []string{"127.0.0.1"},
"ip": "127.0.0.1",
}},
"schema_version": newSchemaVer,
},
wantErr: "",
name: "client_ip",
}, {
in: yobj{
"clients": []yobj{{"mac": "mac"}},
},
want: yobj{
"clients": []yobj{{
"ids": []string{"mac"},
"mac": "mac",
}},
"schema_version": newSchemaVer,
},
wantErr: "",
name: "client_mac",
}, {
in: yobj{
"clients": []yobj{{"ip": "127.0.0.1", "mac": "mac"}},
},
want: yobj{
"clients": []yobj{{
"ids": []string{"127.0.0.1", "mac"},
"ip": "127.0.0.1",
"mac": "mac",
}},
"schema_version": newSchemaVer,
},
wantErr: "",
name: "client_ip_mac",
}, {
in: yobj{
"clients": []yobj{{"ip": 1, "mac": "mac"}},
},
want: yobj{
"clients": []yobj{{"ip": 1, "mac": "mac"}},
"schema_version": newSchemaVer,
},
wantErr: "client.ip is not a string: 1",
name: "inv_client_ip",
}, {
in: yobj{
"clients": []yobj{{"ip": "127.0.0.1", "mac": 1}},
},
want: yobj{
"clients": []yobj{{"ip": "127.0.0.1", "mac": 1}},
"schema_version": newSchemaVer,
},
wantErr: "client.mac is not a string: 1",
name: "inv_client_mac",
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
err := upgradeSchema5to6(tc.in)
testutil.AssertErrorMsg(t, tc.wantErr, err)
assert.Equal(t, tc.want, tc.in)
})
}
}
func TestUpgradeSchema7to8(t *testing.T) {
const host = "1.2.3.4"
oldConf := yobj{

184
internal/querylog/decode.go
View File

@@ -3,19 +3,24 @@ package querylog
import (
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net"
"strings"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/urlfilter/rules"
"github.com/miekg/dns"
)
// logEntryHandler represents a handler for decoding json token to the logEntry
// struct.
type logEntryHandler func(t json.Token, ent *logEntry) error
// logEntryHandlers is the map of log entry decode handlers for various keys.
var logEntryHandlers = map[string]logEntryHandler{
"CID": func(t json.Token, ent *logEntry) error {
v, ok := t.(string)
@@ -166,6 +171,7 @@ var logEntryHandlers = map[string]logEntryHandler{
},
}
// decodeResultRuleKey decodes the token of "Rules" type to logEntry struct.
func decodeResultRuleKey(key string, i int, dec *json.Decoder, ent *logEntry) {
var vToken json.Token
switch key {
@@ -189,6 +195,8 @@ func decodeResultRuleKey(key string, i int, dec *json.Decoder, ent *logEntry) {
}
}
// decodeVTokenAndAddRule decodes the "Rules" toke as [filtering.ResultRule]
// and then adds the decoded object to the slice of result rules.
func decodeVTokenAndAddRule(
key string,
i int,
@@ -213,6 +221,8 @@ func decodeVTokenAndAddRule(
return newRules, vToken
}
// decodeResultRules parses the dec's tokens into logEntry ent interpreting it
// as a slice of the result rules.
func decodeResultRules(dec *json.Decoder, ent *logEntry) {
for {
delimToken, err := dec.Token()
@@ -224,48 +234,53 @@ func decodeResultRules(dec *json.Decoder, ent *logEntry) {
return
}
if d, ok := delimToken.(json.Delim); ok {
if d != '[' {
log.Debug("decodeResultRules: unexpected delim %q", d)
if d, ok := delimToken.(json.Delim); !ok {
return
} else if d != '[' {
log.Debug("decodeResultRules: unexpected delim %q", d)
}
err = decodeResultRuleToken(dec, ent)
if err != nil {
if err != io.EOF && !errors.Is(err, ErrEndOfToken) {
log.Debug("decodeResultRules err: %s", err)
}
} else {
return
}
}
}
i := 0
for {
var keyToken json.Token
keyToken, err = dec.Token()
if err != nil {
if err != io.EOF {
log.Debug("decodeResultRules err: %s", err)
}
return
}
if d, ok := keyToken.(json.Delim); ok {
switch d {
case '}':
i++
case ']':
return
default:
// Go on.
}
continue
}
key, ok := keyToken.(string)
if !ok {
log.Debug("decodeResultRules: keyToken is %T (%[1]v) and not string", keyToken)
return
}
decodeResultRuleKey(key, i, dec, ent)
// decodeResultRuleToken decodes the tokens of "Rules" type to the logEntry ent.
func decodeResultRuleToken(dec *json.Decoder, ent *logEntry) (err error) {
i := 0
for {
var keyToken json.Token
keyToken, err = dec.Token()
if err != nil {
// Don't wrap the error, because it's informative enough as is.
return err
}
if d, ok := keyToken.(json.Delim); ok {
switch d {
case '}':
i++
case ']':
return ErrEndOfToken
default:
// Go on.
}
continue
}
key, ok := keyToken.(string)
if !ok {
return fmt.Errorf("keyToken is %T (%[1]v) and not string", keyToken)
}
decodeResultRuleKey(key, i, dec, ent)
}
}
@@ -322,6 +337,8 @@ func decodeResultReverseHosts(dec *json.Decoder, ent *logEntry) {
}
}
// decodeResultIPList parses the dec's tokens into logEntry ent interpreting it
// as the result IP addresses list.
func decodeResultIPList(dec *json.Decoder, ent *logEntry) {
for {
itemToken, err := dec.Token()
@@ -355,6 +372,8 @@ func decodeResultIPList(dec *json.Decoder, ent *logEntry) {
}
}
// decodeResultDNSRewriteResultKey decodes the token of "DNSRewriteResult" type
// to the logEntry struct.
func decodeResultDNSRewriteResultKey(key string, dec *json.Decoder, ent *logEntry) {
var err error
@@ -395,50 +414,29 @@ func decodeResultDNSRewriteResultKey(key string, dec *json.Decoder, ent *logEntr
log.Debug("decodeResultDNSRewriteResultKey response err: %s", err)
}
for rrType, rrValues := range ent.Result.DNSRewriteResult.Response {
switch rrType {
case
dns.TypeA,
dns.TypeAAAA:
for i, v := range rrValues {
s, _ := v.(string)
rrValues[i] = net.ParseIP(s)
}
default:
// Go on.
}
}
ent.parseDNSRewriteResultIPs()
default:
// Go on.
}
}
// decodeResultDNSRewriteResult parses the dec's tokens into logEntry ent
// interpreting it as the result DNSRewriteResult.
func decodeResultDNSRewriteResult(dec *json.Decoder, ent *logEntry) {
for {
keyToken, err := dec.Token()
key, err := parseKeyToken(dec)
if err != nil {
if err != io.EOF {
log.Debug("decodeResultDNSRewriteResult err: %s", err)
if err != io.EOF && !errors.Is(err, ErrEndOfToken) {
log.Debug("decodeResultDNSRewriteResult: %s", err)
}
return
}
if d, ok := keyToken.(json.Delim); ok {
if d == '}' {
return
}
if key == "" {
continue
}
key, ok := keyToken.(string)
if !ok {
log.Debug("decodeResultDNSRewriteResult: keyToken is %T (%[1]v) and not string", keyToken)
return
}
decodeResultDNSRewriteResultKey(key, dec, ent)
}
}
@@ -474,34 +472,51 @@ func translateResult(ent *logEntry) {
res.IPList = nil
}
// ErrEndOfToken is an error returned by parse key token when the closing
// bracket is found.
const ErrEndOfToken errors.Error = "end of token"
// parseKeyToken parses the dec's token key.
func parseKeyToken(dec *json.Decoder) (key string, err error) {
keyToken, err := dec.Token()
if err != nil {
return "", err
}
if d, ok := keyToken.(json.Delim); ok {
if d == '}' {
return "", ErrEndOfToken
}
return "", nil
}
key, ok := keyToken.(string)
if !ok {
return "", fmt.Errorf("keyToken is %T (%[1]v) and not string", keyToken)
}
return key, nil
}
// decodeResult decodes a token of "Result" type to logEntry struct.
func decodeResult(dec *json.Decoder, ent *logEntry) {
defer translateResult(ent)
for {
keyToken, err := dec.Token()
key, err := parseKeyToken(dec)
if err != nil {
if err != io.EOF {
log.Debug("decodeResult err: %s", err)
if err != io.EOF && !errors.Is(err, ErrEndOfToken) {
log.Debug("decodeResult: %s", err)
}
return
}
if d, ok := keyToken.(json.Delim); ok {
if d == '}' {
return
}
if key == "" {
continue
}
key, ok := keyToken.(string)
if !ok {
log.Debug("decodeResult: keyToken is %T (%[1]v) and not string", keyToken)
return
}
decHandler, ok := resultDecHandlers[key]
if ok {
decHandler(dec, ent)
@@ -527,13 +542,16 @@ func decodeResult(dec *json.Decoder, ent *logEntry) {
}
}
// resultHandlers is the map of log entry decode handlers for various keys.
var resultHandlers = map[string]logEntryHandler{
"IsFiltered": func(t json.Token, ent *logEntry) error {
v, ok := t.(bool)
if !ok {
return nil
}
ent.Result.IsFiltered = v
return nil
},
"Rule": func(t json.Token, ent *logEntry) error {
@@ -578,11 +596,14 @@ var resultHandlers = map[string]logEntryHandler{
if !ok {
return nil
}
i, err := v.Int64()
if err != nil {
return err
}
ent.Result.Reason = filtering.Reason(i)
return nil
},
"ServiceName": func(t json.Token, ent *logEntry) error {
@@ -607,6 +628,7 @@ var resultHandlers = map[string]logEntryHandler{
},
}
// resultDecHandlers is the map of decode handlers for various keys.
var resultDecHandlers = map[string]func(dec *json.Decoder, ent *logEntry){
"ReverseHosts": decodeResultReverseHosts,
"IPList": decodeResultIPList,
@@ -614,9 +636,11 @@ var resultDecHandlers = map[string]func(dec *json.Decoder, ent *logEntry){
"DNSRewriteResult": decodeResultDNSRewriteResult,
}
// decodeLogEntry decodes string str to logEntry ent.
func decodeLogEntry(ent *logEntry, str string) {
dec := json.NewDecoder(strings.NewReader(str))
dec.UseNumber()
for {
keyToken, err := dec.Token()
if err != nil {

3
internal/querylog/decode_test.go
View File

@@ -182,8 +182,7 @@ func TestDecodeLogEntry(t *testing.T) {
if tc.want == "" {
assert.Empty(t, s)
} else {
assert.True(t, strings.HasSuffix(s, tc.want),
"got %q", s)
assert.True(t, strings.HasSuffix(s, tc.want), "got %q", s)
}
logOutput.Reset()

16
internal/querylog/entry.go
View File

@@ -68,3 +68,19 @@ func (e *logEntry) addResponse(resp *dns.Msg, isOrig bool) {
log.Error("querylog: %s", err)
}
}
// parseDNSRewriteResultIPs fills logEntry's DNSRewriteResult response records
// with the IP addresses parsed from the raw strings.
func (e *logEntry) parseDNSRewriteResultIPs() {
for rrType, rrValues := range e.Result.DNSRewriteResult.Response {
switch rrType {
case dns.TypeA, dns.TypeAAAA:
for i, v := range rrValues {
s, _ := v.(string)
rrValues[i] = net.ParseIP(s)
}
default:
// Go on.
}
}
}

94
internal/querylog/qlog.go
View File

@@ -16,32 +16,35 @@ import (
"github.com/miekg/dns"
)
const (
queryLogFileName = "querylog.json" // .gz added during compression
)
// queryLogFileName is a name of the log file. ".gz" extension is added later
// during compression.
const queryLogFileName = "querylog.json"
// queryLog is a structure that writes and reads the DNS query log
// queryLog is a structure that writes and reads the DNS query log.
type queryLog struct {
findClient func(ids []string) (c *Client, err error)
// confMu protects conf.
confMu *sync.RWMutex
conf *Config
conf *Config
anonymizer *aghnet.IPMut
findClient func(ids []string) (c *Client, err error)
// logFile is the path to the log file.
logFile string
// bufferLock protects buffer.
bufferLock sync.RWMutex
// buffer contains recent log entries. The entries in this buffer must not
// be modified.
buffer []*logEntry
fileFlushLock sync.Mutex // synchronize a file-flushing goroutine and main thread
flushPending bool // don't start another goroutine while the previous one is still running
// bufferLock protects buffer.
bufferLock sync.RWMutex
// fileFlushLock synchronizes a file-flushing goroutine and main thread.
fileFlushLock sync.Mutex
fileWriteLock sync.Mutex
anonymizer *aghnet.IPMut
flushPending bool
}
// ClientProto values are names of the client protocols.
@@ -155,6 +158,43 @@ func (l *queryLog) clear() {
log.Debug("querylog: cleared")
}
// newLogEntry creates an instance of logEntry from parameters.
func newLogEntry(params *AddParams) (entry *logEntry) {
q := params.Question.Question[0]
entry = &logEntry{
// TODO(d.kolyshev): Export this timestamp to func params.
Time: time.Now(),
QHost: strings.ToLower(q.Name[:len(q.Name)-1]),
QType: dns.Type(q.Qtype).String(),
QClass: dns.Class(q.Qclass).String(),
ClientID: params.ClientID,
ClientProto: params.ClientProto,
Result: *params.Result,
Upstream: params.Upstream,
IP: params.ClientIP,
Elapsed: params.Elapsed,
Cached: params.Cached,
AuthenticatedData: params.AuthenticatedData,
}
if params.ReqECS != nil {
entry.ReqECS = params.ReqECS.String()
}
entry.addResponse(params.Answer, false)
entry.addResponse(params.OrigAnswer, true)
return entry
}
// Add implements the [QueryLog] interface for *queryLog.
func (l *queryLog) Add(params *AddParams) {
var isEnabled, fileIsEnabled bool
var memSize uint32
@@ -181,35 +221,7 @@ func (l *queryLog) Add(params *AddParams) {
params.Result = &filtering.Result{}
}
now := time.Now()
q := params.Question.Question[0]
entry := &logEntry{
Time: now,
QHost: strings.ToLower(q.Name[:len(q.Name)-1]),
QType: dns.Type(q.Qtype).String(),
QClass: dns.Class(q.Qclass).String(),
ClientID: params.ClientID,
ClientProto: params.ClientProto,
Result: *params.Result,
Upstream: params.Upstream,
IP: params.ClientIP,
Elapsed: params.Elapsed,
Cached: params.Cached,
AuthenticatedData: params.AuthenticatedData,
}
if params.ReqECS != nil {
entry.ReqECS = params.ReqECS.String()
}
entry.addResponse(params.Answer, false)
entry.addResponse(params.OrigAnswer, true)
entry := newLogEntry(params)
needFlush := false
func() {

12
internal/querylog/qlog_test.go
View File

@@ -6,7 +6,6 @@ import (
"testing"
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/dnsproxy/proxyutil"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/AdguardTeam/golibs/testutil"
"github.com/AdguardTeam/golibs/timeutil"
@@ -46,9 +45,10 @@ func TestQueryLog(t *testing.T) {
addEntry(l, "example.com", net.IPv4(1, 1, 1, 4), net.IPv4(2, 2, 2, 4))
type tcAssertion struct {
num int
host string
answer, client net.IP
host string
answer net.IP
client net.IP
num int
}
testCases := []struct {
@@ -367,6 +367,6 @@ func assertLogEntry(t *testing.T, entry *logEntry, host string, answer, client n
require.NoError(t, msg.Unpack(entry.Answer))
require.Len(t, msg.Answer, 1)
ip := proxyutil.IPFromRR(msg.Answer[0]).To16()
assert.Equal(t, answer, ip)
a := testutil.RequireTypeAssert[*dns.A](t, msg.Answer[0])
assert.Equal(t, answer, a.A.To16())
}

292
internal/querylog/qlogfile.go
View File

@@ -12,141 +12,181 @@ import (
"github.com/AdguardTeam/golibs/log"
)
// Timestamp not found errors.
const (
ErrTSNotFound errors.Error = "ts not found"
ErrTSTooLate errors.Error = "ts too late"
ErrTSTooEarly errors.Error = "ts too early"
// Timestamp not found errors.
errTSNotFound errors.Error = "ts not found"
errTSTooLate errors.Error = "ts too late"
errTSTooEarly errors.Error = "ts too early"
// maxEntrySize is a maximum size of the entry.
//
// TODO: Find a way to grow buffer instead of relying on this value when
// reading strings.
maxEntrySize = 16 * 1024
// bufferSize should be enough for at least this number of entries.
bufferSize = 100 * maxEntrySize
)
// TODO: Find a way to grow buffer instead of relying on this value when reading strings
const maxEntrySize = 16 * 1024
// buffer should be enough for at least this number of entries
const bufferSize = 100 * maxEntrySize
// QLogFile represents a single query log file
// It allows reading from the file in the reverse order
// qLogFile represents a single query log file. It allows reading from the
// file in the reverse order.
//
// Please note that this is a stateful object.
// Internally, it contains a pointer to a specific position in the file,
// and it reads lines in reverse order starting from that position.
type QLogFile struct {
file *os.File // the query log file
position int64 // current position in the file
// Please note, that this is a stateful object. Internally, it contains a
// pointer to a specific position in the file, and it reads lines in reverse
// order starting from that position.
type qLogFile struct {
// file is the query log file.
file *os.File
buffer []byte // buffer that we've read from the file
bufferStart int64 // start of the buffer (in the file)
bufferLen int // buffer len
// buffer that we've read from the file.
buffer []byte
lock sync.Mutex // We use mutex to make it thread-safe
// lock is a mutex to make it thread-safe.
lock sync.Mutex
// position is the position in the file.
position int64
// bufferStart is the start of the buffer (in the file).
bufferStart int64
// bufferLen is the length of the buffer.
bufferLen int
}
// NewQLogFile initializes a new instance of the QLogFile
func NewQLogFile(path string) (*QLogFile, error) {
// newQLogFile initializes a new instance of the qLogFile.
func newQLogFile(path string) (qf *qLogFile, err error) {
f, err := os.OpenFile(path, os.O_RDONLY, 0o644)
if err != nil {
return nil, err
}
return &QLogFile{
file: f,
}, nil
return &qLogFile{file: f}, nil
}
// validateQLogLineIdx returns error if the line index is not valid to continue
// search.
func (q *qLogFile) validateQLogLineIdx(lineIdx, lastProbeLineIdx, ts, fSize int64) (err error) {
if lineIdx == lastProbeLineIdx {
if lineIdx == 0 {
return errTSTooEarly
}
// If we're testing the same line twice then most likely the scope is
// too narrow and we won't find anything anymore in any other file.
return fmt.Errorf("looking up timestamp %d in %q: %w", ts, q.file.Name(), errTSNotFound)
} else if lineIdx == fSize {
return errTSTooLate
}
return nil
}
// seekTS performs binary search in the query log file looking for a record
// with the specified timestamp. Once the record is found, it sets
// "position" so that the next ReadNext call returned that record.
// with the specified timestamp. Once the record is found, it sets "position"
// so that the next ReadNext call returned that record.
//
// The algorithm is rather simple:
// 1. It starts with the position in the middle of a file
// 2. Shifts back to the beginning of the line
// 3. Checks the log record timestamp
// 4. If it is lower than the timestamp we are looking for,
// it shifts seek position to 3/4 of the file. Otherwise, to 1/4 of the file.
// 5. It performs the search again, every time the search scope is narrowed twice.
// 1. It starts with the position in the middle of a file.
// 2. Shifts back to the beginning of the line.
// 3. Checks the log record timestamp.
// 4. If it is lower than the timestamp we are looking for, it shifts seek
// position to 3/4 of the file. Otherwise, to 1/4 of the file.
// 5. It performs the search again, every time the search scope is narrowed
// twice.
//
// Returns:
// * It returns the position of the the line with the timestamp we were looking for
// so that when we call "ReadNext" this line was returned.
// * Depth of the search (how many times we compared timestamps).
// * If we could not find it, it returns one of the errors described above.
func (q *QLogFile) seekTS(timestamp int64) (int64, int, error) {
// - It returns the position of the line with the timestamp we were looking
// for so that when we call "ReadNext" this line was returned.
// - Depth of the search (how many times we compared timestamps).
// - If we could not find it, it returns one of the errors described above.
func (q *qLogFile) seekTS(timestamp int64) (pos int64, depth int, err error) {
q.lock.Lock()
defer q.lock.Unlock()
// Empty the buffer
// Empty the buffer.
q.buffer = nil
// First of all, check the file size
// First of all, check the file size.
fileInfo, err := q.file.Stat()
if err != nil {
return 0, 0, err
}
// Define the search scope
start := int64(0) // start of the search interval (position in the file)
end := fileInfo.Size() // end of the search interval (position in the file)
probe := (end - start) / 2 // probe -- approximate index of the line we'll try to check
// Define the search scope.
// Start of the search interval (position in the file).
start := int64(0)
// End of the search interval (position in the file).
end := fileInfo.Size()
// Probe is the approximate index of the line we'll try to check.
probe := (end - start) / 2
var line string
var lineIdx int64 // index of the probe line in the file
// Index of the probe line in the file.
var lineIdx int64
var lineEndIdx int64
var lastProbeLineIdx int64 // index of the last probe line
// Index of the last probe line.
var lastProbeLineIdx int64
lastProbeLineIdx = -1
// Count seek depth in order to detect mistakes
// If depth is too large, we should stop the search
depth := 0
// Count seek depth in order to detect mistakes. If depth is too large,
// we should stop the search.
for {
// Get the line at the specified position
// Get the line at the specified position.
line, lineIdx, lineEndIdx, err = q.readProbeLine(probe)
if err != nil {
return 0, depth, err
}
if lineIdx == lastProbeLineIdx {
if lineIdx == 0 {
return 0, depth, ErrTSTooEarly
}
// If we're testing the same line twice then most likely
// the scope is too narrow and we won't find anything
// anymore in any other file.
return 0, depth, fmt.Errorf("looking up timestamp %d in %q: %w", timestamp, q.file.Name(), ErrTSNotFound)
} else if lineIdx == fileInfo.Size() {
return 0, depth, ErrTSTooLate
// Check if the line index if invalid.
err = q.validateQLogLineIdx(lineIdx, lastProbeLineIdx, timestamp, fileInfo.Size())
if err != nil {
return 0, depth, err
}
// Save the last found idx
// Save the last found idx.
lastProbeLineIdx = lineIdx
// Get the timestamp from the query log record
// Get the timestamp from the query log record.
ts := readQLogTimestamp(line)
if ts == 0 {
return 0, depth, fmt.Errorf("looking up timestamp %d in %q: record %q has empty timestamp", timestamp, q.file.Name(), line)
return 0, depth, fmt.Errorf(
"looking up timestamp %d in %q: record %q has empty timestamp",
timestamp,
q.file.Name(),
line,
)
}
if ts == timestamp {
// Hurray, returning the result
// Hurray, returning the result.
break
}
// Narrow the scope and repeat the search
// Narrow the scope and repeat the search.
if ts > timestamp {
// If the timestamp we're looking for is OLDER than what we found
// Then the line is somewhere on the LEFT side from the current probe position
// If the timestamp we're looking for is OLDER than what we found,
// then the line is somewhere on the LEFT side from the current
// probe position.
end = lineIdx
} else {
// If the timestamp we're looking for is NEWER than what we found
// Then the line is somewhere on the RIGHT side from the current probe position
// If the timestamp we're looking for is NEWER than what we found,
// then the line is somewhere on the RIGHT side from the current
// probe position.
start = lineEndIdx
}
probe = start + (end-start)/2
depth++
if depth >= 100 {
return 0, depth, fmt.Errorf("looking up timestamp %d in %q: depth %d too high: %w", timestamp, q.file.Name(), depth, ErrTSNotFound)
return 0, depth, fmt.Errorf(
"looking up timestamp %d in %q: depth %d too high: %w",
timestamp,
q.file.Name(),
depth,
errTSNotFound,
)
}
}
@@ -154,37 +194,39 @@ func (q *QLogFile) seekTS(timestamp int64) (int64, int, error) {
return q.position, depth, nil
}
// SeekStart changes the current position to the end of the file
// Please note that we're reading query log in the reverse order
// and that's why log start is actually the end of file
// SeekStart changes the current position to the end of the file. Please note,
// that we're reading query log in the reverse order and that's why log start
// is actually the end of file.
//
// Returns nil if we were able to change the current position.
// Returns error in any other case.
func (q *QLogFile) SeekStart() (int64, error) {
// Returns nil if we were able to change the current position. Returns error
// in any other case.
func (q *qLogFile) SeekStart() (int64, error) {
q.lock.Lock()
defer q.lock.Unlock()
// Empty the buffer
// Empty the buffer.
q.buffer = nil
// First of all, check the file size
// First of all, check the file size.
fileInfo, err := q.file.Stat()
if err != nil {
return 0, err
}
// Place the position to the very end of file
// Place the position to the very end of file.
q.position = fileInfo.Size() - 1
if q.position < 0 {
q.position = 0
}
return q.position, nil
}
// ReadNext reads the next line (in the reverse order) from the file
// and shifts the current position left to the next (actually prev) line.
// returns io.EOF if there's nothing to read more
func (q *QLogFile) ReadNext() (string, error) {
// ReadNext reads the next line (in the reverse order) from the file and shifts
// the current position left to the next (actually prev) line.
//
// Returns io.EOF if there's nothing more to read.
func (q *qLogFile) ReadNext() (string, error) {
q.lock.Lock()
defer q.lock.Unlock()
@@ -197,35 +239,34 @@ func (q *QLogFile) ReadNext() (string, error) {
return "", err
}
// Shift position
// Shift position.
if lineIdx == 0 {
q.position = 0
} else {
// there's usually a line break before the line
// so we should shift one more char left from the line
// line\nline
// There's usually a line break before the line, so we should shift one
// more char left from the line "\nline".
q.position = lineIdx - 1
}
return line, err
}
// Close frees the underlying resources
func (q *QLogFile) Close() error {
// Close frees the underlying resources.
func (q *qLogFile) Close() error {
return q.file.Close()
}
// readNextLine reads the next line from the specified position
// this line actually have to END on that position.
// readNextLine reads the next line from the specified position. This line
// actually have to END on that position.
//
// the algorithm is:
// 1. check if we have the buffer initialized
// 2. if it is, scan it and look for the line there
// 3. if we cannot find the line there, read the prev chunk into the buffer
// 4. read the line from the buffer
func (q *QLogFile) readNextLine(position int64) (string, int64, error) {
// The algorithm is:
// 1. Check if we have the buffer initialized.
// 2. If it is so, scan it and look for the line there.
// 3. If we cannot find the line there, read the prev chunk into the buffer.
// 4. Read the line from the buffer.
func (q *qLogFile) readNextLine(position int64) (string, int64, error) {
relativePos := position - q.bufferStart
if q.buffer == nil || (relativePos < maxEntrySize && q.bufferStart != 0) {
// Time to re-init the buffer
// Time to re-init the buffer.
err := q.initBuffer(position)
if err != nil {
return "", 0, err
@@ -233,8 +274,7 @@ func (q *QLogFile) readNextLine(position int64) (string, int64, error) {
relativePos = position - q.bufferStart
}
// Look for the end of the prev line
// This is where we'll read from
// Look for the end of the prev line, this is where we'll read from.
startLine := int64(0)
for i := relativePos - 1; i >= 0; i-- {
if q.buffer[i] == '\n' {
@@ -245,18 +285,19 @@ func (q *QLogFile) readNextLine(position int64) (string, int64, error) {
line := string(q.buffer[startLine:relativePos])
lineIdx := q.bufferStart + startLine
return line, lineIdx, nil
}
// initBuffer initializes the QLogFile buffer.
// the goal is to read a chunk of file that includes the line with the specified position.
func (q *QLogFile) initBuffer(position int64) error {
// initBuffer initializes the qLogFile buffer. The goal is to read a chunk of
// file that includes the line with the specified position.
func (q *qLogFile) initBuffer(position int64) error {
q.bufferStart = int64(0)
if position > bufferSize {
q.bufferStart = position - bufferSize
}
// Seek to this position
// Seek to this position.
_, err := q.file.Seek(q.bufferStart, io.SeekStart)
if err != nil {
return err
@@ -271,34 +312,35 @@ func (q *QLogFile) initBuffer(position int64) error {
return err
}
// readProbeLine reads a line that includes the specified position
// this method is supposed to be used when we use binary search in the Seek method
// in the case of consecutive reads, use readNext (it uses a better buffer)
func (q *QLogFile) readProbeLine(position int64) (string, int64, int64, error) {
// First of all, we should read a buffer that will include the query log line
// In order to do this, we'll define the boundaries
// readProbeLine reads a line that includes the specified position. This
// method is supposed to be used when we use binary search in the Seek method.
// In the case of consecutive reads, use readNext, cause it uses better buffer.
func (q *qLogFile) readProbeLine(position int64) (string, int64, int64, error) {
// First of all, we should read a buffer that will include the query log
// line. In order to do this, we'll define the boundaries.
seekPosition := int64(0)
relativePos := position // position relative to the buffer we're going to read
// Position relative to the buffer we're going to read.
relativePos := position
if position > maxEntrySize {
seekPosition = position - maxEntrySize
relativePos = maxEntrySize
}
// Seek to this position
// Seek to this position.
_, err := q.file.Seek(seekPosition, io.SeekStart)
if err != nil {
return "", 0, 0, err
}
// The buffer size is 2*maxEntrySize
// The buffer size is 2*maxEntrySize.
buffer := make([]byte, maxEntrySize*2)
bufferLen, err := q.file.Read(buffer)
if err != nil {
return "", 0, 0, err
}
// Now start looking for the new line character starting
// from the relativePos and going left
// Now start looking for the new line character starting from the
// relativePos and going left.
startLine := int64(0)
for i := relativePos - 1; i >= 0; i-- {
if buffer[i] == '\n' {
@@ -306,7 +348,7 @@ func (q *QLogFile) readProbeLine(position int64) (string, int64, int64, error) {
break
}
}
// Looking for the end of line now
// Looking for the end of line now.
endLine := int64(bufferLen)
lineEndIdx := endLine + seekPosition
for i := relativePos; i < int64(bufferLen); i++ {
@@ -317,13 +359,13 @@ func (q *QLogFile) readProbeLine(position int64) (string, int64, int64, error) {
}
}
// Finally we can return the string we were looking for
// Finally we can return the string we were looking for.
lineIdx := startLine + seekPosition
return string(buffer[startLine:endLine]), lineIdx, lineEndIdx, nil
}
// readJSONvalue reads a JSON string in form of '"key":"value"'. prefix must be
// of the form '"key":"' to generate less garbage.
// readJSONValue reads a JSON string in form of '"key":"value"'. prefix must
// be of the form '"key":"' to generate less garbage.
func readJSONValue(s, prefix string) string {
i := strings.Index(s, prefix)
if i == -1 {
@@ -340,7 +382,7 @@ func readJSONValue(s, prefix string) string {
return s[start:end]
}
// readQLogTimestamp reads the timestamp field from the query log line
// readQLogTimestamp reads the timestamp field from the query log line.
func readQLogTimestamp(str string) int64 {
val := readJSONValue(str, `"T":"`)
if len(val) == 0 {
@@ -351,10 +393,12 @@ func readQLogTimestamp(str string) int64 {
log.Error("Couldn't find timestamp: %s", str)
return 0
}
tm, err := time.Parse(time.RFC3339Nano, val)
if err != nil {
log.Error("Couldn't parse timestamp: %s", val)
return 0
}
return tm.UnixNano()
}

24
internal/querylog/qlogfile_test.go
View File

@@ -72,15 +72,15 @@ func prepareTestFiles(t *testing.T, filesNum, linesNum int) []string {
return files
}
// newTestQLogFile creates new *QLogFile for tests and registers the required
// newTestQLogFile creates new *qLogFile for tests and registers the required
// cleanup functions.
func newTestQLogFile(t *testing.T, linesNum int) (file *QLogFile) {
func newTestQLogFile(t *testing.T, linesNum int) (file *qLogFile) {
t.Helper()
testFile := prepareTestFiles(t, 1, linesNum)[0]
// Create the new QLogFile instance.
file, err := NewQLogFile(testFile)
// Create the new qLogFile instance.
file, err := newQLogFile(testFile)
require.NoError(t, err)
assert.NotNil(t, file)
@@ -240,7 +240,7 @@ func TestQLogFile_SeekTS_bad(t *testing.T) {
}
}
func getQLogFileLine(q *QLogFile, lineNumber int) (line string, err error) {
func getQLogFileLine(q *qLogFile, lineNumber int) (line string, err error) {
if _, err = q.SeekStart(); err != nil {
return line, err
}
@@ -256,7 +256,7 @@ func getQLogFileLine(q *QLogFile, lineNumber int) (line string, err error) {
// Check adding and loading (with filtering) entries from disk and memory.
func TestQLogFile(t *testing.T) {
// Create the new QLogFile instance.
// Create the new qLogFile instance.
q := newTestQLogFile(t, 2)
// Seek to the start.
@@ -285,7 +285,7 @@ func TestQLogFile(t *testing.T) {
assert.Empty(t, line)
}
func NewTestQLogFileData(t *testing.T, data string) (file *QLogFile) {
func newTestQLogFileData(t *testing.T, data string) (file *qLogFile) {
f, err := os.CreateTemp(t.TempDir(), "*.txt")
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, f.Close)
@@ -293,7 +293,7 @@ func NewTestQLogFileData(t *testing.T, data string) (file *QLogFile) {
_, err = f.WriteString(data)
require.NoError(t, err)
file, err = NewQLogFile(f.Name())
file, err = newQLogFile(f.Name())
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, file.Close)
@@ -309,9 +309,9 @@ func TestQLog_Seek(t *testing.T) {
timestamp, _ := time.Parse(time.RFC3339Nano, "2020-08-31T18:44:25.376690873+03:00")
testCases := []struct {
wantErr error
name string
delta int
wantErr error
wantDepth int
}{{
name: "ok",
@@ -321,12 +321,12 @@ func TestQLog_Seek(t *testing.T) {
}, {
name: "too_late",
delta: 2,
wantErr: ErrTSTooLate,
wantErr: errTSTooLate,
wantDepth: 2,
}, {
name: "too_early",
delta: -2,
wantErr: ErrTSTooEarly,
wantErr: errTSTooEarly,
wantDepth: 1,
}}
@@ -338,7 +338,7 @@ func TestQLog_Seek(t *testing.T) {
timestamp.Add(time.Second).Format(time.RFC3339Nano),
)
q := NewTestQLogFileData(t, data)
q := newTestQLogFileData(t, data)
_, depth, err := q.seekTS(timestamp.Add(time.Second * time.Duration(tc.delta)).UnixNano())
require.Truef(t, errors.Is(err, tc.wantErr), "%v", err)

97
internal/querylog/qlogreader.go
View File

@@ -9,36 +9,36 @@ import (
"github.com/AdguardTeam/golibs/log"
)
// QLogReader allows reading from multiple query log files in the reverse order.
// qLogReader allows reading from multiple query log files in the reverse
// order.
//
// Please note that this is a stateful object.
// Internally, it contains a pointer to a particular query log file, and
// to a specific position in this file, and it reads lines in reverse order
// starting from that position.
type QLogReader struct {
// qFiles - array with the query log files
// The order is - from oldest to newest
qFiles []*QLogFile
// Please note that this is a stateful object. Internally, it contains a
// pointer to a particular query log file, and to a specific position in this
// file, and it reads lines in reverse order starting from that position.
type qLogReader struct {
// qFiles is an array with the query log files. The order is from oldest
// to newest.
qFiles []*qLogFile
currentFile int // Index of the current file
// currentFile is the index of the current file.
currentFile int
}
// NewQLogReader initializes a QLogReader instance
// with the specified files
func NewQLogReader(files []string) (*QLogReader, error) {
qFiles := make([]*QLogFile, 0)
// newQLogReader initializes a qLogReader instance with the specified files.
func newQLogReader(files []string) (*qLogReader, error) {
qFiles := make([]*qLogFile, 0)
for _, f := range files {
q, err := NewQLogFile(f)
q, err := newQLogFile(f)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
continue
}
// Close what we've already opened.
cerr := closeQFiles(qFiles)
if cerr != nil {
log.Debug("querylog: closing files: %s", cerr)
cErr := closeQFiles(qFiles)
if cErr != nil {
log.Debug("querylog: closing files: %s", cErr)
}
return nil, err
@@ -47,31 +47,28 @@ func NewQLogReader(files []string) (*QLogReader, error) {
qFiles = append(qFiles, q)
}
return &QLogReader{
qFiles: qFiles,
currentFile: (len(qFiles) - 1),
}, nil
return &qLogReader{qFiles: qFiles, currentFile: len(qFiles) - 1}, nil
}
// seekTS performs binary search of a query log record with the specified
// timestamp. If the record is found, it sets QLogReader's position to point to
// that line, so that the next ReadNext call returned this line.
func (r *QLogReader) seekTS(timestamp int64) (err error) {
// timestamp. If the record is found, it sets qLogReader's position to point
// to that line, so that the next ReadNext call returned this line.
func (r *qLogReader) seekTS(timestamp int64) (err error) {
for i := len(r.qFiles) - 1; i >= 0; i-- {
q := r.qFiles[i]
_, _, err = q.seekTS(timestamp)
if err != nil {
if errors.Is(err, ErrTSTooEarly) {
if errors.Is(err, errTSTooEarly) {
// Look at the next file, since we've reached the end of this
// one. If there is no next file, it's not found.
err = ErrTSNotFound
err = errTSNotFound
continue
} else if errors.Is(err, ErrTSTooLate) {
} else if errors.Is(err, errTSTooLate) {
// Just seek to the start then. timestamp is probably between
// the end of the previous one and the start of this one.
return r.SeekStart()
} else if errors.Is(err, ErrTSNotFound) {
} else if errors.Is(err, errTSNotFound) {
return err
} else {
return fmt.Errorf("seekts: file at index %d: %w", i, err)
@@ -80,7 +77,7 @@ func (r *QLogReader) seekTS(timestamp int64) (err error) {
// The search is finished, and the searched element has been found.
// Update currentFile only, position is already set properly in
// QLogFile.
// qLogFile.
r.currentFile = i
return nil
@@ -93,13 +90,13 @@ func (r *QLogReader) seekTS(timestamp int64) (err error) {
return nil
}
// SeekStart changes the current position to the end of the newest file
// Please note that we're reading query log in the reverse order
// and that's why log start is actually the end of file
// SeekStart changes the current position to the end of the newest file.
// Please note that we're reading query log in the reverse order and that's why
// the log starts actually at the end of file.
//
// Returns nil if we were able to change the current position.
// Returns error in any other case.
func (r *QLogReader) SeekStart() error {
// Returns nil if we were able to change the current position. Returns error
// in any other cases.
func (r *qLogReader) SeekStart() error {
if len(r.qFiles) == 0 {
return nil
}
@@ -110,10 +107,12 @@ func (r *QLogReader) SeekStart() error {
return err
}
// ReadNext reads the next line (in the reverse order) from the query log files.
// and shifts the current position left to the next (actually prev) line (or the next file).
// returns io.EOF if there's nothing to read more.
func (r *QLogReader) ReadNext() (string, error) {
// ReadNext reads the next line (in the reverse order) from the query log
// files. Then shifts the current position left to the next (actually prev)
// line (or the next file).
//
// Returns io.EOF if there is nothing more to read.
func (r *qLogReader) ReadNext() (string, error) {
if len(r.qFiles) == 0 {
return "", io.EOF
}
@@ -122,7 +121,7 @@ func (r *QLogReader) ReadNext() (string, error) {
q := r.qFiles[r.currentFile]
line, err := q.ReadNext()
if err != nil {
// Shift to the older file
// Shift to the older file.
r.currentFile--
if r.currentFile < 0 {
break
@@ -130,10 +129,10 @@ func (r *QLogReader) ReadNext() (string, error) {
q = r.qFiles[r.currentFile]
// Set it's position to the start right away
// Set its position to the start right away.
_, err = q.SeekStart()
// This is unexpected, return an error right away
// This is unexpected, return an error right away.
if err != nil {
return "", err
}
@@ -142,17 +141,17 @@ func (r *QLogReader) ReadNext() (string, error) {
}
}
// Nothing to read anymore
// Nothing to read anymore.
return "", io.EOF
}
// Close closes the QLogReader
func (r *QLogReader) Close() error {
// Close closes the qLogReader.
func (r *qLogReader) Close() error {
return closeQFiles(r.qFiles)
}
// closeQFiles - helper method to close multiple QLogFile instances
func closeQFiles(qFiles []*QLogFile) error {
// closeQFiles is a helper method to close multiple qLogFile instances.
func closeQFiles(qFiles []*qLogFile) error {
var errs []error
for _, q := range qFiles {
@@ -163,7 +162,7 @@ func closeQFiles(qFiles []*QLogFile) error {
}
if len(errs) > 0 {
return errors.List("error while closing QLogReader", errs...)
return errors.List("error while closing qLogReader", errs...)
}
return nil

16
internal/querylog/qlogreader_test.go
View File

@@ -10,15 +10,15 @@ import (
"github.com/stretchr/testify/require"
)
// newTestQLogReader creates new *QLogReader for tests and registers the
// newTestQLogReader creates new *qLogReader for tests and registers the
// required cleanup functions.
func newTestQLogReader(t *testing.T, filesNum, linesNum int) (reader *QLogReader) {
func newTestQLogReader(t *testing.T, filesNum, linesNum int) (reader *qLogReader) {
t.Helper()
testFiles := prepareTestFiles(t, filesNum, linesNum)
// Create the new QLogReader instance.
reader, err := NewQLogReader(testFiles)
// Create the new qLogReader instance.
reader, err := newQLogReader(testFiles)
require.NoError(t, err)
assert.NotNil(t, reader)
@@ -75,9 +75,9 @@ func TestQLogReader_Seek(t *testing.T) {
r := newTestQLogReader(t, 2, 10000)
testCases := []struct {
want error
name string
time string
want error
}{{
name: "not_too_old",
time: "2020-02-18T22:39:35.920973+03:00",
@@ -97,7 +97,7 @@ func TestQLogReader_Seek(t *testing.T) {
}, {
name: "non-existent_long_ago",
time: "2000-02-19T01:23:16.920973+03:00",
want: ErrTSNotFound,
want: errTSNotFound,
}, {
name: "non-existent_far_ahead",
time: "2100-02-19T01:23:16.920973+03:00",
@@ -105,7 +105,7 @@ func TestQLogReader_Seek(t *testing.T) {
}, {
name: "non-existent_but_could",
time: "2020-02-18T22:36:37.000000+03:00",
want: ErrTSNotFound,
want: errTSNotFound,
}}
for _, tc := range testCases {
@@ -125,9 +125,9 @@ func TestQLogReader_ReadNext(t *testing.T) {
r := newTestQLogReader(t, filesNum, linesNum)
testCases := []struct {
want error
name string
start int
want error
}{{
name: "ok",
start: 0,

138
internal/querylog/search.go
View File

@@ -1,9 +1,11 @@
package querylog
import (
"fmt"
"io"
"time"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"golang.org/x/exp/slices"
)
@@ -134,84 +136,112 @@ func (l *queryLog) search(params *searchParams) (entries []*logEntry, oldest tim
return entries, oldest
}
// searchFiles looks up log records from all log files. It optionally uses the
// client cache, if provided. searchFiles does not scan more than
// maxFileScanEntries so callers may need to call it several times to get all
// results. oldest and total are the time of the oldest processed entry and the
// total number of processed entries, including discarded ones, correspondingly.
func (l *queryLog) searchFiles(
params *searchParams,
cache clientCache,
) (entries []*logEntry, oldest time.Time, total int) {
// seekRecord changes the current position to the next record older than the
// provided parameter.
func (r *qLogReader) seekRecord(olderThan time.Time) (err error) {
if olderThan.IsZero() {
return r.SeekStart()
}
err = r.seekTS(olderThan.UnixNano())
if err == nil {
// Read to the next record, because we only need the one that goes
// after it.
_, err = r.ReadNext()
}
return err
}
// setQLogReader creates a reader with the specified files and sets the
// position to the next record older than the provided parameter.
func (l *queryLog) setQLogReader(olderThan time.Time) (qr *qLogReader, err error) {
files := []string{
l.logFile + ".1",
l.logFile,
}
r, err := NewQLogReader(files)
r, err := newQLogReader(files)
if err != nil {
log.Error("querylog: opening qlog reader: %s", err)
return entries, oldest, 0
}
defer func() {
closeErr := r.Close()
if closeErr != nil {
log.Error("querylog: closing file: %s", err)
}
}()
if params.olderThan.IsZero() {
err = r.SeekStart()
} else {
err = r.seekTS(params.olderThan.UnixNano())
if err == nil {
// Read to the next record, because we only need the one that goes
// after it.
_, err = r.ReadNext()
}
return nil, fmt.Errorf("opening qlog reader: %s", err)
}
err = r.seekRecord(olderThan)
if err != nil {
log.Debug("querylog: cannot seek to %s: %s", params.olderThan, err)
defer func() { err = errors.WithDeferred(err, r.Close()) }()
log.Debug("querylog: cannot seek to %s: %s", olderThan, err)
return entries, oldest, 0
return nil, nil
}
totalLimit := params.offset + params.limit
oldestNano := int64(0)
return r, nil
}
// By default, we do not scan more than maxFileScanEntries at once. The
// idea is to make search calls faster so that the UI could handle it and
// show something quicker. This behavior can be overridden if
// maxFileScanEntries is set to 0.
// readEntries reads entries from the reader to totalLimit. By default, we do
// not scan more than maxFileScanEntries at once. The idea is to make search
// calls faster so that the UI could handle it and show something quicker.
// This behavior can be overridden if maxFileScanEntries is set to 0.
func (l *queryLog) readEntries(
r *qLogReader,
params *searchParams,
cache clientCache,
totalLimit int,
) (entries []*logEntry, oldestNano int64, total int) {
for total < params.maxFileScanEntries || params.maxFileScanEntries <= 0 {
var e *logEntry
var ts int64
e, ts, err = l.readNextEntry(r, params, cache)
if err != nil {
if err == io.EOF {
ent, ts, rErr := l.readNextEntry(r, params, cache)
if rErr != nil {
if rErr == io.EOF {
oldestNano = 0
break
}
log.Error("querylog: reading next entry: %s", err)
log.Error("querylog: reading next entry: %s", rErr)
}
oldestNano = ts
total++
if e != nil {
entries = append(entries, e)
if len(entries) == totalLimit {
break
}
if ent == nil {
continue
}
entries = append(entries, ent)
if len(entries) == totalLimit {
break
}
}
return entries, oldestNano, total
}
// searchFiles looks up log records from all log files. It optionally uses the
// client cache, if provided. searchFiles does not scan more than
// maxFileScanEntries so callers may need to call it several times to get all
// the results. oldest and total are the time of the oldest processed entry
// and the total number of processed entries, including discarded ones,
// correspondingly.
func (l *queryLog) searchFiles(
params *searchParams,
cache clientCache,
) (entries []*logEntry, oldest time.Time, total int) {
r, err := l.setQLogReader(params.olderThan)
if err != nil {
log.Error("querylog: %s", err)
}
if r == nil {
return entries, oldest, 0
}
defer func() {
if closeErr := r.Close(); closeErr != nil {
log.Error("querylog: closing file: %s", closeErr)
}
}()
totalLimit := params.offset + params.limit
entries, oldestNano, total := l.readEntries(r, params, cache, totalLimit)
if oldestNano != 0 {
oldest = time.Unix(0, oldestNano)
}
@@ -243,11 +273,11 @@ func (f quickMatchClientFinder) findClient(clientID, ip string) (c *Client) {
}
// readNextEntry reads the next log entry and checks if it matches the search
// criteria. It optionally uses the client cache, if provided. e is nil if the
// entry doesn't match the search criteria. ts is the timestamp of the
// criteria. It optionally uses the client cache, if provided. e is nil if
// the entry doesn't match the search criteria. ts is the timestamp of the
// processed entry.
func (l *queryLog) readNextEntry(
r *QLogReader,
r *qLogReader,
params *searchParams,
cache clientCache,
) (e *logEntry, ts int64, err error) {

25
internal/querylog/searchparams.go
View File

@@ -2,18 +2,25 @@ package querylog
import "time"
// searchParams represent the search query sent by the client
// searchParams represent the search query sent by the client.
type searchParams struct {
// searchCriteria - list of search criteria that we use to get filter results
searchCriteria []searchCriterion
// olderThen - return entries that are older than this value
// if not set - disregard it and return any value
// olderThen represents a parameter for entries that are older than this
// parameter value. If not set, disregard it and return any value.
olderThan time.Time
offset int // offset for the search
limit int // limit the number of records returned
maxFileScanEntries int // maximum log entries to scan in query log files. if 0 - no limit
// searchCriteria is a list of search criteria that we use to get filter
// results.
searchCriteria []searchCriterion
// offset for the search.
offset int
// limit the number of records returned.
limit int
// maxFileScanEntries is a maximum of log entries to scan in query log
// files. If not set, then no limit.
maxFileScanEntries int
}
// newSearchParams - creates an empty instance of searchParams

14
internal/tools/go.mod
View File

@@ -8,16 +8,16 @@ require (
github.com/gordonklaus/ineffassign v0.0.0-20230107090616-13ace0543b28
github.com/kisielk/errcheck v1.6.3
github.com/kyoh86/looppointer v0.2.1
github.com/securego/gosec/v2 v2.15.0
golang.org/x/tools v0.8.0
golang.org/x/vuln v0.0.0-20230418010118-28ba02ac73db
github.com/securego/gosec/v2 v2.16.0
golang.org/x/tools v0.9.3
golang.org/x/vuln v0.1.0
honnef.co/go/tools v0.4.3
mvdan.cc/gofumpt v0.5.0
mvdan.cc/unparam v0.0.0-20230312165513-e84e2d14e3b8
)
require (
github.com/BurntSushi/toml v1.2.1 // indirect
github.com/BurntSushi/toml v1.3.1 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gookit/color v1.5.3 // indirect
@@ -25,9 +25,9 @@ require (
github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
golang.org/x/exp/typeparams v0.0.0-20230321023759-10a507213a29 // indirect
golang.org/x/exp/typeparams v0.0.0-20230522175609-2e198f4a06a1 // indirect
golang.org/x/mod v0.10.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.7.0 // indirect
golang.org/x/sync v0.2.0 // indirect
golang.org/x/sys v0.8.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

40
internal/tools/go.sum
View File

@@ -1,17 +1,19 @@
github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/toml v1.3.1 h1:rHnDkSK+/g6DlREUK73PkmIs60pqrnuduK+JmP++JmU=
github.com/BurntSushi/toml v1.3.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo=
github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA=
github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/golangci/misspell v0.4.0 h1:KtVB/hTK4bbL/S6bs64rYyk8adjmh1BygbBiaAiX+a0=
github.com/golangci/misspell v0.4.0/go.mod h1:W6O/bwV6lGDxUCChm2ykw9NQdd5bYd1Xkjo88UcWyJc=
github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786 h1:rcv+Ippz6RAtvaGgKxc+8FQIpxHgsF+HBzPyYL2cyVU=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -29,13 +31,13 @@ github.com/kyoh86/nolint v0.0.1 h1:GjNxDEkVn2wAxKHtP7iNTrRxytRZ1wXxLV5j4XzGfRU=
github.com/kyoh86/nolint v0.0.1/go.mod h1:1ZiZZ7qqrZ9dZegU96phwVcdQOMKIqRzFJL3ewq9gtI=
github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 h1:4kuARK6Y6FxaNu/BnU2OAaLF86eTVhP2hjTB6iMvItA=
github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354/go.mod h1:KSVJerMDfblTH7p5MZaTt+8zaT2iEk3AkVb9PQdZuE8=
github.com/onsi/ginkgo/v2 v2.8.0 h1:pAM+oBNPrpXRs+E/8spkeGx9QgekbRVyr74EUvRVOUI=
github.com/onsi/gomega v1.26.0 h1:03cDLK28U6hWvCAns6NeydX3zIm4SF3ci69ulidS32Q=
github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE=
github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/securego/gosec/v2 v2.15.0 h1:v4Ym7FF58/jlykYmmhZ7mTm7FQvN/setNm++0fgIAtw=
github.com/securego/gosec/v2 v2.15.0/go.mod h1:VOjTrZOkUtSDt2QLSJmQBMWnvwiQPEjg0l+5juIqGk8=
github.com/securego/gosec/v2 v2.16.0 h1:Pi0JKoasQQ3NnoRao/ww/N/XdynIB9NRYYZT5CyOs5U=
github.com/securego/gosec/v2 v2.16.0/go.mod h1:xvLcVZqUfo4aAQu56TNv7/Ltz6emAOQAEsrZrt7uGlI=
github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
@@ -49,8 +51,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug=
golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/exp/typeparams v0.0.0-20230321023759-10a507213a29 h1:e7LhZmJ631l59keHP9ssC3sgSn3/oiEHKHKXDkimURY=
golang.org/x/exp/typeparams v0.0.0-20230321023759-10a507213a29/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
golang.org/x/exp/typeparams v0.0.0-20230522175609-2e198f4a06a1 h1:pnP8r+W8Fm7XJ8CWtXi4S9oJmPBTrkfYN/dNbaPj6Y4=
golang.org/x/exp/typeparams v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
@@ -62,12 +64,12 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -77,23 +79,23 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20201007032633-0806396f153e/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
golang.org/x/vuln v0.0.0-20230418010118-28ba02ac73db h1:tLxfII6jPR3mfwEMkyOakawu+Lldo9hIA7vliXnDZYg=
golang.org/x/vuln v0.0.0-20230418010118-28ba02ac73db/go.mod h1:64LpnL2PuSMzFYeCmJjYiRbroOUG9aCZYznINnF5PHE=
golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM=
golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
golang.org/x/vuln v0.1.0 h1:9GRdj6wAIkDrsMevuolY+SXERPjQPp2P1ysYA0jpZe0=
golang.org/x/vuln v0.1.0/go.mod h1:/YuzZYjGbwB8y19CisAppfyw3uTZnuCz3r+qgx/QRzU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=