daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pull request 2273: AG-27492-client-storage-runtime-sources

Browse Source 
Squashed commit of the following:

commit 3191224d6de3d679ed7c3a35a383ebed27d3c181
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Sep 26 18:20:04 2024 +0300

    client: imp tests

commit 6cc4ed53a206657c5626a5894190a6851ebb3b01
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Sep 26 18:04:36 2024 +0300

    client: imp code

commit 79272b299ad16534ba4919e1f0616a875d7a15cb
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Sep 26 16:10:06 2024 +0300

    all: imp code

commit 0a001fffbe4dece198f9343b998d606e85f94703
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Sep 24 20:05:47 2024 +0300

    all: imp tests

commit 80f7e98d302f78332a4e297f243ad1c4a958cadd
Merge: df7492e9d e338214ad
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Sep 24 19:10:13 2024 +0300

    Merge branch 'master' into AG-27492-client-storage-runtime-sources

commit df7492e9de063e3c1b582e9f014554bc6cba7e90
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Sep 24 19:06:37 2024 +0300

    all: imp code

commit 23896ae5a6ee3f2d3a54dc15536d1c9349ba46b5
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Sep 19 21:04:34 2024 +0300

    client: fix typo

commit ba0ba2478ccd50d764d1a6a2d5c4b4b35b9b2a1c
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Sep 19 21:02:13 2024 +0300

    all: imp code

commit f7315be742422dda8d0adede6666b119761e67f4
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Sep 12 14:35:38 2024 +0300

    home: imp code

commit f63d0e80fb34f0d2799be7d51138ad1200449fc5
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Sep 12 14:15:49 2024 +0300

    all: imp code

commit 9feda414b6e469d911c9428d9ce1c5043d27cc93
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Sep 10 17:53:42 2024 +0300

    all: imp code

commit fafd7cbb52d36bea260c9d922b7c4204f73e7a66
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Sep 9 21:13:05 2024 +0300

    all: imp code

commit 2d2b8e02163e1bd03bf46d46ef7dbb6a50c4e844
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Sep 5 20:55:10 2024 +0300

    client: add tests

commit 4d394e6f2162f797c409073cb52e1e0de93a9d5e
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Aug 29 20:40:38 2024 +0300

    all: client storage runtime sources
This commit is contained in:
Stanislav Chzhen
2024-09-30 14:17:42 +03:00
parent e338214ad5
commit d40de33316
14 changed files with 1008 additions and 1176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

470
internal/client/storage.go
View File

@@ -1,30 +1,113 @@
package client
import (
"context"
"fmt"
"net"
"net/netip"
"slices"
"sync"
"time"
"github.com/AdguardTeam/golibs/container"
"github.com/AdguardTeam/AdGuardHome/internal/arpdb"
"github.com/AdguardTeam/AdGuardHome/internal/dhcpsvc"
"github.com/AdguardTeam/AdGuardHome/internal/whois"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/hostsfile"
"github.com/AdguardTeam/golibs/log"
)
// Config is the client storage configuration structure.
//
// TODO(s.chzhen): Expand.
type Config struct {
// AllowedTags is a list of all allowed client tags.
AllowedTags []string
// allowedTags is the list of available client tags.
var allowedTags = []string{
"device_audio",
"device_camera",
"device_gameconsole",
"device_laptop",
"device_nas", // Network-attached Storage
"device_other",
"device_pc",
"device_phone",
"device_printer",
"device_securityalarm",
"device_tablet",
"device_tv",
"os_android",
"os_ios",
"os_linux",
"os_macos",
"os_other",
"os_windows",
"user_admin",
"user_child",
"user_regular",
}
// DHCP is an interface for accessing DHCP lease data the [Storage] needs.
type DHCP interface {
// Leases returns all the DHCP leases.
Leases() (leases []*dhcpsvc.Lease)
// HostByIP returns the hostname of the DHCP client with the given IP
// address. host will be empty if there is no such client, due to an
// assumption that a DHCP client must always have a hostname.
HostByIP(ip netip.Addr) (host string)
// MACByIP returns the MAC address for the given IP address leased. It
// returns nil if there is no such client, due to an assumption that a DHCP
// client must always have a MAC address.
MACByIP(ip netip.Addr) (mac net.HardwareAddr)
}
// EmptyDHCP is the empty [DHCP] implementation that does nothing.
type EmptyDHCP struct{}
// type check
var _ DHCP = EmptyDHCP{}
// Leases implements the [DHCP] interface for emptyDHCP.
func (EmptyDHCP) Leases() (leases []*dhcpsvc.Lease) { return nil }
// HostByIP implements the [DHCP] interface for emptyDHCP.
func (EmptyDHCP) HostByIP(_ netip.Addr) (host string) { return "" }
// MACByIP implements the [DHCP] interface for emptyDHCP.
func (EmptyDHCP) MACByIP(_ netip.Addr) (mac net.HardwareAddr) { return nil }
// HostsContainer is an interface for receiving updates to the system hosts
// file.
type HostsContainer interface {
Upd() (updates <-chan *hostsfile.DefaultStorage)
}
// StorageConfig is the client storage configuration structure.
type StorageConfig struct {
// DHCP is used to match IPs against MACs of persistent clients and update
// [SourceDHCP] runtime client information. It must not be nil.
DHCP DHCP
// EtcHosts is used to update [SourceHostsFile] runtime client information.
EtcHosts HostsContainer
// ARPDB is used to update [SourceARP] runtime client information.
ARPDB arpdb.Interface
// InitialClients is a list of persistent clients parsed from the
// configuration file. Each client must not be nil.
InitialClients []*Persistent
// ARPClientsUpdatePeriod defines how often [SourceARP] runtime client
// information is updated.
ARPClientsUpdatePeriod time.Duration
// RuntimeSourceDHCP specifies whether to update [SourceDHCP] information
// of runtime clients.
RuntimeSourceDHCP bool
}
// Storage contains information about persistent and runtime clients.
type Storage struct {
// allowedTags is a set of all allowed tags.
allowedTags *container.MapSet[string]
// mu protects indexes of persistent and runtime clients.
mu *sync.Mutex
@@ -32,19 +115,250 @@ type Storage struct {
index *index
// runtimeIndex contains information about runtime clients.
runtimeIndex *RuntimeIndex
runtimeIndex *runtimeIndex
// dhcp is used to update [SourceDHCP] runtime client information.
dhcp DHCP
// etcHosts is used to update [SourceHostsFile] runtime client information.
etcHosts HostsContainer
// arpDB is used to update [SourceARP] runtime client information.
arpDB arpdb.Interface
// done is the shutdown signaling channel.
done chan struct{}
// allowedTags is a sorted list of all allowed tags. It must not be
// modified after initialization.
//
// TODO(s.chzhen): Use custom type.
allowedTags []string
// arpClientsUpdatePeriod defines how often [SourceARP] runtime client
// information is updated. It must be greater than zero.
arpClientsUpdatePeriod time.Duration
// runtimeSourceDHCP specifies whether to update [SourceDHCP] information
// of runtime clients.
runtimeSourceDHCP bool
}
// NewStorage returns initialized client storage. conf must not be nil.
func NewStorage(conf *Config) (s *Storage) {
allowedTags := container.NewMapSet(conf.AllowedTags...)
func NewStorage(conf *StorageConfig) (s *Storage, err error) {
tags := slices.Clone(allowedTags)
slices.Sort(tags)
return &Storage{
allowedTags: allowedTags,
mu: &sync.Mutex{},
index: newIndex(),
runtimeIndex: NewRuntimeIndex(),
s = &Storage{
allowedTags: tags,
mu: &sync.Mutex{},
index: newIndex(),
runtimeIndex: newRuntimeIndex(),
dhcp: conf.DHCP,
etcHosts: conf.EtcHosts,
arpDB: conf.ARPDB,
done: make(chan struct{}),
arpClientsUpdatePeriod: conf.ARPClientsUpdatePeriod,
runtimeSourceDHCP: conf.RuntimeSourceDHCP,
}
for i, p := range conf.InitialClients {
err = s.Add(p)
if err != nil {
return nil, fmt.Errorf("adding client %q at index %d: %w", p.Name, i, err)
}
}
s.ReloadARP()
return s, nil
}
// Start starts the goroutines for updating the runtime client information.
//
// TODO(s.chzhen): Pass context.
func (s *Storage) Start(_ context.Context) (err error) {
go s.periodicARPUpdate()
go s.handleHostsUpdates()
return nil
}
// Shutdown gracefully stops the client storage.
//
// TODO(s.chzhen): Pass context.
func (s *Storage) Shutdown(_ context.Context) (err error) {
close(s.done)
return s.closeUpstreams()
}
// periodicARPUpdate periodically reloads runtime clients from ARP. It is
// intended to be used as a goroutine.
func (s *Storage) periodicARPUpdate() {
defer log.OnPanic("storage")
t := time.NewTicker(s.arpClientsUpdatePeriod)
for {
select {
case <-t.C:
s.ReloadARP()
case <-s.done:
return
}
}
}
// ReloadARP reloads runtime clients from ARP, if configured.
func (s *Storage) ReloadARP() {
if s.arpDB != nil {
s.addFromSystemARP()
}
}
// addFromSystemARP adds the IP-hostname pairings from the output of the arp -a
// command.
func (s *Storage) addFromSystemARP() {
s.mu.Lock()
defer s.mu.Unlock()
if err := s.arpDB.Refresh(); err != nil {
s.arpDB = arpdb.Empty{}
log.Error("refreshing arp container: %s", err)
return
}
ns := s.arpDB.Neighbors()
if len(ns) == 0 {
log.Debug("refreshing arp container: the update is empty")
return
}
src := SourceARP
s.runtimeIndex.clearSource(src)
for _, n := range ns {
s.runtimeIndex.setInfo(n.IP, src, []string{n.Name})
}
removed := s.runtimeIndex.removeEmpty()
log.Debug("storage: added %d, removed %d client aliases from arp neighborhood", len(ns), removed)
}
// handleHostsUpdates receives the updates from the hosts container and adds
// them to the clients storage. It is intended to be used as a goroutine.
func (s *Storage) handleHostsUpdates() {
if s.etcHosts == nil {
return
}
defer log.OnPanic("storage")
for {
select {
case upd, ok := <-s.etcHosts.Upd():
if !ok {
return
}
s.addFromHostsFile(upd)
case <-s.done:
return
}
}
}
// addFromHostsFile fills the client-hostname pairing index from the system's
// hosts files.
func (s *Storage) addFromHostsFile(hosts *hostsfile.DefaultStorage) {
s.mu.Lock()
defer s.mu.Unlock()
src := SourceHostsFile
s.runtimeIndex.clearSource(src)
added := 0
hosts.RangeNames(func(addr netip.Addr, names []string) (cont bool) {
// Only the first name of the first record is considered a canonical
// hostname for the IP address.
//
// TODO(e.burkov): Consider using all the names from all the records.
s.runtimeIndex.setInfo(addr, src, []string{names[0]})
added++
return true
})
removed := s.runtimeIndex.removeEmpty()
log.Debug("storage: added %d, removed %d client aliases from system hosts file", added, removed)
}
// type check
var _ AddressUpdater = (*Storage)(nil)
// UpdateAddress implements the [AddressUpdater] interface for *Storage
func (s *Storage) UpdateAddress(ip netip.Addr, host string, info *whois.Info) {
// Common fast path optimization.
if host == "" && info == nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
if host != "" {
s.runtimeIndex.setInfo(ip, SourceRDNS, []string{host})
}
if info != nil {
s.setWHOISInfo(ip, info)
}
}
// UpdateDHCP updates [SourceDHCP] runtime client information.
func (s *Storage) UpdateDHCP() {
if s.dhcp == nil || !s.runtimeSourceDHCP {
return
}
s.mu.Lock()
defer s.mu.Unlock()
src := SourceDHCP
s.runtimeIndex.clearSource(src)
added := 0
for _, l := range s.dhcp.Leases() {
s.runtimeIndex.setInfo(l.IP, src, []string{l.Hostname})
added++
}
removed := s.runtimeIndex.removeEmpty()
log.Debug("storage: added %d, removed %d client aliases from dhcp", added, removed)
}
// setWHOISInfo sets the WHOIS information for a runtime client.
func (s *Storage) setWHOISInfo(ip netip.Addr, wi *whois.Info) {
_, ok := s.index.findByIP(ip)
if ok {
log.Debug("storage: client for %s is already created, ignore whois info", ip)
return
}
rc := s.runtimeIndex.client(ip)
if rc == nil {
rc = NewRuntime(ip)
s.runtimeIndex.add(rc)
}
rc.setWHOIS(wi)
log.Debug("storage: set whois info for runtime client with ip %s: %+v", ip, wi)
}
// Add stores persistent client information or returns an error.
@@ -94,6 +408,9 @@ func (s *Storage) FindByName(name string) (p *Persistent, ok bool) {
// Find finds persistent client by string representation of the client ID, IP
// address, or MAC. And returns its shallow copy.
//
// TODO(s.chzhen): Accept ClientIDData structure instead, which will contain
// the parsed IP address, if any.
func (s *Storage) Find(id string) (p *Persistent, ok bool) {
s.mu.Lock()
defer s.mu.Unlock()
@@ -103,6 +420,16 @@ func (s *Storage) Find(id string) (p *Persistent, ok bool) {
return p.ShallowClone(), ok
}
ip, err := netip.ParseAddr(id)
if err != nil {
return nil, false
}
foundMAC := s.dhcp.MACByIP(ip)
if foundMAC != nil {
return s.FindByMAC(foundMAC)
}
return nil, false
}
@@ -130,11 +457,9 @@ func (s *Storage) FindLoose(ip netip.Addr, id string) (p *Persistent, ok bool) {
return nil, false
}
// FindByMAC finds persistent client by MAC and returns its shallow copy.
// FindByMAC finds persistent client by MAC and returns its shallow copy. s.mu
// is expected to be locked.
func (s *Storage) FindByMAC(mac net.HardwareAddr) (p *Persistent, ok bool) {
s.mu.Lock()
defer s.mu.Unlock()
p, ok = s.index.findByMAC(mac)
if ok {
return p.ShallowClone(), ok
@@ -216,8 +541,8 @@ func (s *Storage) Size() (n int) {
return s.index.size()
}
// CloseUpstreams closes upstream configurations of persistent clients.
func (s *Storage) CloseUpstreams() (err error) {
// closeUpstreams closes upstream configurations of persistent clients.
func (s *Storage) closeUpstreams() (err error) {
s.mu.Lock()
defer s.mu.Unlock()
@@ -226,89 +551,27 @@ func (s *Storage) CloseUpstreams() (err error) {
// ClientRuntime returns a copy of the saved runtime client by ip. If no such
// client exists, returns nil.
//
// TODO(s.chzhen): Use it.
func (s *Storage) ClientRuntime(ip netip.Addr) (rc *Runtime) {
s.mu.Lock()
defer s.mu.Unlock()
return s.runtimeIndex.Client(ip)
}
// UpdateRuntime updates the stored runtime client with information from rc. If
// no such client exists, saves the copy of rc in storage. rc must not be nil.
func (s *Storage) UpdateRuntime(rc *Runtime) (added bool) {
s.mu.Lock()
defer s.mu.Unlock()
return s.updateRuntimeLocked(rc)
}
// updateRuntimeLocked updates the stored runtime client with information from
// rc. rc must not be nil. Storage.mu is expected to be locked.
func (s *Storage) updateRuntimeLocked(rc *Runtime) (added bool) {
stored := s.runtimeIndex.Client(rc.ip)
if stored == nil {
s.runtimeIndex.Add(rc.Clone())
return true
rc = s.runtimeIndex.client(ip)
if rc != nil {
return rc.clone()
}
if rc.whois != nil {
stored.whois = rc.whois.Clone()
if !s.runtimeSourceDHCP {
return nil
}
if rc.arp != nil {
stored.arp = slices.Clone(rc.arp)
host := s.dhcp.HostByIP(ip)
if host == "" {
return nil
}
if rc.rdns != nil {
stored.rdns = slices.Clone(rc.rdns)
}
rc = s.runtimeIndex.setInfo(ip, SourceDHCP, []string{host})
if rc.dhcp != nil {
stored.dhcp = slices.Clone(rc.dhcp)
}
if rc.hostsFile != nil {
stored.hostsFile = slices.Clone(rc.hostsFile)
}
return false
}
// BatchUpdateBySource updates the stored runtime clients information from the
// specified source and returns the number of added and removed clients.
func (s *Storage) BatchUpdateBySource(src Source, rcs []*Runtime) (added, removed int) {
s.mu.Lock()
defer s.mu.Unlock()
for _, rc := range s.runtimeIndex.index {
rc.unset(src)
}
for _, rc := range rcs {
if s.updateRuntimeLocked(rc) {
added++
}
}
for ip, rc := range s.runtimeIndex.index {
if rc.isEmpty() {
delete(s.runtimeIndex.index, ip)
removed++
}
}
return added, removed
}
// SizeRuntime returns the number of the runtime clients.
func (s *Storage) SizeRuntime() (n int) {
s.mu.Lock()
defer s.mu.Unlock()
return s.runtimeIndex.Size()
return rc.clone()
}
// RangeRuntime calls f for each runtime client in an undefined order.
@@ -316,16 +579,11 @@ func (s *Storage) RangeRuntime(f func(rc *Runtime) (cont bool)) {
s.mu.Lock()
defer s.mu.Unlock()
s.runtimeIndex.Range(f)
s.runtimeIndex.rangeClients(f)
}
// DeleteBySource removes all runtime clients that have information only from
// the specified source and returns the number of removed clients.
//
// TODO(s.chzhen): Use it.
func (s *Storage) DeleteBySource(src Source) (n int) {
s.mu.Lock()
defer s.mu.Unlock()
return s.runtimeIndex.DeleteBySource(src)
// AllowedTags returns the list of available client tags. tags must not be
// modified.
func (s *Storage) AllowedTags() (tags []string) {
return s.allowedTags
}