all: sync with master

internal/home/clients.go

@@ -1,7 +1,6 @@
 package home
 
 import (
-	"bytes"
 	"fmt"
 	"net"
 	"net/netip"
@@ -20,6 +19,7 @@ import (
 	"github.com/AdguardTeam/dnsproxy/proxy"
 	"github.com/AdguardTeam/dnsproxy/upstream"
 	"github.com/AdguardTeam/golibs/errors"
+	"github.com/AdguardTeam/golibs/hostsfile"
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/AdguardTeam/golibs/stringutil"
 	"golang.org/x/exp/maps"
@@ -47,11 +47,11 @@ type DHCP interface {
 type clientsContainer struct {
 	// TODO(a.garipov): Perhaps use a number of separate indices for different
 	// types (string, netip.Addr, and so on).
-	list    map[string]*Client // name -> client
-	idIndex map[string]*Client // ID -> client
+	list    map[string]*persistentClient // name -> client
+	idIndex map[string]*persistentClient // ID -> client
 
-	// ipToRC is the IP address to *RuntimeClient map.
-	ipToRC map[netip.Addr]*RuntimeClient
+	// ipToRC maps IP addresses to runtime client information.
+	ipToRC map[netip.Addr]*client.Runtime
 
 	allTags *stringutil.Set
 
@@ -102,9 +102,9 @@ func (clients *clientsContainer) Init(
 		log.Fatal("clients.list != nil")
 	}
 
-	clients.list = make(map[string]*Client)
-	clients.idIndex = make(map[string]*Client)
-	clients.ipToRC = map[netip.Addr]*RuntimeClient{}
+	clients.list = map[string]*persistentClient{}
+	clients.idIndex = map[string]*persistentClient{}
+	clients.ipToRC = map[netip.Addr]*client.Runtime{}
 
 	clients.allTags = stringutil.NewSet(clientTags...)
 
@@ -139,6 +139,9 @@ func (clients *clientsContainer) Init(
 	return nil
 }
 
+// handleHostsUpdates receives the updates from the hosts container and adds
+// them to the clients container.  It's used to be called in a separate
+// goroutine.
 func (clients *clientsContainer) handleHostsUpdates() {
 	for upd := range clients.etcHosts.Upd() {
 		clients.addFromHostsFile(upd)
@@ -185,6 +188,9 @@ type clientObject struct {
 	Tags      []string `yaml:"tags"`
 	Upstreams []string `yaml:"upstreams"`
 
+	// UID is the unique identifier of the persistent client.
+	UID UID `yaml:"uid"`
+
 	// UpstreamsCacheSize is the DNS cache size (in bytes).
 	//
 	// TODO(d.kolyshev): Use [datasize.Bytesize].
@@ -203,66 +209,83 @@ type clientObject struct {
 	IgnoreStatistics bool `yaml:"ignore_statistics"`
 }
 
+// toPersistent returns an initialized persistent client if there are no errors.
+func (o *clientObject) toPersistent(
+	filteringConf *filtering.Config,
+	allTags *stringutil.Set,
+) (cli *persistentClient, err error) {
+	cli = &persistentClient{
+		Name: o.Name,
+
+		Upstreams: o.Upstreams,
+
+		UID: o.UID,
+
+		UseOwnSettings:        !o.UseGlobalSettings,
+		FilteringEnabled:      o.FilteringEnabled,
+		ParentalEnabled:       o.ParentalEnabled,
+		safeSearchConf:        o.SafeSearchConf,
+		SafeBrowsingEnabled:   o.SafeBrowsingEnabled,
+		UseOwnBlockedServices: !o.UseGlobalBlockedServices,
+		IgnoreQueryLog:        o.IgnoreQueryLog,
+		IgnoreStatistics:      o.IgnoreStatistics,
+		UpstreamsCacheEnabled: o.UpstreamsCacheEnabled,
+		UpstreamsCacheSize:    o.UpstreamsCacheSize,
+	}
+
+	err = cli.setIDs(o.IDs)
+	if err != nil {
+		return nil, fmt.Errorf("parsing ids: %w", err)
+	}
+
+	if (cli.UID == UID{}) {
+		cli.UID, err = NewUID()
+		if err != nil {
+			return nil, fmt.Errorf("generating uid: %w", err)
+		}
+	}
+
+	if o.SafeSearchConf.Enabled {
+		o.SafeSearchConf.CustomResolver = safeSearchResolver{}
+
+		err = cli.setSafeSearch(
+			o.SafeSearchConf,
+			filteringConf.SafeSearchCacheSize,
+			time.Minute*time.Duration(filteringConf.CacheTime),
+		)
+		if err != nil {
+			return nil, fmt.Errorf("init safesearch %q: %w", cli.Name, err)
+		}
+	}
+
+	err = o.BlockedServices.Validate()
+	if err != nil {
+		return nil, fmt.Errorf("init blocked services %q: %w", cli.Name, err)
+	}
+
+	cli.BlockedServices = o.BlockedServices.Clone()
+
+	cli.setTags(o.Tags, allTags)
+
+	return cli, nil
+}
+
 // addFromConfig initializes the clients container with objects from the
 // configuration file.
 func (clients *clientsContainer) addFromConfig(
 	objects []*clientObject,
 	filteringConf *filtering.Config,
 ) (err error) {
-	for _, o := range objects {
-		cli := &Client{
-			Name: o.Name,
-
-			IDs:       o.IDs,
-			Upstreams: o.Upstreams,
-
-			UseOwnSettings:        !o.UseGlobalSettings,
-			FilteringEnabled:      o.FilteringEnabled,
-			ParentalEnabled:       o.ParentalEnabled,
-			safeSearchConf:        o.SafeSearchConf,
-			SafeBrowsingEnabled:   o.SafeBrowsingEnabled,
-			UseOwnBlockedServices: !o.UseGlobalBlockedServices,
-			IgnoreQueryLog:        o.IgnoreQueryLog,
-			IgnoreStatistics:      o.IgnoreStatistics,
-			UpstreamsCacheEnabled: o.UpstreamsCacheEnabled,
-			UpstreamsCacheSize:    o.UpstreamsCacheSize,
-		}
-
-		if o.SafeSearchConf.Enabled {
-			o.SafeSearchConf.CustomResolver = safeSearchResolver{}
-
-			err = cli.setSafeSearch(
-				o.SafeSearchConf,
-				filteringConf.SafeSearchCacheSize,
-				time.Minute*time.Duration(filteringConf.CacheTime),
-			)
-			if err != nil {
-				log.Error("clients: init client safesearch %q: %s", cli.Name, err)
-
-				continue
-			}
-		}
-
-		err = o.BlockedServices.Validate()
+	for i, o := range objects {
+		var cli *persistentClient
+		cli, err = o.toPersistent(filteringConf, clients.allTags)
 		if err != nil {
-			return fmt.Errorf("clients: init client blocked services %q: %w", cli.Name, err)
+			return fmt.Errorf("clients: init persistent client at index %d: %w", i, err)
 		}
 
-		cli.BlockedServices = o.BlockedServices.Clone()
-
-		for _, t := range o.Tags {
-			if clients.allTags.Has(t) {
-				cli.Tags = append(cli.Tags, t)
-			} else {
-				log.Info("clients: skipping unknown tag %q", t)
-			}
-		}
-
-		slices.Sort(cli.Tags)
-
-		_, err = clients.Add(cli)
+		_, err = clients.add(cli)
 		if err != nil {
-			log.Error("clients: adding clients %s: %s", cli.Name, err)
+			log.Error("clients: adding client at index %d %s: %s", i, cli.Name, err)
 		}
 	}
 
@@ -282,10 +305,12 @@ func (clients *clientsContainer) forConfig() (objs []*clientObject) {
 
 			BlockedServices: cli.BlockedServices.Clone(),
 
-			IDs:       stringutil.CloneSlice(cli.IDs),
+			IDs:       cli.ids(),
 			Tags:      stringutil.CloneSlice(cli.Tags),
 			Upstreams: stringutil.CloneSlice(cli.Upstreams),
 
+			UID: cli.UID,
+
 			UseGlobalSettings:        !cli.UseOwnSettings,
 			FilteringEnabled:         cli.FilteringEnabled,
 			ParentalEnabled:          cli.ParentalEnabled,
@@ -338,7 +363,7 @@ func (clients *clientsContainer) clientSource(ip netip.Addr) (src client.Source)
 
 	rc, ok := clients.ipToRC[ip]
 	if ok {
-		src = rc.Source
+		src, _ = rc.Info()
 	}
 
 	if src < client.SourceDHCP && clients.dhcp.HostByIP(ip) != "" {
@@ -348,10 +373,10 @@ func (clients *clientsContainer) clientSource(ip netip.Addr) (src client.Source)
 	return src
 }
 
-// findMultiple is a wrapper around Find to make it a valid client finder for
-// the query log.  c is never nil; if no information about the client is found,
-// it returns an artificial client record by only setting the blocking-related
-// fields.  err is always nil.
+// findMultiple is a wrapper around [clientsContainer.find] to make it a valid
+// client finder for the query log.  c is never nil; if no information about the
+// client is found, it returns an artificial client record by only setting the
+// blocking-related fields.  err is always nil.
 func (clients *clientsContainer) findMultiple(ids []string) (c *querylog.Client, err error) {
 	var artClient *querylog.Client
 	var art bool
@@ -385,20 +410,22 @@ func (clients *clientsContainer) clientOrArtificial(
 		}
 	}()
 
-	client, ok := clients.Find(id)
+	cli, ok := clients.find(id)
 	if ok {
 		return &querylog.Client{
-			Name:           client.Name,
-			IgnoreQueryLog: client.IgnoreQueryLog,
+			Name:           cli.Name,
+			IgnoreQueryLog: cli.IgnoreQueryLog,
 		}, false
 	}
 
-	var rc *RuntimeClient
+	var rc *client.Runtime
 	rc, ok = clients.findRuntimeClient(ip)
 	if ok {
+		_, host := rc.Info()
+
 		return &querylog.Client{
-			Name:  rc.Host,
-			WHOIS: rc.WHOIS,
+			Name:  host,
+			WHOIS: rc.WHOIS(),
 		}, false
 	}
 
@@ -407,8 +434,8 @@ func (clients *clientsContainer) clientOrArtificial(
 	}, true
 }
 
-// Find returns a shallow copy of the client if there is one found.
-func (clients *clientsContainer) Find(id string) (c *Client, ok bool) {
+// find returns a shallow copy of the client if there is one found.
+func (clients *clientsContainer) find(id string) (c *persistentClient, ok bool) {
 	clients.lock.Lock()
 	defer clients.lock.Unlock()
 
@@ -417,12 +444,12 @@ func (clients *clientsContainer) Find(id string) (c *Client, ok bool) {
 		return nil, false
 	}
 
-	return c.ShallowClone(), true
+	return c.shallowClone(), true
 }
 
-// shouldCountClient is a wrapper around Find to make it a valid client
-// information finder for the statistics.  If no information about the client
-// is found, it returns true.
+// shouldCountClient is a wrapper around [clientsContainer.find] to make it a
+// valid client information finder for the statistics.  If no information about
+// the client is found, it returns true.
 func (clients *clientsContainer) shouldCountClient(ids []string) (y bool) {
 	clients.lock.Lock()
 	defer clients.lock.Unlock()
@@ -490,7 +517,7 @@ func (clients *clientsContainer) UpstreamConfigByID(
 
 // findLocked searches for a client by its ID.  clients.lock is expected to be
 // locked.
-func (clients *clientsContainer) findLocked(id string) (c *Client, ok bool) {
+func (clients *clientsContainer) findLocked(id string) (c *persistentClient, ok bool) {
 	c, ok = clients.idIndex[id]
 	if ok {
 		return c, true
@@ -502,13 +529,7 @@ func (clients *clientsContainer) findLocked(id string) (c *Client, ok bool) {
 	}
 
 	for _, c = range clients.list {
-		for _, id := range c.IDs {
-			var subnet netip.Prefix
-			subnet, err = netip.ParsePrefix(id)
-			if err != nil {
-				continue
-			}
-
+		for _, subnet := range c.Subnets {
 			if subnet.Contains(ip) {
 				return c, true
 			}
@@ -521,22 +542,16 @@ func (clients *clientsContainer) findLocked(id string) (c *Client, ok bool) {
 
 // findDHCP searches for a client by its MAC, if the DHCP server is active and
 // there is such client.  clients.lock is expected to be locked.
-func (clients *clientsContainer) findDHCP(ip netip.Addr) (c *Client, ok bool) {
+func (clients *clientsContainer) findDHCP(ip netip.Addr) (c *persistentClient, ok bool) {
 	foundMAC := clients.dhcp.MACByIP(ip)
 	if foundMAC == nil {
 		return nil, false
 	}
 
 	for _, c = range clients.list {
-		for _, id := range c.IDs {
-			mac, err := net.ParseMAC(id)
-			if err != nil {
-				continue
-			}
-
-			if bytes.Equal(mac, foundMAC) {
-				return c, true
-			}
+		_, found := slices.BinarySearchFunc(c.MACs, foundMAC, slices.Compare[net.HardwareAddr])
+		if found {
+			return c, true
 		}
 	}
 
@@ -545,7 +560,7 @@ func (clients *clientsContainer) findDHCP(ip netip.Addr) (c *Client, ok bool) {
 
 // runtimeClient returns a runtime client from internal index.  Note that it
 // doesn't include DHCP clients.
-func (clients *clientsContainer) runtimeClient(ip netip.Addr) (rc *RuntimeClient, ok bool) {
+func (clients *clientsContainer) runtimeClient(ip netip.Addr) (rc *client.Runtime, ok bool) {
 	if ip == (netip.Addr{}) {
 		return nil, false
 	}
@@ -559,52 +574,43 @@ func (clients *clientsContainer) runtimeClient(ip netip.Addr) (rc *RuntimeClient
 }
 
 // findRuntimeClient finds a runtime client by their IP.
-func (clients *clientsContainer) findRuntimeClient(ip netip.Addr) (rc *RuntimeClient, ok bool) {
-	if rc, ok = clients.runtimeClient(ip); ok && rc.Source > client.SourceDHCP {
-		return rc, ok
-	}
-
+func (clients *clientsContainer) findRuntimeClient(ip netip.Addr) (rc *client.Runtime, ok bool) {
+	rc, ok = clients.runtimeClient(ip)
 	host := clients.dhcp.HostByIP(ip)
-	if host == "" {
-		return rc, ok
+
+	if host != "" {
+		if !ok {
+			rc = &client.Runtime{}
+		}
+
+		rc.SetInfo(client.SourceDHCP, []string{host})
+
+		return rc, true
 	}
 
-	return &RuntimeClient{
-		Host:   host,
-		Source: client.SourceDHCP,
-		WHOIS:  &whois.Info{},
-	}, true
+	return rc, ok
 }
 
-// check validates the client.
-func (clients *clientsContainer) check(c *Client) (err error) {
+// check validates the client.  It also sorts the client tags.
+func (clients *clientsContainer) check(c *persistentClient) (err error) {
 	switch {
 	case c == nil:
 		return errors.Error("client is nil")
 	case c.Name == "":
 		return errors.Error("invalid name")
-	case len(c.IDs) == 0:
+	case c.idsLen() == 0:
 		return errors.Error("id required")
 	default:
 		// Go on.
 	}
 
-	for i, id := range c.IDs {
-		var norm string
-		norm, err = normalizeClientIdentifier(id)
-		if err != nil {
-			return fmt.Errorf("client at index %d: %w", i, err)
-		}
-
-		c.IDs[i] = norm
-	}
-
 	for _, t := range c.Tags {
 		if !clients.allTags.Has(t) {
 			return fmt.Errorf("invalid tag: %q", t)
 		}
 	}
 
+	// TODO(s.chzhen):  Move to the constructor.
 	slices.Sort(c.Tags)
 
 	err = dnsforward.ValidateUpstreams(c.Upstreams)
@@ -615,38 +621,9 @@ func (clients *clientsContainer) check(c *Client) (err error) {
 	return nil
 }
 
-// normalizeClientIdentifier returns a normalized version of idStr.  If idStr
-// cannot be normalized, it returns an error.
-func normalizeClientIdentifier(idStr string) (norm string, err error) {
-	if idStr == "" {
-		return "", errors.Error("clientid is empty")
-	}
-
-	var ip netip.Addr
-	if ip, err = netip.ParseAddr(idStr); err == nil {
-		return ip.String(), nil
-	}
-
-	var subnet netip.Prefix
-	if subnet, err = netip.ParsePrefix(idStr); err == nil {
-		return subnet.String(), nil
-	}
-
-	var mac net.HardwareAddr
-	if mac, err = net.ParseMAC(idStr); err == nil {
-		return mac.String(), nil
-	}
-
-	if err = dnsforward.ValidateClientID(idStr); err == nil {
-		return strings.ToLower(idStr), nil
-	}
-
-	return "", fmt.Errorf("bad client identifier %q", idStr)
-}
-
-// Add adds a new client object.  ok is false if such client already exists or
+// add adds a new client object.  ok is false if such client already exists or
 // if an error occurred.
-func (clients *clientsContainer) Add(c *Client) (ok bool, err error) {
+func (clients *clientsContainer) add(c *persistentClient) (ok bool, err error) {
 	err = clients.check(c)
 	if err != nil {
 		return false, err
@@ -662,50 +639,52 @@ func (clients *clientsContainer) Add(c *Client) (ok bool, err error) {
 	}
 
 	// check ID index
-	for _, id := range c.IDs {
-		var c2 *Client
+	ids := c.ids()
+	for _, id := range ids {
+		var c2 *persistentClient
 		c2, ok = clients.idIndex[id]
 		if ok {
 			return false, fmt.Errorf("another client uses the same ID (%q): %q", id, c2.Name)
 		}
 	}
 
-	clients.add(c)
+	clients.addLocked(c)
 
-	log.Debug("clients: added %q: ID:%q [%d]", c.Name, c.IDs, len(clients.list))
+	log.Debug("clients: added %q: ID:%q [%d]", c.Name, ids, len(clients.list))
 
 	return true, nil
 }
 
-// add c to the indexes. clients.lock is expected to be locked.
-func (clients *clientsContainer) add(c *Client) {
+// addLocked c to the indexes.  clients.lock is expected to be locked.
+func (clients *clientsContainer) addLocked(c *persistentClient) {
 	// update Name index
 	clients.list[c.Name] = c
 
 	// update ID index
-	for _, id := range c.IDs {
+	for _, id := range c.ids() {
 		clients.idIndex[id] = c
 	}
 }
 
-// Del removes a client.  ok is false if there is no such client.
-func (clients *clientsContainer) Del(name string) (ok bool) {
+// remove removes a client.  ok is false if there is no such client.
+func (clients *clientsContainer) remove(name string) (ok bool) {
 	clients.lock.Lock()
 	defer clients.lock.Unlock()
 
-	var c *Client
+	var c *persistentClient
 	c, ok = clients.list[name]
 	if !ok {
 		return false
 	}
 
-	clients.del(c)
+	clients.removeLocked(c)
 
 	return true
 }
 
-// del removes c from the indexes. clients.lock is expected to be locked.
-func (clients *clientsContainer) del(c *Client) {
+// removeLocked removes c from the indexes.  clients.lock is expected to be
+// locked.
+func (clients *clientsContainer) removeLocked(c *persistentClient) {
 	if err := c.closeUpstreams(); err != nil {
 		log.Error("client container: removing client %s: %s", c.Name, err)
 	}
@@ -714,13 +693,13 @@ func (clients *clientsContainer) del(c *Client) {
 	delete(clients.list, c.Name)
 
 	// Update the ID index.
-	for _, id := range c.IDs {
+	for _, id := range c.ids() {
 		delete(clients.idIndex, id)
 	}
 }
 
-// Update updates a client by its name.
-func (clients *clientsContainer) Update(prev, c *Client) (err error) {
+// update updates a client by its name.
+func (clients *clientsContainer) update(prev, c *persistentClient) (err error) {
 	err = clients.check(c)
 	if err != nil {
 		// Don't wrap the error since it's informative enough as is.
@@ -738,18 +717,23 @@ func (clients *clientsContainer) Update(prev, c *Client) (err error) {
 		}
 	}
 
+	if c.equalIDs(prev) {
+		clients.removeLocked(prev)
+		clients.addLocked(c)
+
+		return nil
+	}
+
 	// Check the ID index.
-	if !slices.Equal(prev.IDs, c.IDs) {
-		for _, id := range c.IDs {
-			existing, ok := clients.idIndex[id]
-			if ok && existing != prev {
-				return fmt.Errorf("id %q is used by client with name %q", id, existing.Name)
-			}
+	for _, id := range c.ids() {
+		existing, ok := clients.idIndex[id]
+		if ok && existing != prev {
+			return fmt.Errorf("id %q is used by client with name %q", id, existing.Name)
 		}
 	}
 
-	clients.del(prev)
-	clients.add(c)
+	clients.removeLocked(prev)
+	clients.addLocked(c)
 
 	return nil
 }
@@ -764,23 +748,20 @@ func (clients *clientsContainer) setWHOISInfo(ip netip.Addr, wi *whois.Info) {
 		return
 	}
 
-	// TODO(e.burkov):  Consider storing WHOIS information separately and
-	// potentially get rid of [RuntimeClient].
 	rc, ok := clients.ipToRC[ip]
 	if !ok {
 		// Create a RuntimeClient implicitly so that we don't do this check
 		// again.
-		rc = &RuntimeClient{
-			Source: client.SourceWHOIS,
-		}
+		rc = &client.Runtime{}
 		clients.ipToRC[ip] = rc
 
 		log.Debug("clients: set whois info for runtime client with ip %s: %+v", ip, wi)
 	} else {
-		log.Debug("clients: set whois info for runtime client %s: %+v", rc.Host, wi)
+		host, _ := rc.Info()
+		log.Debug("clients: set whois info for runtime client %s: %+v", host, wi)
 	}
 
-	rc.WHOIS = wi
+	rc.SetWHOIS(wi)
 }
 
 // addHost adds a new IP-hostname pairing.  The priorities of the sources are
@@ -839,18 +820,13 @@ func (clients *clientsContainer) addHostLocked(
 			}
 		}
 
-		rc = &RuntimeClient{
-			WHOIS: &whois.Info{},
-		}
+		rc = &client.Runtime{}
 		clients.ipToRC[ip] = rc
-	} else if src < rc.Source {
-		return false
 	}
 
-	rc.Host = host
-	rc.Source = src
+	rc.SetInfo(src, []string{host})
 
-	log.Debug("clients: added %s -> %q [%d]", ip, host, len(clients.ipToRC))
+	log.Debug("clients: adding client info %s -> %q %q [%d]", ip, src, host, len(clients.ipToRC))
 
 	return true
 }
@@ -859,7 +835,8 @@ func (clients *clientsContainer) addHostLocked(
 func (clients *clientsContainer) rmHostsBySrc(src client.Source) {
 	n := 0
 	for ip, rc := range clients.ipToRC {
-		if rc.Source == src {
+		rc.Unset(src)
+		if rc.IsEmpty() {
 			delete(clients.ipToRC, ip)
 			n++
 		}
@@ -870,21 +847,24 @@ func (clients *clientsContainer) rmHostsBySrc(src client.Source) {
 
 // addFromHostsFile fills the client-hostname pairing index from the system's
 // hosts files.
-func (clients *clientsContainer) addFromHostsFile(hosts aghnet.Hosts) {
+func (clients *clientsContainer) addFromHostsFile(hosts *hostsfile.DefaultStorage) {
 	clients.lock.Lock()
 	defer clients.lock.Unlock()
 
 	clients.rmHostsBySrc(client.SourceHostsFile)
 
 	n := 0
-	for addr, rec := range hosts {
+	hosts.RangeNames(func(addr netip.Addr, names []string) (cont bool) {
 		// Only the first name of the first record is considered a canonical
 		// hostname for the IP address.
 		//
 		// TODO(e.burkov):  Consider using all the names from all the records.
-		clients.addHostLocked(addr, rec[0].Names[0], client.SourceHostsFile)
-		n++
-	}
+		if clients.addHostLocked(addr, names[0], client.SourceHostsFile) {
+			n++
+		}
+
+		return true
+	})
 
 	log.Debug("clients: added %d client aliases from system hosts file", n)
 }
@@ -926,7 +906,7 @@ func (clients *clientsContainer) addFromSystemARP() {
 // the persistent clients.
 func (clients *clientsContainer) close() (err error) {
 	persistent := maps.Values(clients.list)
-	slices.SortFunc(persistent, func(a, b *Client) (res int) {
+	slices.SortFunc(persistent, func(a, b *persistentClient) (res int) {
 		return strings.Compare(a.Name, b.Name)
 	})