daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pull request: querylog: search clients by name, enrich http resp

Browse Source 
Updates #1273.

Squashed commit of the following:

commit 55b78153b1b775c855e759011141bbbe6d4b962c
Author: Artem Baskal <a.baskal@adguard.com>
Date:   Fri Apr 2 16:55:39 2021 +0300

    Update client_info in case of null

commit 5c80c1438ed9d961af11617831b704d6ae15cc34
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Apr 2 16:24:14 2021 +0300

    querylog: always set client_info

commit b48efd64d757cc0bcf5b34de22fdd0b0464d98a6
Merge: 4ed7eab5 23c9f528
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Apr 2 16:22:08 2021 +0300

    Merge branch 'master' into 1273-querylog-client-name

commit 4ed7eab52b6b5b0c0ddb5aa5a3225a62d1f9265b
Merge: dbf990eb 70d4c70e
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Apr 2 12:57:17 2021 +0300

    Merge branch 'master' into 1273-querylog-client-name

commit dbf990eb881116754554270e7b691b5db8e9ee34
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Apr 2 12:56:13 2021 +0300

    home: imp names

commit c2cfdef494ca26fff62b9fa008f1b389d9d4d46b
Author: Artem Baskal <a.baskal@adguard.com>
Date:   Thu Apr 1 19:26:04 2021 +0300

    Rename to whois

commit e3cc4a68ee576770b1922680155308e33bed31e8
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Apr 1 19:03:42 2021 +0300

    home: imp whois more

commit 3b8ef8691c298aff35946b35923ef2e5b1f9bbbe
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Apr 1 18:51:14 2021 +0300

    home: imp whois resp

commit fb97e0d74976723a512d6ff4c69e830fe59c8df8
Author: Artem Baskal <a.baskal@adguard.com>
Date:   Thu Apr 1 18:00:03 2021 +0300

    Fix client_info ids prop types

commit 298005189e372651ceff453e88aca19ee925a138
Author: Artem Baskal <a.baskal@adguard.com>
Date:   Thu Apr 1 17:58:14 2021 +0300

    Adapt changes on client

commit aa1769f64197d865478a66271da483babfc5dfd0
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Apr 1 17:18:36 2021 +0300

    all: add more fields to querylog client

commit 4b2a2dbd380ec410f3068d15ea16430912e03e33
Merge: cda92c3f 2e4e2f62
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Apr 1 16:57:26 2021 +0300

    Merge branch 'master' into 1273-querylog-client-name

commit cda92c3f0331cbac252f3163d31457f716bc7f2c
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Mar 29 18:03:51 2021 +0300

    querylog: fix windows tests

commit 5a56f0a32608869ed93a38f18f63ea3a20f7bde2
Merge: 627e4958 e710ce11
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Mar 29 17:45:53 2021 +0300

    Merge branch 'master' into 1273-querylog-client-name

commit 627e495828e82d44cc77aa393536479f23cc68b7
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Mar 29 17:44:49 2021 +0300

    querylog: add tests, imp code, docs

commit 6dec468a2f0c29357875ff99458e0e8f8e580e6d
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Mar 26 16:10:47 2021 +0300

    querylog: search clients by name, enrich http resp
This commit is contained in:
Ainar Garipov
2021-04-02 17:30:39 +03:00
parent 23c9f528db
commit 7c35d208b1
30 changed files with 713 additions and 367 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
internal/home/clients.go
View File

@@ -15,6 +15,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/dhcpd"
"github.com/AdguardTeam/AdGuardHome/internal/dnsfilter"
"github.com/AdguardTeam/AdGuardHome/internal/dnsforward"
"github.com/AdguardTeam/AdGuardHome/internal/querylog"
"github.com/AdguardTeam/AdGuardHome/internal/util"
"github.com/AdguardTeam/dnsproxy/proxy"
"github.com/AdguardTeam/dnsproxy/upstream"
@@ -60,19 +61,26 @@ const (
ClientSourceHostsFile
)
// ClientHost information
type ClientHost struct {
// RuntimeClient information
type RuntimeClient struct {
Host string
Source clientSource
WhoisInfo [][]string // [[key,value], ...]
WhoisInfo *RuntimeClientWhoisInfo
}
// RuntimeClientWhoisInfo is the filtered WHOIS data for a runtime client.
type RuntimeClientWhoisInfo struct {
City string `json:"city,omitempty"`
Country string `json:"country,omitempty"`
Orgname string `json:"orgname,omitempty"`
}
type clientsContainer struct {
// TODO(a.garipov): Perhaps use a number of separate indices for
// different types (string, net.IP, and so on).
list map[string]*Client // name -> client
idIndex map[string]*Client // ID -> client
ipHost map[string]*ClientHost // IP -> Hostname
list map[string]*Client // name -> client
idIndex map[string]*Client // ID -> client
ipToRC map[string]*RuntimeClient // IP -> runtime client
lock sync.Mutex
allTags map[string]bool
@@ -97,7 +105,7 @@ func (clients *clientsContainer) Init(objects []clientObject, dhcpServer *dhcpd.
}
clients.list = make(map[string]*Client)
clients.idIndex = make(map[string]*Client)
clients.ipHost = make(map[string]*ClientHost)
clients.ipToRC = make(map[string]*RuntimeClient)
clients.allTags = make(map[string]bool)
for _, t := range clientTags {
@@ -128,7 +136,7 @@ func (clients *clientsContainer) Start() {
}
}
// Reload - reload auto-clients
// Reload reloads runtime clients.
func (clients *clientsContainer) Reload() {
clients.addFromSystemARP()
}
@@ -248,21 +256,70 @@ func (clients *clientsContainer) Exists(id string, source clientSource) (ok bool
return true
}
var ch *ClientHost
ch, ok = clients.ipHost[id]
var rc *RuntimeClient
rc, ok = clients.ipToRC[id]
if !ok {
return false
}
// Return false if the new source has higher priority.
return source <= ch.Source
return source <= rc.Source
}
func copyStrings(a []string) (b []string) {
return append(b, a...)
}
// Find searches for a client by its ID.
func toQueryLogWhois(wi *RuntimeClientWhoisInfo) (cw *querylog.ClientWhois) {
if wi == nil {
return &querylog.ClientWhois{}
}
return &querylog.ClientWhois{
City: wi.City,
Country: wi.Country,
Orgname: wi.Orgname,
}
}
// findMultiple is a wrapper around Find to make it a valid client finder for
// the query log. err is always nil.
func (clients *clientsContainer) findMultiple(ids []string) (c *querylog.Client, err error) {
for _, id := range ids {
var name string
var foundIDs []string
whois := &querylog.ClientWhois{}
c, ok := clients.Find(id)
if ok {
name = c.Name
foundIDs = c.IDs
} else {
var rc RuntimeClient
rc, ok = clients.FindRuntimeClient(id)
if !ok {
continue
}
foundIDs = []string{rc.Host}
whois = toQueryLogWhois(rc.WhoisInfo)
}
ip := net.ParseIP(id)
disallowed, disallowedRule := clients.dnsServer.IsBlockedIP(ip)
return &querylog.Client{
Name: name,
DisallowedRule: disallowedRule,
Whois: whois,
IDs: foundIDs,
Disallowed: disallowed,
}, nil
}
return nil, nil
}
func (clients *clientsContainer) Find(id string) (c *Client, ok bool) {
clients.lock.Lock()
defer clients.lock.Unlock()
@@ -361,21 +418,22 @@ func (clients *clientsContainer) findLocked(id string) (c *Client, ok bool) {
return nil, false
}
// FindAutoClient - search for an auto-client by IP
func (clients *clientsContainer) FindAutoClient(ip string) (ClientHost, bool) {
// FindRuntimeClient finds a runtime client by their IP.
func (clients *clientsContainer) FindRuntimeClient(ip string) (RuntimeClient, bool) {
ipAddr := net.ParseIP(ip)
if ipAddr == nil {
return ClientHost{}, false
return RuntimeClient{}, false
}
clients.lock.Lock()
defer clients.lock.Unlock()
ch, ok := clients.ipHost[ip]
rc, ok := clients.ipToRC[ip]
if ok {
return *ch, true
return *rc, true
}
return ClientHost{}, false
return RuntimeClient{}, false
}
// check validates the client.
@@ -558,9 +616,7 @@ func (clients *clientsContainer) Update(name string, c *Client) (err error) {
}
// SetWhoisInfo sets the WHOIS information for a client.
//
// TODO(a.garipov): Perhaps replace [][]string with map[string]string.
func (clients *clientsContainer) SetWhoisInfo(ip string, info [][]string) {
func (clients *clientsContainer) SetWhoisInfo(ip string, wi *RuntimeClientWhoisInfo) {
clients.lock.Lock()
defer clients.lock.Unlock()
@@ -570,21 +626,24 @@ func (clients *clientsContainer) SetWhoisInfo(ip string, info [][]string) {
return
}
ch, ok := clients.ipHost[ip]
rc, ok := clients.ipToRC[ip]
if ok {
ch.WhoisInfo = info
log.Debug("clients: set whois info for auto-client %s: %q", ch.Host, info)
rc.WhoisInfo = wi
log.Debug("clients: set whois info for runtime client %s: %+v", rc.Host, wi)
return
}
// Create a ClientHost implicitly so that we don't do this check again
ch = &ClientHost{
// Create a RuntimeClient implicitly so that we don't do this check
// again.
rc = &RuntimeClient{
Source: ClientSourceWHOIS,
}
ch.WhoisInfo = info
clients.ipHost[ip] = ch
log.Debug("clients: set whois info for auto-client with IP %s: %q", ip, info)
rc.WhoisInfo = wi
clients.ipToRC[ip] = rc
log.Debug("clients: set whois info for runtime client with ip %s: %+v", ip, wi)
}
// AddHost adds a new IP-hostname pairing. The priorities of the sources is
@@ -600,24 +659,25 @@ func (clients *clientsContainer) AddHost(ip, host string, src clientSource) (ok
// addHostLocked adds a new IP-hostname pairing. For internal use only.
func (clients *clientsContainer) addHostLocked(ip, host string, src clientSource) (ok bool) {
var ch *ClientHost
ch, ok = clients.ipHost[ip]
var rc *RuntimeClient
rc, ok = clients.ipToRC[ip]
if ok {
if ch.Source > src {
if rc.Source > src {
return false
}
ch.Source = src
rc.Source = src
} else {
ch = &ClientHost{
Host: host,
Source: src,
rc = &RuntimeClient{
Host: host,
Source: src,
WhoisInfo: &RuntimeClientWhoisInfo{},
}
clients.ipHost[ip] = ch
clients.ipToRC[ip] = rc
}
log.Debug("clients: added %q -> %q [%d]", ip, host, len(clients.ipHost))
log.Debug("clients: added %q -> %q [%d]", ip, host, len(clients.ipToRC))
return true
}
@@ -625,9 +685,9 @@ func (clients *clientsContainer) addHostLocked(ip, host string, src clientSource
// rmHostsBySrc removes all entries that match the specified source.
func (clients *clientsContainer) rmHostsBySrc(src clientSource) {
n := 0
for k, v := range clients.ipHost {
for k, v := range clients.ipToRC {
if v.Source == src {
delete(clients.ipHost, k)
delete(clients.ipToRC, k)
n++
}
}

26
internal/home/clients_test.go
View File

@@ -163,17 +163,20 @@ func TestClientsWhois(t *testing.T) {
testing: true,
}
clients.Init(nil, nil, nil)
whois := [][]string{{"orgname", "orgname-val"}, {"country", "country-val"}}
whois := &RuntimeClientWhoisInfo{
Country: "AU",
Orgname: "Example Org",
}
t.Run("new_client", func(t *testing.T) {
clients.SetWhoisInfo("1.1.1.255", whois)
require.NotNil(t, clients.ipHost["1.1.1.255"])
h := clients.ipHost["1.1.1.255"]
require.NotNil(t, clients.ipToRC["1.1.1.255"])
require.Len(t, h.WhoisInfo, 2)
require.Len(t, h.WhoisInfo[0], 2)
assert.Equal(t, "orgname-val", h.WhoisInfo[0][1])
h := clients.ipToRC["1.1.1.255"]
require.NotNil(t, h)
assert.Equal(t, h.WhoisInfo, whois)
})
t.Run("existing_auto-client", func(t *testing.T) {
@@ -183,12 +186,11 @@ func TestClientsWhois(t *testing.T) {
clients.SetWhoisInfo("1.1.1.1", whois)
require.NotNil(t, clients.ipHost["1.1.1.1"])
h := clients.ipHost["1.1.1.1"]
require.NotNil(t, clients.ipToRC["1.1.1.1"])
h := clients.ipToRC["1.1.1.1"]
require.NotNil(t, h)
require.Len(t, h.WhoisInfo, 2)
require.Len(t, h.WhoisInfo[0], 2)
assert.Equal(t, "orgname-val", h.WhoisInfo[0][1])
assert.Equal(t, h.WhoisInfo, whois)
})
t.Run("can't_set_manually-added", func(t *testing.T) {
@@ -200,7 +202,7 @@ func TestClientsWhois(t *testing.T) {
assert.True(t, ok)
clients.SetWhoisInfo("1.1.1.2", whois)
require.Nil(t, clients.ipHost["1.1.1.2"])
require.Nil(t, clients.ipToRC["1.1.1.2"])
assert.True(t, clients.Del("client1"))
})
}

58
internal/home/clientshttp.go
View File

@@ -22,7 +22,7 @@ type clientJSON struct {
Upstreams []string `json:"upstreams"`
WhoisInfo map[string]string `json:"whois_info"`
WhoisInfo *RuntimeClientWhoisInfo `json:"whois_info"`
// Disallowed - if true -- client's IP is not disallowed
// Otherwise, it is blocked.
@@ -34,18 +34,18 @@ type clientJSON struct {
DisallowedRule string `json:"disallowed_rule"`
}
type clientHostJSON struct {
type runtimeClientJSON struct {
WhoisInfo *RuntimeClientWhoisInfo `json:"whois_info"`
IP string `json:"ip"`
Name string `json:"name"`
Source string `json:"source"`
WhoisInfo map[string]string `json:"whois_info"`
}
type clientListJSON struct {
Clients []clientJSON `json:"clients"`
AutoClients []clientHostJSON `json:"auto_clients"`
Tags []string `json:"supported_tags"`
Clients []clientJSON `json:"clients"`
RuntimeClients []runtimeClientJSON `json:"auto_clients"`
Tags []string `json:"supported_tags"`
}
// respond with information about configured clients
@@ -53,18 +53,21 @@ func (clients *clientsContainer) handleGetClients(w http.ResponseWriter, _ *http
data := clientListJSON{}
clients.lock.Lock()
defer clients.lock.Unlock()
for _, c := range clients.list {
cj := clientToJSON(c)
data.Clients = append(data.Clients, cj)
}
for ip, ch := range clients.ipHost {
cj := clientHostJSON{
IP: ip,
Name: ch.Host,
for ip, rc := range clients.ipToRC {
cj := runtimeClientJSON{
IP: ip,
Name: rc.Host,
WhoisInfo: rc.WhoisInfo,
}
cj.Source = "etc/hosts"
switch ch.Source {
switch rc.Source {
case ClientSourceDHCP:
cj.Source = "DHCP"
case ClientSourceRDNS:
@@ -75,14 +78,8 @@ func (clients *clientsContainer) handleGetClients(w http.ResponseWriter, _ *http
cj.Source = "WHOIS"
}
cj.WhoisInfo = map[string]string{}
for _, wi := range ch.WhoisInfo {
cj.WhoisInfo[wi[0]] = wi[1]
}
data.AutoClients = append(data.AutoClients, cj)
data.RuntimeClients = append(data.RuntimeClients, cj)
}
clients.lock.Unlock()
data.Tags = clientTags
@@ -129,21 +126,21 @@ func clientToJSON(c *Client) clientJSON {
BlockedServices: c.BlockedServices,
Upstreams: c.Upstreams,
WhoisInfo: &RuntimeClientWhoisInfo{},
}
return cj
}
// Convert ClientHost object to JSON
func clientHostToJSON(ip string, ch ClientHost) clientJSON {
cj := clientJSON{
Name: ch.Host,
IDs: []string{ip},
// runtimeClientToJSON converts a RuntimeClient into a JSON struct.
func runtimeClientToJSON(ip string, rc RuntimeClient) (cj clientJSON) {
cj = clientJSON{
Name: rc.Host,
IDs: []string{ip},
WhoisInfo: rc.WhoisInfo,
}
cj.WhoisInfo = map[string]string{}
for _, wi := range ch.WhoisInfo {
cj.WhoisInfo[wi[0]] = wi[1]
}
return cj
}
@@ -268,7 +265,7 @@ func (clients *clientsContainer) findTemporary(ip net.IP, idStr string) (cj clie
return cj, false
}
ch, ok := clients.FindAutoClient(idStr)
rc, ok := clients.FindRuntimeClient(idStr)
if !ok {
// It is still possible that the IP used to be in the runtime
// clients list, but then the server was reloaded. So, check
@@ -284,12 +281,13 @@ func (clients *clientsContainer) findTemporary(ip net.IP, idStr string) (cj clie
IDs: []string{idStr},
Disallowed: disallowed,
DisallowedRule: rule,
WhoisInfo: &RuntimeClientWhoisInfo{},
}
return cj, true
}
cj = clientHostToJSON(idStr, ch)
cj = runtimeClientToJSON(idStr, rc)
cj.Disallowed, cj.DisallowedRule = clients.dnsServer.IsBlockedIP(ip)
return cj, true

2
internal/home/config.go
View File

@@ -285,7 +285,7 @@ func (c *configuration) write() error {
Context.queryLog.WriteDiskConfig(&dc)
config.DNS.QueryLogEnabled = dc.Enabled
config.DNS.QueryLogFileEnabled = dc.FileEnabled
config.DNS.QueryLogInterval = dc.Interval
config.DNS.QueryLogInterval = dc.RotationIvl
config.DNS.QueryLogMemSize = dc.MemSize
config.DNS.AnonymizeClientIP = dc.AnonymizeClientIP
}

14
internal/home/dns.go
View File

@@ -42,15 +42,17 @@ func initDNSServer() error {
if err != nil {
return fmt.Errorf("couldn't initialize statistics module")
}
conf := querylog.Config{
Enabled: config.DNS.QueryLogEnabled,
FileEnabled: config.DNS.QueryLogFileEnabled,
BaseDir: baseDir,
Interval: config.DNS.QueryLogInterval,
MemSize: config.DNS.QueryLogMemSize,
AnonymizeClientIP: config.DNS.AnonymizeClientIP,
ConfigModified: onConfigModified,
HTTPRegister: httpRegister,
FindClient: Context.clients.findMultiple,
BaseDir: baseDir,
RotationIvl: config.DNS.QueryLogInterval,
MemSize: config.DNS.QueryLogMemSize,
Enabled: config.DNS.QueryLogEnabled,
FileEnabled: config.DNS.QueryLogFileEnabled,
AnonymizeClientIP: config.DNS.AnonymizeClientIP,
}
Context.queryLog = querylog.New(conf)

4
internal/home/rdns_test.go
View File

@@ -84,7 +84,7 @@ func TestRDNS_Begin(t *testing.T) {
clients: &clientsContainer{
list: map[string]*Client{},
idIndex: tc.cliIDIndex,
ipHost: map[string]*ClientHost{},
ipToRC: map[string]*RuntimeClient{},
allTags: map[string]bool{},
},
}
@@ -229,7 +229,7 @@ func TestRDNS_WorkerLoop(t *testing.T) {
cc := &clientsContainer{
list: map[string]*Client{},
idIndex: map[string]*Client{},
ipHost: map[string]*ClientHost{},
ipToRC: map[string]*RuntimeClient{},
allTags: map[string]bool{},
}
ch := make(chan net.IP)

32
internal/home/whois.go
View File

@@ -182,29 +182,31 @@ func (w *Whois) queryAll(ctx context.Context, target string) (string, error) {
}
// Request WHOIS information
func (w *Whois) process(ctx context.Context, ip net.IP) [][]string {
data := [][]string{}
func (w *Whois) process(ctx context.Context, ip net.IP) (wi *RuntimeClientWhoisInfo) {
resp, err := w.queryAll(ctx, ip.String())
if err != nil {
log.Debug("Whois: error: %s IP:%s", err, ip)
return data
return nil
}
log.Debug("Whois: IP:%s response: %d bytes", ip, len(resp))
m := whoisParse(resp)
keys := []string{"orgname", "country", "city"}
for _, k := range keys {
v, found := m[k]
if !found {
continue
}
pair := []string{k, v}
data = append(data, pair)
wi = &RuntimeClientWhoisInfo{
City: m["city"],
Country: m["country"],
Orgname: m["orgname"],
}
return data
// Don't return an empty struct so that the frontend doesn't get
// confused.
if *wi == (RuntimeClientWhoisInfo{}) {
return nil
}
return wi
}
// Begin - begin requesting WHOIS info
@@ -234,11 +236,9 @@ func (w *Whois) Begin(ip net.IP) {
// workerLoop processes the IP addresses it got from the channel and associates
// the retrieving WHOIS info with a client.
func (w *Whois) workerLoop() {
for {
ip := <-w.ipChan
for ip := range w.ipChan {
info := w.process(context.Background(), ip)
if len(info) == 0 {
if info == nil {
continue
}