daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pull request 1736: 4299-querylog-stats-api

Browse Source 
Merge in DNS/adguard-home from 4299-querylog-stats-api to master

Updates #1717.
Updates #4299.

Squashed commit of the following:

commit 5b706b7997a536bc4fd2c532fb89ca5ab3536848
Merge: 48b62b0f 306c1983
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 22 13:53:09 2023 +0300

    Merge branch 'master' into 4299-querylog-stats-api

commit 48b62b0f1882f1ad120c6cdd90cd7dd8cb8a7738
Author: Vladislav Abdulmyanov <v.abdulmyanov@adguard.com>
Date:   Wed Mar 22 12:25:04 2023 +0200

    client: fix styles, add titles and descrs

commit 97e31cff70d05b51bd0e5ea2d20e8e7a251a7e41
Author: Vladislav Abdulmyanov <v.abdulmyanov@adguard.com>
Date:   Tue Mar 21 18:38:12 2023 +0200

    client: add ignored domains for querylog

commit 24d75c4376382205ae6b8f731b1cd23d517772c9
Author: Vladislav Abdulmyanov <v.abdulmyanov@adguard.com>
Date:   Tue Mar 21 18:21:13 2023 +0200

    client: add ignore domains for stats

commit eefc3891d01f90af79fdac9ba8eea06d4d54a0bc
Merge: 978675ea 1daabb97
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Mar 21 10:53:35 2023 +0300

    Merge branch 'master' into 4299-querylog-stats-api

commit 978675ea2c07bf248b4c8f26ebdf78cf59a12ef5
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Mar 21 10:53:11 2023 +0300

    openapi: fix chlog

commit 2ed33007aade115d38b0ca582206cc10678b084c
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Mar 20 17:49:07 2023 +0300

    home: fix tests

commit 6af11520c164553ee9fce8f214ea169672188d7e
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Mar 20 17:40:16 2023 +0300

    home: fix typo

commit 56acdfde5b1ee8d16b232c1293b91affbe319ad1
Merge: 319da34d 48431f8b
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Mar 20 17:32:58 2023 +0300

    Merge branch 'master' into 4299-querylog-stats-api

commit 319da34de41ec84310b23bba2ad79c8a3a4c14ff
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Mar 3 17:34:38 2023 +0300

    querylog: fix docs

commit d5a8f24d5b336e7bdbbca18069f6ede8c96bcc2c
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Mar 3 11:42:00 2023 +0300

    stats: fix docs

commit e0cbfc1c4078180a05835ce7587e9f45484adc81
Merge: 4743c810 012e5beb
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 1 18:45:17 2023 +0300

    Merge branch 'master' into 4299-querylog-stats-api

commit 4743c81038052b9e0ca29ae5f1565021d36ca1ef
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 1 18:14:16 2023 +0300

    all: imp code; fix time conversion

commit 34310cffd7e331d098c535590245387051674fa8
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 1 12:34:11 2023 +0300

    chlog: restore order

commit cadd864a66655242948f1cb16e6d4945c0235d7e
Merge: 2f3e25be bb226434
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 1 12:26:06 2023 +0300

    Merge branch 'master' into 4299-querylog-stats-api

commit 2f3e25bee56d2c6ddcf4aa2fc6a1dc51ed9b06e1
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 1 12:25:14 2023 +0300

    all: fix fmt

commit d54022baa6c8a3d0d3c308a9b6b1a6a9dc6ac7b6
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Feb 28 16:16:40 2023 +0300

    all: imp code; fix chlog

commit df22de91f59a51194c55e7bcbe5bc3fcc60cb8e3
Merge: e1ea4797 a772212d
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Feb 27 17:24:09 2023 +0300

    Merge branch 'master' into 4299-querylog-stats-api

commit e1ea4797af974c36f06683ffc6eaaae917921a43
Merge: d7db0a5a bb80a7c2
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Feb 27 17:23:20 2023 +0300

    Merge branch 'master' into 4299-querylog-stats-api

commit d7db0a5af1e1f49f6174c1c42e6d9306f2381d16
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Feb 27 17:12:20 2023 +0300

    all: imp docs

... and 15 more commits
This commit is contained in:
Stanislav Chzhen
2023-03-23 13:46:57 +03:00
parent 306c1983a2
commit 143616ca6e
29 changed files with 995 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
internal/querylog/http.go
View File

@@ -13,9 +13,11 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/AdguardTeam/golibs/timeutil"
"golang.org/x/exp/slices"
"golang.org/x/net/idna"
)
@@ -25,8 +27,8 @@ type configJSON struct {
// fractional numbers and not mess the API users by changing the units.
Interval float64 `json:"interval"`
// Enabled shows if the querylog is enabled. It is an [aghalg.NullBool]
// to be able to tell when it's set without using pointers.
// Enabled shows if the querylog is enabled. It is an aghalg.NullBool to
// be able to tell when it's set without using pointers.
Enabled aghalg.NullBool `json:"enabled"`
// AnonymizeClientIP shows if the clients' IP addresses must be anonymized.
@@ -35,12 +37,39 @@ type configJSON struct {
AnonymizeClientIP aghalg.NullBool `json:"anonymize_client_ip"`
}
// getConfigResp is the JSON structure for the querylog configuration.
type getConfigResp struct {
// Ignored is the list of host names, which should not be written to log.
Ignored []string `json:"ignored"`
// Interval is the querylog rotation interval in milliseconds.
Interval float64 `json:"interval"`
// Enabled shows if the querylog is enabled. It is an aghalg.NullBool to
// be able to tell when it's set without using pointers.
Enabled aghalg.NullBool `json:"enabled"`
// AnonymizeClientIP shows if the clients' IP addresses must be anonymized.
// It is an aghalg.NullBool to be able to tell when it's set without using
// pointers.
//
// TODO(a.garipov): Consider using separate setting for statistics.
AnonymizeClientIP aghalg.NullBool `json:"anonymize_client_ip"`
}
// Register web handlers
func (l *queryLog) initWeb() {
l.conf.HTTPRegister(http.MethodGet, "/control/querylog", l.handleQueryLog)
l.conf.HTTPRegister(http.MethodGet, "/control/querylog_info", l.handleQueryLogInfo)
l.conf.HTTPRegister(http.MethodPost, "/control/querylog_clear", l.handleQueryLogClear)
l.conf.HTTPRegister(http.MethodPost, "/control/querylog_config", l.handleQueryLogConfig)
l.conf.HTTPRegister(http.MethodGet, "/control/querylog/config", l.handleGetQueryLogConfig)
l.conf.HTTPRegister(
http.MethodPut,
"/control/querylog/config/update",
l.handlePutQueryLogConfig,
)
}
func (l *queryLog) handleQueryLog(w http.ResponseWriter, r *http.Request) {
@@ -64,11 +93,41 @@ func (l *queryLog) handleQueryLogClear(_ http.ResponseWriter, _ *http.Request) {
l.clear()
}
// Get configuration
// handleQueryLogInfo handles requests to the GET /control/querylog_info
// endpoint.
//
// Deprecated: Remove it when migration to the new API is over.
func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
l.lock.Lock()
defer l.lock.Unlock()
ivl := l.conf.RotationIvl
if !checkInterval(ivl) {
// NOTE: If interval is custom we set it to 90 days for compatibility
// with old API.
ivl = timeutil.Day * 90
}
_ = aghhttp.WriteJSONResponse(w, r, configJSON{
Enabled: aghalg.BoolToNullBool(l.conf.Enabled),
Interval: l.conf.RotationIvl.Hours() / 24,
Interval: ivl.Hours() / 24,
AnonymizeClientIP: aghalg.BoolToNullBool(l.conf.AnonymizeClientIP),
})
}
// handleGetQueryLogConfig handles requests to the GET /control/querylog/config
// endpoint.
func (l *queryLog) handleGetQueryLogConfig(w http.ResponseWriter, r *http.Request) {
l.lock.Lock()
defer l.lock.Unlock()
ignored := l.conf.Ignored.Values()
slices.Sort(ignored)
_ = aghhttp.WriteJSONResponse(w, r, getConfigResp{
Ignored: ignored,
Interval: float64(l.conf.RotationIvl.Milliseconds()),
Enabled: aghalg.BoolToNullBool(l.conf.Enabled),
AnonymizeClientIP: aghalg.BoolToNullBool(l.conf.AnonymizeClientIP),
})
}
@@ -88,6 +147,8 @@ func AnonymizeIP(ip net.IP) {
}
// handleQueryLogConfig handles the POST /control/querylog_config queries.
//
// Deprecated: Remove it when migration to the new API is over.
func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request) {
// Set NaN as initial value to be able to know if it changed later by
// comparing it to NaN.
@@ -103,6 +164,7 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request)
}
ivl := time.Duration(float64(timeutil.Day) * newConf.Interval)
hasIvl := !math.IsNaN(newConf.Interval)
if hasIvl && !checkInterval(ivl) {
aghhttp.Error(r, w, http.StatusBadRequest, "unsupported interval")
@@ -115,8 +177,6 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request)
l.lock.Lock()
defer l.lock.Unlock()
// Copy data, modify it, then activate. Other threads (readers) don't need
// to use this lock.
conf := *l.conf
if newConf.Enabled != aghalg.NBNull {
conf.Enabled = newConf.Enabled == aghalg.NBTrue
@@ -138,6 +198,65 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request)
l.conf = &conf
}
// handlePutQueryLogConfig handles the PUT /control/querylog/config/update
// queries.
func (l *queryLog) handlePutQueryLogConfig(w http.ResponseWriter, r *http.Request) {
newConf := &getConfigResp{}
err := json.NewDecoder(r.Body).Decode(newConf)
if err != nil {
aghhttp.Error(r, w, http.StatusBadRequest, "%s", err)
return
}
set, err := aghnet.NewDomainNameSet(newConf.Ignored)
if err != nil {
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "ignored: %s", err)
return
}
ivl := time.Duration(newConf.Interval) * time.Millisecond
err = validateIvl(ivl)
if err != nil {
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "unsupported interval: %s", err)
return
}
if newConf.Enabled == aghalg.NBNull {
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "enabled is null")
return
}
if newConf.AnonymizeClientIP == aghalg.NBNull {
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "anonymize_client_ip is null")
return
}
defer l.conf.ConfigModified()
l.lock.Lock()
defer l.lock.Unlock()
conf := *l.conf
conf.Ignored = set
conf.RotationIvl = ivl
conf.Enabled = newConf.Enabled == aghalg.NBTrue
conf.AnonymizeClientIP = newConf.AnonymizeClientIP == aghalg.NBTrue
if conf.AnonymizeClientIP {
l.anonymizer.Store(AnonymizeIP)
} else {
l.anonymizer.Store(nil)
}
l.conf = &conf
}
// "value" -> value, return TRUE
func getDoubleQuotesEnclosedValue(s *string) bool {
t := *s

17
internal/querylog/qlog.go
View File

@@ -132,6 +132,20 @@ func checkInterval(ivl time.Duration) (ok bool) {
return ivl == quarterDay || ivl == day || ivl == week || ivl == month || ivl == threeMonths
}
// validateIvl returns an error if ivl is less than an hour or more than a
// year.
func validateIvl(ivl time.Duration) (err error) {
if ivl < time.Hour {
return errors.Error("less than an hour")
}
if ivl > timeutil.Day*365 {
return errors.Error("more than a year")
}
return nil
}
func (l *queryLog) WriteDiskConfig(c *Config) {
*c = *l.conf
}
@@ -258,6 +272,9 @@ func (l *queryLog) Add(params *AddParams) {
// ShouldLog returns true if request for the host should be logged.
func (l *queryLog) ShouldLog(host string, _, _ uint16) bool {
l.lock.Lock()
defer l.lock.Unlock()
return !l.isIgnored(host)
}

15
internal/querylog/qlog_test.go
View File

@@ -22,13 +22,14 @@ func TestMain(m *testing.M) {
// TestQueryLog tests adding and loading (with filtering) entries from disk and
// memory.
func TestQueryLog(t *testing.T) {
l := newQueryLog(Config{
l, err := newQueryLog(Config{
Enabled: true,
FileEnabled: true,
RotationIvl: timeutil.Day,
MemSize: 100,
BaseDir: t.TempDir(),
})
require.NoError(t, err)
// Add disk entries.
addEntry(l, "example.org", net.IPv4(1, 1, 1, 1), net.IPv4(2, 2, 2, 1))
@@ -125,12 +126,13 @@ func TestQueryLog(t *testing.T) {
}
func TestQueryLogOffsetLimit(t *testing.T) {
l := newQueryLog(Config{
l, err := newQueryLog(Config{
Enabled: true,
RotationIvl: timeutil.Day,
MemSize: 100,
BaseDir: t.TempDir(),
})
require.NoError(t, err)
const (
entNum = 10
@@ -199,13 +201,14 @@ func TestQueryLogOffsetLimit(t *testing.T) {
}
func TestQueryLogMaxFileScanEntries(t *testing.T) {
l := newQueryLog(Config{
l, err := newQueryLog(Config{
Enabled: true,
FileEnabled: true,
RotationIvl: timeutil.Day,
MemSize: 100,
BaseDir: t.TempDir(),
})
require.NoError(t, err)
const entNum = 10
// Add entries to the log.
@@ -227,13 +230,14 @@ func TestQueryLogMaxFileScanEntries(t *testing.T) {
}
func TestQueryLogFileDisabled(t *testing.T) {
l := newQueryLog(Config{
l, err := newQueryLog(Config{
Enabled: true,
FileEnabled: false,
RotationIvl: timeutil.Day,
MemSize: 2,
BaseDir: t.TempDir(),
})
require.NoError(t, err)
addEntry(l, "example1.org", net.IPv4(1, 1, 1, 1), net.IPv4(2, 2, 2, 1))
addEntry(l, "example2.org", net.IPv4(1, 1, 1, 1), net.IPv4(2, 2, 2, 1))
@@ -254,13 +258,14 @@ func TestQueryLogShouldLog(t *testing.T) {
)
set := stringutil.NewSet(ignored1, ignored2)
l := newQueryLog(Config{
l, err := newQueryLog(Config{
Enabled: true,
RotationIvl: timeutil.Day,
MemSize: 100,
BaseDir: t.TempDir(),
Ignored: set,
})
require.NoError(t, err)
testCases := []struct {
name string

18
internal/querylog/querylog.go
View File

@@ -1,6 +1,7 @@
package querylog
import (
"fmt"
"net"
"path/filepath"
"time"
@@ -9,9 +10,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/AdguardTeam/golibs/timeutil"
"github.com/miekg/dns"
)
@@ -135,12 +134,12 @@ func (p *AddParams) validate() (err error) {
}
// New creates a new instance of the query log.
func New(conf Config) (ql QueryLog) {
func New(conf Config) (ql QueryLog, err error) {
return newQueryLog(conf)
}
// newQueryLog crates a new queryLog.
func newQueryLog(conf Config) (l *queryLog) {
func newQueryLog(conf Config) (l *queryLog, err error) {
findClient := conf.FindClient
if findClient == nil {
findClient = func(_ []string) (_ *Client, _ error) {
@@ -158,13 +157,10 @@ func newQueryLog(conf Config) (l *queryLog) {
l.conf = &Config{}
*l.conf = conf
if !checkInterval(conf.RotationIvl) {
log.Info(
"querylog: warning: unsupported rotation interval %s, setting to 1 day",
conf.RotationIvl,
)
l.conf.RotationIvl = timeutil.Day
err = validateIvl(conf.RotationIvl)
if err != nil {
return nil, fmt.Errorf("unsupported interval: %w", err)
}
return l
return l, nil
}

3
internal/querylog/search_test.go
View File

@@ -35,7 +35,7 @@ func TestQueryLog_Search_findClient(t *testing.T) {
return nil, nil
}
l := newQueryLog(Config{
l, err := newQueryLog(Config{
FindClient: findClient,
BaseDir: t.TempDir(),
RotationIvl: timeutil.Day,
@@ -44,6 +44,7 @@ func TestQueryLog_Search_findClient(t *testing.T) {
FileEnabled: true,
AnonymizeClientIP: false,
})
require.NoError(t, err)
t.Cleanup(l.Close)
q := &dns.Msg{