Pull request 1736: 4299-querylog-stats-api
Merge in DNS/adguard-home from 4299-querylog-stats-api to master Updates #1717. Updates #4299. Squashed commit of the following: commit 5b706b7997a536bc4fd2c532fb89ca5ab3536848 Merge: 48b62b0f306c1983Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Wed Mar 22 13:53:09 2023 +0300 Merge branch 'master' into 4299-querylog-stats-api commit 48b62b0f1882f1ad120c6cdd90cd7dd8cb8a7738 Author: Vladislav Abdulmyanov <v.abdulmyanov@adguard.com> Date: Wed Mar 22 12:25:04 2023 +0200 client: fix styles, add titles and descrs commit 97e31cff70d05b51bd0e5ea2d20e8e7a251a7e41 Author: Vladislav Abdulmyanov <v.abdulmyanov@adguard.com> Date: Tue Mar 21 18:38:12 2023 +0200 client: add ignored domains for querylog commit 24d75c4376382205ae6b8f731b1cd23d517772c9 Author: Vladislav Abdulmyanov <v.abdulmyanov@adguard.com> Date: Tue Mar 21 18:21:13 2023 +0200 client: add ignore domains for stats commit eefc3891d01f90af79fdac9ba8eea06d4d54a0bc Merge: 978675ea1daabb97Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Tue Mar 21 10:53:35 2023 +0300 Merge branch 'master' into 4299-querylog-stats-api commit 978675ea2c07bf248b4c8f26ebdf78cf59a12ef5 Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Tue Mar 21 10:53:11 2023 +0300 openapi: fix chlog commit 2ed33007aade115d38b0ca582206cc10678b084c Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Mon Mar 20 17:49:07 2023 +0300 home: fix tests commit 6af11520c164553ee9fce8f214ea169672188d7e Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Mon Mar 20 17:40:16 2023 +0300 home: fix typo commit 56acdfde5b1ee8d16b232c1293b91affbe319ad1 Merge: 319da34d48431f8bAuthor: Stanislav Chzhen <s.chzhen@adguard.com> Date: Mon Mar 20 17:32:58 2023 +0300 Merge branch 'master' into 4299-querylog-stats-api commit 319da34de41ec84310b23bba2ad79c8a3a4c14ff Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Fri Mar 3 17:34:38 2023 +0300 querylog: fix docs commit d5a8f24d5b336e7bdbbca18069f6ede8c96bcc2c Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Fri Mar 3 11:42:00 2023 +0300 stats: fix docs commit e0cbfc1c4078180a05835ce7587e9f45484adc81 Merge: 4743c810012e5bebAuthor: Stanislav Chzhen <s.chzhen@adguard.com> Date: Wed Mar 1 18:45:17 2023 +0300 Merge branch 'master' into 4299-querylog-stats-api commit 4743c81038052b9e0ca29ae5f1565021d36ca1ef Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Wed Mar 1 18:14:16 2023 +0300 all: imp code; fix time conversion commit 34310cffd7e331d098c535590245387051674fa8 Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Wed Mar 1 12:34:11 2023 +0300 chlog: restore order commit cadd864a66655242948f1cb16e6d4945c0235d7e Merge: 2f3e25bebb226434Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Wed Mar 1 12:26:06 2023 +0300 Merge branch 'master' into 4299-querylog-stats-api commit 2f3e25bee56d2c6ddcf4aa2fc6a1dc51ed9b06e1 Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Wed Mar 1 12:25:14 2023 +0300 all: fix fmt commit d54022baa6c8a3d0d3c308a9b6b1a6a9dc6ac7b6 Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Tue Feb 28 16:16:40 2023 +0300 all: imp code; fix chlog commit df22de91f59a51194c55e7bcbe5bc3fcc60cb8e3 Merge: e1ea4797a772212dAuthor: Stanislav Chzhen <s.chzhen@adguard.com> Date: Mon Feb 27 17:24:09 2023 +0300 Merge branch 'master' into 4299-querylog-stats-api commit e1ea4797af974c36f06683ffc6eaaae917921a43 Merge: d7db0a5abb80a7c2Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Mon Feb 27 17:23:20 2023 +0300 Merge branch 'master' into 4299-querylog-stats-api commit d7db0a5af1e1f49f6174c1c42e6d9306f2381d16 Author: Stanislav Chzhen <s.chzhen@adguard.com> Date: Mon Feb 27 17:12:20 2023 +0300 all: imp docs ... and 15 more commits
This commit is contained in:
@@ -1,8 +1,13 @@
|
||||
package aghnet
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
)
|
||||
|
||||
// The maximum lengths of generated hostnames for different IP versions.
|
||||
@@ -59,3 +64,27 @@ func GenerateHostname(ip net.IP) (hostname string) {
|
||||
|
||||
return generateIPv6Hostname(ip)
|
||||
}
|
||||
|
||||
// NewDomainNameSet returns nil and error, if list has duplicate or empty
|
||||
// domain name. Otherwise returns a set, which contains non-FQDN domain names,
|
||||
// and nil error.
|
||||
func NewDomainNameSet(list []string) (set *stringutil.Set, err error) {
|
||||
set = stringutil.NewSet()
|
||||
|
||||
for i, v := range list {
|
||||
host := strings.ToLower(strings.TrimSuffix(v, "."))
|
||||
// TODO(a.garipov): Think about ignoring empty (".") names in the
|
||||
// future.
|
||||
if host == "" {
|
||||
return nil, errors.Error("host name is empty")
|
||||
}
|
||||
|
||||
if set.Has(host) {
|
||||
return nil, fmt.Errorf("duplicate host name %q at index %d", host, i)
|
||||
}
|
||||
|
||||
set.Add(host)
|
||||
}
|
||||
|
||||
return set, nil
|
||||
}
|
||||
|
||||
@@ -228,34 +228,32 @@ type tlsConfigSettings struct {
|
||||
}
|
||||
|
||||
type queryLogConfig struct {
|
||||
// Ignored is the list of host names, which should not be written to log.
|
||||
Ignored []string `yaml:"ignored"`
|
||||
|
||||
// Interval is the interval for query log's files rotation.
|
||||
Interval timeutil.Duration `yaml:"interval"`
|
||||
|
||||
// MemSize is the number of entries kept in memory before they are flushed
|
||||
// to disk.
|
||||
MemSize uint32 `yaml:"size_memory"`
|
||||
|
||||
// Enabled defines if the query log is enabled.
|
||||
Enabled bool `yaml:"enabled"`
|
||||
|
||||
// FileEnabled defines, if the query log is written to the file.
|
||||
FileEnabled bool `yaml:"file_enabled"`
|
||||
|
||||
// Interval is the interval for query log's files rotation.
|
||||
Interval timeutil.Duration `yaml:"interval"`
|
||||
|
||||
// MemSize is the number of entries kept in memory before they are
|
||||
// flushed to disk.
|
||||
MemSize uint32 `yaml:"size_memory"`
|
||||
|
||||
// Ignored is the list of host names, which should not be written to
|
||||
// log.
|
||||
Ignored []string `yaml:"ignored"`
|
||||
}
|
||||
|
||||
type statsConfig struct {
|
||||
// Enabled defines if the statistics are enabled.
|
||||
Enabled bool `yaml:"enabled"`
|
||||
|
||||
// Interval is the time interval for flushing statistics to the disk in
|
||||
// days.
|
||||
Interval uint32 `yaml:"interval"`
|
||||
|
||||
// Ignored is the list of host names, which should not be counted.
|
||||
Ignored []string `yaml:"ignored"`
|
||||
|
||||
// Interval is the retention interval for statistics.
|
||||
Interval timeutil.Duration `yaml:"interval"`
|
||||
|
||||
// Enabled defines if the statistics are enabled.
|
||||
Enabled bool `yaml:"enabled"`
|
||||
}
|
||||
|
||||
// config is the global configuration structure.
|
||||
@@ -322,7 +320,7 @@ var config = &configuration{
|
||||
},
|
||||
Stats: statsConfig{
|
||||
Enabled: true,
|
||||
Interval: 1,
|
||||
Interval: timeutil.Duration{Duration: 1 * timeutil.Day},
|
||||
Ignored: []string{},
|
||||
},
|
||||
// NOTE: Keep these parameters in sync with the one put into
|
||||
@@ -503,7 +501,7 @@ func (c *configuration) write() (err error) {
|
||||
if Context.stats != nil {
|
||||
statsConf := stats.Config{}
|
||||
Context.stats.WriteDiskConfig(&statsConf)
|
||||
config.Stats.Interval = statsConf.LimitDays
|
||||
config.Stats.Interval = timeutil.Duration{Duration: statsConf.Limit}
|
||||
config.Stats.Enabled = statsConf.Enabled
|
||||
config.Stats.Ignored = statsConf.Ignored.Values()
|
||||
slices.Sort(config.Stats.Ignored)
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
|
||||
@@ -22,7 +21,6 @@ import (
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/netutil"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
"github.com/ameshkov/dnscrypt/v2"
|
||||
yaml "gopkg.in/yaml.v3"
|
||||
)
|
||||
@@ -54,13 +52,13 @@ func initDNS() (err error) {
|
||||
|
||||
statsConf := stats.Config{
|
||||
Filename: filepath.Join(baseDir, "stats.db"),
|
||||
LimitDays: config.Stats.Interval,
|
||||
Limit: config.Stats.Interval.Duration,
|
||||
ConfigModified: onConfigModified,
|
||||
HTTPRegister: httpRegister,
|
||||
Enabled: config.Stats.Enabled,
|
||||
}
|
||||
|
||||
set, err := nonDupEmptyHostNames(config.Stats.Ignored)
|
||||
set, err := aghnet.NewDomainNameSet(config.Stats.Ignored)
|
||||
if err != nil {
|
||||
return fmt.Errorf("statistics: ignored list: %w", err)
|
||||
}
|
||||
@@ -84,13 +82,16 @@ func initDNS() (err error) {
|
||||
FileEnabled: config.QueryLog.FileEnabled,
|
||||
}
|
||||
|
||||
set, err = nonDupEmptyHostNames(config.QueryLog.Ignored)
|
||||
set, err = aghnet.NewDomainNameSet(config.QueryLog.Ignored)
|
||||
if err != nil {
|
||||
return fmt.Errorf("querylog: ignored list: %w", err)
|
||||
}
|
||||
|
||||
conf.Ignored = set
|
||||
Context.queryLog = querylog.New(conf)
|
||||
Context.queryLog, err = querylog.New(conf)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init querylog: %w", err)
|
||||
}
|
||||
|
||||
Context.filters, err = filtering.New(config.DNS.DnsfilterConf, nil)
|
||||
if err != nil {
|
||||
@@ -535,30 +536,6 @@ func closeDNSServer() {
|
||||
log.Debug("all dns modules are closed")
|
||||
}
|
||||
|
||||
// nonDupEmptyHostNames returns nil and error, if list has duplicate or empty
|
||||
// host name. Otherwise returns a set, which contains lowercase host names
|
||||
// without dot at the end, and nil error.
|
||||
func nonDupEmptyHostNames(list []string) (set *stringutil.Set, err error) {
|
||||
set = stringutil.NewSet()
|
||||
|
||||
for _, v := range list {
|
||||
host := strings.ToLower(strings.TrimSuffix(v, "."))
|
||||
// TODO(a.garipov): Think about ignoring empty (".") names in
|
||||
// the future.
|
||||
if host == "" {
|
||||
return nil, errors.Error("host name is empty")
|
||||
}
|
||||
|
||||
if set.Has(host) {
|
||||
return nil, fmt.Errorf("duplicate host name %q", host)
|
||||
}
|
||||
|
||||
set.Add(host)
|
||||
}
|
||||
|
||||
return set, nil
|
||||
}
|
||||
|
||||
// safeSearchResolver is a [filtering.Resolver] implementation used for safe
|
||||
// search.
|
||||
type safeSearchResolver struct{}
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
)
|
||||
|
||||
// currentSchemaVersion is the current schema version.
|
||||
const currentSchemaVersion = 19
|
||||
const currentSchemaVersion = 20
|
||||
|
||||
// These aliases are provided for convenience.
|
||||
type (
|
||||
@@ -92,6 +92,7 @@ func upgradeConfigSchema(oldVersion int, diskConf yobj) (err error) {
|
||||
upgradeSchema16to17,
|
||||
upgradeSchema17to18,
|
||||
upgradeSchema18to19,
|
||||
upgradeSchema19to20,
|
||||
}
|
||||
|
||||
n := 0
|
||||
@@ -1064,6 +1065,47 @@ func upgradeSchema18to19(diskConf yobj) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// upgradeSchema19to20 performs the following changes:
|
||||
//
|
||||
// # BEFORE:
|
||||
// 'statistics':
|
||||
// 'interval': 1
|
||||
//
|
||||
// # AFTER:
|
||||
// 'statistics':
|
||||
// 'interval': 24h
|
||||
func upgradeSchema19to20(diskConf yobj) (err error) {
|
||||
log.Printf("Upgrade yaml: 19 to 20")
|
||||
diskConf["schema_version"] = 20
|
||||
|
||||
statsVal, ok := diskConf["statistics"]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
var stats yobj
|
||||
stats, ok = statsVal.(yobj)
|
||||
if !ok {
|
||||
return fmt.Errorf("unexpected type of stats: %T", statsVal)
|
||||
}
|
||||
|
||||
const field = "interval"
|
||||
|
||||
// Set the initial value from the global configuration structure.
|
||||
statsIvl := 1
|
||||
statsIvlVal, ok := stats[field]
|
||||
if ok {
|
||||
statsIvl, ok = statsIvlVal.(int)
|
||||
if !ok {
|
||||
return fmt.Errorf("unexpected type of %s: %T", field, statsIvlVal)
|
||||
}
|
||||
}
|
||||
|
||||
stats[field] = timeutil.Duration{Duration: time.Duration(statsIvl) * timeutil.Day}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TODO(a.garipov): Replace with log.Output when we port it to our logging
|
||||
// package.
|
||||
func funcName() string {
|
||||
|
||||
@@ -951,3 +951,98 @@ func TestUpgradeSchema18to19(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpgradeSchema19to20(t *testing.T) {
|
||||
testCases := []struct {
|
||||
ivl any
|
||||
want any
|
||||
wantErr string
|
||||
name string
|
||||
}{{
|
||||
ivl: 1,
|
||||
want: timeutil.Duration{Duration: timeutil.Day},
|
||||
wantErr: "",
|
||||
name: "success",
|
||||
}, {
|
||||
ivl: 0.25,
|
||||
want: 0,
|
||||
wantErr: "unexpected type of interval: float64",
|
||||
name: "fail",
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
conf := yobj{
|
||||
"statistics": yobj{
|
||||
"interval": tc.ivl,
|
||||
},
|
||||
"schema_version": 19,
|
||||
}
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := upgradeSchema19to20(conf)
|
||||
|
||||
if tc.wantErr != "" {
|
||||
require.Error(t, err)
|
||||
|
||||
assert.Equal(t, tc.wantErr, err.Error())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, conf["schema_version"], 20)
|
||||
|
||||
statsVal, ok := conf["statistics"]
|
||||
require.True(t, ok)
|
||||
|
||||
var stats yobj
|
||||
stats, ok = statsVal.(yobj)
|
||||
require.True(t, ok)
|
||||
|
||||
var newIvl timeutil.Duration
|
||||
newIvl, ok = stats["interval"].(timeutil.Duration)
|
||||
require.True(t, ok)
|
||||
|
||||
assert.Equal(t, tc.want, newIvl)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("no_stats", func(t *testing.T) {
|
||||
err := upgradeSchema19to20(yobj{})
|
||||
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("bad_stats", func(t *testing.T) {
|
||||
err := upgradeSchema19to20(yobj{
|
||||
"statistics": 0,
|
||||
})
|
||||
|
||||
testutil.AssertErrorMsg(t, "unexpected type of stats: int", err)
|
||||
})
|
||||
|
||||
t.Run("no_field", func(t *testing.T) {
|
||||
conf := yobj{
|
||||
"statistics": yobj{},
|
||||
}
|
||||
|
||||
err := upgradeSchema19to20(conf)
|
||||
require.NoError(t, err)
|
||||
|
||||
statsVal, ok := conf["statistics"]
|
||||
require.True(t, ok)
|
||||
|
||||
var stats yobj
|
||||
stats, ok = statsVal.(yobj)
|
||||
require.True(t, ok)
|
||||
|
||||
var ivl any
|
||||
ivl, ok = stats["interval"]
|
||||
require.True(t, ok)
|
||||
|
||||
var ivlVal timeutil.Duration
|
||||
ivlVal, ok = ivl.(timeutil.Duration)
|
||||
require.True(t, ok)
|
||||
|
||||
assert.Equal(t, 24*time.Hour, ivlVal.Duration)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -13,9 +13,11 @@ import (
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"golang.org/x/exp/slices"
|
||||
"golang.org/x/net/idna"
|
||||
)
|
||||
|
||||
@@ -25,8 +27,8 @@ type configJSON struct {
|
||||
// fractional numbers and not mess the API users by changing the units.
|
||||
Interval float64 `json:"interval"`
|
||||
|
||||
// Enabled shows if the querylog is enabled. It is an [aghalg.NullBool]
|
||||
// to be able to tell when it's set without using pointers.
|
||||
// Enabled shows if the querylog is enabled. It is an aghalg.NullBool to
|
||||
// be able to tell when it's set without using pointers.
|
||||
Enabled aghalg.NullBool `json:"enabled"`
|
||||
|
||||
// AnonymizeClientIP shows if the clients' IP addresses must be anonymized.
|
||||
@@ -35,12 +37,39 @@ type configJSON struct {
|
||||
AnonymizeClientIP aghalg.NullBool `json:"anonymize_client_ip"`
|
||||
}
|
||||
|
||||
// getConfigResp is the JSON structure for the querylog configuration.
|
||||
type getConfigResp struct {
|
||||
// Ignored is the list of host names, which should not be written to log.
|
||||
Ignored []string `json:"ignored"`
|
||||
|
||||
// Interval is the querylog rotation interval in milliseconds.
|
||||
Interval float64 `json:"interval"`
|
||||
|
||||
// Enabled shows if the querylog is enabled. It is an aghalg.NullBool to
|
||||
// be able to tell when it's set without using pointers.
|
||||
Enabled aghalg.NullBool `json:"enabled"`
|
||||
|
||||
// AnonymizeClientIP shows if the clients' IP addresses must be anonymized.
|
||||
// It is an aghalg.NullBool to be able to tell when it's set without using
|
||||
// pointers.
|
||||
//
|
||||
// TODO(a.garipov): Consider using separate setting for statistics.
|
||||
AnonymizeClientIP aghalg.NullBool `json:"anonymize_client_ip"`
|
||||
}
|
||||
|
||||
// Register web handlers
|
||||
func (l *queryLog) initWeb() {
|
||||
l.conf.HTTPRegister(http.MethodGet, "/control/querylog", l.handleQueryLog)
|
||||
l.conf.HTTPRegister(http.MethodGet, "/control/querylog_info", l.handleQueryLogInfo)
|
||||
l.conf.HTTPRegister(http.MethodPost, "/control/querylog_clear", l.handleQueryLogClear)
|
||||
l.conf.HTTPRegister(http.MethodPost, "/control/querylog_config", l.handleQueryLogConfig)
|
||||
|
||||
l.conf.HTTPRegister(http.MethodGet, "/control/querylog/config", l.handleGetQueryLogConfig)
|
||||
l.conf.HTTPRegister(
|
||||
http.MethodPut,
|
||||
"/control/querylog/config/update",
|
||||
l.handlePutQueryLogConfig,
|
||||
)
|
||||
}
|
||||
|
||||
func (l *queryLog) handleQueryLog(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -64,11 +93,41 @@ func (l *queryLog) handleQueryLogClear(_ http.ResponseWriter, _ *http.Request) {
|
||||
l.clear()
|
||||
}
|
||||
|
||||
// Get configuration
|
||||
// handleQueryLogInfo handles requests to the GET /control/querylog_info
|
||||
// endpoint.
|
||||
//
|
||||
// Deprecated: Remove it when migration to the new API is over.
|
||||
func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
|
||||
l.lock.Lock()
|
||||
defer l.lock.Unlock()
|
||||
|
||||
ivl := l.conf.RotationIvl
|
||||
|
||||
if !checkInterval(ivl) {
|
||||
// NOTE: If interval is custom we set it to 90 days for compatibility
|
||||
// with old API.
|
||||
ivl = timeutil.Day * 90
|
||||
}
|
||||
|
||||
_ = aghhttp.WriteJSONResponse(w, r, configJSON{
|
||||
Enabled: aghalg.BoolToNullBool(l.conf.Enabled),
|
||||
Interval: l.conf.RotationIvl.Hours() / 24,
|
||||
Interval: ivl.Hours() / 24,
|
||||
AnonymizeClientIP: aghalg.BoolToNullBool(l.conf.AnonymizeClientIP),
|
||||
})
|
||||
}
|
||||
|
||||
// handleGetQueryLogConfig handles requests to the GET /control/querylog/config
|
||||
// endpoint.
|
||||
func (l *queryLog) handleGetQueryLogConfig(w http.ResponseWriter, r *http.Request) {
|
||||
l.lock.Lock()
|
||||
defer l.lock.Unlock()
|
||||
|
||||
ignored := l.conf.Ignored.Values()
|
||||
slices.Sort(ignored)
|
||||
_ = aghhttp.WriteJSONResponse(w, r, getConfigResp{
|
||||
Ignored: ignored,
|
||||
Interval: float64(l.conf.RotationIvl.Milliseconds()),
|
||||
Enabled: aghalg.BoolToNullBool(l.conf.Enabled),
|
||||
AnonymizeClientIP: aghalg.BoolToNullBool(l.conf.AnonymizeClientIP),
|
||||
})
|
||||
}
|
||||
@@ -88,6 +147,8 @@ func AnonymizeIP(ip net.IP) {
|
||||
}
|
||||
|
||||
// handleQueryLogConfig handles the POST /control/querylog_config queries.
|
||||
//
|
||||
// Deprecated: Remove it when migration to the new API is over.
|
||||
func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request) {
|
||||
// Set NaN as initial value to be able to know if it changed later by
|
||||
// comparing it to NaN.
|
||||
@@ -103,6 +164,7 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
|
||||
ivl := time.Duration(float64(timeutil.Day) * newConf.Interval)
|
||||
|
||||
hasIvl := !math.IsNaN(newConf.Interval)
|
||||
if hasIvl && !checkInterval(ivl) {
|
||||
aghhttp.Error(r, w, http.StatusBadRequest, "unsupported interval")
|
||||
@@ -115,8 +177,6 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request)
|
||||
l.lock.Lock()
|
||||
defer l.lock.Unlock()
|
||||
|
||||
// Copy data, modify it, then activate. Other threads (readers) don't need
|
||||
// to use this lock.
|
||||
conf := *l.conf
|
||||
if newConf.Enabled != aghalg.NBNull {
|
||||
conf.Enabled = newConf.Enabled == aghalg.NBTrue
|
||||
@@ -138,6 +198,65 @@ func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request)
|
||||
l.conf = &conf
|
||||
}
|
||||
|
||||
// handlePutQueryLogConfig handles the PUT /control/querylog/config/update
|
||||
// queries.
|
||||
func (l *queryLog) handlePutQueryLogConfig(w http.ResponseWriter, r *http.Request) {
|
||||
newConf := &getConfigResp{}
|
||||
err := json.NewDecoder(r.Body).Decode(newConf)
|
||||
if err != nil {
|
||||
aghhttp.Error(r, w, http.StatusBadRequest, "%s", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
set, err := aghnet.NewDomainNameSet(newConf.Ignored)
|
||||
if err != nil {
|
||||
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "ignored: %s", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
ivl := time.Duration(newConf.Interval) * time.Millisecond
|
||||
err = validateIvl(ivl)
|
||||
if err != nil {
|
||||
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "unsupported interval: %s", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if newConf.Enabled == aghalg.NBNull {
|
||||
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "enabled is null")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if newConf.AnonymizeClientIP == aghalg.NBNull {
|
||||
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "anonymize_client_ip is null")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
defer l.conf.ConfigModified()
|
||||
|
||||
l.lock.Lock()
|
||||
defer l.lock.Unlock()
|
||||
|
||||
conf := *l.conf
|
||||
|
||||
conf.Ignored = set
|
||||
conf.RotationIvl = ivl
|
||||
conf.Enabled = newConf.Enabled == aghalg.NBTrue
|
||||
|
||||
conf.AnonymizeClientIP = newConf.AnonymizeClientIP == aghalg.NBTrue
|
||||
if conf.AnonymizeClientIP {
|
||||
l.anonymizer.Store(AnonymizeIP)
|
||||
} else {
|
||||
l.anonymizer.Store(nil)
|
||||
}
|
||||
|
||||
l.conf = &conf
|
||||
}
|
||||
|
||||
// "value" -> value, return TRUE
|
||||
func getDoubleQuotesEnclosedValue(s *string) bool {
|
||||
t := *s
|
||||
|
||||
@@ -132,6 +132,20 @@ func checkInterval(ivl time.Duration) (ok bool) {
|
||||
return ivl == quarterDay || ivl == day || ivl == week || ivl == month || ivl == threeMonths
|
||||
}
|
||||
|
||||
// validateIvl returns an error if ivl is less than an hour or more than a
|
||||
// year.
|
||||
func validateIvl(ivl time.Duration) (err error) {
|
||||
if ivl < time.Hour {
|
||||
return errors.Error("less than an hour")
|
||||
}
|
||||
|
||||
if ivl > timeutil.Day*365 {
|
||||
return errors.Error("more than a year")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *queryLog) WriteDiskConfig(c *Config) {
|
||||
*c = *l.conf
|
||||
}
|
||||
@@ -258,6 +272,9 @@ func (l *queryLog) Add(params *AddParams) {
|
||||
|
||||
// ShouldLog returns true if request for the host should be logged.
|
||||
func (l *queryLog) ShouldLog(host string, _, _ uint16) bool {
|
||||
l.lock.Lock()
|
||||
defer l.lock.Unlock()
|
||||
|
||||
return !l.isIgnored(host)
|
||||
}
|
||||
|
||||
|
||||
@@ -22,13 +22,14 @@ func TestMain(m *testing.M) {
|
||||
// TestQueryLog tests adding and loading (with filtering) entries from disk and
|
||||
// memory.
|
||||
func TestQueryLog(t *testing.T) {
|
||||
l := newQueryLog(Config{
|
||||
l, err := newQueryLog(Config{
|
||||
Enabled: true,
|
||||
FileEnabled: true,
|
||||
RotationIvl: timeutil.Day,
|
||||
MemSize: 100,
|
||||
BaseDir: t.TempDir(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add disk entries.
|
||||
addEntry(l, "example.org", net.IPv4(1, 1, 1, 1), net.IPv4(2, 2, 2, 1))
|
||||
@@ -125,12 +126,13 @@ func TestQueryLog(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestQueryLogOffsetLimit(t *testing.T) {
|
||||
l := newQueryLog(Config{
|
||||
l, err := newQueryLog(Config{
|
||||
Enabled: true,
|
||||
RotationIvl: timeutil.Day,
|
||||
MemSize: 100,
|
||||
BaseDir: t.TempDir(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
const (
|
||||
entNum = 10
|
||||
@@ -199,13 +201,14 @@ func TestQueryLogOffsetLimit(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestQueryLogMaxFileScanEntries(t *testing.T) {
|
||||
l := newQueryLog(Config{
|
||||
l, err := newQueryLog(Config{
|
||||
Enabled: true,
|
||||
FileEnabled: true,
|
||||
RotationIvl: timeutil.Day,
|
||||
MemSize: 100,
|
||||
BaseDir: t.TempDir(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
const entNum = 10
|
||||
// Add entries to the log.
|
||||
@@ -227,13 +230,14 @@ func TestQueryLogMaxFileScanEntries(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestQueryLogFileDisabled(t *testing.T) {
|
||||
l := newQueryLog(Config{
|
||||
l, err := newQueryLog(Config{
|
||||
Enabled: true,
|
||||
FileEnabled: false,
|
||||
RotationIvl: timeutil.Day,
|
||||
MemSize: 2,
|
||||
BaseDir: t.TempDir(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
addEntry(l, "example1.org", net.IPv4(1, 1, 1, 1), net.IPv4(2, 2, 2, 1))
|
||||
addEntry(l, "example2.org", net.IPv4(1, 1, 1, 1), net.IPv4(2, 2, 2, 1))
|
||||
@@ -254,13 +258,14 @@ func TestQueryLogShouldLog(t *testing.T) {
|
||||
)
|
||||
set := stringutil.NewSet(ignored1, ignored2)
|
||||
|
||||
l := newQueryLog(Config{
|
||||
l, err := newQueryLog(Config{
|
||||
Enabled: true,
|
||||
RotationIvl: timeutil.Day,
|
||||
MemSize: 100,
|
||||
BaseDir: t.TempDir(),
|
||||
Ignored: set,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package querylog
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"path/filepath"
|
||||
"time"
|
||||
@@ -9,9 +10,7 @@ import (
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"github.com/miekg/dns"
|
||||
)
|
||||
|
||||
@@ -135,12 +134,12 @@ func (p *AddParams) validate() (err error) {
|
||||
}
|
||||
|
||||
// New creates a new instance of the query log.
|
||||
func New(conf Config) (ql QueryLog) {
|
||||
func New(conf Config) (ql QueryLog, err error) {
|
||||
return newQueryLog(conf)
|
||||
}
|
||||
|
||||
// newQueryLog crates a new queryLog.
|
||||
func newQueryLog(conf Config) (l *queryLog) {
|
||||
func newQueryLog(conf Config) (l *queryLog, err error) {
|
||||
findClient := conf.FindClient
|
||||
if findClient == nil {
|
||||
findClient = func(_ []string) (_ *Client, _ error) {
|
||||
@@ -158,13 +157,10 @@ func newQueryLog(conf Config) (l *queryLog) {
|
||||
l.conf = &Config{}
|
||||
*l.conf = conf
|
||||
|
||||
if !checkInterval(conf.RotationIvl) {
|
||||
log.Info(
|
||||
"querylog: warning: unsupported rotation interval %s, setting to 1 day",
|
||||
conf.RotationIvl,
|
||||
)
|
||||
l.conf.RotationIvl = timeutil.Day
|
||||
err = validateIvl(conf.RotationIvl)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unsupported interval: %w", err)
|
||||
}
|
||||
|
||||
return l
|
||||
return l, nil
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ func TestQueryLog_Search_findClient(t *testing.T) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
l := newQueryLog(Config{
|
||||
l, err := newQueryLog(Config{
|
||||
FindClient: findClient,
|
||||
BaseDir: t.TempDir(),
|
||||
RotationIvl: timeutil.Day,
|
||||
@@ -44,6 +44,7 @@ func TestQueryLog_Search_findClient(t *testing.T) {
|
||||
FileEnabled: true,
|
||||
AnonymizeClientIP: false,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(l.Close)
|
||||
|
||||
q := &dns.Msg{
|
||||
|
||||
@@ -7,8 +7,12 @@ import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
// topAddrs is an alias for the types of the TopFoo fields of statsResponse.
|
||||
@@ -44,7 +48,7 @@ func (s *StatsCtx) handleStats(w http.ResponseWriter, r *http.Request) {
|
||||
defer s.lock.Unlock()
|
||||
|
||||
start := time.Now()
|
||||
resp, ok := s.getData(s.limitHours)
|
||||
resp, ok := s.getData(uint32(s.limit.Hours()))
|
||||
log.Debug("stats: prepared data in %v", time.Since(start))
|
||||
|
||||
if !ok {
|
||||
@@ -63,20 +67,62 @@ type configResp struct {
|
||||
IntervalDays uint32 `json:"interval"`
|
||||
}
|
||||
|
||||
// getConfigResp is the response to the GET /control/stats_info.
|
||||
type getConfigResp struct {
|
||||
// Ignored is the list of host names, which should not be counted.
|
||||
Ignored []string `json:"ignored"`
|
||||
|
||||
// Interval is the statistics rotation interval in milliseconds.
|
||||
Interval float64 `json:"interval"`
|
||||
|
||||
// Enabled shows if statistics are enabled. It is an aghalg.NullBool to be
|
||||
// able to tell when it's set without using pointers.
|
||||
Enabled aghalg.NullBool `json:"enabled"`
|
||||
}
|
||||
|
||||
// handleStatsInfo handles requests to the GET /control/stats_info endpoint.
|
||||
//
|
||||
// Deprecated: Remove it when migration to the new API is over.
|
||||
func (s *StatsCtx) handleStatsInfo(w http.ResponseWriter, r *http.Request) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
resp := configResp{IntervalDays: s.limitHours / 24}
|
||||
days := uint32(s.limit / timeutil.Day)
|
||||
ok := checkInterval(days)
|
||||
if !ok || (s.enabled && days == 0) {
|
||||
// NOTE: If interval is custom we set it to 90 days for compatibility
|
||||
// with old API.
|
||||
days = 90
|
||||
}
|
||||
|
||||
resp := configResp{IntervalDays: days}
|
||||
if !s.enabled {
|
||||
resp.IntervalDays = 0
|
||||
}
|
||||
_ = aghhttp.WriteJSONResponse(w, r, resp)
|
||||
}
|
||||
|
||||
// handleGetStatsConfig handles requests to the GET /control/stats/config
|
||||
// endpoint.
|
||||
func (s *StatsCtx) handleGetStatsConfig(w http.ResponseWriter, r *http.Request) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
ignored := s.ignored.Values()
|
||||
slices.Sort(ignored)
|
||||
|
||||
resp := getConfigResp{
|
||||
Ignored: ignored,
|
||||
Interval: float64(s.limit.Milliseconds()),
|
||||
Enabled: aghalg.BoolToNullBool(s.enabled),
|
||||
}
|
||||
_ = aghhttp.WriteJSONResponse(w, r, resp)
|
||||
}
|
||||
|
||||
// handleStatsConfig handles requests to the POST /control/stats_config
|
||||
// endpoint.
|
||||
//
|
||||
// Deprecated: Remove it when migration to the new API is over.
|
||||
func (s *StatsCtx) handleStatsConfig(w http.ResponseWriter, r *http.Request) {
|
||||
reqData := configResp{}
|
||||
err := json.NewDecoder(r.Body).Decode(&reqData)
|
||||
@@ -92,8 +138,55 @@ func (s *StatsCtx) handleStatsConfig(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
s.setLimit(int(reqData.IntervalDays))
|
||||
s.configModified()
|
||||
defer s.configModified()
|
||||
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
limit := time.Duration(reqData.IntervalDays) * timeutil.Day
|
||||
s.setLimit(limit)
|
||||
}
|
||||
|
||||
// handlePutStatsConfig handles requests to the PUT /control/stats/config/update
|
||||
// endpoint.
|
||||
func (s *StatsCtx) handlePutStatsConfig(w http.ResponseWriter, r *http.Request) {
|
||||
reqData := getConfigResp{}
|
||||
err := json.NewDecoder(r.Body).Decode(&reqData)
|
||||
if err != nil {
|
||||
aghhttp.Error(r, w, http.StatusBadRequest, "json decode: %s", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
set, err := aghnet.NewDomainNameSet(reqData.Ignored)
|
||||
if err != nil {
|
||||
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "ignored: %s", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
ivl := time.Duration(reqData.Interval) * time.Millisecond
|
||||
err = validateIvl(ivl)
|
||||
if err != nil {
|
||||
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "unsupported interval: %s", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if reqData.Enabled == aghalg.NBNull {
|
||||
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "enabled is null")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
defer s.configModified()
|
||||
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
s.ignored = set
|
||||
s.limit = ivl
|
||||
s.enabled = reqData.Enabled == aghalg.NBTrue
|
||||
}
|
||||
|
||||
// handleStatsReset handles requests to the POST /control/stats_reset endpoint.
|
||||
@@ -114,4 +207,7 @@ func (s *StatsCtx) initWeb() {
|
||||
s.httpRegister(http.MethodPost, "/control/stats_reset", s.handleStatsReset)
|
||||
s.httpRegister(http.MethodPost, "/control/stats_config", s.handleStatsConfig)
|
||||
s.httpRegister(http.MethodGet, "/control/stats_info", s.handleStatsInfo)
|
||||
|
||||
s.httpRegister(http.MethodGet, "/control/stats/config", s.handleGetStatsConfig)
|
||||
s.httpRegister(http.MethodPut, "/control/stats/config/update", s.handlePutStatsConfig)
|
||||
}
|
||||
|
||||
152
internal/stats/http_test.go
Normal file
152
internal/stats/http_test.go
Normal file
@@ -0,0 +1,152 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
|
||||
"github.com/AdguardTeam/golibs/testutil"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestHandleStatsConfig(t *testing.T) {
|
||||
const (
|
||||
smallIvl = 1 * time.Minute
|
||||
minIvl = 1 * time.Hour
|
||||
maxIvl = 365 * timeutil.Day
|
||||
)
|
||||
|
||||
conf := Config{
|
||||
Filename: filepath.Join(t.TempDir(), "stats.db"),
|
||||
Limit: time.Hour * 24,
|
||||
Enabled: true,
|
||||
UnitID: func() (id uint32) { return 0 },
|
||||
ConfigModified: func() {},
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
body getConfigResp
|
||||
wantCode int
|
||||
wantErr string
|
||||
}{{
|
||||
name: "set_ivl_1_minIvl",
|
||||
body: getConfigResp{
|
||||
Enabled: aghalg.NBTrue,
|
||||
Interval: float64(minIvl.Milliseconds()),
|
||||
Ignored: []string{},
|
||||
},
|
||||
wantCode: http.StatusOK,
|
||||
wantErr: "",
|
||||
}, {
|
||||
name: "small_interval",
|
||||
body: getConfigResp{
|
||||
Enabled: aghalg.NBTrue,
|
||||
Interval: float64(smallIvl.Milliseconds()),
|
||||
Ignored: []string{},
|
||||
},
|
||||
wantCode: http.StatusUnprocessableEntity,
|
||||
wantErr: "unsupported interval: less than an hour\n",
|
||||
}, {
|
||||
name: "big_interval",
|
||||
body: getConfigResp{
|
||||
Enabled: aghalg.NBTrue,
|
||||
Interval: float64(maxIvl.Milliseconds() + minIvl.Milliseconds()),
|
||||
Ignored: []string{},
|
||||
},
|
||||
wantCode: http.StatusUnprocessableEntity,
|
||||
wantErr: "unsupported interval: more than a year\n",
|
||||
}, {
|
||||
name: "set_ignored_ivl_1_maxIvl",
|
||||
body: getConfigResp{
|
||||
Enabled: aghalg.NBTrue,
|
||||
Interval: float64(maxIvl.Milliseconds()),
|
||||
Ignored: []string{
|
||||
"ignor.ed",
|
||||
},
|
||||
},
|
||||
wantCode: http.StatusOK,
|
||||
wantErr: "",
|
||||
}, {
|
||||
name: "ignored_duplicate",
|
||||
body: getConfigResp{
|
||||
Enabled: aghalg.NBTrue,
|
||||
Interval: float64(minIvl.Milliseconds()),
|
||||
Ignored: []string{
|
||||
"ignor.ed",
|
||||
"ignor.ed",
|
||||
},
|
||||
},
|
||||
wantCode: http.StatusUnprocessableEntity,
|
||||
wantErr: "ignored: duplicate host name \"ignor.ed\" at index 1\n",
|
||||
}, {
|
||||
name: "ignored_empty",
|
||||
body: getConfigResp{
|
||||
Enabled: aghalg.NBTrue,
|
||||
Interval: float64(minIvl.Milliseconds()),
|
||||
Ignored: []string{
|
||||
"",
|
||||
},
|
||||
},
|
||||
wantCode: http.StatusUnprocessableEntity,
|
||||
wantErr: "ignored: host name is empty\n",
|
||||
}, {
|
||||
name: "enabled_is_null",
|
||||
body: getConfigResp{
|
||||
Enabled: aghalg.NBNull,
|
||||
Interval: float64(minIvl.Milliseconds()),
|
||||
Ignored: []string{},
|
||||
},
|
||||
wantCode: http.StatusUnprocessableEntity,
|
||||
wantErr: "enabled is null\n",
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s, err := New(conf)
|
||||
require.NoError(t, err)
|
||||
|
||||
s.Start()
|
||||
testutil.CleanupAndRequireSuccess(t, s.Close)
|
||||
|
||||
buf, err := json.Marshal(tc.body)
|
||||
require.NoError(t, err)
|
||||
|
||||
const (
|
||||
configGet = "/control/stats/config"
|
||||
configPut = "/control/stats/config/update"
|
||||
)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPut, configPut, bytes.NewReader(buf))
|
||||
rw := httptest.NewRecorder()
|
||||
|
||||
s.handlePutStatsConfig(rw, req)
|
||||
require.Equal(t, tc.wantCode, rw.Code)
|
||||
|
||||
if tc.wantCode != http.StatusOK {
|
||||
assert.Equal(t, tc.wantErr, rw.Body.String())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
resp := httptest.NewRequest(http.MethodGet, configGet, nil)
|
||||
rw = httptest.NewRecorder()
|
||||
|
||||
s.handleGetStatsConfig(rw, resp)
|
||||
require.Equal(t, http.StatusOK, rw.Code)
|
||||
|
||||
ans := getConfigResp{}
|
||||
err = json.Unmarshal(rw.Body.Bytes(), &ans)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, tc.body, ans)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
@@ -25,6 +26,20 @@ func checkInterval(days uint32) (ok bool) {
|
||||
return days == 0 || days == 1 || days == 7 || days == 30 || days == 90
|
||||
}
|
||||
|
||||
// validateIvl returns an error if ivl is less than an hour or more than a
|
||||
// year.
|
||||
func validateIvl(ivl time.Duration) (err error) {
|
||||
if ivl < time.Hour {
|
||||
return errors.Error("less than an hour")
|
||||
}
|
||||
|
||||
if ivl > timeutil.Day*365 {
|
||||
return errors.Error("more than a year")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Config is the configuration structure for the statistics collecting.
|
||||
type Config struct {
|
||||
// UnitID is the function to generate the identifier for current unit. If
|
||||
@@ -42,9 +57,8 @@ type Config struct {
|
||||
// Filename is the name of the database file.
|
||||
Filename string
|
||||
|
||||
// LimitDays is the maximum number of days to collect statistics into the
|
||||
// current unit.
|
||||
LimitDays uint32
|
||||
// Limit is an upper limit for collecting statistics.
|
||||
Limit time.Duration
|
||||
|
||||
// Enabled tells if the statistics are enabled.
|
||||
Enabled bool
|
||||
@@ -105,11 +119,8 @@ type StatsCtx struct {
|
||||
// enabled tells if the statistics are enabled.
|
||||
enabled bool
|
||||
|
||||
// limitHours is the maximum number of hours to collect statistics into the
|
||||
// current unit.
|
||||
//
|
||||
// TODO(s.chzhen): Rewrite to use time.Duration.
|
||||
limitHours uint32
|
||||
// limit is an upper limit for collecting statistics.
|
||||
limit time.Duration
|
||||
|
||||
// ignored is the list of host names, which should not be counted.
|
||||
ignored *stringutil.Set
|
||||
@@ -128,9 +139,14 @@ func New(conf Config) (s *StatsCtx, err error) {
|
||||
httpRegister: conf.HTTPRegister,
|
||||
ignored: conf.Ignored,
|
||||
}
|
||||
if s.limitHours = conf.LimitDays * 24; !checkInterval(conf.LimitDays) {
|
||||
s.limitHours = 24
|
||||
|
||||
err = validateIvl(conf.Limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unsupported interval: %w", err)
|
||||
}
|
||||
|
||||
s.limit = conf.Limit
|
||||
|
||||
if s.unitIDGen = newUnitID; conf.UnitID != nil {
|
||||
s.unitIDGen = conf.UnitID
|
||||
}
|
||||
@@ -150,7 +166,7 @@ func New(conf Config) (s *StatsCtx, err error) {
|
||||
return nil, fmt.Errorf("stats: opening a transaction: %w", err)
|
||||
}
|
||||
|
||||
deleted := deleteOldUnits(tx, id-s.limitHours-1)
|
||||
deleted := deleteOldUnits(tx, id-uint32(s.limit.Hours())-1)
|
||||
udb = loadUnitFromDB(tx, id)
|
||||
|
||||
err = finishTxn(tx, deleted > 0)
|
||||
@@ -231,7 +247,7 @@ func (s *StatsCtx) Update(e Entry) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
if !s.enabled || s.limitHours == 0 {
|
||||
if !s.enabled || s.limit == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -263,7 +279,7 @@ func (s *StatsCtx) WriteDiskConfig(dc *Config) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
dc.LimitDays = s.limitHours / 24
|
||||
dc.Limit = s.limit
|
||||
dc.Enabled = s.enabled
|
||||
dc.Ignored = s.ignored
|
||||
}
|
||||
@@ -273,7 +289,7 @@ func (s *StatsCtx) TopClientsIP(maxCount uint) (ips []netip.Addr) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
limit := s.limitHours
|
||||
limit := uint32(s.limit.Hours())
|
||||
if !s.enabled || limit == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -377,7 +393,7 @@ func (s *StatsCtx) flush() (cont bool, sleepFor time.Duration) {
|
||||
return false, 0
|
||||
}
|
||||
|
||||
limit := s.limitHours
|
||||
limit := uint32(s.limit.Hours())
|
||||
if limit == 0 || ptr.id == id {
|
||||
return true, time.Second
|
||||
}
|
||||
@@ -436,14 +452,14 @@ func (s *StatsCtx) periodicFlush() {
|
||||
log.Debug("periodic flushing finished")
|
||||
}
|
||||
|
||||
func (s *StatsCtx) setLimit(limitDays int) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
if limitDays != 0 {
|
||||
// setLimit sets the limit. s.lock is expected to be locked.
|
||||
//
|
||||
// TODO(s.chzhen): Remove it when migration to the new API is over.
|
||||
func (s *StatsCtx) setLimit(limit time.Duration) {
|
||||
if limit != 0 {
|
||||
s.enabled = true
|
||||
s.limitHours = uint32(24 * limitDays)
|
||||
log.Debug("stats: set limit: %d days", limitDays)
|
||||
s.limit = limit
|
||||
log.Debug("stats: set limit: %d days", limit/timeutil.Day)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/golibs/testutil"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -35,9 +36,9 @@ func TestStats_races(t *testing.T) {
|
||||
var r uint32
|
||||
idGen := func() (id uint32) { return atomic.LoadUint32(&r) }
|
||||
conf := Config{
|
||||
UnitID: idGen,
|
||||
Filename: filepath.Join(t.TempDir(), "./stats.db"),
|
||||
LimitDays: 1,
|
||||
UnitID: idGen,
|
||||
Filename: filepath.Join(t.TempDir(), "./stats.db"),
|
||||
Limit: timeutil.Day,
|
||||
}
|
||||
|
||||
s, err := New(conf)
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/stats"
|
||||
"github.com/AdguardTeam/golibs/netutil"
|
||||
"github.com/AdguardTeam/golibs/testutil"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -51,10 +52,10 @@ func TestStats(t *testing.T) {
|
||||
|
||||
handlers := map[string]http.Handler{}
|
||||
conf := stats.Config{
|
||||
Filename: filepath.Join(t.TempDir(), "stats.db"),
|
||||
LimitDays: 1,
|
||||
Enabled: true,
|
||||
UnitID: constUnitID,
|
||||
Filename: filepath.Join(t.TempDir(), "stats.db"),
|
||||
Limit: timeutil.Day,
|
||||
Enabled: true,
|
||||
UnitID: constUnitID,
|
||||
HTTPRegister: func(_, url string, handler http.HandlerFunc) {
|
||||
handlers[url] = handler
|
||||
},
|
||||
@@ -158,7 +159,7 @@ func TestLargeNumbers(t *testing.T) {
|
||||
|
||||
conf := stats.Config{
|
||||
Filename: filepath.Join(t.TempDir(), "stats.db"),
|
||||
LimitDays: 1,
|
||||
Limit: timeutil.Day,
|
||||
Enabled: true,
|
||||
UnitID: func() (id uint32) { return atomic.LoadUint32(&curHour) },
|
||||
HTTPRegister: func(_, url string, handler http.HandlerFunc) { handlers[url] = handler },
|
||||
|
||||
Reference in New Issue
Block a user