daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pull request 1736: 4299-querylog-stats-api

Browse Source 
Merge in DNS/adguard-home from 4299-querylog-stats-api to master

Updates #1717.
Updates #4299.

Squashed commit of the following:

commit 5b706b7997a536bc4fd2c532fb89ca5ab3536848
Merge: 48b62b0f 306c1983
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 22 13:53:09 2023 +0300

    Merge branch 'master' into 4299-querylog-stats-api

commit 48b62b0f1882f1ad120c6cdd90cd7dd8cb8a7738
Author: Vladislav Abdulmyanov <v.abdulmyanov@adguard.com>
Date:   Wed Mar 22 12:25:04 2023 +0200

    client: fix styles, add titles and descrs

commit 97e31cff70d05b51bd0e5ea2d20e8e7a251a7e41
Author: Vladislav Abdulmyanov <v.abdulmyanov@adguard.com>
Date:   Tue Mar 21 18:38:12 2023 +0200

    client: add ignored domains for querylog

commit 24d75c4376382205ae6b8f731b1cd23d517772c9
Author: Vladislav Abdulmyanov <v.abdulmyanov@adguard.com>
Date:   Tue Mar 21 18:21:13 2023 +0200

    client: add ignore domains for stats

commit eefc3891d01f90af79fdac9ba8eea06d4d54a0bc
Merge: 978675ea 1daabb97
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Mar 21 10:53:35 2023 +0300

    Merge branch 'master' into 4299-querylog-stats-api

commit 978675ea2c07bf248b4c8f26ebdf78cf59a12ef5
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Mar 21 10:53:11 2023 +0300

    openapi: fix chlog

commit 2ed33007aade115d38b0ca582206cc10678b084c
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Mar 20 17:49:07 2023 +0300

    home: fix tests

commit 6af11520c164553ee9fce8f214ea169672188d7e
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Mar 20 17:40:16 2023 +0300

    home: fix typo

commit 56acdfde5b1ee8d16b232c1293b91affbe319ad1
Merge: 319da34d 48431f8b
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Mar 20 17:32:58 2023 +0300

    Merge branch 'master' into 4299-querylog-stats-api

commit 319da34de41ec84310b23bba2ad79c8a3a4c14ff
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Mar 3 17:34:38 2023 +0300

    querylog: fix docs

commit d5a8f24d5b336e7bdbbca18069f6ede8c96bcc2c
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Mar 3 11:42:00 2023 +0300

    stats: fix docs

commit e0cbfc1c4078180a05835ce7587e9f45484adc81
Merge: 4743c810 012e5beb
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 1 18:45:17 2023 +0300

    Merge branch 'master' into 4299-querylog-stats-api

commit 4743c81038052b9e0ca29ae5f1565021d36ca1ef
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 1 18:14:16 2023 +0300

    all: imp code; fix time conversion

commit 34310cffd7e331d098c535590245387051674fa8
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 1 12:34:11 2023 +0300

    chlog: restore order

commit cadd864a66655242948f1cb16e6d4945c0235d7e
Merge: 2f3e25be bb226434
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 1 12:26:06 2023 +0300

    Merge branch 'master' into 4299-querylog-stats-api

commit 2f3e25bee56d2c6ddcf4aa2fc6a1dc51ed9b06e1
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Mar 1 12:25:14 2023 +0300

    all: fix fmt

commit d54022baa6c8a3d0d3c308a9b6b1a6a9dc6ac7b6
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Feb 28 16:16:40 2023 +0300

    all: imp code; fix chlog

commit df22de91f59a51194c55e7bcbe5bc3fcc60cb8e3
Merge: e1ea4797 a772212d
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Feb 27 17:24:09 2023 +0300

    Merge branch 'master' into 4299-querylog-stats-api

commit e1ea4797af974c36f06683ffc6eaaae917921a43
Merge: d7db0a5a bb80a7c2
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Feb 27 17:23:20 2023 +0300

    Merge branch 'master' into 4299-querylog-stats-api

commit d7db0a5af1e1f49f6174c1c42e6d9306f2381d16
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Feb 27 17:12:20 2023 +0300

    all: imp docs

... and 15 more commits
This commit is contained in:
Stanislav Chzhen
2023-03-23 13:46:57 +03:00
parent 306c1983a2
commit 143616ca6e
29 changed files with 995 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
internal/home/config.go
View File

@@ -228,34 +228,32 @@ type tlsConfigSettings struct {
}
type queryLogConfig struct {
// Ignored is the list of host names, which should not be written to log.
Ignored []string `yaml:"ignored"`
// Interval is the interval for query log's files rotation.
Interval timeutil.Duration `yaml:"interval"`
// MemSize is the number of entries kept in memory before they are flushed
// to disk.
MemSize uint32 `yaml:"size_memory"`
// Enabled defines if the query log is enabled.
Enabled bool `yaml:"enabled"`
// FileEnabled defines, if the query log is written to the file.
FileEnabled bool `yaml:"file_enabled"`
// Interval is the interval for query log's files rotation.
Interval timeutil.Duration `yaml:"interval"`
// MemSize is the number of entries kept in memory before they are
// flushed to disk.
MemSize uint32 `yaml:"size_memory"`
// Ignored is the list of host names, which should not be written to
// log.
Ignored []string `yaml:"ignored"`
}
type statsConfig struct {
// Enabled defines if the statistics are enabled.
Enabled bool `yaml:"enabled"`
// Interval is the time interval for flushing statistics to the disk in
// days.
Interval uint32 `yaml:"interval"`
// Ignored is the list of host names, which should not be counted.
Ignored []string `yaml:"ignored"`
// Interval is the retention interval for statistics.
Interval timeutil.Duration `yaml:"interval"`
// Enabled defines if the statistics are enabled.
Enabled bool `yaml:"enabled"`
}
// config is the global configuration structure.
@@ -322,7 +320,7 @@ var config = &configuration{
},
Stats: statsConfig{
Enabled: true,
Interval: 1,
Interval: timeutil.Duration{Duration: 1 * timeutil.Day},
Ignored: []string{},
},
// NOTE: Keep these parameters in sync with the one put into
@@ -503,7 +501,7 @@ func (c *configuration) write() (err error) {
if Context.stats != nil {
statsConf := stats.Config{}
Context.stats.WriteDiskConfig(&statsConf)
config.Stats.Interval = statsConf.LimitDays
config.Stats.Interval = timeutil.Duration{Duration: statsConf.Limit}
config.Stats.Enabled = statsConf.Enabled
config.Stats.Ignored = statsConf.Ignored.Values()
slices.Sort(config.Stats.Ignored)

37
internal/home/dns.go
View File

@@ -8,7 +8,6 @@ import (
"net/url"
"os"
"path/filepath"
"strings"
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
@@ -22,7 +21,6 @@ import (
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/ameshkov/dnscrypt/v2"
yaml "gopkg.in/yaml.v3"
)
@@ -54,13 +52,13 @@ func initDNS() (err error) {
statsConf := stats.Config{
Filename: filepath.Join(baseDir, "stats.db"),
LimitDays: config.Stats.Interval,
Limit: config.Stats.Interval.Duration,
ConfigModified: onConfigModified,
HTTPRegister: httpRegister,
Enabled: config.Stats.Enabled,
}
set, err := nonDupEmptyHostNames(config.Stats.Ignored)
set, err := aghnet.NewDomainNameSet(config.Stats.Ignored)
if err != nil {
return fmt.Errorf("statistics: ignored list: %w", err)
}
@@ -84,13 +82,16 @@ func initDNS() (err error) {
FileEnabled: config.QueryLog.FileEnabled,
}
set, err = nonDupEmptyHostNames(config.QueryLog.Ignored)
set, err = aghnet.NewDomainNameSet(config.QueryLog.Ignored)
if err != nil {
return fmt.Errorf("querylog: ignored list: %w", err)
}
conf.Ignored = set
Context.queryLog = querylog.New(conf)
Context.queryLog, err = querylog.New(conf)
if err != nil {
return fmt.Errorf("init querylog: %w", err)
}
Context.filters, err = filtering.New(config.DNS.DnsfilterConf, nil)
if err != nil {
@@ -535,30 +536,6 @@ func closeDNSServer() {
log.Debug("all dns modules are closed")
}
// nonDupEmptyHostNames returns nil and error, if list has duplicate or empty
// host name. Otherwise returns a set, which contains lowercase host names
// without dot at the end, and nil error.
func nonDupEmptyHostNames(list []string) (set *stringutil.Set, err error) {
set = stringutil.NewSet()
for _, v := range list {
host := strings.ToLower(strings.TrimSuffix(v, "."))
// TODO(a.garipov): Think about ignoring empty (".") names in
// the future.
if host == "" {
return nil, errors.Error("host name is empty")
}
if set.Has(host) {
return nil, fmt.Errorf("duplicate host name %q", host)
}
set.Add(host)
}
return set, nil
}
// safeSearchResolver is a [filtering.Resolver] implementation used for safe
// search.
type safeSearchResolver struct{}

44
internal/home/upgrade.go
View File

@@ -22,7 +22,7 @@ import (
)
// currentSchemaVersion is the current schema version.
const currentSchemaVersion = 19
const currentSchemaVersion = 20
// These aliases are provided for convenience.
type (
@@ -92,6 +92,7 @@ func upgradeConfigSchema(oldVersion int, diskConf yobj) (err error) {
upgradeSchema16to17,
upgradeSchema17to18,
upgradeSchema18to19,
upgradeSchema19to20,
}
n := 0
@@ -1064,6 +1065,47 @@ func upgradeSchema18to19(diskConf yobj) (err error) {
return nil
}
// upgradeSchema19to20 performs the following changes:
//
// # BEFORE:
// 'statistics':
// 'interval': 1
//
// # AFTER:
// 'statistics':
// 'interval': 24h
func upgradeSchema19to20(diskConf yobj) (err error) {
log.Printf("Upgrade yaml: 19 to 20")
diskConf["schema_version"] = 20
statsVal, ok := diskConf["statistics"]
if !ok {
return nil
}
var stats yobj
stats, ok = statsVal.(yobj)
if !ok {
return fmt.Errorf("unexpected type of stats: %T", statsVal)
}
const field = "interval"
// Set the initial value from the global configuration structure.
statsIvl := 1
statsIvlVal, ok := stats[field]
if ok {
statsIvl, ok = statsIvlVal.(int)
if !ok {
return fmt.Errorf("unexpected type of %s: %T", field, statsIvlVal)
}
}
stats[field] = timeutil.Duration{Duration: time.Duration(statsIvl) * timeutil.Day}
return nil
}
// TODO(a.garipov): Replace with log.Output when we port it to our logging
// package.
func funcName() string {

95
internal/home/upgrade_test.go
View File

@@ -951,3 +951,98 @@ func TestUpgradeSchema18to19(t *testing.T) {
})
}
}
func TestUpgradeSchema19to20(t *testing.T) {
testCases := []struct {
ivl any
want any
wantErr string
name string
}{{
ivl: 1,
want: timeutil.Duration{Duration: timeutil.Day},
wantErr: "",
name: "success",
}, {
ivl: 0.25,
want: 0,
wantErr: "unexpected type of interval: float64",
name: "fail",
}}
for _, tc := range testCases {
conf := yobj{
"statistics": yobj{
"interval": tc.ivl,
},
"schema_version": 19,
}
t.Run(tc.name, func(t *testing.T) {
err := upgradeSchema19to20(conf)
if tc.wantErr != "" {
require.Error(t, err)
assert.Equal(t, tc.wantErr, err.Error())
return
}
require.NoError(t, err)
require.Equal(t, conf["schema_version"], 20)
statsVal, ok := conf["statistics"]
require.True(t, ok)
var stats yobj
stats, ok = statsVal.(yobj)
require.True(t, ok)
var newIvl timeutil.Duration
newIvl, ok = stats["interval"].(timeutil.Duration)
require.True(t, ok)
assert.Equal(t, tc.want, newIvl)
})
}
t.Run("no_stats", func(t *testing.T) {
err := upgradeSchema19to20(yobj{})
assert.NoError(t, err)
})
t.Run("bad_stats", func(t *testing.T) {
err := upgradeSchema19to20(yobj{
"statistics": 0,
})
testutil.AssertErrorMsg(t, "unexpected type of stats: int", err)
})
t.Run("no_field", func(t *testing.T) {
conf := yobj{
"statistics": yobj{},
}
err := upgradeSchema19to20(conf)
require.NoError(t, err)
statsVal, ok := conf["statistics"]
require.True(t, ok)
var stats yobj
stats, ok = statsVal.(yobj)
require.True(t, ok)
var ivl any
ivl, ok = stats["interval"]
require.True(t, ok)
var ivlVal timeutil.Duration
ivlVal, ok = ivl.(timeutil.Duration)
require.True(t, ok)
assert.Equal(t, 24*time.Hour, ivlVal.Duration)
})
}