daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pull request 1731: 4299-stats-ignore

Browse Source 
Merge in DNS/adguard-home from 4299-stats-ignore to master

Updates #1717.
Updates #4299.

Squashed commit of the following:

commit 1d1212d088c944e995deae2fd599eccb0a075033
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Feb 13 17:53:36 2023 +0300

    fix changelog

commit 5f56852c21d794bd87c13192d3857757be10f9b2
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Feb 13 17:39:02 2023 +0300

    add todo; fix data race

commit 89b8b16ddf5a43ebf68174cbaf9e8a53365f8cbe
Merge: e0a6bb49 ec19a85e
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 17:21:38 2023 +0300

    Merge branch 'master' into 4299-stats-ignore

commit e0a6bb490b651d1cf31589a7f17095fff4cb4dbb
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 17:21:06 2023 +0300

    interval under mutex

commit c569c7bc237f11b23fe47c98a20a1c5cb36751cb
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 16:19:35 2023 +0300

    fix mutex

commit 9374cf0c54dccc2fbfc38765b52c64e1c479137c
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 16:03:17 2023 +0300

    fix typo

commit 1f4fd1e7ab1b3c2f8e9c3d32ef7e4958f99abb47
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 15:55:44 2023 +0300

    add mutex

commit 2148048ce9ad228381cbb51a806c9b9cc21458fd
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 12:27:36 2023 +0300

    add key check

commit a19350977c463f888aea70d0dace26dff0173a65
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Feb 9 18:34:36 2023 +0300

    fix changelog

commit 23c3b6da162dfd513884b460c265ba4cafeb9727
Merge: 8fccc0b8 b89105e3
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Feb 9 13:28:59 2023 +0300

    Merge branch 'master' into 4299-stats-ignore

commit 8fccc0b8ec670a37e5209d795f35c43dd64afeb3
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Feb 9 13:27:42 2023 +0300

    add changelog

commit 0416c71742795b2fb8adb0173dcd6a99d9d9c676
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Feb 8 14:31:55 2023 +0300

    all: stats ignore
This commit is contained in:
Stanislav Chzhen
2023-02-13 18:15:33 +03:00
parent ec19a85ed0
commit ff04b2a7d3
14 changed files with 320 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
internal/home/config.go
View File

@@ -116,6 +116,7 @@ type configuration struct {
DNS dnsConfig `yaml:"dns"`
TLS tlsConfigSettings `yaml:"tls"`
QueryLog queryLogConfig `yaml:"querylog"`
Stats statsConfig `yaml:"statistics"`
// Filters reflects the filters from [filtering.Config]. It's cloned to the
// config used in the filtering module at the startup. Afterwards it's
@@ -149,10 +150,6 @@ type dnsConfig struct {
BindHosts []netip.Addr `yaml:"bind_hosts"`
Port int `yaml:"port"`
// StatsInterval is the time interval for flushing statistics to the disk in
// days.
StatsInterval uint32 `yaml:"statistics_interval"`
// AnonymizeClientIP defines if clients' IP addresses should be anonymized
// in query log and statistics.
AnonymizeClientIP bool `yaml:"anonymize_client_ip"`
@@ -234,8 +231,20 @@ type queryLogConfig struct {
// flushed to disk.
MemSize uint32 `yaml:"size_memory"`
// Ignored is the list of host names, which are should not be written
// to log.
// Ignored is the list of host names, which should not be written to
// log.
Ignored []string `yaml:"ignored"`
}
type statsConfig struct {
// Enabled defines if the statistics are enabled.
Enabled bool `yaml:"enabled"`
// Interval is the time interval for flushing statistics to the disk in
// days.
Interval uint32 `yaml:"interval"`
// Ignored is the list of host names, which should not be counted.
Ignored []string `yaml:"ignored"`
}
@@ -249,9 +258,8 @@ var config = &configuration{
AuthBlockMin: 15,
WebSessionTTLHours: 30 * 24,
DNS: dnsConfig{
BindHosts: []netip.Addr{netip.IPv4Unspecified()},
Port: defaultPortDNS,
StatsInterval: 1,
BindHosts: []netip.Addr{netip.IPv4Unspecified()},
Port: defaultPortDNS,
FilteringConfig: dnsforward.FilteringConfig{
ProtectionEnabled: true, // whether or not use any of filtering features
BlockingMode: dnsforward.BlockingModeDefault,
@@ -296,6 +304,11 @@ var config = &configuration{
MemSize: 1000,
Ignored: []string{},
},
Stats: statsConfig{
Enabled: true,
Interval: 1,
Ignored: []string{},
},
// NOTE: Keep these parameters in sync with the one put into
// client/src/helpers/filters/filters.js by scripts/vetted-filters.
//
@@ -472,9 +485,12 @@ func (c *configuration) write() (err error) {
}
if Context.stats != nil {
sdc := stats.DiskConfig{}
Context.stats.WriteDiskConfig(&sdc)
config.DNS.StatsInterval = sdc.Interval
statsConf := stats.Config{}
Context.stats.WriteDiskConfig(&statsConf)
config.Stats.Interval = statsConf.LimitDays
config.Stats.Enabled = statsConf.Enabled
config.Stats.Ignored = statsConf.Ignored.Values()
sort.Strings(config.Stats.Ignored)
}
if Context.queryLog != nil {

46
internal/home/dns.go
View File

@@ -53,10 +53,18 @@ func initDNS() (err error) {
statsConf := stats.Config{
Filename: filepath.Join(baseDir, "stats.db"),
LimitDays: config.DNS.StatsInterval,
LimitDays: config.Stats.Interval,
ConfigModified: onConfigModified,
HTTPRegister: httpRegister,
Enabled: config.Stats.Enabled,
}
set, err := nonDupEmptyHostNames(config.Stats.Ignored)
if err != nil {
return fmt.Errorf("statistics: ignored list: %w", err)
}
statsConf.Ignored = set
Context.stats, err = stats.New(statsConf)
if err != nil {
return fmt.Errorf("init stats: %w", err)
@@ -73,16 +81,14 @@ func initDNS() (err error) {
MemSize: config.QueryLog.MemSize,
Enabled: config.QueryLog.Enabled,
FileEnabled: config.QueryLog.FileEnabled,
Ignored: stringutil.NewSet(),
}
for _, v := range config.QueryLog.Ignored {
host := strings.ToLower(strings.TrimSuffix(v, "."))
if conf.Ignored.Has(host) {
return fmt.Errorf("duplicate ignored host %s", host)
}
conf.Ignored.Add(host)
set, err = nonDupEmptyHostNames(config.QueryLog.Ignored)
if err != nil {
return fmt.Errorf("querylog: ignored list: %w", err)
}
conf.Ignored = set
Context.queryLog = querylog.New(conf)
Context.filters, err = filtering.New(config.DNS.DnsfilterConf, nil)
@@ -526,3 +532,27 @@ func closeDNSServer() {
log.Debug("all dns modules are closed")
}
// nonDupEmptyHostNames returns nil and error, if list has duplicate or empty
// host name. Otherwise returns a set, which contains lowercase host names
// without dot at the end, and nil error.
func nonDupEmptyHostNames(list []string) (set *stringutil.Set, err error) {
set = stringutil.NewSet()
for _, v := range list {
host := strings.ToLower(strings.TrimSuffix(v, "."))
// TODO(a.garipov): Think about ignoring empty (".") names in
// the future.
if host == "" {
return nil, errors.Error("host name is empty")
}
if set.Has(host) {
return nil, fmt.Errorf("duplicate host name %q", host)
}
set.Add(host)
}
return set, nil
}

55
internal/home/upgrade.go
View File

@@ -22,7 +22,7 @@ import (
)
// currentSchemaVersion is the current schema version.
const currentSchemaVersion = 15
const currentSchemaVersion = 16
// These aliases are provided for convenience.
type (
@@ -88,6 +88,7 @@ func upgradeConfigSchema(oldVersion int, diskConf yobj) (err error) {
upgradeSchema12to13,
upgradeSchema13to14,
upgradeSchema14to15,
upgradeSchema15to16,
}
n := 0
@@ -823,8 +824,12 @@ func upgradeSchema14to15(diskConf yobj) (err error) {
log.Printf("Upgrade yaml: 14 to 15")
diskConf["schema_version"] = 15
dnsVal := diskConf["dns"]
dns, ok := dnsVal.(map[string]any)
dnsVal, ok := diskConf["dns"]
if !ok {
return nil
}
dns, ok := dnsVal.(yobj)
if !ok {
return fmt.Errorf("unexpected type of dns: %T", dnsVal)
}
@@ -856,6 +861,50 @@ func upgradeSchema14to15(diskConf yobj) (err error) {
return nil
}
// upgradeSchema15to16 performs the following changes:
//
// # BEFORE:
// 'dns':
// 'statistics_interval': 1
//
// # AFTER:
// 'statistics':
// 'enabled': true
// 'interval': 1
// 'ignored': []
func upgradeSchema15to16(diskConf yobj) (err error) {
log.Printf("Upgrade yaml: 15 to 16")
diskConf["schema_version"] = 16
dnsVal, ok := diskConf["dns"]
if !ok {
return nil
}
dns, ok := dnsVal.(yobj)
if !ok {
return fmt.Errorf("unexpected type of dns: %T", dnsVal)
}
stats := map[string]any{
"enabled": true,
"interval": 1,
"ignored": []any{},
}
k := "statistics_interval"
v, has := dns[k]
if has {
stats["enabled"] = v != 0
stats["interval"] = v
}
delete(dns, k)
diskConf["statistics"] = stats
return nil
}
// TODO(a.garipov): Replace with log.Output when we port it to our logging
// package.
func funcName() string {

59
internal/home/upgrade_test.go
View File

@@ -688,3 +688,62 @@ func TestUpgradeSchema14to15(t *testing.T) {
})
}
}
func TestUpgradeSchema15to16(t *testing.T) {
const newSchemaVer = 16
defaultWantObj := yobj{
"statistics": map[string]any{
"enabled": true,
"interval": 1,
"ignored": []any{},
},
"dns": map[string]any{},
"schema_version": newSchemaVer,
}
testCases := []struct {
in yobj
want yobj
name string
}{{
in: yobj{
"dns": map[string]any{
"statistics_interval": 1,
},
},
want: defaultWantObj,
name: "basic",
}, {
in: yobj{
"dns": map[string]any{},
},
want: defaultWantObj,
name: "default_values",
}, {
in: yobj{
"dns": map[string]any{
"statistics_interval": 0,
},
},
want: yobj{
"statistics": map[string]any{
"enabled": false,
"interval": 0,
"ignored": []any{},
},
"dns": map[string]any{},
"schema_version": newSchemaVer,
},
name: "stats_disabled",
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
err := upgradeSchema15to16(tc.in)
require.NoError(t, err)
assert.Equal(t, tc.want, tc.in)
})
}
}