daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pull request 1731: 4299-stats-ignore

Browse Source 
Merge in DNS/adguard-home from 4299-stats-ignore to master

Updates #1717.
Updates #4299.

Squashed commit of the following:

commit 1d1212d088c944e995deae2fd599eccb0a075033
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Feb 13 17:53:36 2023 +0300

    fix changelog

commit 5f56852c21d794bd87c13192d3857757be10f9b2
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Feb 13 17:39:02 2023 +0300

    add todo; fix data race

commit 89b8b16ddf5a43ebf68174cbaf9e8a53365f8cbe
Merge: e0a6bb49 ec19a85e
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 17:21:38 2023 +0300

    Merge branch 'master' into 4299-stats-ignore

commit e0a6bb490b651d1cf31589a7f17095fff4cb4dbb
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 17:21:06 2023 +0300

    interval under mutex

commit c569c7bc237f11b23fe47c98a20a1c5cb36751cb
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 16:19:35 2023 +0300

    fix mutex

commit 9374cf0c54dccc2fbfc38765b52c64e1c479137c
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 16:03:17 2023 +0300

    fix typo

commit 1f4fd1e7ab1b3c2f8e9c3d32ef7e4958f99abb47
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 15:55:44 2023 +0300

    add mutex

commit 2148048ce9ad228381cbb51a806c9b9cc21458fd
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Feb 10 12:27:36 2023 +0300

    add key check

commit a19350977c463f888aea70d0dace26dff0173a65
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Feb 9 18:34:36 2023 +0300

    fix changelog

commit 23c3b6da162dfd513884b460c265ba4cafeb9727
Merge: 8fccc0b8 b89105e3
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Feb 9 13:28:59 2023 +0300

    Merge branch 'master' into 4299-stats-ignore

commit 8fccc0b8ec670a37e5209d795f35c43dd64afeb3
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Feb 9 13:27:42 2023 +0300

    add changelog

commit 0416c71742795b2fb8adb0173dcd6a99d9d9c676
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Feb 8 14:31:55 2023 +0300

    all: stats ignore
This commit is contained in:
Stanislav Chzhen
2023-02-13 18:15:33 +03:00
parent ec19a85ed0
commit ff04b2a7d3
14 changed files with 320 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
internal/stats/http.go
View File

@@ -5,7 +5,6 @@ package stats
import (
"encoding/json"
"net/http"
"sync/atomic"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
@@ -41,10 +40,11 @@ type StatsResp struct {
// handleStats handles requests to the GET /control/stats endpoint.
func (s *StatsCtx) handleStats(w http.ResponseWriter, r *http.Request) {
limit := atomic.LoadUint32(&s.limitHours)
s.lock.Lock()
defer s.lock.Unlock()
start := time.Now()
resp, ok := s.getData(limit)
resp, ok := s.getData(s.limitHours)
log.Debug("stats: prepared data in %v", time.Since(start))
if !ok {
@@ -65,7 +65,13 @@ type configResp struct {
// handleStatsInfo handles requests to the GET /control/stats_info endpoint.
func (s *StatsCtx) handleStatsInfo(w http.ResponseWriter, r *http.Request) {
resp := configResp{IntervalDays: atomic.LoadUint32(&s.limitHours) / 24}
s.lock.Lock()
defer s.lock.Unlock()
resp := configResp{IntervalDays: s.limitHours / 24}
if !s.enabled {
resp.IntervalDays = 0
}
_ = aghhttp.WriteJSONResponse(w, r, resp)
}

100
internal/stats/stats.go
View File

@@ -15,16 +15,10 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/stringutil"
"go.etcd.io/bbolt"
)
// DiskConfig is the configuration structure that is stored in file.
type DiskConfig struct {
// Interval is the number of days for which the statistics are collected
// before flushing to the database.
Interval uint32 `yaml:"statistics_interval"`
}
// checkInterval returns true if days is valid to be used as statistics
// retention interval. The valid values are 0, 1, 7, 30 and 90.
func checkInterval(days uint32) (ok bool) {
@@ -51,6 +45,12 @@ type Config struct {
// LimitDays is the maximum number of days to collect statistics into the
// current unit.
LimitDays uint32
// Enabled tells if the statistics are enabled.
Enabled bool
// Ignored is the list of host names, which should not be counted.
Ignored *stringutil.Set
}
// Interface is the statistics interface to be used by other packages.
@@ -68,19 +68,15 @@ type Interface interface {
TopClientsIP(limit uint) []netip.Addr
// WriteDiskConfig puts the Interface's configuration to the dc.
WriteDiskConfig(dc *DiskConfig)
WriteDiskConfig(dc *Config)
// ShouldCount returns true if request for the host should be counted.
ShouldCount(host string, qType, qClass uint16) bool
}
// StatsCtx collects the statistics and flushes it to the database. Its default
// flushing interval is one hour.
type StatsCtx struct {
// limitHours is the maximum number of hours to collect statistics into the
// current unit.
//
// It is of type uint32 to be accessed by atomic. It's arranged at the
// beginning of the structure to keep 64-bit alignment.
limitHours uint32
// currMu protects curr.
currMu *sync.RWMutex
// curr is the actual statistics collection result.
@@ -102,6 +98,21 @@ type StatsCtx struct {
// filename is the name of database file.
filename string
// lock protects all the fields below.
lock sync.Mutex
// enabled tells if the statistics are enabled.
enabled bool
// limitHours is the maximum number of hours to collect statistics into the
// current unit.
//
// TODO(s.chzhen): Rewrite to use time.Duration.
limitHours uint32
// ignored is the list of host names, which should not be counted.
ignored *stringutil.Set
}
// New creates s from conf and properly initializes it. Don't use s before
@@ -110,10 +121,12 @@ func New(conf Config) (s *StatsCtx, err error) {
defer withRecovered(&err)
s = &StatsCtx{
enabled: conf.Enabled,
currMu: &sync.RWMutex{},
filename: conf.Filename,
configModified: conf.ConfigModified,
httpRegister: conf.HTTPRegister,
ignored: conf.Ignored,
}
if s.limitHours = conf.LimitDays * 24; !checkInterval(conf.LimitDays) {
s.limitHours = 24
@@ -215,7 +228,10 @@ func (s *StatsCtx) Close() (err error) {
// Update implements the Interface interface for *StatsCtx.
func (s *StatsCtx) Update(e Entry) {
if atomic.LoadUint32(&s.limitHours) == 0 {
s.lock.Lock()
defer s.lock.Unlock()
if !s.enabled || s.limitHours == 0 {
return
}
@@ -243,14 +259,22 @@ func (s *StatsCtx) Update(e Entry) {
}
// WriteDiskConfig implements the Interface interface for *StatsCtx.
func (s *StatsCtx) WriteDiskConfig(dc *DiskConfig) {
dc.Interval = atomic.LoadUint32(&s.limitHours) / 24
func (s *StatsCtx) WriteDiskConfig(dc *Config) {
s.lock.Lock()
defer s.lock.Unlock()
dc.LimitDays = s.limitHours / 24
dc.Enabled = s.enabled
dc.Ignored = s.ignored
}
// TopClientsIP implements the [Interface] interface for *StatsCtx.
func (s *StatsCtx) TopClientsIP(maxCount uint) (ips []netip.Addr) {
limit := atomic.LoadUint32(&s.limitHours)
if limit == 0 {
s.lock.Lock()
defer s.lock.Unlock()
limit := s.limitHours
if !s.enabled || limit == 0 {
return nil
}
@@ -342,6 +366,9 @@ func (s *StatsCtx) openDB() (err error) {
func (s *StatsCtx) flush() (cont bool, sleepFor time.Duration) {
id := s.unitIDGen()
s.lock.Lock()
defer s.lock.Unlock()
s.currMu.Lock()
defer s.currMu.Unlock()
@@ -350,7 +377,7 @@ func (s *StatsCtx) flush() (cont bool, sleepFor time.Duration) {
return false, 0
}
limit := atomic.LoadUint32(&s.limitHours)
limit := s.limitHours
if limit == 0 || ptr.id == id {
return true, time.Second
}
@@ -410,14 +437,23 @@ func (s *StatsCtx) periodicFlush() {
}
func (s *StatsCtx) setLimit(limitDays int) {
atomic.StoreUint32(&s.limitHours, uint32(24*limitDays))
if limitDays == 0 {
if err := s.clear(); err != nil {
log.Error("stats: %s", err)
}
s.lock.Lock()
defer s.lock.Unlock()
if limitDays != 0 {
s.enabled = true
s.limitHours = uint32(24 * limitDays)
log.Debug("stats: set limit: %d days", limitDays)
return
}
log.Debug("stats: set limit: %d days", limitDays)
s.enabled = false
log.Debug("stats: disabled")
if err := s.clear(); err != nil {
log.Error("stats: %s", err)
}
}
// Reset counters and clear database
@@ -520,3 +556,13 @@ func (s *StatsCtx) loadUnits(limit uint32) (units []*unitDB, firstID uint32) {
return units, firstID
}
// ShouldCount returns true if request for the host should be counted.
func (s *StatsCtx) ShouldCount(host string, _, _ uint16) bool {
return !s.isIgnored(host)
}
// isIgnored returns true if the host is in the Ignored list.
func (s *StatsCtx) isIgnored(host string) bool {
return s.ignored.Has(host)
}

2
internal/stats/stats_test.go
View File

@@ -53,6 +53,7 @@ func TestStats(t *testing.T) {
conf := stats.Config{
Filename: filepath.Join(t.TempDir(), "stats.db"),
LimitDays: 1,
Enabled: true,
UnitID: constUnitID,
HTTPRegister: func(_, url string, handler http.HandlerFunc) {
handlers[url] = handler
@@ -158,6 +159,7 @@ func TestLargeNumbers(t *testing.T) {
conf := stats.Config{
Filename: filepath.Join(t.TempDir(), "stats.db"),
LimitDays: 1,
Enabled: true,
UnitID: func() (id uint32) { return atomic.LoadUint32(&curHour) },
HTTPRegister: func(_, url string, handler http.HandlerFunc) { handlers[url] = handler },
}

13
internal/stats/unit.go
View File

@@ -10,6 +10,7 @@ import (
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/stringutil"
"go.etcd.io/bbolt"
)
@@ -341,11 +342,13 @@ type pairsGetter func(u *unitDB) (pairs []countPair)
// topsCollector collects statistics about highest values from the given *unitDB
// slice using pg to retrieve data.
func topsCollector(units []*unitDB, max int, pg pairsGetter) []map[string]uint64 {
func topsCollector(units []*unitDB, max int, ignored *stringutil.Set, pg pairsGetter) []map[string]uint64 {
m := map[string]uint64{}
for _, u := range units {
for _, cp := range pg(u) {
m[cp.Name] += cp.Count
if !ignored.Has(cp.Name) {
m[cp.Name] += cp.Count
}
}
}
a2 := convertMapToSlice(m, max)
@@ -408,9 +411,9 @@ func (s *StatsCtx) getData(limit uint32) (StatsResp, bool) {
BlockedFiltering: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RFiltered] }),
ReplacedSafebrowsing: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RSafeBrowsing] }),
ReplacedParental: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RParental] }),
TopQueried: topsCollector(units, maxDomains, func(u *unitDB) (pairs []countPair) { return u.Domains }),
TopBlocked: topsCollector(units, maxDomains, func(u *unitDB) (pairs []countPair) { return u.BlockedDomains }),
TopClients: topsCollector(units, maxClients, func(u *unitDB) (pairs []countPair) { return u.Clients }),
TopQueried: topsCollector(units, maxDomains, s.ignored, func(u *unitDB) (pairs []countPair) { return u.Domains }),
TopBlocked: topsCollector(units, maxDomains, s.ignored, func(u *unitDB) (pairs []countPair) { return u.BlockedDomains }),
TopClients: topsCollector(units, maxClients, nil, func(u *unitDB) (pairs []countPair) { return u.Clients }),
}
// Total counters: