all: sync with master; upd chlog
This commit is contained in:
@@ -7,8 +7,12 @@ import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
// topAddrs is an alias for the types of the TopFoo fields of statsResponse.
|
||||
@@ -38,13 +42,21 @@ type StatsResp struct {
|
||||
AvgProcessingTime float64 `json:"avg_processing_time"`
|
||||
}
|
||||
|
||||
// handleStats handles requests to the GET /control/stats endpoint.
|
||||
// handleStats is the handler for the GET /control/stats HTTP API.
|
||||
func (s *StatsCtx) handleStats(w http.ResponseWriter, r *http.Request) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
start := time.Now()
|
||||
resp, ok := s.getData(s.limitHours)
|
||||
|
||||
var (
|
||||
resp StatsResp
|
||||
ok bool
|
||||
)
|
||||
func() {
|
||||
s.confMu.RLock()
|
||||
defer s.confMu.RUnlock()
|
||||
|
||||
resp, ok = s.getData(uint32(s.limit.Hours()))
|
||||
}()
|
||||
|
||||
log.Debug("stats: prepared data in %v", time.Since(start))
|
||||
|
||||
if !ok {
|
||||
@@ -63,20 +75,73 @@ type configResp struct {
|
||||
IntervalDays uint32 `json:"interval"`
|
||||
}
|
||||
|
||||
// handleStatsInfo handles requests to the GET /control/stats_info endpoint.
|
||||
func (s *StatsCtx) handleStatsInfo(w http.ResponseWriter, r *http.Request) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
// getConfigResp is the response to the GET /control/stats_info.
|
||||
type getConfigResp struct {
|
||||
// Ignored is the list of host names, which should not be counted.
|
||||
Ignored []string `json:"ignored"`
|
||||
|
||||
resp := configResp{IntervalDays: s.limitHours / 24}
|
||||
if !s.enabled {
|
||||
// Interval is the statistics rotation interval in milliseconds.
|
||||
Interval float64 `json:"interval"`
|
||||
|
||||
// Enabled shows if statistics are enabled. It is an aghalg.NullBool to be
|
||||
// able to tell when it's set without using pointers.
|
||||
Enabled aghalg.NullBool `json:"enabled"`
|
||||
}
|
||||
|
||||
// handleStatsInfo is the handler for the GET /control/stats_info HTTP API.
|
||||
//
|
||||
// Deprecated: Remove it when migration to the new API is over.
|
||||
func (s *StatsCtx) handleStatsInfo(w http.ResponseWriter, r *http.Request) {
|
||||
var (
|
||||
enabled bool
|
||||
limit time.Duration
|
||||
)
|
||||
func() {
|
||||
s.confMu.RLock()
|
||||
defer s.confMu.RUnlock()
|
||||
|
||||
enabled, limit = s.enabled, s.limit
|
||||
}()
|
||||
|
||||
days := uint32(limit / timeutil.Day)
|
||||
ok := checkInterval(days)
|
||||
if !ok || (enabled && days == 0) {
|
||||
// NOTE: If interval is custom we set it to 90 days for compatibility
|
||||
// with old API.
|
||||
days = 90
|
||||
}
|
||||
|
||||
resp := configResp{IntervalDays: days}
|
||||
if !enabled {
|
||||
resp.IntervalDays = 0
|
||||
}
|
||||
|
||||
_ = aghhttp.WriteJSONResponse(w, r, resp)
|
||||
}
|
||||
|
||||
// handleStatsConfig handles requests to the POST /control/stats_config
|
||||
// endpoint.
|
||||
// handleGetStatsConfig is the handler for the GET /control/stats/config HTTP
|
||||
// API.
|
||||
func (s *StatsCtx) handleGetStatsConfig(w http.ResponseWriter, r *http.Request) {
|
||||
var resp *getConfigResp
|
||||
func() {
|
||||
s.confMu.RLock()
|
||||
defer s.confMu.RUnlock()
|
||||
|
||||
resp = &getConfigResp{
|
||||
Ignored: s.ignored.Values(),
|
||||
Interval: float64(s.limit.Milliseconds()),
|
||||
Enabled: aghalg.BoolToNullBool(s.enabled),
|
||||
}
|
||||
}()
|
||||
|
||||
slices.Sort(resp.Ignored)
|
||||
|
||||
_ = aghhttp.WriteJSONResponse(w, r, resp)
|
||||
}
|
||||
|
||||
// handleStatsConfig is the handler for the POST /control/stats_config HTTP API.
|
||||
//
|
||||
// Deprecated: Remove it when migration to the new API is over.
|
||||
func (s *StatsCtx) handleStatsConfig(w http.ResponseWriter, r *http.Request) {
|
||||
reqData := configResp{}
|
||||
err := json.NewDecoder(r.Body).Decode(&reqData)
|
||||
@@ -92,11 +157,59 @@ func (s *StatsCtx) handleStatsConfig(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
s.setLimit(int(reqData.IntervalDays))
|
||||
s.configModified()
|
||||
limit := time.Duration(reqData.IntervalDays) * timeutil.Day
|
||||
|
||||
defer s.configModified()
|
||||
|
||||
s.confMu.Lock()
|
||||
defer s.confMu.Unlock()
|
||||
|
||||
s.setLimit(limit)
|
||||
}
|
||||
|
||||
// handleStatsReset handles requests to the POST /control/stats_reset endpoint.
|
||||
// handlePutStatsConfig is the handler for the PUT /control/stats/config/update
|
||||
// HTTP API.
|
||||
func (s *StatsCtx) handlePutStatsConfig(w http.ResponseWriter, r *http.Request) {
|
||||
reqData := getConfigResp{}
|
||||
err := json.NewDecoder(r.Body).Decode(&reqData)
|
||||
if err != nil {
|
||||
aghhttp.Error(r, w, http.StatusBadRequest, "json decode: %s", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
set, err := aghnet.NewDomainNameSet(reqData.Ignored)
|
||||
if err != nil {
|
||||
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "ignored: %s", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
ivl := time.Duration(reqData.Interval) * time.Millisecond
|
||||
err = validateIvl(ivl)
|
||||
if err != nil {
|
||||
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "unsupported interval: %s", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if reqData.Enabled == aghalg.NBNull {
|
||||
aghhttp.Error(r, w, http.StatusUnprocessableEntity, "enabled is null")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
defer s.configModified()
|
||||
|
||||
s.confMu.Lock()
|
||||
defer s.confMu.Unlock()
|
||||
|
||||
s.ignored = set
|
||||
s.limit = ivl
|
||||
s.enabled = reqData.Enabled == aghalg.NBTrue
|
||||
}
|
||||
|
||||
// handleStatsReset is the handler for the POST /control/stats_reset HTTP API.
|
||||
func (s *StatsCtx) handleStatsReset(w http.ResponseWriter, r *http.Request) {
|
||||
err := s.clear()
|
||||
if err != nil {
|
||||
@@ -112,6 +225,10 @@ func (s *StatsCtx) initWeb() {
|
||||
|
||||
s.httpRegister(http.MethodGet, "/control/stats", s.handleStats)
|
||||
s.httpRegister(http.MethodPost, "/control/stats_reset", s.handleStatsReset)
|
||||
s.httpRegister(http.MethodPost, "/control/stats_config", s.handleStatsConfig)
|
||||
s.httpRegister(http.MethodGet, "/control/stats/config", s.handleGetStatsConfig)
|
||||
s.httpRegister(http.MethodPut, "/control/stats/config/update", s.handlePutStatsConfig)
|
||||
|
||||
// Deprecated handlers.
|
||||
s.httpRegister(http.MethodGet, "/control/stats_info", s.handleStatsInfo)
|
||||
s.httpRegister(http.MethodPost, "/control/stats_config", s.handleStatsConfig)
|
||||
}
|
||||
|
||||
153
internal/stats/http_test.go
Normal file
153
internal/stats/http_test.go
Normal file
@@ -0,0 +1,153 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
|
||||
"github.com/AdguardTeam/golibs/testutil"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestHandleStatsConfig(t *testing.T) {
|
||||
const (
|
||||
smallIvl = 1 * time.Minute
|
||||
minIvl = 1 * time.Hour
|
||||
maxIvl = 365 * timeutil.Day
|
||||
)
|
||||
|
||||
conf := Config{
|
||||
UnitID: func() (id uint32) { return 0 },
|
||||
ConfigModified: func() {},
|
||||
ShouldCountClient: func([]string) bool { return true },
|
||||
Filename: filepath.Join(t.TempDir(), "stats.db"),
|
||||
Limit: time.Hour * 24,
|
||||
Enabled: true,
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
wantErr string
|
||||
body getConfigResp
|
||||
wantCode int
|
||||
}{{
|
||||
name: "set_ivl_1_minIvl",
|
||||
body: getConfigResp{
|
||||
Enabled: aghalg.NBTrue,
|
||||
Interval: float64(minIvl.Milliseconds()),
|
||||
Ignored: []string{},
|
||||
},
|
||||
wantCode: http.StatusOK,
|
||||
wantErr: "",
|
||||
}, {
|
||||
name: "small_interval",
|
||||
body: getConfigResp{
|
||||
Enabled: aghalg.NBTrue,
|
||||
Interval: float64(smallIvl.Milliseconds()),
|
||||
Ignored: []string{},
|
||||
},
|
||||
wantCode: http.StatusUnprocessableEntity,
|
||||
wantErr: "unsupported interval: less than an hour\n",
|
||||
}, {
|
||||
name: "big_interval",
|
||||
body: getConfigResp{
|
||||
Enabled: aghalg.NBTrue,
|
||||
Interval: float64(maxIvl.Milliseconds() + minIvl.Milliseconds()),
|
||||
Ignored: []string{},
|
||||
},
|
||||
wantCode: http.StatusUnprocessableEntity,
|
||||
wantErr: "unsupported interval: more than a year\n",
|
||||
}, {
|
||||
name: "set_ignored_ivl_1_maxIvl",
|
||||
body: getConfigResp{
|
||||
Enabled: aghalg.NBTrue,
|
||||
Interval: float64(maxIvl.Milliseconds()),
|
||||
Ignored: []string{
|
||||
"ignor.ed",
|
||||
},
|
||||
},
|
||||
wantCode: http.StatusOK,
|
||||
wantErr: "",
|
||||
}, {
|
||||
name: "ignored_duplicate",
|
||||
body: getConfigResp{
|
||||
Enabled: aghalg.NBTrue,
|
||||
Interval: float64(minIvl.Milliseconds()),
|
||||
Ignored: []string{
|
||||
"ignor.ed",
|
||||
"ignor.ed",
|
||||
},
|
||||
},
|
||||
wantCode: http.StatusUnprocessableEntity,
|
||||
wantErr: "ignored: duplicate host name \"ignor.ed\" at index 1\n",
|
||||
}, {
|
||||
name: "ignored_empty",
|
||||
body: getConfigResp{
|
||||
Enabled: aghalg.NBTrue,
|
||||
Interval: float64(minIvl.Milliseconds()),
|
||||
Ignored: []string{
|
||||
"",
|
||||
},
|
||||
},
|
||||
wantCode: http.StatusUnprocessableEntity,
|
||||
wantErr: "ignored: host name is empty\n",
|
||||
}, {
|
||||
name: "enabled_is_null",
|
||||
body: getConfigResp{
|
||||
Enabled: aghalg.NBNull,
|
||||
Interval: float64(minIvl.Milliseconds()),
|
||||
Ignored: []string{},
|
||||
},
|
||||
wantCode: http.StatusUnprocessableEntity,
|
||||
wantErr: "enabled is null\n",
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s, err := New(conf)
|
||||
require.NoError(t, err)
|
||||
|
||||
s.Start()
|
||||
testutil.CleanupAndRequireSuccess(t, s.Close)
|
||||
|
||||
buf, err := json.Marshal(tc.body)
|
||||
require.NoError(t, err)
|
||||
|
||||
const (
|
||||
configGet = "/control/stats/config"
|
||||
configPut = "/control/stats/config/update"
|
||||
)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPut, configPut, bytes.NewReader(buf))
|
||||
rw := httptest.NewRecorder()
|
||||
|
||||
s.handlePutStatsConfig(rw, req)
|
||||
require.Equal(t, tc.wantCode, rw.Code)
|
||||
|
||||
if tc.wantCode != http.StatusOK {
|
||||
assert.Equal(t, tc.wantErr, rw.Body.String())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
resp := httptest.NewRequest(http.MethodGet, configGet, nil)
|
||||
rw = httptest.NewRecorder()
|
||||
|
||||
s.handleGetStatsConfig(rw, resp)
|
||||
require.Equal(t, http.StatusOK, rw.Code)
|
||||
|
||||
ans := getConfigResp{}
|
||||
err = json.Unmarshal(rw.Body.Bytes(), &ans)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, tc.body, ans)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/AdguardTeam/golibs/errors"
|
||||
"github.com/AdguardTeam/golibs/log"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
@@ -25,7 +26,23 @@ func checkInterval(days uint32) (ok bool) {
|
||||
return days == 0 || days == 1 || days == 7 || days == 30 || days == 90
|
||||
}
|
||||
|
||||
// validateIvl returns an error if ivl is less than an hour or more than a
|
||||
// year.
|
||||
func validateIvl(ivl time.Duration) (err error) {
|
||||
if ivl < time.Hour {
|
||||
return errors.Error("less than an hour")
|
||||
}
|
||||
|
||||
if ivl > timeutil.Day*365 {
|
||||
return errors.Error("more than a year")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Config is the configuration structure for the statistics collecting.
|
||||
//
|
||||
// Do not alter any fields of this structure after using it.
|
||||
type Config struct {
|
||||
// UnitID is the function to generate the identifier for current unit. If
|
||||
// nil, the default function is used, see newUnitID.
|
||||
@@ -35,22 +52,24 @@ type Config struct {
|
||||
// interface.
|
||||
ConfigModified func()
|
||||
|
||||
// ShouldCountClient returns client's ignore setting.
|
||||
ShouldCountClient func([]string) bool
|
||||
|
||||
// HTTPRegister is the function that registers handlers for the stats
|
||||
// endpoints.
|
||||
HTTPRegister aghhttp.RegisterFunc
|
||||
|
||||
// Ignored is the list of host names, which should not be counted.
|
||||
Ignored *stringutil.Set
|
||||
|
||||
// Filename is the name of the database file.
|
||||
Filename string
|
||||
|
||||
// LimitDays is the maximum number of days to collect statistics into the
|
||||
// current unit.
|
||||
LimitDays uint32
|
||||
// Limit is an upper limit for collecting statistics.
|
||||
Limit time.Duration
|
||||
|
||||
// Enabled tells if the statistics are enabled.
|
||||
Enabled bool
|
||||
|
||||
// Ignored is the list of host names, which should not be counted.
|
||||
Ignored *stringutil.Set
|
||||
}
|
||||
|
||||
// Interface is the statistics interface to be used by other packages.
|
||||
@@ -71,7 +90,7 @@ type Interface interface {
|
||||
WriteDiskConfig(dc *Config)
|
||||
|
||||
// ShouldCount returns true if request for the host should be counted.
|
||||
ShouldCount(host string, qType, qClass uint16) bool
|
||||
ShouldCount(host string, qType, qClass uint16, ids []string) bool
|
||||
}
|
||||
|
||||
// StatsCtx collects the statistics and flushes it to the database. Its default
|
||||
@@ -96,23 +115,23 @@ type StatsCtx struct {
|
||||
// interface.
|
||||
configModified func()
|
||||
|
||||
// filename is the name of database file.
|
||||
filename string
|
||||
|
||||
// lock protects all the fields below.
|
||||
lock sync.Mutex
|
||||
|
||||
// enabled tells if the statistics are enabled.
|
||||
enabled bool
|
||||
|
||||
// limitHours is the maximum number of hours to collect statistics into the
|
||||
// current unit.
|
||||
//
|
||||
// TODO(s.chzhen): Rewrite to use time.Duration.
|
||||
limitHours uint32
|
||||
// confMu protects ignored, limit, and enabled.
|
||||
confMu *sync.RWMutex
|
||||
|
||||
// ignored is the list of host names, which should not be counted.
|
||||
ignored *stringutil.Set
|
||||
|
||||
// shouldCountClient returns client's ignore setting.
|
||||
shouldCountClient func([]string) bool
|
||||
|
||||
// filename is the name of database file.
|
||||
filename string
|
||||
|
||||
// limit is an upper limit for collecting statistics.
|
||||
limit time.Duration
|
||||
|
||||
// enabled tells if the statistics are enabled.
|
||||
enabled bool
|
||||
}
|
||||
|
||||
// New creates s from conf and properly initializes it. Don't use s before
|
||||
@@ -120,17 +139,28 @@ type StatsCtx struct {
|
||||
func New(conf Config) (s *StatsCtx, err error) {
|
||||
defer withRecovered(&err)
|
||||
|
||||
err = validateIvl(conf.Limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unsupported interval: %w", err)
|
||||
}
|
||||
|
||||
if conf.ShouldCountClient == nil {
|
||||
return nil, errors.Error("should count client is unspecified")
|
||||
}
|
||||
|
||||
s = &StatsCtx{
|
||||
enabled: conf.Enabled,
|
||||
currMu: &sync.RWMutex{},
|
||||
filename: conf.Filename,
|
||||
configModified: conf.ConfigModified,
|
||||
httpRegister: conf.HTTPRegister,
|
||||
ignored: conf.Ignored,
|
||||
}
|
||||
if s.limitHours = conf.LimitDays * 24; !checkInterval(conf.LimitDays) {
|
||||
s.limitHours = 24
|
||||
configModified: conf.ConfigModified,
|
||||
filename: conf.Filename,
|
||||
|
||||
confMu: &sync.RWMutex{},
|
||||
ignored: conf.Ignored,
|
||||
shouldCountClient: conf.ShouldCountClient,
|
||||
limit: conf.Limit,
|
||||
enabled: conf.Enabled,
|
||||
}
|
||||
|
||||
if s.unitIDGen = newUnitID; conf.UnitID != nil {
|
||||
s.unitIDGen = conf.UnitID
|
||||
}
|
||||
@@ -150,7 +180,7 @@ func New(conf Config) (s *StatsCtx, err error) {
|
||||
return nil, fmt.Errorf("stats: opening a transaction: %w", err)
|
||||
}
|
||||
|
||||
deleted := deleteOldUnits(tx, id-s.limitHours-1)
|
||||
deleted := deleteOldUnits(tx, id-uint32(s.limit.Hours())-1)
|
||||
udb = loadUnitFromDB(tx, id)
|
||||
|
||||
err = finishTxn(tx, deleted > 0)
|
||||
@@ -228,10 +258,10 @@ func (s *StatsCtx) Close() (err error) {
|
||||
|
||||
// Update implements the Interface interface for *StatsCtx.
|
||||
func (s *StatsCtx) Update(e Entry) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
s.confMu.Lock()
|
||||
defer s.confMu.Unlock()
|
||||
|
||||
if !s.enabled || s.limitHours == 0 {
|
||||
if !s.enabled || s.limit == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -260,20 +290,20 @@ func (s *StatsCtx) Update(e Entry) {
|
||||
|
||||
// WriteDiskConfig implements the Interface interface for *StatsCtx.
|
||||
func (s *StatsCtx) WriteDiskConfig(dc *Config) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
s.confMu.RLock()
|
||||
defer s.confMu.RUnlock()
|
||||
|
||||
dc.LimitDays = s.limitHours / 24
|
||||
dc.Ignored = s.ignored.Clone()
|
||||
dc.Limit = s.limit
|
||||
dc.Enabled = s.enabled
|
||||
dc.Ignored = s.ignored
|
||||
}
|
||||
|
||||
// TopClientsIP implements the [Interface] interface for *StatsCtx.
|
||||
func (s *StatsCtx) TopClientsIP(maxCount uint) (ips []netip.Addr) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
s.confMu.RLock()
|
||||
defer s.confMu.RUnlock()
|
||||
|
||||
limit := s.limitHours
|
||||
limit := uint32(s.limit.Hours())
|
||||
if !s.enabled || limit == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -366,8 +396,8 @@ func (s *StatsCtx) openDB() (err error) {
|
||||
func (s *StatsCtx) flush() (cont bool, sleepFor time.Duration) {
|
||||
id := s.unitIDGen()
|
||||
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
s.confMu.Lock()
|
||||
defer s.confMu.Unlock()
|
||||
|
||||
s.currMu.Lock()
|
||||
defer s.currMu.Unlock()
|
||||
@@ -377,7 +407,7 @@ func (s *StatsCtx) flush() (cont bool, sleepFor time.Duration) {
|
||||
return false, 0
|
||||
}
|
||||
|
||||
limit := s.limitHours
|
||||
limit := uint32(s.limit.Hours())
|
||||
if limit == 0 || ptr.id == id {
|
||||
return true, time.Second
|
||||
}
|
||||
@@ -436,14 +466,14 @@ func (s *StatsCtx) periodicFlush() {
|
||||
log.Debug("periodic flushing finished")
|
||||
}
|
||||
|
||||
func (s *StatsCtx) setLimit(limitDays int) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
if limitDays != 0 {
|
||||
// setLimit sets the limit. s.lock is expected to be locked.
|
||||
//
|
||||
// TODO(s.chzhen): Remove it when migration to the new API is over.
|
||||
func (s *StatsCtx) setLimit(limit time.Duration) {
|
||||
if limit != 0 {
|
||||
s.enabled = true
|
||||
s.limitHours = uint32(24 * limitDays)
|
||||
log.Debug("stats: set limit: %d days", limitDays)
|
||||
s.limit = limit
|
||||
log.Debug("stats: set limit: %d days", limit/timeutil.Day)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -558,11 +588,19 @@ func (s *StatsCtx) loadUnits(limit uint32) (units []*unitDB, firstID uint32) {
|
||||
}
|
||||
|
||||
// ShouldCount returns true if request for the host should be counted.
|
||||
func (s *StatsCtx) ShouldCount(host string, _, _ uint16) bool {
|
||||
func (s *StatsCtx) ShouldCount(host string, _, _ uint16, ids []string) bool {
|
||||
s.confMu.RLock()
|
||||
defer s.confMu.RUnlock()
|
||||
|
||||
if !s.shouldCountClient(ids) {
|
||||
return false
|
||||
}
|
||||
|
||||
return !s.isIgnored(host)
|
||||
}
|
||||
|
||||
// isIgnored returns true if the host is in the Ignored list.
|
||||
// isIgnored returns true if the host is in the ignored domains list. It
|
||||
// assumes that s.confMu is locked for reading.
|
||||
func (s *StatsCtx) isIgnored(host string) bool {
|
||||
return s.ignored.Has(host)
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/AdguardTeam/golibs/testutil"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -35,9 +36,10 @@ func TestStats_races(t *testing.T) {
|
||||
var r uint32
|
||||
idGen := func() (id uint32) { return atomic.LoadUint32(&r) }
|
||||
conf := Config{
|
||||
UnitID: idGen,
|
||||
Filename: filepath.Join(t.TempDir(), "./stats.db"),
|
||||
LimitDays: 1,
|
||||
ShouldCountClient: func([]string) bool { return true },
|
||||
UnitID: idGen,
|
||||
Filename: filepath.Join(t.TempDir(), "./stats.db"),
|
||||
Limit: timeutil.Day,
|
||||
}
|
||||
|
||||
s, err := New(conf)
|
||||
|
||||
@@ -12,7 +12,10 @@ import (
|
||||
|
||||
"github.com/AdguardTeam/AdGuardHome/internal/stats"
|
||||
"github.com/AdguardTeam/golibs/netutil"
|
||||
"github.com/AdguardTeam/golibs/stringutil"
|
||||
"github.com/AdguardTeam/golibs/testutil"
|
||||
"github.com/AdguardTeam/golibs/timeutil"
|
||||
"github.com/miekg/dns"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -51,10 +54,11 @@ func TestStats(t *testing.T) {
|
||||
|
||||
handlers := map[string]http.Handler{}
|
||||
conf := stats.Config{
|
||||
Filename: filepath.Join(t.TempDir(), "stats.db"),
|
||||
LimitDays: 1,
|
||||
Enabled: true,
|
||||
UnitID: constUnitID,
|
||||
ShouldCountClient: func([]string) bool { return true },
|
||||
Filename: filepath.Join(t.TempDir(), "stats.db"),
|
||||
Limit: timeutil.Day,
|
||||
Enabled: true,
|
||||
UnitID: constUnitID,
|
||||
HTTPRegister: func(_, url string, handler http.HandlerFunc) {
|
||||
handlers[url] = handler
|
||||
},
|
||||
@@ -157,11 +161,12 @@ func TestLargeNumbers(t *testing.T) {
|
||||
handlers := map[string]http.Handler{}
|
||||
|
||||
conf := stats.Config{
|
||||
Filename: filepath.Join(t.TempDir(), "stats.db"),
|
||||
LimitDays: 1,
|
||||
Enabled: true,
|
||||
UnitID: func() (id uint32) { return atomic.LoadUint32(&curHour) },
|
||||
HTTPRegister: func(_, url string, handler http.HandlerFunc) { handlers[url] = handler },
|
||||
ShouldCountClient: func([]string) bool { return true },
|
||||
Filename: filepath.Join(t.TempDir(), "stats.db"),
|
||||
Limit: timeutil.Day,
|
||||
Enabled: true,
|
||||
UnitID: func() (id uint32) { return atomic.LoadUint32(&curHour) },
|
||||
HTTPRegister: func(_, url string, handler http.HandlerFunc) { handlers[url] = handler },
|
||||
}
|
||||
|
||||
s, err := stats.New(conf)
|
||||
@@ -196,3 +201,60 @@ func TestLargeNumbers(t *testing.T) {
|
||||
assertSuccessAndUnmarshal(t, data, handlers["/control/stats"], req)
|
||||
assert.Equal(t, hoursNum*cliNumPerHour, int(data.NumDNSQueries))
|
||||
}
|
||||
|
||||
func TestShouldCount(t *testing.T) {
|
||||
const (
|
||||
ignored1 = "ignor.ed"
|
||||
ignored2 = "ignored.to"
|
||||
)
|
||||
set := stringutil.NewSet(ignored1, ignored2)
|
||||
|
||||
s, err := stats.New(stats.Config{
|
||||
Enabled: true,
|
||||
Filename: filepath.Join(t.TempDir(), "stats.db"),
|
||||
Limit: timeutil.Day,
|
||||
Ignored: set,
|
||||
ShouldCountClient: func(ids []string) (a bool) {
|
||||
return ids[0] != "no_count"
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
s.Start()
|
||||
testutil.CleanupAndRequireSuccess(t, s.Close)
|
||||
|
||||
testCases := []struct {
|
||||
wantCount assert.BoolAssertionFunc
|
||||
name string
|
||||
host string
|
||||
ids []string
|
||||
}{{
|
||||
name: "count",
|
||||
host: "example.com",
|
||||
ids: []string{"whatever"},
|
||||
wantCount: assert.True,
|
||||
}, {
|
||||
name: "no_count_ignored_1",
|
||||
host: ignored1,
|
||||
ids: []string{"whatever"},
|
||||
wantCount: assert.False,
|
||||
}, {
|
||||
name: "no_count_ignored_2",
|
||||
host: ignored2,
|
||||
ids: []string{"whatever"},
|
||||
wantCount: assert.False,
|
||||
}, {
|
||||
name: "no_count_client_ignore",
|
||||
host: "example.com",
|
||||
ids: []string{"no_count"},
|
||||
wantCount: assert.False,
|
||||
}}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
res := s.ShouldCount(tc.host, dns.TypeA, dns.ClassINET, tc.ids)
|
||||
|
||||
tc.wantCount(t, res)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,6 +72,19 @@ type Entry struct {
|
||||
|
||||
// unit collects the statistics data for a specific period of time.
|
||||
type unit struct {
|
||||
// domains stores the number of requests for each domain.
|
||||
domains map[string]uint64
|
||||
|
||||
// blockedDomains stores the number of requests for each domain that has
|
||||
// been blocked.
|
||||
blockedDomains map[string]uint64
|
||||
|
||||
// clients stores the number of requests from each client.
|
||||
clients map[string]uint64
|
||||
|
||||
// nResult stores the number of requests grouped by it's result.
|
||||
nResult []uint64
|
||||
|
||||
// id is the unique unit's identifier. It's set to an absolute hour number
|
||||
// since the beginning of UNIX time by the default ID generating function.
|
||||
//
|
||||
@@ -81,29 +94,20 @@ type unit struct {
|
||||
|
||||
// nTotal stores the total number of requests.
|
||||
nTotal uint64
|
||||
// nResult stores the number of requests grouped by it's result.
|
||||
nResult []uint64
|
||||
|
||||
// timeSum stores the sum of processing time in milliseconds of each request
|
||||
// written by the unit.
|
||||
timeSum uint64
|
||||
|
||||
// domains stores the number of requests for each domain.
|
||||
domains map[string]uint64
|
||||
// blockedDomains stores the number of requests for each domain that has
|
||||
// been blocked.
|
||||
blockedDomains map[string]uint64
|
||||
// clients stores the number of requests from each client.
|
||||
clients map[string]uint64
|
||||
}
|
||||
|
||||
// newUnit allocates the new *unit.
|
||||
func newUnit(id uint32) (u *unit) {
|
||||
return &unit{
|
||||
id: id,
|
||||
domains: map[string]uint64{},
|
||||
blockedDomains: map[string]uint64{},
|
||||
clients: map[string]uint64{},
|
||||
nResult: make([]uint64, resultLast),
|
||||
domains: make(map[string]uint64),
|
||||
blockedDomains: make(map[string]uint64),
|
||||
clients: make(map[string]uint64),
|
||||
id: id,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,19 +119,25 @@ type countPair struct {
|
||||
}
|
||||
|
||||
// unitDB is the structure for serializing statistics data into the database.
|
||||
//
|
||||
// NOTE: Do not change the names or types of fields, as this structure is used
|
||||
// for GOB encoding.
|
||||
type unitDB struct {
|
||||
// NTotal is the total number of requests.
|
||||
NTotal uint64
|
||||
// NResult is the number of requests by the result's kind.
|
||||
NResult []uint64
|
||||
|
||||
// Domains is the number of requests for each domain name.
|
||||
Domains []countPair
|
||||
|
||||
// BlockedDomains is the number of requests blocked for each domain name.
|
||||
BlockedDomains []countPair
|
||||
|
||||
// Clients is the number of requests from each client.
|
||||
Clients []countPair
|
||||
|
||||
// NTotal is the total number of requests.
|
||||
NTotal uint64
|
||||
|
||||
// TimeAvg is the average of processing times in milliseconds of all the
|
||||
// requests in the unit.
|
||||
TimeAvg uint32
|
||||
|
||||
Reference in New Issue
Block a user