daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cherry-pick: 4358 stats races

Browse Source 
Merge in DNS/adguard-home from 4358-stats-races to master

Updates #4358

Squashed commit of the following:

commit 162d17b04d95adad21fb9b3c5a6fb64df2e037ec
Merge: 17732cfa d4c3a43b
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Aug 17 14:04:20 2022 +0300

    Merge branch 'master' into 4358-stats-races

commit 17732cfa0f3b2589bf2c252697eee1d6b358a66c
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Aug 17 13:53:42 2022 +0300

    stats: imp docs, locking

commit 4ee090869af0fa2b777c12027c3b77d5acd6e4de
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Tue Aug 16 20:26:19 2022 +0300

    stats: revert const

commit a7681a1b882cef04511fcd5d569f5abe2f955239
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Tue Aug 16 20:23:00 2022 +0300

    stats: imp concurrency

commit a6c6c1a0572e4201cd24644fd3f86f51fc27f633
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Tue Aug 16 19:51:30 2022 +0300

    stats: imp code, tests, docs

commit 954196b49f5ad91d91f445ff656e63c318e4124c
Merge: 281e00da 6e63757f
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Tue Aug 16 13:07:32 2022 +0300

    Merge branch 'master' into 4358-stats-races

commit 281e00daf781d045269584ce0158eed1d77918df
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Fri Aug 12 16:22:18 2022 +0300

    stats: imp closing

commit ed036d9aa7e25498869edfb866b6e923538970eb
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Fri Aug 12 16:11:12 2022 +0300

    stats: imp tests more

commit f848a12487ecd2afc8416e800510090cc1be7330
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Fri Aug 12 13:54:19 2022 +0300

    stats: imp tests, code

commit 60e11f042d51ec68850143129e61c701c5e4f3a4
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Aug 11 16:36:07 2022 +0300

    stats: fix test

commit 6d97f1db093b5ce0d37984ff96a9ef6f4e02dba1
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Aug 11 14:53:21 2022 +0300

    stats: imp code, docs

commit 20c70c2847b0de6c7f9271a8d9a831175ed0c499
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Aug 10 20:53:36 2022 +0300

    stats: imp shared memory safety

commit 8b3945670a190bab070171e6b4976edab1e3e2a2
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Aug 10 17:22:55 2022 +0300

    stats: imp code
This commit is contained in:
Eugene Burkov
2022-08-17 14:09:13 +03:00
committed by Ainar Garipov
parent 7bb9b2416b
commit 993a3fc42c
6 changed files with 822 additions and 739 deletions
Download Patch File Download Diff File Expand all files Collapse all files

263
internal/stats/stats_test.go
View File

@@ -1,13 +1,17 @@
package stats
package stats_test
import (
"encoding/json"
"fmt"
"net"
"os"
"net/http"
"net/http/httptest"
"path/filepath"
"sync/atomic"
"testing"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/AdGuardHome/internal/stats"
"github.com/AdguardTeam/golibs/testutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -17,147 +21,176 @@ func TestMain(m *testing.M) {
aghtest.DiscardLogOutput(m)
}
func UIntArrayEquals(a, b []uint64) bool {
if len(a) != len(b) {
return false
// constUnitID is the UnitIDGenFunc which always return 0.
func constUnitID() (id uint32) { return 0 }
func assertSuccessAndUnmarshal(t *testing.T, to any, handler http.Handler, req *http.Request) {
t.Helper()
require.NotNil(t, handler)
rw := httptest.NewRecorder()
handler.ServeHTTP(rw, req)
require.Equal(t, http.StatusOK, rw.Code)
data := rw.Body.Bytes()
if to == nil {
assert.Empty(t, data)
return
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
err := json.Unmarshal(data, to)
require.NoError(t, err)
}
func TestStats(t *testing.T) {
conf := Config{
Filename: "./stats.db",
cliIP := net.IP{127, 0, 0, 1}
cliIPStr := cliIP.String()
handlers := map[string]http.Handler{}
conf := stats.Config{
Filename: filepath.Join(t.TempDir(), "stats.db"),
LimitDays: 1,
UnitID: constUnitID,
HTTPRegister: func(_, url string, handler http.HandlerFunc) {
handlers[url] = handler
},
}
s, err := New(conf)
s, err := stats.New(conf)
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, func() (err error) {
s.clear()
s.Close()
return os.Remove(conf.Filename)
s.Start()
testutil.CleanupAndRequireSuccess(t, s.Close)
t.Run("data", func(t *testing.T) {
const reqDomain = "domain"
entries := []stats.Entry{{
Domain: reqDomain,
Client: cliIPStr,
Result: stats.RFiltered,
Time: 123456,
}, {
Domain: reqDomain,
Client: cliIPStr,
Result: stats.RNotFiltered,
Time: 123456,
}}
wantData := &stats.StatsResp{
TimeUnits: "hours",
TopQueried: []map[string]uint64{0: {reqDomain: 1}},
TopClients: []map[string]uint64{0: {cliIPStr: 2}},
TopBlocked: []map[string]uint64{0: {reqDomain: 1}},
DNSQueries: []uint64{
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2,
},
BlockedFiltering: []uint64{
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,
},
ReplacedSafebrowsing: []uint64{
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
},
ReplacedParental: []uint64{
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
},
NumDNSQueries: 2,
NumBlockedFiltering: 1,
NumReplacedSafebrowsing: 0,
NumReplacedSafesearch: 0,
NumReplacedParental: 0,
AvgProcessingTime: 0.123456,
}
for _, e := range entries {
s.Update(e)
}
data := &stats.StatsResp{}
req := httptest.NewRequest(http.MethodGet, "/control/stats", nil)
assertSuccessAndUnmarshal(t, data, handlers["/control/stats"], req)
assert.Equal(t, wantData, data)
})
s.Update(Entry{
Domain: "domain",
Client: "127.0.0.1",
Result: RFiltered,
Time: 123456,
})
s.Update(Entry{
Domain: "domain",
Client: "127.0.0.1",
Result: RNotFiltered,
Time: 123456,
t.Run("tops", func(t *testing.T) {
topClients := s.TopClientsIP(2)
require.NotEmpty(t, topClients)
assert.True(t, cliIP.Equal(topClients[0]))
})
d, ok := s.getData()
require.True(t, ok)
t.Run("reset", func(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/control/stats_reset", nil)
assertSuccessAndUnmarshal(t, nil, handlers["/control/stats_reset"], req)
a := []uint64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}
assert.True(t, UIntArrayEquals(d.DNSQueries, a))
_24zeroes := [24]uint64{}
emptyData := &stats.StatsResp{
TimeUnits: "hours",
TopQueried: []map[string]uint64{},
TopClients: []map[string]uint64{},
TopBlocked: []map[string]uint64{},
DNSQueries: _24zeroes[:],
BlockedFiltering: _24zeroes[:],
ReplacedSafebrowsing: _24zeroes[:],
ReplacedParental: _24zeroes[:],
}
a = []uint64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}
assert.True(t, UIntArrayEquals(d.BlockedFiltering, a))
req = httptest.NewRequest(http.MethodGet, "/control/stats", nil)
data := &stats.StatsResp{}
a = []uint64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
assert.True(t, UIntArrayEquals(d.ReplacedSafebrowsing, a))
a = []uint64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
assert.True(t, UIntArrayEquals(d.ReplacedParental, a))
m := d.TopQueried
require.NotEmpty(t, m)
assert.EqualValues(t, 1, m[0]["domain"])
m = d.TopBlocked
require.NotEmpty(t, m)
assert.EqualValues(t, 1, m[0]["domain"])
m = d.TopClients
require.NotEmpty(t, m)
assert.EqualValues(t, 2, m[0]["127.0.0.1"])
assert.EqualValues(t, 2, d.NumDNSQueries)
assert.EqualValues(t, 1, d.NumBlockedFiltering)
assert.EqualValues(t, 0, d.NumReplacedSafebrowsing)
assert.EqualValues(t, 0, d.NumReplacedSafesearch)
assert.EqualValues(t, 0, d.NumReplacedParental)
assert.EqualValues(t, 0.123456, d.AvgProcessingTime)
topClients := s.GetTopClientsIP(2)
require.NotEmpty(t, topClients)
assert.True(t, net.IP{127, 0, 0, 1}.Equal(topClients[0]))
assertSuccessAndUnmarshal(t, data, handlers["/control/stats"], req)
assert.Equal(t, emptyData, data)
})
}
func TestLargeNumbers(t *testing.T) {
var hour int32 = 0
newID := func() uint32 {
// Use "atomic" to make go race detector happy.
return uint32(atomic.LoadInt32(&hour))
var curHour uint32 = 1
handlers := map[string]http.Handler{}
conf := stats.Config{
Filename: filepath.Join(t.TempDir(), "stats.db"),
LimitDays: 1,
UnitID: func() (id uint32) { return atomic.LoadUint32(&curHour) },
HTTPRegister: func(_, url string, handler http.HandlerFunc) { handlers[url] = handler },
}
conf := Config{
Filename: "./stats.db",
LimitDays: 1,
UnitID: newID,
}
s, err := New(conf)
s, err := stats.New(conf)
require.NoError(t, err)
testutil.CleanupAndRequireSuccess(t, func() (err error) {
s.Close()
return os.Remove(conf.Filename)
})
s.Start()
testutil.CleanupAndRequireSuccess(t, s.Close)
// Number of distinct clients and domains every hour.
const n = 1000
const (
hoursNum = 12
cliNumPerHour = 1000
)
for h := 0; h < 12; h++ {
atomic.AddInt32(&hour, 1)
for i := 0; i < n; i++ {
s.Update(Entry{
Domain: fmt.Sprintf("domain%d", i),
Client: net.IP{
127,
0,
byte((i & 0xff00) >> 8),
byte(i & 0xff),
}.String(),
Result: RNotFiltered,
req := httptest.NewRequest(http.MethodGet, "/control/stats", nil)
for h := 0; h < hoursNum; h++ {
atomic.AddUint32(&curHour, 1)
for i := 0; i < cliNumPerHour; i++ {
ip := net.IP{127, 0, byte((i & 0xff00) >> 8), byte(i & 0xff)}
e := stats.Entry{
Domain: fmt.Sprintf("domain%d.hour%d", i, h),
Client: ip.String(),
Result: stats.RNotFiltered,
Time: 123456,
})
}
s.Update(e)
}
}
d, ok := s.getData()
require.True(t, ok)
assert.EqualValues(t, hour*n, d.NumDNSQueries)
}
func TestStatsCollector(t *testing.T) {
ng := func(_ *unitDB) uint64 {
return 0
}
units := make([]*unitDB, 720)
t.Run("hours", func(t *testing.T) {
statsData := statsCollector(units, 0, Hours, ng)
assert.Len(t, statsData, 720)
})
t.Run("days", func(t *testing.T) {
for i := 0; i != 25; i++ {
statsData := statsCollector(units, uint32(i), Days, ng)
require.Lenf(t, statsData, 30, "i=%d", i)
}
})
data := &stats.StatsResp{}
assertSuccessAndUnmarshal(t, data, handlers["/control/stats"], req)
assert.Equal(t, hoursNum*cliNumPerHour, int(data.NumDNSQueries))
}