daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pull request: 2639 use testify require vol.2

Browse Source 
Merge in DNS/adguard-home from 2639-testify-require-2 to master

Updates #2639.

Squashed commit of the following:

commit 31cc29a166e2e48a73956853cbc6d6dd681ab6da
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Tue Feb 9 18:48:31 2021 +0300

    all: deal with t.Run

commit 484f477fbfedd03aca4d322bc1cc9e131f30e1ce
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Tue Feb 9 17:44:02 2021 +0300

    all: fix readability, imp tests

commit 1231a825b353c16e43eae1b660dbb4c87805f564
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Tue Feb 9 16:06:29 2021 +0300

    all: imp tests
This commit is contained in:
Eugene Burkov
2021-02-09 19:38:31 +03:00
parent 6471504555
commit a3dddd72c1
10 changed files with 345 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
internal/stats/stats_test.go
View File

@@ -9,6 +9,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMain(m *testing.M) {
@@ -34,28 +35,30 @@ func TestStats(t *testing.T) {
Filename: "./stats.db",
LimitDays: 1,
}
s, err := createObject(conf)
require.Nil(t, err)
t.Cleanup(func() {
s.clear()
s.Close()
assert.Nil(t, os.Remove(conf.Filename))
})
s, _ := createObject(conf)
e := Entry{}
e.Domain = "domain"
e.Client = "127.0.0.1"
e.Result = RFiltered
e.Time = 123456
s.Update(e)
e.Domain = "domain"
e.Client = "127.0.0.1"
e.Result = RNotFiltered
e.Time = 123456
s.Update(e)
s.Update(Entry{
Domain: "domain",
Client: "127.0.0.1",
Result: RFiltered,
Time: 123456,
})
s.Update(Entry{
Domain: "domain",
Client: "127.0.0.1",
Result: RNotFiltered,
Time: 123456,
})
d, ok := s.getData()
assert.True(t, ok)
require.True(t, ok)
a := []uint64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}
assert.True(t, UIntArrayEquals(d.DNSQueries, a))
@@ -70,12 +73,15 @@ func TestStats(t *testing.T) {
assert.True(t, UIntArrayEquals(d.ReplacedParental, a))
m := d.TopQueried
require.NotEmpty(t, m)
assert.EqualValues(t, 1, m[0]["domain"])
m = d.TopBlocked
require.NotEmpty(t, m)
assert.EqualValues(t, 1, m[0]["domain"])
m = d.TopClients
require.NotEmpty(t, m)
assert.EqualValues(t, 2, m[0]["127.0.0.1"])
assert.EqualValues(t, 2, d.NumDNSQueries)
@@ -86,81 +92,69 @@ func TestStats(t *testing.T) {
assert.EqualValues(t, 0.123456, d.AvgProcessingTime)
topClients := s.GetTopClientsIP(2)
require.NotEmpty(t, topClients)
assert.True(t, net.IP{127, 0, 0, 1}.Equal(topClients[0]))
s.clear()
s.Close()
}
func TestLargeNumbers(t *testing.T) {
var hour int32 = 1
var hour int32 = 0
newID := func() uint32 {
// use "atomic" to make Go race detector happy
// Use "atomic" to make go race detector happy.
return uint32(atomic.LoadInt32(&hour))
}
// log.SetLevel(log.DEBUG)
conf := Config{
Filename: "./stats.db",
LimitDays: 1,
UnitID: newID,
}
s, err := createObject(conf)
require.Nil(t, err)
t.Cleanup(func() {
s.Close()
assert.Nil(t, os.Remove(conf.Filename))
})
s, _ := createObject(conf)
e := Entry{}
// Number of distinct clients and domains every hour.
const n = 1000
n := 1000 // number of distinct clients and domains every hour
for h := 0; h != 12; h++ {
if h != 0 {
atomic.AddInt32(&hour, 1)
}
for i := 0; i != n; i++ {
e.Domain = fmt.Sprintf("domain%d", i)
ip := net.IP{127, 0, 0, 1}
ip[2] = byte((i & 0xff00) >> 8)
ip[3] = byte(i & 0xff)
e.Client = ip.String()
e.Result = RNotFiltered
e.Time = 123456
s.Update(e)
for h := 0; h < 12; h++ {
atomic.AddInt32(&hour, 1)
for i := 0; i < n; i++ {
s.Update(Entry{
Domain: fmt.Sprintf("domain%d", i),
Client: net.IP{
127,
0,
byte((i & 0xff00) >> 8),
byte(i & 0xff),
}.String(),
Result: RNotFiltered,
Time: 123456,
})
}
}
d, ok := s.getData()
assert.True(t, ok)
assert.EqualValues(t, int(hour)*n, d.NumDNSQueries)
s.Close()
require.True(t, ok)
assert.EqualValues(t, hour*n, d.NumDNSQueries)
}
// this code is a chunk copied from getData() that generates aggregate data per day
func aggregateDataPerDay(firstID uint32) int {
firstDayID := (firstID + 24 - 1) / 24 * 24 // align_ceil(24)
a := []uint64{}
var sum uint64
id := firstDayID
nextDayID := firstDayID + 24
for i := firstDayID - firstID; int(i) != 720; i++ {
sum++
if id == nextDayID {
a = append(a, sum)
sum = 0
nextDayID += 24
func TestStatsCollector(t *testing.T) {
ng := func(_ *unitDB) uint64 {
return 0
}
units := make([]*unitDB, 720)
t.Run("hours", func(t *testing.T) {
statsData := statsCollector(units, 0, Hours, ng)
assert.Len(t, statsData, 720)
})
t.Run("days", func(t *testing.T) {
for i := 0; i != 25; i++ {
statsData := statsCollector(units, uint32(i), Days, ng)
require.Lenf(t, statsData, 30, "i=%d", i)
}
id++
}
if id <= nextDayID {
a = append(a, sum)
}
return len(a)
}
func TestAggregateDataPerTimeUnit(t *testing.T) {
for i := 0; i != 25; i++ {
alen := aggregateDataPerDay(uint32(i))
assert.Equalf(t, 30, alen, "i=%d", i)
}
})
}

113
internal/stats/unit.go
View File

@@ -528,6 +528,57 @@ func (s *statsCtx) loadUnits(limit uint32) ([]*unitDB, uint32) {
return units, firstID
}
// numsGetter is a signature for statsCollector argument.
type numsGetter func(u *unitDB) (num uint64)
// statsCollector collects statisctics for the given *unitDB slice by specified
// timeUnit using ng to retrieve data.
func statsCollector(units []*unitDB, firstID uint32, timeUnit TimeUnit, ng numsGetter) (nums []uint64) {
if timeUnit == Hours {
for _, u := range units {
nums = append(nums, ng(u))
}
} else {
// Per time unit counters: 720 hours may span 31 days, so we
// skip data for the first day in this case.
// align_ceil(24)
firstDayID := (firstID + 24 - 1) / 24 * 24
var sum uint64
id := firstDayID
nextDayID := firstDayID + 24
for i := int(firstDayID - firstID); i != len(units); i++ {
sum += ng(units[i])
if id == nextDayID {
nums = append(nums, sum)
sum = 0
nextDayID += 24
}
id++
}
if id <= nextDayID {
nums = append(nums, sum)
}
}
return nums
}
// pairsGetter is a signature for topsCollector argument.
type pairsGetter func(u *unitDB) (pairs []countPair)
// topsCollector collects statistics about highest values fro the given *unitDB
// slice using pg to retrieve data.
func topsCollector(units []*unitDB, max int, pg pairsGetter) []map[string]uint64 {
m := map[string]uint64{}
for _, u := range units {
for _, it := range pg(u) {
m[it.Name] += it.Count
}
}
a2 := convertMapToSlice(m, max)
return convertTopSlice(a2)
}
/* Algorithm:
. Prepare array of N units, where N is the value of "limit" configuration setting
. Load data for the most recent units from file
@@ -568,65 +619,25 @@ func (s *statsCtx) getData() (statsResponse, bool) {
return statsResponse{}, false
}
// per time unit counters:
// 720 hours may span 31 days, so we skip data for the first day in this case
firstDayID := (firstID + 24 - 1) / 24 * 24 // align_ceil(24)
statsCollector := func(numsGetter func(u *unitDB) (num uint64)) (nums []uint64) {
if timeUnit == Hours {
for _, u := range units {
nums = append(nums, numsGetter(u))
}
} else {
var sum uint64
id := firstDayID
nextDayID := firstDayID + 24
for i := int(firstDayID - firstID); i != len(units); i++ {
sum += numsGetter(units[i])
if id == nextDayID {
nums = append(nums, sum)
sum = 0
nextDayID += 24
}
id++
}
if id <= nextDayID {
nums = append(nums, sum)
}
}
return nums
}
topsCollector := func(max int, pairsGetter func(u *unitDB) (pairs []countPair)) []map[string]uint64 {
m := map[string]uint64{}
for _, u := range units {
for _, it := range pairsGetter(u) {
m[it.Name] += it.Count
}
}
a2 := convertMapToSlice(m, max)
return convertTopSlice(a2)
}
dnsQueries := statsCollector(func(u *unitDB) (num uint64) { return u.NTotal })
dnsQueries := statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NTotal })
if timeUnit != Hours && len(dnsQueries) != int(limit/24) {
log.Fatalf("len(dnsQueries) != limit: %d %d", len(dnsQueries), limit)
}
data := statsResponse{
DNSQueries: dnsQueries,
BlockedFiltering: statsCollector(func(u *unitDB) (num uint64) { return u.NResult[RFiltered] }),
ReplacedSafebrowsing: statsCollector(func(u *unitDB) (num uint64) { return u.NResult[RSafeBrowsing] }),
ReplacedParental: statsCollector(func(u *unitDB) (num uint64) { return u.NResult[RParental] }),
TopQueried: topsCollector(maxDomains, func(u *unitDB) (pairs []countPair) { return u.Domains }),
TopBlocked: topsCollector(maxDomains, func(u *unitDB) (pairs []countPair) { return u.BlockedDomains }),
TopClients: topsCollector(maxClients, func(u *unitDB) (pairs []countPair) { return u.Clients }),
BlockedFiltering: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RFiltered] }),
ReplacedSafebrowsing: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RSafeBrowsing] }),
ReplacedParental: statsCollector(units, firstID, timeUnit, func(u *unitDB) (num uint64) { return u.NResult[RParental] }),
TopQueried: topsCollector(units, maxDomains, func(u *unitDB) (pairs []countPair) { return u.Domains }),
TopBlocked: topsCollector(units, maxDomains, func(u *unitDB) (pairs []countPair) { return u.BlockedDomains }),
TopClients: topsCollector(units, maxClients, func(u *unitDB) (pairs []countPair) { return u.Clients }),
}
// total counters:
sum := unitDB{}
sum.NResult = make([]uint64, rLast)
// Total counters:
sum := unitDB{
NResult: make([]uint64, rLast),
}
timeN := 0
for _, u := range units {
sum.NTotal += u.NTotal