daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pull request: all: client id support

Browse Source 
Merge in DNS/adguard-home from 1383-client-id to master

Updates #1383.

Squashed commit of the following:

commit ebe2678bfa9bf651a2cb1e64499b38edcf19a7ad
Author: Ildar Kamalov <ik@adguard.com>
Date:   Wed Jan 27 17:51:59 2021 +0300

    - client: check if IP is valid

commit 0c330585a170ea149ee75e43dfa65211e057299c
Author: Ildar Kamalov <ik@adguard.com>
Date:   Wed Jan 27 17:07:50 2021 +0300

    - client: find clients by client_id

commit 71c9593ee35d996846f061e114b7867c3aa3c978
Merge: 9104f161 3e9edd9e
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Jan 27 16:09:45 2021 +0300

    Merge branch 'master' into 1383-client-id

commit 9104f1615d2d462606c52017df25a422df872cea
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Jan 27 13:28:50 2021 +0300

    dnsforward: imp tests

commit ed47f26e611ade625a2cc2c2f71a291b796bbf8f
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Jan 27 12:39:52 2021 +0300

    dnsforward: fix address

commit 98b222ba69a5d265f620c180c960d01c84a1fb3b
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Jan 26 19:50:31 2021 +0300

    home: imp code

commit 4f3966548a2d8437d0b68207dd108dd1a6cb7d20
Merge: 199fdc05 c215b820
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Jan 26 19:45:13 2021 +0300

    Merge branch 'master' into 1383-client-id

commit 199fdc056f8a8be5500584f3aaee32865188aedc
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Jan 26 19:20:37 2021 +0300

    all: imp tests, logging, etc

commit 35ff14f4d534251aecb2ea60baba225f3eed8a3e
Author: Ildar Kamalov <ik@adguard.com>
Date:   Tue Jan 26 18:55:19 2021 +0300

    + client: remove block button from clients with client_id

commit 32991a0b4c56583a02fb5e00bba95d96000bce20
Author: Ildar Kamalov <ik@adguard.com>
Date:   Tue Jan 26 18:54:25 2021 +0300

    + client: add requests count for client_id

commit 2d68df4d2eac4a296d7469923e601dad4575c1a1
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Jan 26 15:49:50 2021 +0300

    stats: handle client ids

commit 4e14ab3590328f93a8cd6e9cbe1665baf74f220b
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Jan 26 13:45:25 2021 +0300

    openapi: fix example

commit ca9cf3f744fe197cace2c28ddc5bc68f71dad1f3
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Jan 26 13:37:10 2021 +0300

    openapi: improve clients find api docs

commit f79876e550c424558b704bc316a4cd04f25db011
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Jan 26 13:18:52 2021 +0300

    home: accept ids in clients find

commit 5b72595122aa0bd64debadfd753ed8a0e0840629
Merge: 607e241f abf8f65f
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Jan 25 18:34:56 2021 +0300

    Merge branch 'master' into 1383-client-id

commit 607e241f1c339dd6397218f70b8301e3de6a1ee0
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Jan 25 18:30:39 2021 +0300

    dnsforward: fix quic

commit f046352fef93e46234c2bbe8ae316d21034260e5
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Jan 25 16:53:09 2021 +0300

    all: remove wildcard requirement

commit 3b679489bae82c54177372be453fe184d8f0bab6
Author: Andrey Meshkov <am@adguard.com>
Date:   Mon Jan 25 16:02:28 2021 +0300

    workDir now supports symlinks

commit 0647ab4f113de2223f6949df001f42ecab05c995
Author: Ildar Kamalov <ik@adguard.com>
Date:   Mon Jan 25 14:59:46 2021 +0300

    - client: remove wildcard from domain validation

commit b1aec04a4ecadc9d65648ed6d284188fecce01c3
Author: Ildar Kamalov <ik@adguard.com>
Date:   Mon Jan 25 14:55:39 2021 +0300

    + client: add form to download mobileconfig

... and 12 more commits
This commit is contained in:
Ainar Garipov
2021-01-27 18:32:13 +03:00
parent 3e9edd9eac
commit fc9ddcf941
56 changed files with 1735 additions and 623 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/stats/stats.go
View File

@@ -76,10 +76,14 @@ const (
rLast
)
// Entry - data to add
// Entry is a statistics data entry.
type Entry struct {
// Clients is the client's primary ID.
//
// TODO(a.garipov): Make this a {net.IP, string} enum?
Client string
Domain string
Client net.IP
Result Result
Time uint32 // processing time (msec)
}

11
internal/stats/stats_test.go
View File

@@ -39,13 +39,13 @@ func TestStats(t *testing.T) {
e := Entry{}
e.Domain = "domain"
e.Client = net.IP{127, 0, 0, 1}
e.Client = "127.0.0.1"
e.Result = RFiltered
e.Time = 123456
s.Update(e)
e.Domain = "domain"
e.Client = net.IP{127, 0, 0, 1}
e.Client = "127.0.0.1"
e.Result = RNotFiltered
e.Time = 123456
s.Update(e)
@@ -113,9 +113,10 @@ func TestLargeNumbers(t *testing.T) {
}
for i := 0; i != n; i++ {
e.Domain = fmt.Sprintf("domain%d", i)
e.Client = net.IP{127, 0, 0, 1}
e.Client[2] = byte((i & 0xff00) >> 8)
e.Client[3] = byte(i & 0xff)
ip := net.IP{127, 0, 0, 1}
ip[2] = byte((i & 0xff00) >> 8)
ip[3] = byte(i & 0xff)
e.Client = ip.String()
e.Result = RNotFiltered
e.Time = 123456
s.Update(e)

44
internal/stats/unit.go
View File

@@ -223,6 +223,7 @@ func (s *statsCtx) periodicFlush() {
s.unitLock.Lock()
ptr := s.unit
s.unitLock.Unlock()
if ptr == nil {
break
}
@@ -230,6 +231,7 @@ func (s *statsCtx) periodicFlush() {
id := s.conf.UnitID()
if ptr.id == id {
time.Sleep(time.Second)
continue
}
@@ -243,6 +245,7 @@ func (s *statsCtx) periodicFlush() {
if tx == nil {
continue
}
ok1 := s.flushUnitToDB(tx, u.id, udb)
ok2 := s.deleteUnit(tx, id-s.conf.limit)
if ok1 || ok2 {
@@ -251,6 +254,7 @@ func (s *statsCtx) periodicFlush() {
_ = tx.Rollback()
}
}
log.Tracef("periodicFlush() exited")
}
@@ -265,7 +269,7 @@ func (s *statsCtx) deleteUnit(tx *bolt.Tx, id uint32) bool {
return true
}
func convertMapToArray(m map[string]uint64, max int) []countPair {
func convertMapToSlice(m map[string]uint64, max int) []countPair {
a := []countPair{}
for k, v := range m {
pair := countPair{}
@@ -283,7 +287,7 @@ func convertMapToArray(m map[string]uint64, max int) []countPair {
return a[:max]
}
func convertArrayToMap(a []countPair) map[string]uint64 {
func convertSliceToMap(a []countPair) map[string]uint64 {
m := map[string]uint64{}
for _, it := range a {
m[it.Name] = it.Count
@@ -301,9 +305,9 @@ func serialize(u *unit) *unitDB {
udb.TimeAvg = uint32(u.timeSum / u.nTotal)
}
udb.Domains = convertMapToArray(u.domains, maxDomains)
udb.BlockedDomains = convertMapToArray(u.blockedDomains, maxDomains)
udb.Clients = convertMapToArray(u.clients, maxClients)
udb.Domains = convertMapToSlice(u.domains, maxDomains)
udb.BlockedDomains = convertMapToSlice(u.blockedDomains, maxDomains)
udb.Clients = convertMapToSlice(u.clients, maxClients)
return &udb
}
@@ -319,9 +323,9 @@ func deserialize(u *unit, udb *unitDB) {
u.nResult[i] = udb.NResult[i]
}
u.domains = convertArrayToMap(udb.Domains)
u.blockedDomains = convertArrayToMap(udb.BlockedDomains)
u.clients = convertArrayToMap(udb.Clients)
u.domains = convertSliceToMap(udb.Domains)
u.blockedDomains = convertSliceToMap(udb.BlockedDomains)
u.clients = convertSliceToMap(udb.Clients)
u.timeSum = uint64(udb.TimeAvg) * u.nTotal
}
@@ -372,7 +376,7 @@ func (s *statsCtx) loadUnitFromDB(tx *bolt.Tx, id uint32) *unitDB {
return &udb
}
func convertTopArray(a []countPair) []map[string]uint64 {
func convertTopSlice(a []countPair) []map[string]uint64 {
m := []map[string]uint64{}
for _, it := range a {
ent := map[string]uint64{}
@@ -461,13 +465,20 @@ func (s *statsCtx) getClientIP(ip net.IP) (clientIP net.IP) {
func (s *statsCtx) Update(e Entry) {
if e.Result == 0 ||
e.Result >= rLast ||
len(e.Domain) == 0 ||
!(len(e.Client) == 4 || len(e.Client) == 16) {
e.Domain == "" ||
e.Client == "" {
return
}
client := s.getClientIP(e.Client)
clientID := e.Client
if ip := net.ParseIP(clientID); ip != nil {
ip = s.getClientIP(ip)
clientID = ip.String()
}
s.unitLock.Lock()
defer s.unitLock.Unlock()
u := s.unit
u.nResult[e.Result]++
@@ -478,10 +489,9 @@ func (s *statsCtx) Update(e Entry) {
u.blockedDomains[e.Domain]++
}
u.clients[client.String()]++
u.clients[clientID]++
u.timeSum += uint64(e.Time)
u.nTotal++
s.unitLock.Unlock()
}
func (s *statsCtx) loadUnits(limit uint32) ([]*unitDB, uint32) {
@@ -594,8 +604,8 @@ func (s *statsCtx) getData() (statsResponse, bool) {
m[it.Name] += it.Count
}
}
a2 := convertMapToArray(m, max)
return convertTopArray(a2)
a2 := convertMapToSlice(m, max)
return convertTopSlice(a2)
}
dnsQueries := statsCollector(func(u *unitDB) (num uint64) { return u.NTotal })
@@ -661,7 +671,7 @@ func (s *statsCtx) GetTopClientsIP(maxCount uint) []net.IP {
m[it.Name] += it.Count
}
}
a := convertMapToArray(m, int(maxCount))
a := convertMapToSlice(m, int(maxCount))
d := []net.IP{}
for _, it := range a {
d = append(d, net.ParseIP(it.Name))