daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pull request: 2508 ip conversion vol.2

Browse Source 
Merge in DNS/adguard-home from 2508-ip-conversion-vol2 to master

Closes #2508.

Squashed commit of the following:

commit 5b9d33f9cd352756831f63e34c4aea48674628c1
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Wed Jan 20 17:15:17 2021 +0300

    util: replace net.IPNet with pointer

commit 680126de7d59464077f9edf1bbaa925dd3fcee19
Merge: d3ba6a6c 5a50efad
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Wed Jan 20 17:02:41 2021 +0300

    Merge branch 'master' into 2508-ip-conversion-vol2

commit d3ba6a6cdd01c0aa736418fdb86ed40120169fe9
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Tue Jan 19 18:29:54 2021 +0300

    all: remove last conversion

commit 88b63f11a6c3f8705d7fa0c448c50dd646cc9214
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Tue Jan 19 14:12:45 2021 +0300

    all: improve code quality

commit 71af60c70a0dbaf55e2221023d6d2e4993c9e9a7
Merge: 98af3784 9f75725d
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Mon Jan 18 17:13:27 2021 +0300

    Merge branch 'master' into 2508-ip-conversion-vol2

commit 98af3784ce44d0993d171653c13d6e83bb8d1e6a
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Mon Jan 18 16:32:53 2021 +0300

    all: log changes

commit e99595a172bae1e844019d344544be84ddd65e4e
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Mon Jan 18 16:06:49 2021 +0300

    all: fix or remove remaining net.IP <-> string conversions

commit 7fd0634ce945f7e4c9b856684c5199f8a84a543e
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Fri Jan 15 15:36:17 2021 +0300

    all: remove redundant net.IP <-> string converions

commit 5df8af030421237d41b67ed659f83526cc258199
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Thu Jan 14 16:35:25 2021 +0300

    stats: remove redundant net.IP <-> string conversion

commit fbe4e3fc015e6898063543a90c04401d76dbb18f
Author: Eugene Burkov <e.burkov@adguard.com>
Date:   Thu Jan 14 16:20:35 2021 +0300

    querylog: remove redundant net.IP <-> string conversion
This commit is contained in:
Eugene Burkov
2021-01-20 17:27:53 +03:00
parent 5a50efadb2
commit 7fab31beae
45 changed files with 324 additions and 302 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/querylog/decode.go
View File

@@ -22,9 +22,11 @@ var logEntryHandlers = map[string]logEntryHandler{
if !ok {
return nil
}
if len(ent.IP) == 0 {
ent.IP = v
if ent.IP == nil {
ent.IP = net.ParseIP(v)
}
return nil
},
"T": func(t json.Token, ent *logEntry) error {

2
internal/querylog/decode_test.go
View File

@@ -47,7 +47,7 @@ func TestDecodeLogEntry(t *testing.T) {
assert.Nil(t, err)
want := &logEntry{
IP: "127.0.0.1",
IP: net.IPv4(127, 0, 0, 1),
Time: time.Date(2020, 11, 25, 15, 55, 56, 519796000, time.UTC),
QHost: "an.yandex.ru",
QType: "A",

27
internal/querylog/json.go
View File

@@ -14,22 +14,19 @@ import (
// TODO(a.garipov): Use a proper structured approach here.
// Get Client IP address
func (l *queryLog) getClientIP(clientIP string) string {
if l.conf.AnonymizeClientIP {
ip := net.ParseIP(clientIP)
if ip != nil {
ip4 := ip.To4()
const AnonymizeClientIP4Mask = 16
const AnonymizeClientIP6Mask = 112
if ip4 != nil {
clientIP = ip4.Mask(net.CIDRMask(AnonymizeClientIP4Mask, 32)).String()
} else {
clientIP = ip.Mask(net.CIDRMask(AnonymizeClientIP6Mask, 128)).String()
}
func (l *queryLog) getClientIP(ip net.IP) (clientIP net.IP) {
if l.conf.AnonymizeClientIP && ip != nil {
const AnonymizeClientIPv4Mask = 16
const AnonymizeClientIPv6Mask = 112
if ip.To4() != nil {
return ip.Mask(net.CIDRMask(AnonymizeClientIPv4Mask, 32))
}
return ip.Mask(net.CIDRMask(AnonymizeClientIPv6Mask, 128))
}
return clientIP
return ip
}
// jobject is a JSON object alias.
@@ -153,9 +150,9 @@ func answerToMap(a *dns.Msg) (answers []jobject) {
// try most common record types
switch v := k.(type) {
case *dns.A:
answer["value"] = v.A.String()
answer["value"] = v.A
case *dns.AAAA:
answer["value"] = v.AAAA.String()
answer["value"] = v.AAAA
case *dns.MX:
answer["value"] = fmt.Sprintf("%v %v", v.Preference, v.Mx)
case *dns.CNAME:

5
internal/querylog/qlog.go
View File

@@ -3,6 +3,7 @@ package querylog
import (
"fmt"
"net"
"os"
"path/filepath"
"strings"
@@ -60,7 +61,7 @@ func NewClientProto(s string) (cp ClientProto, err error) {
// logEntry - represents a single log entry
type logEntry struct {
IP string `json:"IP"` // Client IP
IP net.IP `json:"IP"` // Client IP
Time time.Time `json:"T"`
QHost string `json:"QH"`
@@ -147,7 +148,7 @@ func (l *queryLog) Add(params AddParams) {
now := time.Now()
entry := logEntry{
IP: l.getClientIP(params.ClientIP.String()),
IP: l.getClientIP(params.ClientIP),
Time: now,
Result: *params.Result,

58
internal/querylog/qlog_test.go
View File

@@ -40,27 +40,27 @@ func TestQueryLog(t *testing.T) {
l := newQueryLog(conf)
// add disk entries
addEntry(l, "example.org", "1.1.1.1", "2.2.2.1")
addEntry(l, "example.org", net.IPv4(1, 1, 1, 1), net.IPv4(2, 2, 2, 1))
// write to disk (first file)
_ = l.flushLogBuffer(true)
// start writing to the second file
_ = l.rotate()
// add disk entries
addEntry(l, "example.org", "1.1.1.2", "2.2.2.2")
addEntry(l, "example.org", net.IPv4(1, 1, 1, 2), net.IPv4(2, 2, 2, 2))
// write to disk
_ = l.flushLogBuffer(true)
// add memory entries
addEntry(l, "test.example.org", "1.1.1.3", "2.2.2.3")
addEntry(l, "example.com", "1.1.1.4", "2.2.2.4")
addEntry(l, "test.example.org", net.IPv4(1, 1, 1, 3), net.IPv4(2, 2, 2, 3))
addEntry(l, "example.com", net.IPv4(1, 1, 1, 4), net.IPv4(2, 2, 2, 4))
// get all entries
params := newSearchParams()
entries, _ := l.search(params)
assert.Len(t, entries, 4)
assertLogEntry(t, entries[0], "example.com", "1.1.1.4", "2.2.2.4")
assertLogEntry(t, entries[1], "test.example.org", "1.1.1.3", "2.2.2.3")
assertLogEntry(t, entries[2], "example.org", "1.1.1.2", "2.2.2.2")
assertLogEntry(t, entries[3], "example.org", "1.1.1.1", "2.2.2.1")
assertLogEntry(t, entries[0], "example.com", net.IPv4(1, 1, 1, 4), net.IPv4(2, 2, 2, 4))
assertLogEntry(t, entries[1], "test.example.org", net.IPv4(1, 1, 1, 3), net.IPv4(2, 2, 2, 3))
assertLogEntry(t, entries[2], "example.org", net.IPv4(1, 1, 1, 2), net.IPv4(2, 2, 2, 2))
assertLogEntry(t, entries[3], "example.org", net.IPv4(1, 1, 1, 1), net.IPv4(2, 2, 2, 1))
// search by domain (strict)
params = newSearchParams()
@@ -71,7 +71,7 @@ func TestQueryLog(t *testing.T) {
})
entries, _ = l.search(params)
assert.Len(t, entries, 1)
assertLogEntry(t, entries[0], "test.example.org", "1.1.1.3", "2.2.2.3")
assertLogEntry(t, entries[0], "test.example.org", net.IPv4(1, 1, 1, 3), net.IPv4(2, 2, 2, 3))
// search by domain (not strict)
params = newSearchParams()
@@ -82,9 +82,9 @@ func TestQueryLog(t *testing.T) {
})
entries, _ = l.search(params)
assert.Len(t, entries, 3)
assertLogEntry(t, entries[0], "test.example.org", "1.1.1.3", "2.2.2.3")
assertLogEntry(t, entries[1], "example.org", "1.1.1.2", "2.2.2.2")
assertLogEntry(t, entries[2], "example.org", "1.1.1.1", "2.2.2.1")
assertLogEntry(t, entries[0], "test.example.org", net.IPv4(1, 1, 1, 3), net.IPv4(2, 2, 2, 3))
assertLogEntry(t, entries[1], "example.org", net.IPv4(1, 1, 1, 2), net.IPv4(2, 2, 2, 2))
assertLogEntry(t, entries[2], "example.org", net.IPv4(1, 1, 1, 1), net.IPv4(2, 2, 2, 1))
// search by client IP (strict)
params = newSearchParams()
@@ -95,7 +95,7 @@ func TestQueryLog(t *testing.T) {
})
entries, _ = l.search(params)
assert.Len(t, entries, 1)
assertLogEntry(t, entries[0], "example.org", "1.1.1.2", "2.2.2.2")
assertLogEntry(t, entries[0], "example.org", net.IPv4(1, 1, 1, 2), net.IPv4(2, 2, 2, 2))
// search by client IP (part of)
params = newSearchParams()
@@ -106,10 +106,10 @@ func TestQueryLog(t *testing.T) {
})
entries, _ = l.search(params)
assert.Len(t, entries, 4)
assertLogEntry(t, entries[0], "example.com", "1.1.1.4", "2.2.2.4")
assertLogEntry(t, entries[1], "test.example.org", "1.1.1.3", "2.2.2.3")
assertLogEntry(t, entries[2], "example.org", "1.1.1.2", "2.2.2.2")
assertLogEntry(t, entries[3], "example.org", "1.1.1.1", "2.2.2.1")
assertLogEntry(t, entries[0], "example.com", net.IPv4(1, 1, 1, 4), net.IPv4(2, 2, 2, 4))
assertLogEntry(t, entries[1], "test.example.org", net.IPv4(1, 1, 1, 3), net.IPv4(2, 2, 2, 3))
assertLogEntry(t, entries[2], "example.org", net.IPv4(1, 1, 1, 2), net.IPv4(2, 2, 2, 2))
assertLogEntry(t, entries[3], "example.org", net.IPv4(1, 1, 1, 1), net.IPv4(2, 2, 2, 1))
}
func TestQueryLogOffsetLimit(t *testing.T) {
@@ -124,13 +124,13 @@ func TestQueryLogOffsetLimit(t *testing.T) {
// add 10 entries to the log
for i := 0; i < 10; i++ {
addEntry(l, "second.example.org", "1.1.1.1", "2.2.2.1")
addEntry(l, "second.example.org", net.IPv4(1, 1, 1, 1), net.IPv4(2, 2, 2, 1))
}
// write them to disk (first file)
_ = l.flushLogBuffer(true)
// add 10 more entries to the log (memory)
for i := 0; i < 10; i++ {
addEntry(l, "first.example.org", "1.1.1.1", "2.2.2.1")
addEntry(l, "first.example.org", net.IPv4(1, 1, 1, 1), net.IPv4(2, 2, 2, 1))
}
// First page
@@ -178,7 +178,7 @@ func TestQueryLogMaxFileScanEntries(t *testing.T) {
// add 10 entries to the log
for i := 0; i < 10; i++ {
addEntry(l, "example.org", "1.1.1.1", "2.2.2.1")
addEntry(l, "example.org", net.IPv4(1, 1, 1, 1), net.IPv4(2, 2, 2, 1))
}
// write them to disk (first file)
_ = l.flushLogBuffer(true)
@@ -204,9 +204,9 @@ func TestQueryLogFileDisabled(t *testing.T) {
defer func() { _ = os.RemoveAll(conf.BaseDir) }()
l := newQueryLog(conf)
addEntry(l, "example1.org", "1.1.1.1", "2.2.2.1")
addEntry(l, "example2.org", "1.1.1.1", "2.2.2.1")
addEntry(l, "example3.org", "1.1.1.1", "2.2.2.1")
addEntry(l, "example1.org", net.IPv4(1, 1, 1, 1), net.IPv4(2, 2, 2, 1))
addEntry(l, "example2.org", net.IPv4(1, 1, 1, 1), net.IPv4(2, 2, 2, 1))
addEntry(l, "example3.org", net.IPv4(1, 1, 1, 1), net.IPv4(2, 2, 2, 1))
// the oldest entry is now removed from mem buffer
params := newSearchParams()
@@ -216,7 +216,7 @@ func TestQueryLogFileDisabled(t *testing.T) {
assert.Equal(t, "example2.org", ll[1].QHost)
}
func addEntry(l *queryLog, host, answerStr, client string) {
func addEntry(l *queryLog, host string, answerStr, client net.IP) {
q := dns.Msg{}
q.Question = append(q.Question, dns.Question{
Name: host + ".",
@@ -232,7 +232,7 @@ func addEntry(l *queryLog, host, answerStr, client string) {
Rrtype: dns.TypeA,
Class: dns.ClassINET,
}
answer.A = net.ParseIP(answerStr)
answer.A = answerStr
a.Answer = append(a.Answer, answer)
res := dnsfilter.Result{
IsFiltered: true,
@@ -248,13 +248,13 @@ func addEntry(l *queryLog, host, answerStr, client string) {
Answer: &a,
OrigAnswer: &a,
Result: &res,
ClientIP: net.ParseIP(client),
ClientIP: client,
Upstream: "upstream",
}
l.Add(params)
}
func assertLogEntry(t *testing.T, entry *logEntry, host, answer, client string) bool {
func assertLogEntry(t *testing.T, entry *logEntry, host string, answer, client net.IP) bool {
assert.Equal(t, host, entry.QHost)
assert.Equal(t, client, entry.IP)
assert.Equal(t, "A", entry.QType)
@@ -263,9 +263,9 @@ func assertLogEntry(t *testing.T, entry *logEntry, host, answer, client string)
msg := new(dns.Msg)
assert.Nil(t, msg.Unpack(entry.Answer))
assert.Len(t, msg.Answer, 1)
ip := proxyutil.GetIPFromDNSRecord(msg.Answer[0])
ip := proxyutil.GetIPFromDNSRecord(msg.Answer[0]).To16()
assert.NotNil(t, ip)
assert.Equal(t, answer, ip.String())
assert.Equal(t, answer, ip)
return true
}

8
internal/querylog/searchcriteria.go
View File

@@ -94,16 +94,20 @@ func (c *searchCriteria) ctDomainOrClientCase(entry *logEntry) bool {
if c.strict && qhost == searchVal {
return true
}
if !c.strict && strings.Contains(qhost, searchVal) {
return true
}
if c.strict && entry.IP == c.value {
ipStr := entry.IP.String()
if c.strict && ipStr == c.value {
return true
}
if !c.strict && strings.Contains(entry.IP, c.value) {
if !c.strict && strings.Contains(ipStr, c.value) {
return true
}
return false
}