daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pull request #886: all: allow multiple rules in dns filter results

Browse Source 
Merge in DNS/adguard-home from 2102-rules-result to master

Updates #2102.

Squashed commit of the following:

commit 47b2aa94c56b37be492c3c01e8111054612d9722
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Dec 17 13:12:27 2020 +0300

    querylog: remove pre-v0.99.3 compatibility code

commit 2af0ee43c2444a7d842fcff057f2ba02f300244b
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Dec 17 13:00:27 2020 +0300

    all: improve documentation

commit 3add300a42f0aa67bb315a448e294636c85d0b3b
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Dec 16 18:30:01 2020 +0300

    all: improve changelog

commit e04ef701fc2de7f4453729e617641c47e0883679
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Dec 16 17:56:53 2020 +0300

    all: improve code and documentation

commit 4f04845ae275ae4291869e00c62e4ff81b01eaa3
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Dec 16 17:01:08 2020 +0300

    all: document changes, improve api

commit bc59b7656a402d0c65f13bd74a71d8dda6a8a65d
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Dec 15 18:22:01 2020 +0300

    all: allow multiple rules in dns filter results
This commit is contained in:
Ainar Garipov
2020-12-17 13:32:46 +03:00
parent 2c56a68597
commit 2e8352d31c
28 changed files with 610 additions and 356 deletions
Download Patch File Download Diff File Expand all files Collapse all files

153
internal/querylog/decode.go
View File

@@ -9,7 +9,6 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/dnsfilter"
"github.com/AdguardTeam/golibs/log"
"github.com/miekg/dns"
)
type logEntryHandler (func(t json.Token, ent *logEntry) error)
@@ -85,6 +84,29 @@ var logEntryHandlers = map[string]logEntryHandler{
ent.OrigAnswer, err = base64.StdEncoding.DecodeString(v)
return err
},
"Upstream": func(t json.Token, ent *logEntry) error {
v, ok := t.(string)
if !ok {
return nil
}
ent.Upstream = v
return nil
},
"Elapsed": func(t json.Token, ent *logEntry) error {
v, ok := t.(json.Number)
if !ok {
return nil
}
i, err := v.Int64()
if err != nil {
return err
}
ent.Elapsed = time.Duration(i)
return nil
},
}
var resultHandlers = map[string]logEntryHandler{
"IsFiltered": func(t json.Token, ent *logEntry) error {
v, ok := t.(bool)
if !ok {
@@ -94,23 +116,40 @@ var logEntryHandlers = map[string]logEntryHandler{
return nil
},
"Rule": func(t json.Token, ent *logEntry) error {
v, ok := t.(string)
s, ok := t.(string)
if !ok {
return nil
}
ent.Result.Rule = v
l := len(ent.Result.Rules)
if l == 0 {
ent.Result.Rules = []*dnsfilter.ResultRule{{}}
l++
}
ent.Result.Rules[l-1].Text = s
return nil
},
"FilterID": func(t json.Token, ent *logEntry) error {
v, ok := t.(json.Number)
n, ok := t.(json.Number)
if !ok {
return nil
}
i, err := v.Int64()
i, err := n.Int64()
if err != nil {
return err
}
ent.Result.FilterID = i
l := len(ent.Result.Rules)
if l == 0 {
ent.Result.Rules = []*dnsfilter.ResultRule{{}}
l++
}
ent.Result.Rules[l-1].FilterListID = i
return nil
},
"Reason": func(t json.Token, ent *logEntry) error {
@@ -133,62 +172,50 @@ var logEntryHandlers = map[string]logEntryHandler{
ent.Result.ServiceName = v
return nil
},
"Upstream": func(t json.Token, ent *logEntry) error {
v, ok := t.(string)
if !ok {
return nil
}
ent.Upstream = v
return nil
},
"Elapsed": func(t json.Token, ent *logEntry) error {
v, ok := t.(json.Number)
if !ok {
return nil
}
i, err := v.Int64()
}
func decodeResult(dec *json.Decoder, ent *logEntry) {
for {
keyToken, err := dec.Token()
if err != nil {
return err
if err != io.EOF {
log.Debug("decodeResult err: %s", err)
}
return
}
ent.Elapsed = time.Duration(i)
return nil
},
"Result": func(json.Token, *logEntry) error {
return nil
},
"Question": func(t json.Token, ent *logEntry) error {
v, ok := t.(string)
if d, ok := keyToken.(json.Delim); ok {
if d == '}' {
return
}
continue
}
key, ok := keyToken.(string)
if !ok {
return nil
log.Debug("decodeResult: keyToken is %T (%[1]v) and not string", keyToken)
return
}
var qstr []byte
qstr, err := base64.StdEncoding.DecodeString(v)
if err != nil {
return err
}
q := new(dns.Msg)
err = q.Unpack(qstr)
if err != nil {
return err
}
ent.QHost = q.Question[0].Name
if len(ent.QHost) == 0 {
return nil // nil???
}
ent.QHost = ent.QHost[:len(ent.QHost)-1]
ent.QType = dns.TypeToString[q.Question[0].Qtype]
ent.QClass = dns.ClassToString[q.Question[0].Qclass]
return nil
},
"Time": func(t json.Token, ent *logEntry) error {
v, ok := t.(string)
handler, ok := resultHandlers[key]
if !ok {
return nil
continue
}
var err error
ent.Time, err = time.Parse(time.RFC3339, v)
return err
},
val, err := dec.Token()
if err != nil {
return
}
if err = handler(val, ent); err != nil {
log.Debug("decodeResult handler err: %s", err)
return
}
}
}
func decodeLogEntry(ent *logEntry, str string) {
@@ -200,18 +227,27 @@ func decodeLogEntry(ent *logEntry, str string) {
if err != io.EOF {
log.Debug("decodeLogEntry err: %s", err)
}
return
}
if _, ok := keyToken.(json.Delim); ok {
continue
}
key, ok := keyToken.(string)
if !ok {
log.Debug("decodeLogEntry: keyToken is %T and not string", keyToken)
log.Debug("decodeLogEntry: keyToken is %T (%[1]v) and not string", keyToken)
return
}
if key == "Result" {
decodeResult(dec, ent)
continue
}
handler, ok := logEntryHandlers[key]
if !ok {
continue
@@ -223,7 +259,8 @@ func decodeLogEntry(ent *logEntry, str string) {
}
if err = handler(val, ent); err != nil {
log.Debug("decodeLogEntry err: %s", err)
log.Debug("decodeLogEntry handler err: %s", err)
return
}
}

26
internal/querylog/decode_test.go
View File

@@ -21,29 +21,17 @@ func TestDecode_decodeQueryLog(t *testing.T) {
log string
want string
}{{
name: "back_compatibility_all_right",
log: `{"Question":"ULgBAAABAAAAAAAAC2FkZ3VhcmR0ZWFtBmdpdGh1YgJpbwAAHAAB","Answer":"ULiBgAABAAAAAQAAC2FkZ3VhcmR0ZWFtBmdpdGh1YgJpbwAAHAABwBgABgABAAADQgBLB25zLTE2MjIJYXdzZG5zLTEwAmNvAnVrABFhd3NkbnMtaG9zdG1hc3RlcgZhbWF6b24DY29tAAAAAAEAABwgAAADhAASdQAAAVGA","Result":{},"Time":"2020-11-13T12:41:25.970861+03:00","Elapsed":244066501,"IP":"127.0.0.1","Upstream":"https://1.1.1.1:443/dns-query"}`,
want: "default",
}, {
name: "back_compatibility_bad_msg",
log: `{"Question":"","Answer":"ULiBgAABAAAAAQAAC2FkZ3VhcmR0ZWFtBmdpdGh1YgJpbwAAHAABwBgABgABAAADQgBLB25zLTE2MjIJYXdzZG5zLTEwAmNvAnVrABFhd3NkbnMtaG9zdG1hc3RlcgZhbWF6b24DY29tAAAAAAEAABwgAAADhAASdQAAAVGA","Result":{},"Time":"2020-11-13T12:41:25.970861+03:00","Elapsed":244066501,"IP":"127.0.0.1","Upstream":"https://1.1.1.1:443/dns-query"}`,
want: "decodeLogEntry err: dns: overflow unpacking uint16\n",
}, {
name: "back_compatibility_bad_decoding",
log: `{"Question":"LgBAAABAAAAAAAAC2FkZ3VhcmR0ZWFtBmdpdGh1YgJpbwAAHAAB","Answer":"ULiBgAABAAAAAQAAC2FkZ3VhcmR0ZWFtBmdpdGh1YgJpbwAAHAABwBgABgABAAADQgBLB25zLTE2MjIJYXdzZG5zLTEwAmNvAnVrABFhd3NkbnMtaG9zdG1hc3RlcgZhbWF6b24DY29tAAAAAAEAABwgAAADhAASdQAAAVGA","Result":{},"Time":"2020-11-13T12:41:25.970861+03:00","Elapsed":244066501,"IP":"127.0.0.1","Upstream":"https://1.1.1.1:443/dns-query"}`,
want: "decodeLogEntry err: illegal base64 data at input byte 48\n",
}, {
name: "modern_all_right",
name: "all_right",
log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`,
want: "default",
}, {
name: "bad_filter_id",
log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1.5},"Elapsed":837429}`,
want: "decodeLogEntry err: strconv.ParseInt: parsing \"1.5\": invalid syntax\n",
want: "decodeResult handler err: strconv.ParseInt: parsing \"1.5\": invalid syntax\n",
}, {
name: "bad_is_filtered",
log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":trooe,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`,
want: "default",
want: "decodeLogEntry err: invalid character 'o' in literal true (expecting 'u')\n",
}, {
name: "bad_elapsed",
log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":-1}`,
@@ -55,7 +43,7 @@ func TestDecode_decodeQueryLog(t *testing.T) {
}, {
name: "bad_time",
log: `{"IP":"127.0.0.1","T":"12/09/1998T15:00:00.000000+05:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`,
want: "decodeLogEntry err: parsing time \"12/09/1998T15:00:00.000000+05:00\" as \"2006-01-02T15:04:05Z07:00\": cannot parse \"9/1998T15:00:00.000000+05:00\" as \"2006\"\n",
want: "decodeLogEntry handler err: parsing time \"12/09/1998T15:00:00.000000+05:00\" as \"2006-01-02T15:04:05Z07:00\": cannot parse \"9/1998T15:00:00.000000+05:00\" as \"2006\"\n",
}, {
name: "bad_host",
log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":6,"QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`,
@@ -75,7 +63,7 @@ func TestDecode_decodeQueryLog(t *testing.T) {
}, {
name: "very_bad_client_proto",
log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"dog","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`,
want: "decodeLogEntry err: invalid client proto: \"dog\"\n",
want: "decodeLogEntry handler err: invalid client proto: \"dog\"\n",
}, {
name: "bad_answer",
log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":0.9,"Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`,
@@ -83,7 +71,7 @@ func TestDecode_decodeQueryLog(t *testing.T) {
}, {
name: "very_bad_answer",
log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":"||an.yandex.","FilterID":1},"Elapsed":837429}`,
want: "decodeLogEntry err: illegal base64 data at input byte 61\n",
want: "decodeLogEntry handler err: illegal base64 data at input byte 61\n",
}, {
name: "bad_rule",
log: `{"IP":"127.0.0.1","T":"2020-11-25T18:55:56.519796+03:00","QH":"an.yandex.ru","QT":"A","QC":"IN","CP":"","Answer":"Qz+BgAABAAEAAAAAAmFuBnlhbmRleAJydQAAAQABwAwAAQABAAAACgAEAAAAAA==","Result":{"IsFiltered":true,"Reason":3,"Rule":false,"FilterID":1},"Elapsed":837429}`,
@@ -102,7 +90,7 @@ func TestDecode_decodeQueryLog(t *testing.T) {
l := &logEntry{}
decodeLogEntry(l, tc.log)
assert.True(t, strings.HasSuffix(logOutput.String(), tc.want), logOutput.String())
assert.True(t, strings.HasSuffix(logOutput.String(), tc.want), "%q\ndoes not end with\n%q", logOutput.String(), tc.want)
logOutput.Reset()
})

70
internal/querylog/json.go
View File

@@ -6,10 +6,13 @@ import (
"strconv"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/dnsfilter"
"github.com/AdguardTeam/golibs/log"
"github.com/miekg/dns"
)
// TODO(a.garipov): Use a proper structured approach here.
// Get Client IP address
func (l *queryLog) getClientIP(clientIP string) string {
if l.conf.AnonymizeClientIP {
@@ -29,10 +32,12 @@ func (l *queryLog) getClientIP(clientIP string) string {
return clientIP
}
// entriesToJSON - converts log entries to JSON
func (l *queryLog) entriesToJSON(entries []*logEntry, oldest time.Time) map[string]interface{} {
// init the response object
data := []map[string]interface{}{}
// jobject is a JSON object alias.
type jobject = map[string]interface{}
// entriesToJSON converts query log entries to JSON.
func (l *queryLog) entriesToJSON(entries []*logEntry, oldest time.Time) (res jobject) {
data := []jobject{}
// the elements order is already reversed (from newer to older)
for i := 0; i < len(entries); i++ {
@@ -41,17 +46,18 @@ func (l *queryLog) entriesToJSON(entries []*logEntry, oldest time.Time) map[stri
data = append(data, jsonEntry)
}
result := map[string]interface{}{}
result["oldest"] = ""
if !oldest.IsZero() {
result["oldest"] = oldest.Format(time.RFC3339Nano)
res = jobject{
"data": data,
"oldest": "",
}
if !oldest.IsZero() {
res["oldest"] = oldest.Format(time.RFC3339Nano)
}
result["data"] = data
return result
return res
}
func (l *queryLog) logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
func (l *queryLog) logEntryToJSONEntry(entry *logEntry) (jsonEntry jobject) {
var msg *dns.Msg
if len(entry.Answer) > 0 {
@@ -62,17 +68,18 @@ func (l *queryLog) logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
}
}
jsonEntry := map[string]interface{}{
jsonEntry = jobject{
"reason": entry.Result.Reason.String(),
"elapsedMs": strconv.FormatFloat(entry.Elapsed.Seconds()*1000, 'f', -1, 64),
"time": entry.Time.Format(time.RFC3339Nano),
"client": l.getClientIP(entry.IP),
"client_proto": entry.ClientProto,
}
jsonEntry["question"] = map[string]interface{}{
"host": entry.QHost,
"type": entry.QType,
"class": entry.QClass,
"upstream": entry.Upstream,
"question": jobject{
"host": entry.QHost,
"type": entry.QType,
"class": entry.QClass,
},
}
if msg != nil {
@@ -83,12 +90,15 @@ func (l *queryLog) logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
if opt != nil {
dnssecOk = opt.Do()
}
jsonEntry["answer_dnssec"] = dnssecOk
}
if len(entry.Result.Rule) > 0 {
jsonEntry["rule"] = entry.Result.Rule
jsonEntry["filterId"] = entry.Result.FilterID
jsonEntry["rules"] = resultRulesToJSONRules(entry.Result.Rules)
if len(entry.Result.Rules) > 0 && len(entry.Result.Rules[0].Text) > 0 {
jsonEntry["rule"] = entry.Result.Rules[0].Text
jsonEntry["filterId"] = entry.Result.Rules[0].FilterListID
}
if len(entry.Result.ServiceName) != 0 {
@@ -113,20 +123,30 @@ func (l *queryLog) logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
}
}
jsonEntry["upstream"] = entry.Upstream
return jsonEntry
}
func answerToMap(a *dns.Msg) []map[string]interface{} {
func resultRulesToJSONRules(rules []*dnsfilter.ResultRule) (jsonRules []jobject) {
jsonRules = make([]jobject, len(rules))
for i, r := range rules {
jsonRules[i] = jobject{
"filter_list_id": r.FilterListID,
"text": r.Text,
}
}
return jsonRules
}
func answerToMap(a *dns.Msg) (answers []jobject) {
if a == nil || len(a.Answer) == 0 {
return nil
}
answers := []map[string]interface{}{}
answers = []jobject{}
for _, k := range a.Answer {
header := k.Header()
answer := map[string]interface{}{
answer := jobject{
"type": dns.TypeToString[header.Rrtype],
"ttl": header.Ttl,
}

6
internal/querylog/qlog_test.go
View File

@@ -236,10 +236,12 @@ func addEntry(l *queryLog, host, answerStr, client string) {
a.Answer = append(a.Answer, answer)
res := dnsfilter.Result{
IsFiltered: true,
Rule: "SomeRule",
Reason: dnsfilter.ReasonRewrite,
ServiceName: "SomeService",
FilterID: 1,
Rules: []*dnsfilter.ResultRule{{
FilterListID: 1,
Text: "SomeRule",
}},
}
params := AddParams{
Question: &q,

4
internal/querylog/searchcriteria.go
View File

@@ -117,7 +117,7 @@ func (c *searchCriteria) ctFilteringStatusCase(res dnsfilter.Result) bool {
res.Reason.In(
dnsfilter.NotFilteredWhiteList,
dnsfilter.ReasonRewrite,
dnsfilter.RewriteEtcHosts,
dnsfilter.RewriteAutoHosts,
)
case filteringStatusBlocked:
@@ -137,7 +137,7 @@ func (c *searchCriteria) ctFilteringStatusCase(res dnsfilter.Result) bool {
return res.Reason == dnsfilter.NotFilteredWhiteList
case filteringStatusRewritten:
return res.Reason.In(dnsfilter.ReasonRewrite, dnsfilter.RewriteEtcHosts)
return res.Reason.In(dnsfilter.ReasonRewrite, dnsfilter.RewriteAutoHosts)
case filteringStatusSafeSearch:
return res.IsFiltered && res.Reason == dnsfilter.FilteredSafeSearch