daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

all: sync with master; upd chlog

Browse Source
This commit is contained in:
Ainar Garipov
2023-03-09 15:39:35 +03:00
parent 4f928be393
commit a21558f418
98 changed files with 2687 additions and 24734 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
internal/querylog/decode.go
View File

@@ -247,55 +247,20 @@ var resultHandlers = map[string]logEntryHandler{
}
func decodeResultRuleKey(key string, i int, dec *json.Decoder, ent *logEntry) {
var vToken json.Token
switch key {
case "FilterListID":
vToken, err := dec.Token()
if err != nil {
if err != io.EOF {
log.Debug("decodeResultRuleKey %s err: %s", key, err)
}
return
}
if len(ent.Result.Rules) < i+1 {
ent.Result.Rules = append(ent.Result.Rules, &filtering.ResultRule{})
}
ent.Result.Rules, vToken = decodeVTokenAndAddRule(key, i, dec, ent.Result.Rules)
if n, ok := vToken.(json.Number); ok {
ent.Result.Rules[i].FilterListID, _ = n.Int64()
}
case "IP":
vToken, err := dec.Token()
if err != nil {
if err != io.EOF {
log.Debug("decodeResultRuleKey %s err: %s", key, err)
}
return
}
if len(ent.Result.Rules) < i+1 {
ent.Result.Rules = append(ent.Result.Rules, &filtering.ResultRule{})
}
ent.Result.Rules, vToken = decodeVTokenAndAddRule(key, i, dec, ent.Result.Rules)
if ipStr, ok := vToken.(string); ok {
ent.Result.Rules[i].IP = net.ParseIP(ipStr)
}
case "Text":
vToken, err := dec.Token()
if err != nil {
if err != io.EOF {
log.Debug("decodeResultRuleKey %s err: %s", key, err)
}
return
}
if len(ent.Result.Rules) < i+1 {
ent.Result.Rules = append(ent.Result.Rules, &filtering.ResultRule{})
}
ent.Result.Rules, vToken = decodeVTokenAndAddRule(key, i, dec, ent.Result.Rules)
if s, ok := vToken.(string); ok {
ent.Result.Rules[i].Text = s
}
@@ -304,6 +269,30 @@ func decodeResultRuleKey(key string, i int, dec *json.Decoder, ent *logEntry) {
}
}
func decodeVTokenAndAddRule(
key string,
i int,
dec *json.Decoder,
rules []*filtering.ResultRule,
) (newRules []*filtering.ResultRule, vToken json.Token) {
newRules = rules
vToken, err := dec.Token()
if err != nil {
if err != io.EOF {
log.Debug("decodeResultRuleKey %s err: %s", key, err)
}
return newRules, nil
}
if len(rules) < i+1 {
newRules = append(newRules, &filtering.ResultRule{})
}
return newRules, vToken
}
func decodeResultRules(dec *json.Decoder, ent *logEntry) {
for {
delimToken, err := dec.Token()

3
internal/querylog/http.go
View File

@@ -54,10 +54,7 @@ func (l *queryLog) handleQueryLog(w http.ResponseWriter, r *http.Request) {
return
}
// search for the log entries
entries, oldest := l.search(params)
// convert log entries to JSON
data := l.entriesToJSON(entries, oldest)
_ = aghhttp.WriteJSONResponse(w, r, data)

60
internal/querylog/json.go
View File

@@ -1,7 +1,6 @@
package querylog
import (
"fmt"
"strconv"
"strings"
"time"
@@ -117,7 +116,7 @@ func (l *queryLog) setMsgData(entry *logEntry, jsonEntry jobject) {
// it from there as well.
jsonEntry["answer_dnssec"] = entry.AuthenticatedData || msg.AuthenticatedData
if a := answerToMap(msg); a != nil {
if a := answerToJSON(msg); a != nil {
jsonEntry["answer"] = a
}
}
@@ -136,7 +135,7 @@ func (l *queryLog) setOrigAns(entry *logEntry, jsonEntry jobject) {
return
}
if a := answerToMap(orig); a != nil {
if a := answerToJSON(orig); a != nil {
jsonEntry["original_answer"] = a
}
}
@@ -159,55 +158,24 @@ type dnsAnswer struct {
TTL uint32 `json:"ttl"`
}
func answerToMap(a *dns.Msg) (answers []*dnsAnswer) {
if a == nil || len(a.Answer) == 0 {
// answerToJSON converts the answer records of msg, if any, to their JSON form.
func answerToJSON(msg *dns.Msg) (answers []*dnsAnswer) {
if msg == nil || len(msg.Answer) == 0 {
return nil
}
answers = make([]*dnsAnswer, 0, len(a.Answer))
for _, k := range a.Answer {
header := k.Header()
answer := &dnsAnswer{
answers = make([]*dnsAnswer, 0, len(msg.Answer))
for _, rr := range msg.Answer {
header := rr.Header()
a := &dnsAnswer{
Type: dns.TypeToString[header.Rrtype],
TTL: header.Ttl,
// Remove the header string from the answer value since it's mostly
// unnecessary in the log.
Value: strings.TrimPrefix(rr.String(), header.String()),
TTL: header.Ttl,
}
// Some special treatment for some well-known types.
//
// TODO(a.garipov): Consider just calling String() for everyone
// instead.
switch v := k.(type) {
case nil:
// Probably unlikely, but go on.
case *dns.A:
answer.Value = v.A.String()
case *dns.AAAA:
answer.Value = v.AAAA.String()
case *dns.MX:
answer.Value = fmt.Sprintf("%v %v", v.Preference, v.Mx)
case *dns.CNAME:
answer.Value = v.Target
case *dns.NS:
answer.Value = v.Ns
case *dns.SPF:
answer.Value = strings.Join(v.Txt, "\n")
case *dns.TXT:
answer.Value = strings.Join(v.Txt, "\n")
case *dns.PTR:
answer.Value = v.Ptr
case *dns.SOA:
answer.Value = fmt.Sprintf("%v %v %v %v %v %v %v", v.Ns, v.Mbox, v.Serial, v.Refresh, v.Retry, v.Expire, v.Minttl)
case *dns.CAA:
answer.Value = fmt.Sprintf("%v %v \"%v\"", v.Flag, v.Tag, v.Value)
case *dns.HINFO:
answer.Value = fmt.Sprintf("\"%v\" \"%v\"", v.Cpu, v.Os)
case *dns.RRSIG:
answer.Value = fmt.Sprintf("%v %v %v %v %v %v %v %v %v", dns.TypeToString[v.TypeCovered], v.Algorithm, v.Labels, v.OrigTtl, v.Expiration, v.Inception, v.KeyTag, v.SignerName, v.Signature)
default:
answer.Value = v.String()
}
answers = append(answers, answer)
answers = append(answers, a)
}
return answers

10
internal/querylog/qlog.go
View File

@@ -31,7 +31,8 @@ type queryLog struct {
// bufferLock protects buffer.
bufferLock sync.RWMutex
// buffer contains recent log entries.
// buffer contains recent log entries. The entries in this buffer must not
// be modified.
buffer []*logEntry
fileFlushLock sync.Mutex // synchronize a file-flushing goroutine and main thread
@@ -100,6 +101,13 @@ type logEntry struct {
AuthenticatedData bool `json:"AD,omitempty"`
}
// shallowClone returns a shallow clone of e.
func (e *logEntry) shallowClone() (clone *logEntry) {
cloneVal := *e
return &cloneVal
}
func (l *queryLog) Start() {
if l.conf.HTTPRegister != nil {
l.initWeb()

72
internal/querylog/qlog_test.go
View File

@@ -2,11 +2,8 @@ package querylog
import (
"fmt"
"math/rand"
"net"
"sort"
"testing"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/dnsproxy/proxyutil"
@@ -352,72 +349,3 @@ func assertLogEntry(t *testing.T, entry *logEntry, host string, answer, client n
ip := proxyutil.IPFromRR(msg.Answer[0]).To16()
assert.Equal(t, answer, ip)
}
func testEntries() (entries []*logEntry) {
rsrc := rand.NewSource(time.Now().UnixNano())
rgen := rand.New(rsrc)
entries = make([]*logEntry, 1000)
for i := range entries {
min := rgen.Intn(60)
sec := rgen.Intn(60)
entries[i] = &logEntry{
Time: time.Date(2020, 1, 1, 0, min, sec, 0, time.UTC),
}
}
return entries
}
// logEntriesByTimeDesc is a wrapper over []*logEntry for sorting.
//
// NOTE(a.garipov): Weirdly enough, on my machine this gets consistently
// outperformed by sort.Slice, see the benchmark below. I'm leaving this
// implementation here, in tests, in case we want to make sure it outperforms on
// most machines, but for now this is unused in the actual code.
type logEntriesByTimeDesc []*logEntry
// Len implements the sort.Interface interface for logEntriesByTimeDesc.
func (les logEntriesByTimeDesc) Len() (n int) { return len(les) }
// Less implements the sort.Interface interface for logEntriesByTimeDesc.
func (les logEntriesByTimeDesc) Less(i, j int) (less bool) {
return les[i].Time.After(les[j].Time)
}
// Swap implements the sort.Interface interface for logEntriesByTimeDesc.
func (les logEntriesByTimeDesc) Swap(i, j int) { les[i], les[j] = les[j], les[i] }
func BenchmarkLogEntry_sort(b *testing.B) {
b.Run("methods", func(b *testing.B) {
for i := 0; i < b.N; i++ {
b.StopTimer()
entries := testEntries()
b.StartTimer()
sort.Stable(logEntriesByTimeDesc(entries))
}
})
b.Run("reflect", func(b *testing.B) {
for i := 0; i < b.N; i++ {
b.StopTimer()
entries := testEntries()
b.StartTimer()
sort.SliceStable(entries, func(i, j int) (less bool) {
return entries[i].Time.After(entries[j].Time)
})
}
})
}
func TestLogEntriesByTime_sort(t *testing.T) {
entries := testEntries()
sort.Sort(logEntriesByTimeDesc(entries))
for i := range entries[1:] {
assert.False(t, entries[i+1].Time.After(entries[i].Time),
"%s %s", entries[i+1].Time, entries[i].Time)
}
}

12
internal/querylog/search.go
View File

@@ -2,10 +2,10 @@ package querylog
import (
"io"
"sort"
"time"
"github.com/AdguardTeam/golibs/log"
"golang.org/x/exp/slices"
)
// client finds the client info, if any, by its ClientID and IP address,
@@ -52,7 +52,9 @@ func (l *queryLog) searchMemory(params *searchParams, cache clientCache) (entrie
// Go through the buffer in the reverse order, from newer to older.
var err error
for i := len(l.buffer) - 1; i >= 0; i-- {
e := l.buffer[i]
// A shallow clone is enough, since the only thing that this loop
// modifies is the client field.
e := l.buffer[i].shallowClone()
e.client, err = l.client(e.ClientID, e.IP.String(), cache)
if err != nil {
@@ -98,8 +100,8 @@ func (l *queryLog) search(params *searchParams) (entries []*logEntry, oldest tim
// weird on the frontend.
//
// See https://github.com/AdguardTeam/AdGuardHome/issues/2293.
sort.SliceStable(entries, func(i, j int) (less bool) {
return entries[i].Time.After(entries[j].Time)
slices.SortStableFunc(entries, func(a, b *logEntry) (sortsBefore bool) {
return a.Time.After(b.Time)
})
if params.offset > 0 {
@@ -130,7 +132,7 @@ func (l *queryLog) search(params *searchParams) (entries []*logEntry, oldest tim
// searchFiles looks up log records from all log files. It optionally uses the
// client cache, if provided. searchFiles does not scan more than
// maxFileScanEntries so callers may need to call it several times to get all
// results. oldset and total are the time of the oldest processed entry and the
// results. oldest and total are the time of the oldest processed entry and the
// total number of processed entries, including discarded ones, correspondingly.
func (l *queryLog) searchFiles(
params *searchParams,

85
internal/querylog/searchcriterion.go
View File

@@ -1,6 +1,7 @@
package querylog
import (
"fmt"
"strings"
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
@@ -118,7 +119,7 @@ func (c *searchCriterion) match(entry *logEntry) bool {
case ctTerm:
return c.ctDomainOrClientCase(entry)
case ctFilteringStatus:
return c.ctFilteringStatusCase(entry.Result)
return c.ctFilteringStatusCase(entry.Result.Reason, entry.Result.IsFiltered)
}
return false
@@ -141,54 +142,70 @@ func (c *searchCriterion) ctDomainOrClientCase(e *logEntry) bool {
return ctDomainOrClientCaseNonStrict(c.value, c.asciiVal, clientID, name, host, ip)
}
func (c *searchCriterion) ctFilteringStatusCase(res filtering.Result) bool {
// ctFilteringStatusCase returns true if the result matches the value.
func (c *searchCriterion) ctFilteringStatusCase(
reason filtering.Reason,
isFiltered bool,
) (matched bool) {
switch c.value {
case filteringStatusAll:
return true
case filteringStatusFiltered:
return res.IsFiltered ||
res.Reason.In(
filtering.NotFilteredAllowList,
filtering.Rewritten,
filtering.RewrittenAutoHosts,
filtering.RewrittenRule,
)
case filteringStatusBlocked:
return res.IsFiltered &&
res.Reason.In(filtering.FilteredBlockList, filtering.FilteredBlockedService)
case filteringStatusBlockedService:
return res.IsFiltered && res.Reason == filtering.FilteredBlockedService
case filteringStatusBlockedParental:
return res.IsFiltered && res.Reason == filtering.FilteredParental
case filteringStatusBlockedSafebrowsing:
return res.IsFiltered && res.Reason == filtering.FilteredSafeBrowsing
case
filteringStatusBlocked,
filteringStatusBlockedParental,
filteringStatusBlockedSafebrowsing,
filteringStatusBlockedService,
filteringStatusFiltered,
filteringStatusSafeSearch:
return isFiltered && c.isFilteredWithReason(reason)
case filteringStatusWhitelisted:
return res.Reason == filtering.NotFilteredAllowList
return reason == filtering.NotFilteredAllowList
case filteringStatusRewritten:
return res.Reason.In(
return reason.In(
filtering.Rewritten,
filtering.RewrittenAutoHosts,
filtering.RewrittenRule,
)
case filteringStatusSafeSearch:
return res.IsFiltered && res.Reason == filtering.FilteredSafeSearch
case filteringStatusProcessed:
return !res.Reason.In(
return !reason.In(
filtering.FilteredBlockList,
filtering.FilteredBlockedService,
filtering.NotFilteredAllowList,
)
default:
return false
}
}
// isFilteredWithReason returns true if reason matches the criterion value.
// c.value must be one of:
//
// - filteringStatusBlocked
// - filteringStatusBlockedParental
// - filteringStatusBlockedSafebrowsing
// - filteringStatusBlockedService
// - filteringStatusFiltered
// - filteringStatusSafeSearch
func (c *searchCriterion) isFilteredWithReason(reason filtering.Reason) (matched bool) {
switch c.value {
case filteringStatusBlocked:
return reason.In(filtering.FilteredBlockList, filtering.FilteredBlockedService)
case filteringStatusBlockedParental:
return reason == filtering.FilteredParental
case filteringStatusBlockedSafebrowsing:
return reason == filtering.FilteredSafeBrowsing
case filteringStatusBlockedService:
return reason == filtering.FilteredBlockedService
case filteringStatusFiltered:
return reason.In(
filtering.NotFilteredAllowList,
filtering.Rewritten,
filtering.RewrittenAutoHosts,
filtering.RewrittenRule,
)
case filteringStatusSafeSearch:
return reason == filtering.FilteredSafeSearch
default:
panic(fmt.Errorf("unexpected value %q", c.value))
}
}