daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

* querylog: move code to a separate package

Browse Source 
+ config: "querylog_interval" setting
/control/querylog_config, /control/querylog_info
+ POST /control/querylog_clear
This commit is contained in:
Simon Zolin
2019-08-26 11:54:38 +03:00
parent 8f9ca4cba7
commit 8104c902ee
11 changed files with 457 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
dnsforward/dnsforward.go
View File

@@ -11,6 +11,7 @@ import (
"time"
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/AdguardTeam/AdGuardHome/querylog"
"github.com/AdguardTeam/AdGuardHome/stats"
"github.com/AdguardTeam/dnsproxy/proxy"
"github.com/AdguardTeam/dnsproxy/upstream"
@@ -40,7 +41,7 @@ const (
type Server struct {
dnsProxy *proxy.Proxy // DNS proxy instance
dnsFilter *dnsfilter.Dnsfilter // DNS filter instance
queryLog *queryLog // Query log instance
queryLog querylog.QueryLog // Query log instance
stats stats.Stats
AllowedClients map[string]bool // IP addresses of whitelist clients
@@ -54,16 +55,11 @@ type Server struct {
}
// NewServer creates a new instance of the dnsforward.Server
// baseDir is the base directory for query logs
// Note: this function must be called only once
func NewServer(baseDir string, stats stats.Stats) *Server {
s := &Server{
queryLog: newQueryLog(baseDir),
}
func NewServer(stats stats.Stats, queryLog querylog.QueryLog) *Server {
s := &Server{}
s.stats = stats
log.Printf("Start DNS server periodic jobs")
go s.queryLog.periodicQueryLogRotate()
s.queryLog = queryLog
return s
}
@@ -75,6 +71,7 @@ type FilteringConfig struct {
BlockingMode string `yaml:"blocking_mode"` // mode how to answer filtered requests
BlockedResponseTTL uint32 `yaml:"blocked_response_ttl"` // if 0, then default is used (3600)
QueryLogEnabled bool `yaml:"querylog_enabled"` // if true, query log is enabled
QueryLogInterval uint32 `yaml:"querylog_interval"` // time interval for query log (in days)
Ratelimit int `yaml:"ratelimit"` // max number of requests per second from a given IP (0 to disable)
RatelimitWhitelist []string `yaml:"ratelimit_whitelist"` // a list of whitelisted client IP addresses
RefuseAny bool `yaml:"refuse_any"` // if true, refuse ANY requests
@@ -303,8 +300,7 @@ func (s *Server) stopInternal() error {
s.dnsFilter = nil
}
// flush remainder to file
return s.queryLog.flushLogBuffer(true)
return nil
}
// IsRunning returns true if the DNS server is running
@@ -343,13 +339,6 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.RUnlock()
}
// GetQueryLog returns a map with the current query log ready to be converted to a JSON
func (s *Server) GetQueryLog() []map[string]interface{} {
s.RLock()
defer s.RUnlock()
return s.queryLog.getQueryLog()
}
// Return TRUE if this client should be blocked
func (s *Server) isBlockedIP(ip string) bool {
if len(s.AllowedClients) != 0 || len(s.AllowedClientsIPNet) != 0 {
@@ -469,12 +458,12 @@ func (s *Server) handleDNSRequest(p *proxy.Proxy, d *proxy.DNSContext) error {
}
elapsed := time.Since(start)
if s.conf.QueryLogEnabled && shouldLog {
if s.conf.QueryLogEnabled && shouldLog && s.queryLog != nil {
upstreamAddr := ""
if d.Upstream != nil {
upstreamAddr = d.Upstream.Address()
}
_ = s.queryLog.logRequest(msg, d.Res, res, elapsed, d.Addr, upstreamAddr)
s.queryLog.Add(msg, d.Res, res, elapsed, d.Addr, upstreamAddr)
}
s.updateStats(d, elapsed, *res)

66
dnsforward/dnsforward_test.go
View File

@@ -10,7 +10,6 @@ import (
"encoding/pem"
"math/big"
"net"
"os"
"sync"
"testing"
"time"
@@ -18,18 +17,15 @@ import (
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/AdguardTeam/dnsproxy/proxy"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
)
const (
tlsServerName = "testdns.adguard.com"
dataDir = "testData"
testMessagesCount = 10
)
func TestServer(t *testing.T) {
s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@@ -45,10 +41,6 @@ func TestServer(t *testing.T) {
}
assertGoogleAResponse(t, reply)
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
// message over TCP
req = createGoogleATestMessage()
addr = s.dnsProxy.Addr("tcp")
@@ -59,10 +51,6 @@ func TestServer(t *testing.T) {
}
assertGoogleAResponse(t, reply)
// check query log and stats again
log = s.GetQueryLog()
assert.Equal(t, 2, len(log), "Log size")
err = s.Stop()
if err != nil {
t.Fatalf("DNS server failed to stop: %s", err)
@@ -72,7 +60,6 @@ func TestServer(t *testing.T) {
func TestServerWithProtectionDisabled(t *testing.T) {
s := createTestServer(t)
s.conf.ProtectionEnabled = false
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@@ -88,10 +75,6 @@ func TestServerWithProtectionDisabled(t *testing.T) {
}
assertGoogleAResponse(t, reply)
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
err = s.Stop()
if err != nil {
t.Fatalf("DNS server failed to stop: %s", err)
@@ -102,7 +85,6 @@ func TestDotServer(t *testing.T) {
// Prepare the proxy server
_, certPem, keyPem := createServerTLSConfig(t)
s := createTestServer(t)
defer removeDataDir(t)
s.conf.TLSConfig = TLSConfig{
TLSListenAddr: &net.TCPAddr{Port: 0},
@@ -143,7 +125,6 @@ func TestDotServer(t *testing.T) {
func TestServerRace(t *testing.T) {
s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@@ -168,7 +149,6 @@ func TestServerRace(t *testing.T) {
func TestSafeSearch(t *testing.T) {
s := createTestServer(t)
s.conf.SafeSearchEnabled = true
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@@ -210,7 +190,6 @@ func TestSafeSearch(t *testing.T) {
func TestInvalidRequest(t *testing.T) {
s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@@ -229,11 +208,6 @@ func TestInvalidRequest(t *testing.T) {
t.Fatalf("got a response to an invalid query")
}
// check query log and stats
// invalid requests aren't written to the query log
log := s.GetQueryLog()
assert.Equal(t, 0, len(log), "Log size")
err = s.Stop()
if err != nil {
t.Fatalf("DNS server failed to stop: %s", err)
@@ -242,7 +216,6 @@ func TestInvalidRequest(t *testing.T) {
func TestBlockedRequest(t *testing.T) {
s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@@ -267,10 +240,6 @@ func TestBlockedRequest(t *testing.T) {
t.Fatalf("Wrong response: %s", reply.String())
}
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
err = s.Stop()
if err != nil {
t.Fatalf("DNS server failed to stop: %s", err)
@@ -280,7 +249,6 @@ func TestBlockedRequest(t *testing.T) {
func TestNullBlockedRequest(t *testing.T) {
s := createTestServer(t)
s.conf.FilteringConfig.BlockingMode = "null_ip"
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@@ -312,10 +280,6 @@ func TestNullBlockedRequest(t *testing.T) {
t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0])
}
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
err = s.Stop()
if err != nil {
t.Fatalf("DNS server failed to stop: %s", err)
@@ -324,7 +288,6 @@ func TestNullBlockedRequest(t *testing.T) {
func TestBlockedByHosts(t *testing.T) {
s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@@ -356,10 +319,6 @@ func TestBlockedByHosts(t *testing.T) {
t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0])
}
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
err = s.Stop()
if err != nil {
t.Fatalf("DNS server failed to stop: %s", err)
@@ -368,7 +327,6 @@ func TestBlockedByHosts(t *testing.T) {
func TestBlockedBySafeBrowsing(t *testing.T) {
s := createTestServer(t)
defer removeDataDir(t)
err := s.Start(nil)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
@@ -411,10 +369,6 @@ func TestBlockedBySafeBrowsing(t *testing.T) {
t.Fatalf("DNS server %s returned wrong answer type instead of A: %v", addr, reply.Answer[0])
}
// check query log and stats
log := s.GetQueryLog()
assert.Equal(t, 1, len(log), "Log size")
err = s.Stop()
if err != nil {
t.Fatalf("DNS server failed to stop: %s", err)
@@ -422,7 +376,7 @@ func TestBlockedBySafeBrowsing(t *testing.T) {
}
func createTestServer(t *testing.T) *Server {
s := NewServer(createDataDir(t), nil)
s := NewServer(nil, nil)
s.conf.UDPListenAddr = &net.UDPAddr{Port: 0}
s.conf.TCPListenAddr = &net.TCPAddr{Port: 0}
@@ -489,21 +443,6 @@ func createServerTLSConfig(t *testing.T) (*tls.Config, []byte, []byte) {
return &tls.Config{Certificates: []tls.Certificate{cert}, ServerName: tlsServerName, MinVersion: tls.VersionTLS12}, certPem, keyPem
}
func createDataDir(t *testing.T) string {
err := os.MkdirAll(dataDir, 0755)
if err != nil {
t.Fatalf("Cannot create %s: %s", dataDir, err)
}
return dataDir
}
func removeDataDir(t *testing.T) {
err := os.RemoveAll(dataDir)
if err != nil {
t.Fatalf("Cannot remove %s: %s", dataDir, err)
}
}
func sendTestMessageAsync(t *testing.T, conn *dns.Conn, g *sync.WaitGroup) {
defer func() {
g.Done()
@@ -607,7 +546,6 @@ func TestIsBlockedIPAllowed(t *testing.T) {
s.conf.AllowedClients = []string{"1.1.1.1", "2.2.0.0/16"}
err := s.Start(nil)
defer removeDataDir(t)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
}
@@ -631,7 +569,6 @@ func TestIsBlockedIPDisallowed(t *testing.T) {
s.conf.DisallowedClients = []string{"1.1.1.1", "2.2.0.0/16"}
err := s.Start(nil)
defer removeDataDir(t)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
}
@@ -655,7 +592,6 @@ func TestIsBlockedIPBlockedDomain(t *testing.T) {
s.conf.BlockedHosts = []string{"host1", "host2"}
err := s.Start(nil)
defer removeDataDir(t)
if err != nil {
t.Fatalf("Failed to start server: %s", err)
}

240
dnsforward/querylog.go
View File

@@ -1,240 +0,0 @@
package dnsforward
import (
"fmt"
"net"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/AdguardTeam/golibs/log"
"github.com/miekg/dns"
)
const (
logBufferCap = 5000 // maximum capacity of logBuffer before it's flushed to disk
queryLogTimeLimit = time.Hour * 24 // how far in the past we care about querylogs
queryLogRotationPeriod = time.Hour * 24 // rotate the log every 24 hours
queryLogFileName = "querylog.json" // .gz added during compression
queryLogSize = 5000 // maximum API response for /querylog
queryLogTopSize = 500 // Keep in memory only top N values
)
// queryLog is a structure that writes and reads the DNS query log
type queryLog struct {
logFile string // path to the log file
logBufferLock sync.RWMutex
logBuffer []*logEntry
fileFlushLock sync.Mutex // synchronize a file-flushing goroutine and main thread
flushPending bool // don't start another goroutine while the previous one is still running
queryLogCache []*logEntry
queryLogLock sync.RWMutex
}
// newQueryLog creates a new instance of the query log
func newQueryLog(baseDir string) *queryLog {
l := &queryLog{
logFile: filepath.Join(baseDir, queryLogFileName),
}
return l
}
type logEntry struct {
Question []byte
Answer []byte `json:",omitempty"` // sometimes empty answers happen like binerdunt.top or rev2.globalrootservers.net
Result dnsfilter.Result
Time time.Time
Elapsed time.Duration
IP string
Upstream string `json:",omitempty"` // if empty, means it was cached
}
func (l *queryLog) logRequest(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string) *logEntry {
var q []byte
var a []byte
var err error
ip := GetIPString(addr)
if question != nil {
q, err = question.Pack()
if err != nil {
log.Printf("failed to pack question for querylog: %s", err)
return nil
}
}
if answer != nil {
a, err = answer.Pack()
if err != nil {
log.Printf("failed to pack answer for querylog: %s", err)
return nil
}
}
if result == nil {
result = &dnsfilter.Result{}
}
now := time.Now()
entry := logEntry{
Question: q,
Answer: a,
Result: *result,
Time: now,
Elapsed: elapsed,
IP: ip,
Upstream: upstream,
}
l.logBufferLock.Lock()
l.logBuffer = append(l.logBuffer, &entry)
needFlush := false
if !l.flushPending {
needFlush = len(l.logBuffer) >= logBufferCap
if needFlush {
l.flushPending = true
}
}
l.logBufferLock.Unlock()
l.queryLogLock.Lock()
l.queryLogCache = append(l.queryLogCache, &entry)
if len(l.queryLogCache) > queryLogSize {
toremove := len(l.queryLogCache) - queryLogSize
l.queryLogCache = l.queryLogCache[toremove:]
}
l.queryLogLock.Unlock()
// if buffer needs to be flushed to disk, do it now
if needFlush {
// write to file
// do it in separate goroutine -- we are stalling DNS response this whole time
go l.flushLogBuffer(false) // nolint
}
return &entry
}
// getQueryLogJson returns a map with the current query log ready to be converted to a JSON
func (l *queryLog) getQueryLog() []map[string]interface{} {
l.queryLogLock.RLock()
values := make([]*logEntry, len(l.queryLogCache))
copy(values, l.queryLogCache)
l.queryLogLock.RUnlock()
// reverse it so that newest is first
for left, right := 0, len(values)-1; left < right; left, right = left+1, right-1 {
values[left], values[right] = values[right], values[left]
}
// iterate
var data = []map[string]interface{}{}
for _, entry := range values {
var q *dns.Msg
var a *dns.Msg
if len(entry.Question) > 0 {
q = new(dns.Msg)
if err := q.Unpack(entry.Question); err != nil {
// ignore, log and move on
log.Printf("Failed to unpack dns message question: %s", err)
q = nil
}
}
if len(entry.Answer) > 0 {
a = new(dns.Msg)
if err := a.Unpack(entry.Answer); err != nil {
// ignore, log and move on
log.Printf("Failed to unpack dns message question: %s", err)
a = nil
}
}
jsonEntry := map[string]interface{}{
"reason": entry.Result.Reason.String(),
"elapsedMs": strconv.FormatFloat(entry.Elapsed.Seconds()*1000, 'f', -1, 64),
"time": entry.Time.Format(time.RFC3339),
"client": entry.IP,
}
if q != nil {
jsonEntry["question"] = map[string]interface{}{
"host": strings.ToLower(strings.TrimSuffix(q.Question[0].Name, ".")),
"type": dns.Type(q.Question[0].Qtype).String(),
"class": dns.Class(q.Question[0].Qclass).String(),
}
}
if a != nil {
jsonEntry["status"] = dns.RcodeToString[a.Rcode]
}
if len(entry.Result.Rule) > 0 {
jsonEntry["rule"] = entry.Result.Rule
jsonEntry["filterId"] = entry.Result.FilterID
}
if len(entry.Result.ServiceName) != 0 {
jsonEntry["service_name"] = entry.Result.ServiceName
}
answers := answerToMap(a)
if answers != nil {
jsonEntry["answer"] = answers
}
data = append(data, jsonEntry)
}
return data
}
func answerToMap(a *dns.Msg) []map[string]interface{} {
if a == nil || len(a.Answer) == 0 {
return nil
}
var answers = []map[string]interface{}{}
for _, k := range a.Answer {
header := k.Header()
answer := map[string]interface{}{
"type": dns.TypeToString[header.Rrtype],
"ttl": header.Ttl,
}
// try most common record types
switch v := k.(type) {
case *dns.A:
answer["value"] = v.A
case *dns.AAAA:
answer["value"] = v.AAAA
case *dns.MX:
answer["value"] = fmt.Sprintf("%v %v", v.Preference, v.Mx)
case *dns.CNAME:
answer["value"] = v.Target
case *dns.NS:
answer["value"] = v.Ns
case *dns.SPF:
answer["value"] = v.Txt
case *dns.TXT:
answer["value"] = v.Txt
case *dns.PTR:
answer["value"] = v.Ptr
case *dns.SOA:
answer["value"] = fmt.Sprintf("%v %v %v %v %v %v %v", v.Ns, v.Mbox, v.Serial, v.Refresh, v.Retry, v.Expire, v.Minttl)
case *dns.CAA:
answer["value"] = fmt.Sprintf("%v %v \"%v\"", v.Flag, v.Tag, v.Value)
case *dns.HINFO:
answer["value"] = fmt.Sprintf("\"%v\" \"%v\"", v.Cpu, v.Os)
case *dns.RRSIG:
answer["value"] = fmt.Sprintf("%v %v %v %v %v %v %v %v %v", dns.TypeToString[v.TypeCovered], v.Algorithm, v.Labels, v.OrigTtl, v.Expiration, v.Inception, v.KeyTag, v.SignerName, v.Signature)
default:
// type unknown, marshall it as-is
answer["value"] = v
}
answers = append(answers, answer)
}
return answers
}

180
dnsforward/querylog_file.go
View File

@@ -1,180 +0,0 @@
package dnsforward
import (
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"os"
"sync"
"time"
"github.com/AdguardTeam/golibs/log"
"github.com/go-test/deep"
)
var (
fileWriteLock sync.Mutex
)
const enableGzip = false
// flushLogBuffer flushes the current buffer to file and resets the current buffer
func (l *queryLog) flushLogBuffer(fullFlush bool) error {
l.fileFlushLock.Lock()
defer l.fileFlushLock.Unlock()
// flush remainder to file
l.logBufferLock.Lock()
needFlush := len(l.logBuffer) >= logBufferCap
if !needFlush && !fullFlush {
l.logBufferLock.Unlock()
return nil
}
flushBuffer := l.logBuffer
l.logBuffer = nil
l.flushPending = false
l.logBufferLock.Unlock()
err := l.flushToFile(flushBuffer)
if err != nil {
log.Error("Saving querylog to file failed: %s", err)
return err
}
return nil
}
// flushToFile saves the specified log entries to the query log file
func (l *queryLog) flushToFile(buffer []*logEntry) error {
if len(buffer) == 0 {
log.Debug("querylog: there's nothing to write to a file")
return nil
}
start := time.Now()
var b bytes.Buffer
e := json.NewEncoder(&b)
for _, entry := range buffer {
err := e.Encode(entry)
if err != nil {
log.Error("Failed to marshal entry: %s", err)
return err
}
}
elapsed := time.Since(start)
log.Debug("%d elements serialized via json in %v: %d kB, %v/entry, %v/entry", len(buffer), elapsed, b.Len()/1024, float64(b.Len())/float64(len(buffer)), elapsed/time.Duration(len(buffer)))
err := checkBuffer(buffer, b)
if err != nil {
log.Error("failed to check buffer: %s", err)
return err
}
var zb bytes.Buffer
filename := l.logFile
// gzip enabled?
if enableGzip {
filename += ".gz"
zw := gzip.NewWriter(&zb)
zw.Name = l.logFile
zw.ModTime = time.Now()
_, err = zw.Write(b.Bytes())
if err != nil {
log.Error("Couldn't compress to gzip: %s", err)
zw.Close()
return err
}
if err = zw.Close(); err != nil {
log.Error("Couldn't close gzip writer: %s", err)
return err
}
} else {
zb = b
}
fileWriteLock.Lock()
defer fileWriteLock.Unlock()
f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
log.Error("failed to create file \"%s\": %s", filename, err)
return err
}
defer f.Close()
n, err := f.Write(zb.Bytes())
if err != nil {
log.Error("Couldn't write to file: %s", err)
return err
}
log.Debug("ok \"%s\": %v bytes written", filename, n)
return nil
}
func checkBuffer(buffer []*logEntry, b bytes.Buffer) error {
l := len(buffer)
d := json.NewDecoder(&b)
i := 0
for d.More() {
entry := &logEntry{}
err := d.Decode(entry)
if err != nil {
log.Error("Failed to decode: %s", err)
return err
}
if diff := deep.Equal(entry, buffer[i]); diff != nil {
log.Error("decoded buffer differs: %s", diff)
return fmt.Errorf("decoded buffer differs: %s", diff)
}
i++
}
if i != l {
err := fmt.Errorf("check fail: %d vs %d entries", l, i)
log.Error("%v", err)
return err
}
log.Debug("check ok: %d entries", i)
return nil
}
func (l *queryLog) rotateQueryLog() error {
from := l.logFile
to := l.logFile + ".1"
if enableGzip {
from = l.logFile + ".gz"
to = l.logFile + ".gz.1"
}
if _, err := os.Stat(from); os.IsNotExist(err) {
// do nothing, file doesn't exist
return nil
}
err := os.Rename(from, to)
if err != nil {
log.Error("Failed to rename querylog: %s", err)
return err
}
log.Debug("Rotated from %s to %s successfully", from, to)
return nil
}
func (l *queryLog) periodicQueryLogRotate() {
for range time.Tick(queryLogRotationPeriod) {
err := l.rotateQueryLog()
if err != nil {
log.Error("Failed to rotate querylog: %s", err)
// do nothing, continue rotating
}
}
}