daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge: + new query logs API

Browse Source 
* commit '59c4a2886a97143e3a36912ec895dc1a06be88cc':
  openapi
  If there are no more older entries, `"oldest":""` is returned.
  fix search by "whitelisted", "rewritten"
  doc
  fix whois test
  + "dot"
  openapi
  * minor
  + client_proto
  * openapi
  + new query logs API
This commit is contained in:
Andrey Meshkov
2020-06-18 00:23:52 +03:00
parent 4870da7f94 59c4a2886a
commit b1c951fb2c
13 changed files with 116 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
querylog/decode.go
View File

@@ -37,6 +37,9 @@ func decodeLogEntry(ent *logEntry, str string) {
case "QC":
ent.QClass = v
case "CP":
ent.ClientProto = v
case "Answer":
ent.Answer, err = base64.StdEncoding.DecodeString(v)
case "OrigAnswer":

11
querylog/json.go
View File

@@ -63,10 +63,11 @@ func (l *queryLog) logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
}
jsonEntry := map[string]interface{}{
"reason": entry.Result.Reason.String(),
"elapsedMs": strconv.FormatFloat(entry.Elapsed.Seconds()*1000, 'f', -1, 64),
"time": entry.Time.Format(time.RFC3339Nano),
"client": l.getClientIP(entry.IP),
"reason": entry.Result.Reason.String(),
"elapsedMs": strconv.FormatFloat(entry.Elapsed.Seconds()*1000, 'f', -1, 64),
"time": entry.Time.Format(time.RFC3339Nano),
"client": l.getClientIP(entry.IP),
"client_proto": entry.ClientProto,
}
jsonEntry["question"] = map[string]interface{}{
"host": entry.QHost,
@@ -112,6 +113,8 @@ func (l *queryLog) logEntryToJSONEntry(entry *logEntry) map[string]interface{} {
}
}
jsonEntry["upstream"] = entry.Upstream
return jsonEntry
}

9
querylog/qlog.go
View File

@@ -38,6 +38,8 @@ type logEntry struct {
QType string `json:"QT"`
QClass string `json:"QC"`
ClientProto string `json:"CP"` // "" or "doh"
Answer []byte `json:",omitempty"` // sometimes empty answers happen like binerdunt.top or rev2.globalrootservers.net
OrigAnswer []byte `json:",omitempty"`
@@ -119,9 +121,10 @@ func (l *queryLog) Add(params AddParams) {
IP: l.getClientIP(params.ClientIP.String()),
Time: now,
Result: *params.Result,
Elapsed: params.Elapsed,
Upstream: params.Upstream,
Result: *params.Result,
Elapsed: params.Elapsed,
Upstream: params.Upstream,
ClientProto: params.ClientProto,
}
q := params.Question.Question[0]
entry.QHost = strings.ToLower(q.Name[:len(q.Name)-1]) // remove the last dot

10
querylog/qlog_http.go
View File

@@ -142,10 +142,6 @@ func (l *queryLog) parseSearchCriteria(q url.Values, name string, ct criteriaTyp
c.strict = true
}
if ct == ctClient && l.conf.AnonymizeClientIP {
c.value = l.getClientIP(c.value)
}
if ct == ctFilteringStatus && !util.ContainsString(filteringStatusValues, c.value) {
return false, c, fmt.Errorf("invalid value %s", c.value)
}
@@ -180,10 +176,8 @@ func (l *queryLog) parseSearchParams(r *http.Request) (*searchParams, error) {
}
paramNames := map[string]criteriaType{
"filter_domain": ctDomain,
"filter_client": ctClient,
"filter_question_type": ctQuestionType,
"filter_response_status": ctFilteringStatus,
"search": ctDomainOrClient,
"response_status": ctFilteringStatus,
}
for k, v := range paramNames {

8
querylog/qlog_test.go
View File

@@ -57,7 +57,7 @@ func TestQueryLog(t *testing.T) {
// search by domain (strict)
params = newSearchParams()
params.searchCriteria = append(params.searchCriteria, searchCriteria{
criteriaType: ctDomain,
criteriaType: ctDomainOrClient,
strict: true,
value: "test.example.org",
})
@@ -68,7 +68,7 @@ func TestQueryLog(t *testing.T) {
// search by domain (not strict)
params = newSearchParams()
params.searchCriteria = append(params.searchCriteria, searchCriteria{
criteriaType: ctDomain,
criteriaType: ctDomainOrClient,
strict: false,
value: "example.org",
})
@@ -81,7 +81,7 @@ func TestQueryLog(t *testing.T) {
// search by client IP (strict)
params = newSearchParams()
params.searchCriteria = append(params.searchCriteria, searchCriteria{
criteriaType: ctClient,
criteriaType: ctDomainOrClient,
strict: true,
value: "2.2.2.2",
})
@@ -92,7 +92,7 @@ func TestQueryLog(t *testing.T) {
// search by client IP (part of)
params = newSearchParams()
params.searchCriteria = append(params.searchCriteria, searchCriteria{
criteriaType: ctClient,
criteriaType: ctDomainOrClient,
strict: false,
value: "2.2.2",
})

15
querylog/querylog.go
View File

@@ -41,13 +41,14 @@ type Config struct {
// AddParams - parameters for Add()
type AddParams struct {
Question *dns.Msg
Answer *dns.Msg // The response we sent to the client (optional)
OrigAnswer *dns.Msg // The response from an upstream server (optional)
Result *dnsfilter.Result // Filtering result (optional)
Elapsed time.Duration // Time spent for processing the request
ClientIP net.IP
Upstream string
Question *dns.Msg
Answer *dns.Msg // The response we sent to the client (optional)
OrigAnswer *dns.Msg // The response from an upstream server (optional)
Result *dnsfilter.Result // Filtering result (optional)
Elapsed time.Duration // Time spent for processing the request
ClientIP net.IP
Upstream string // Upstream server URL
ClientProto string // Protocol for the client connection: "" (plain), "doh", "dot"
}
// New - create a new instance of the query log

4
querylog/querylog_search.go
View File

@@ -129,7 +129,9 @@ func (l *queryLog) searchFiles(params *searchParams) ([]*logEntry, time.Time, in
}
}
oldest = time.Unix(0, oldestNano)
if oldestNano != 0 {
oldest = time.Unix(0, oldestNano)
}
return entries, oldest, total
}

50
querylog/search_criteria.go
View File

@@ -9,9 +9,7 @@ import (
type criteriaType int
const (
ctDomain criteriaType = iota // domain name
ctClient // client IP address
ctQuestionType // question type
ctDomainOrClient criteriaType = iota // domain name or client IP address
ctFilteringStatus // filtering status
)
@@ -25,6 +23,7 @@ const (
filteringStatusWhitelisted = "whitelisted" // whitelisted
filteringStatusRewritten = "rewritten" // all kinds of rewrites
filteringStatusSafeSearch = "safe_search" // enforced safe search
filteringStatusProcessed = "processed" // not blocked, not white-listed entries
)
// filteringStatusValues -- array with all possible filteringStatus values
@@ -32,6 +31,7 @@ var filteringStatusValues = []string{
filteringStatusAll, filteringStatusFiltered, filteringStatusBlocked,
filteringStatusBlockedSafebrowsing, filteringStatusBlockedParental,
filteringStatusWhitelisted, filteringStatusRewritten, filteringStatusSafeSearch,
filteringStatusProcessed,
}
// searchCriteria - every search request may contain a list of different search criteria
@@ -48,12 +48,9 @@ func (c *searchCriteria) quickMatch(line string) bool {
// note that we do this only for a limited set of criteria
switch c.criteriaType {
case ctDomain:
return c.quickMatchJSONValue(line, "QH")
case ctClient:
return c.quickMatchJSONValue(line, "IP")
case ctQuestionType:
return c.quickMatchJSONValue(line, "QT")
case ctDomainOrClient:
return c.quickMatchJSONValue(line, "QH") ||
c.quickMatchJSONValue(line, "IP")
default:
return true
}
@@ -80,29 +77,23 @@ func (c *searchCriteria) quickMatchJSONValue(line string, propertyName string) b
// nolint (gocyclo)
func (c *searchCriteria) match(entry *logEntry) bool {
switch c.criteriaType {
case ctDomain:
case ctDomainOrClient:
if c.strict && entry.QHost == c.value {
return true
}
if !c.strict && strings.Contains(entry.QHost, c.value) {
return true
}
return false
case ctClient:
if c.strict && entry.IP == c.value {
return true
}
if !c.strict && strings.Contains(entry.IP, c.value) {
return true
}
return false
case ctQuestionType:
if c.strict && entry.QType == c.value {
return true
}
if !c.strict && strings.Contains(entry.QType, c.value) {
return true
}
case ctFilteringStatus:
res := entry.Result
@@ -110,7 +101,10 @@ func (c *searchCriteria) match(entry *logEntry) bool {
case filteringStatusAll:
return true
case filteringStatusFiltered:
return res.IsFiltered
return res.IsFiltered ||
res.Reason == dnsfilter.NotFilteredWhiteList ||
res.Reason == dnsfilter.ReasonRewrite ||
res.Reason == dnsfilter.RewriteEtcHosts
case filteringStatusBlocked:
return res.IsFiltered &&
(res.Reason == dnsfilter.FilteredBlackList ||
@@ -120,19 +114,21 @@ func (c *searchCriteria) match(entry *logEntry) bool {
case filteringStatusBlockedSafebrowsing:
return res.IsFiltered && res.Reason == dnsfilter.FilteredSafeBrowsing
case filteringStatusWhitelisted:
return res.IsFiltered && res.Reason == dnsfilter.NotFilteredWhiteList
return res.Reason == dnsfilter.NotFilteredWhiteList
case filteringStatusRewritten:
return res.IsFiltered &&
(res.Reason == dnsfilter.ReasonRewrite ||
res.Reason == dnsfilter.RewriteEtcHosts)
return (res.Reason == dnsfilter.ReasonRewrite ||
res.Reason == dnsfilter.RewriteEtcHosts)
case filteringStatusSafeSearch:
return res.IsFiltered && res.Reason == dnsfilter.FilteredSafeSearch
case filteringStatusProcessed:
return !(res.Reason == dnsfilter.FilteredBlackList ||
res.Reason == dnsfilter.FilteredBlockedService ||
res.Reason == dnsfilter.NotFilteredWhiteList)
default:
return false
}
default:
return false
}
return false