daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pull request: querylog imp code

Browse Source 
Merge in DNS/adguard-home from querylog-imp-code to master

Squashed commit of the following:

commit a58ad36508a2355b686d314dec51ac0b5e357281
Merge: df5494f2c 941eb1dd7
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Wed May 24 15:26:55 2023 +0300

    Merge remote-tracking branch 'origin/master' into querylog-imp-code

commit df5494f2c337736690a3c2a547c2d71858d0378f
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Wed May 24 15:24:43 2023 +0300

    querylog: imp code

commit 8c3c2b76dd5858e7b107f222c112e9cde2477fb3
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Wed May 24 12:14:15 2023 +0300

    all: lint script

commit be04a4decfaf20a1649d32ecaab3c1c6bb205ffd
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Wed May 24 12:03:12 2023 +0300

    querylog: imp code

commit fe7beacff3a5cfcf2332c4998b9c65820284eaf7
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Wed May 24 11:57:33 2023 +0300

    querylog: imp docs

commit 2ae239c57d12524fbc092f582842af2ad726c1d0
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Wed May 24 11:46:54 2023 +0300

    querylog: imp code

commit 417216cefbf154fa870f8f43468f35e0e345971f
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Wed May 24 11:25:44 2023 +0300

    querylog: imp code

commit 514b6ee99113844a4e0dad30dc53703e3220c289
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Wed May 24 11:14:13 2023 +0300

    querylog: imp docs

commit 321351a3abb524208daacd5a3a7fbf5f07ab259d
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Mon May 22 16:38:31 2023 +0300

    querylog: imp code

commit ee91de5c43210b5bc213f933d411adb894d2e586
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Mon May 22 16:01:32 2023 +0300

    querylog: imp code

commit 862ff12177fb769d5cb2ec250eaee538dc91d70a
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Mon May 22 15:07:24 2023 +0300

    querylog: imp code

commit cc62c1c4ae8b813d03ccf51b596ba1ebf44d9a1f
Merge: 37ace34e9 24b41100c
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Mon May 22 13:09:10 2023 +0300

    Merge remote-tracking branch 'origin/master' into querylog-imp-code

commit 37ace34e91e5189bef6e774db960f40cdaa18270
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Mon May 22 11:23:08 2023 +0300

    querylog: imp code

commit 8417815a6349f10b5dbad410ce28aab98bc479fa
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Mon May 22 11:08:29 2023 +0300

    querylog: imp docs

commit 4e5cde74d25713f78675aa3e18083b4fb5e619f3
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri May 19 16:41:34 2023 +0300

    querylog: imp code

commit 3494eab7006240f652a0217d305ac916bd6c3c83
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri May 19 16:13:08 2023 +0300

    all: lint script

commit 704534ce6278e7d9b1bef30a3acc4e59f25693bc
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri May 19 16:12:04 2023 +0300

    querylog: imp code

commit 48510102a2fa5187f78067d2b9157dac62f8bb56
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri May 19 15:52:57 2023 +0300

    querylog: imp code

commit 89c273aea0e6758eb749a2d3bbaf1bc385a57797
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri May 19 15:40:50 2023 +0300

    querylog: imp code

commit 0057fe64553ad38de0fda10efb9d3512c9a00e45
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri May 19 13:54:46 2023 +0300

    querylog: imp code

... and 1 more commit
This commit is contained in:
Dimitry Kolyshev
2023-05-24 16:33:15 +03:00
parent 941eb1dd73
commit cbc7985e75
11 changed files with 515 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
internal/querylog/search.go
View File

@@ -1,9 +1,11 @@
package querylog
import (
"fmt"
"io"
"time"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"golang.org/x/exp/slices"
)
@@ -134,84 +136,112 @@ func (l *queryLog) search(params *searchParams) (entries []*logEntry, oldest tim
return entries, oldest
}
// searchFiles looks up log records from all log files. It optionally uses the
// client cache, if provided. searchFiles does not scan more than
// maxFileScanEntries so callers may need to call it several times to get all
// results. oldest and total are the time of the oldest processed entry and the
// total number of processed entries, including discarded ones, correspondingly.
func (l *queryLog) searchFiles(
params *searchParams,
cache clientCache,
) (entries []*logEntry, oldest time.Time, total int) {
// seekRecord changes the current position to the next record older than the
// provided parameter.
func (r *qLogReader) seekRecord(olderThan time.Time) (err error) {
if olderThan.IsZero() {
return r.SeekStart()
}
err = r.seekTS(olderThan.UnixNano())
if err == nil {
// Read to the next record, because we only need the one that goes
// after it.
_, err = r.ReadNext()
}
return err
}
// setQLogReader creates a reader with the specified files and sets the
// position to the next record older than the provided parameter.
func (l *queryLog) setQLogReader(olderThan time.Time) (qr *qLogReader, err error) {
files := []string{
l.logFile + ".1",
l.logFile,
}
r, err := NewQLogReader(files)
r, err := newQLogReader(files)
if err != nil {
log.Error("querylog: opening qlog reader: %s", err)
return entries, oldest, 0
}
defer func() {
closeErr := r.Close()
if closeErr != nil {
log.Error("querylog: closing file: %s", err)
}
}()
if params.olderThan.IsZero() {
err = r.SeekStart()
} else {
err = r.seekTS(params.olderThan.UnixNano())
if err == nil {
// Read to the next record, because we only need the one that goes
// after it.
_, err = r.ReadNext()
}
return nil, fmt.Errorf("opening qlog reader: %s", err)
}
err = r.seekRecord(olderThan)
if err != nil {
log.Debug("querylog: cannot seek to %s: %s", params.olderThan, err)
defer func() { err = errors.WithDeferred(err, r.Close()) }()
log.Debug("querylog: cannot seek to %s: %s", olderThan, err)
return entries, oldest, 0
return nil, nil
}
totalLimit := params.offset + params.limit
oldestNano := int64(0)
return r, nil
}
// By default, we do not scan more than maxFileScanEntries at once. The
// idea is to make search calls faster so that the UI could handle it and
// show something quicker. This behavior can be overridden if
// maxFileScanEntries is set to 0.
// readEntries reads entries from the reader to totalLimit. By default, we do
// not scan more than maxFileScanEntries at once. The idea is to make search
// calls faster so that the UI could handle it and show something quicker.
// This behavior can be overridden if maxFileScanEntries is set to 0.
func (l *queryLog) readEntries(
r *qLogReader,
params *searchParams,
cache clientCache,
totalLimit int,
) (entries []*logEntry, oldestNano int64, total int) {
for total < params.maxFileScanEntries || params.maxFileScanEntries <= 0 {
var e *logEntry
var ts int64
e, ts, err = l.readNextEntry(r, params, cache)
if err != nil {
if err == io.EOF {
ent, ts, rErr := l.readNextEntry(r, params, cache)
if rErr != nil {
if rErr == io.EOF {
oldestNano = 0
break
}
log.Error("querylog: reading next entry: %s", err)
log.Error("querylog: reading next entry: %s", rErr)
}
oldestNano = ts
total++
if e != nil {
entries = append(entries, e)
if len(entries) == totalLimit {
break
}
if ent == nil {
continue
}
entries = append(entries, ent)
if len(entries) == totalLimit {
break
}
}
return entries, oldestNano, total
}
// searchFiles looks up log records from all log files. It optionally uses the
// client cache, if provided. searchFiles does not scan more than
// maxFileScanEntries so callers may need to call it several times to get all
// the results. oldest and total are the time of the oldest processed entry
// and the total number of processed entries, including discarded ones,
// correspondingly.
func (l *queryLog) searchFiles(
params *searchParams,
cache clientCache,
) (entries []*logEntry, oldest time.Time, total int) {
r, err := l.setQLogReader(params.olderThan)
if err != nil {
log.Error("querylog: %s", err)
}
if r == nil {
return entries, oldest, 0
}
defer func() {
if closeErr := r.Close(); closeErr != nil {
log.Error("querylog: closing file: %s", closeErr)
}
}()
totalLimit := params.offset + params.limit
entries, oldestNano, total := l.readEntries(r, params, cache, totalLimit)
if oldestNano != 0 {
oldest = time.Unix(0, oldestNano)
}
@@ -243,11 +273,11 @@ func (f quickMatchClientFinder) findClient(clientID, ip string) (c *Client) {
}
// readNextEntry reads the next log entry and checks if it matches the search
// criteria. It optionally uses the client cache, if provided. e is nil if the
// entry doesn't match the search criteria. ts is the timestamp of the
// criteria. It optionally uses the client cache, if provided. e is nil if
// the entry doesn't match the search criteria. ts is the timestamp of the
// processed entry.
func (l *queryLog) readNextEntry(
r *QLogReader,
r *qLogReader,
params *searchParams,
cache clientCache,
) (e *logEntry, ts int64, err error) {