Compare commits

..

18 Commits

Author SHA1 Message Date
Ainar Garipov
6d3b5c364b all: add tests; imp addrproc, docs 2023-07-14 17:39:42 +03:00
Ainar Garipov
9f93a21bf6 all: imp client resolving 2023-07-13 18:18:49 +03:00
Ainar Garipov
f22d893845 Pull request 1921: 6003-relax-scan-limit
Updates #6003.

Squashed commit of the following:

commit 1cc42303c29edc621802fc182ccb5701e412f099
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Jul 13 13:47:41 2023 +0300

    all: fix chlog

commit e835084c7aac6384ea7b0886e6b3b1d614438baa
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Jul 13 13:40:45 2023 +0300

    rulelist: imp longer line handling
2023-07-13 13:57:32 +03:00
Ainar Garipov
de63eeabfa Pull request 1920: 5985-client-text
Updates #5985.

Squashed commit of the following:

commit 925b55df066cb44eb37851491034f65727efcc79
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Jul 12 20:39:04 2023 +0300

    client: imp text more

commit f4094eeeab01c168362366450169f7806faba198
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Jul 12 20:24:33 2023 +0300

    client: fix text

commit 4e3a76da7c5f3d12716dbf5a9d5472e0c9b744c0
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Jul 12 20:22:31 2023 +0300

    client: imp runtime client text
2023-07-12 21:05:29 +03:00
Ainar Garipov
1aaffd1b72 Pull request 1919: rm-ports
Squashed commit of the following:

commit 892cb403112f8b816d99e645d7419bfd49ad3c33
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Jul 12 17:30:28 2023 +0300

    docker: rm 784, 8853 ports
2023-07-12 17:52:29 +03:00
Ainar Garipov
eb97e7dc01 Pull request 1918: upd-chlog
Squashed commit of the following:

commit d5d21a8dd3ca892b8c9ba3d6c2154a99933d6dc3
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Jul 12 16:07:53 2023 +0300

    all: upd chlog
2023-07-12 16:14:23 +03:00
Stanislav Chzhen
55335c4061 Pull request 1908: AG-23497-scripts-download-languages
Squashed commit of the following:

commit 874e847fc9bbfaeb8af1c02eb0ba1dbb98bd008f
Merge: 4becdd809 a79deda66
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Jul 12 16:01:45 2023 +0300

    Merge branch 'master' into AG-23497-scripts-download-languages

commit 4becdd8092558b15d783674f5b9d1e9c151e3a8c
Merge: 1e5385c33 40884624c
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Jul 12 13:34:34 2023 +0300

    Merge branch 'master' into AG-23497-scripts-download-languages

commit 1e5385c33a298b0b8563fee6704f6bb3ded12d60
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Jul 11 19:56:29 2023 +0300

    all: upd golibs, imp code

commit 0498960b00be21b1294f8b71108b234554e5847f
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Jul 7 19:05:58 2023 +0300

    scripts: imp naming

commit 6e36ed83c6bec2fe6159442a9e6805c0720e27f5
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Jul 6 16:37:13 2023 +0300

    scripts: separate files

commit 55027cfa1c04b0a36e5267b024b53a45f26dd974
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Wed Jul 5 13:51:40 2023 +0300

    scripts: add download languages
2023-07-12 16:06:17 +03:00
Ainar Garipov
a79deda665 Pull request 1917: upd-go
Squashed commit of the following:

commit 72423458d6589027221d340a53af607622678b23
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Jul 12 14:24:02 2023 +0300

    all: upd go
2023-07-12 14:37:05 +03:00
Ainar Garipov
40884624c2 Pull request 1916: 5990-root-ignore
Updates #5990.

Squashed commit of the following:

commit 1d5d3451c855681a631b85652417ee1bebadab01
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Jul 11 20:11:45 2023 +0300

    all: allow ignoring root in querylog and stats
2023-07-11 20:42:40 +03:00
Ainar Garipov
0a1887a854 Pull request 1914: upd-flts
Squashed commit of the following:

commit a8932f56fad583ecfcb7efae36fc516454bc6610
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Jul 11 16:20:55 2023 +0300

    filtering: fix docs; upd svcs
2023-07-11 16:27:20 +03:00
Dimitry Kolyshev
65b526b969 Pull request: 5972-ip-dupl-ans
Updates #5972.

Squashed commit of the following:

commit 0e089f9ff8fd7e6d7cb53aa7c3b92435d1d41a81
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Tue Jul 11 15:33:16 2023 +0300

    dnsforward: imp code

commit 39527c078fd9ad6ea4906659e185d54e74ef6465
Merge: 03641b0b5 61ed74374
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Tue Jul 11 11:29:19 2023 +0300

    Merge remote-tracking branch 'origin/master' into 5972-ip-dupl-ans

    # Conflicts:
    #	CHANGELOG.md

commit 03641b0b511f8e48d386be76d0a4776296cf047d
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Mon Jul 10 14:03:28 2023 +0300

    all: dupl ips in answer
2023-07-11 15:46:01 +03:00
Ainar Garipov
61ed743748 Pull request 1913: parental-cache-size
Squashed commit of the following:

commit 6e7dcf0c59c478869e65cb6945d8d262b9eb1879
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Jul 10 19:44:49 2023 +0300

    home: fix parental cache size
2023-07-10 20:00:29 +03:00
Ainar Garipov
c02a14117d Pull request 1912: 5896-safe-browsing-ptr
Updates #5896.

Squashed commit of the following:

commit 49340544a2a8762283397cdb54b91ed534591fa0
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Jul 10 17:59:45 2023 +0300

    hashprefix: fix loop pointer
2023-07-10 19:04:31 +03:00
Ainar Garipov
7b92d53b84 Pull request 1910: new-rulelist-parser
Squashed commit of the following:

commit bac0da6818388d67840b8fe9b633ce0804964ed9
Merge: cb6759b63 f7dd83251
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Jul 7 18:22:40 2023 +0300

    Merge branch 'master' into new-rulelist-parser

commit cb6759b63546b35074ec0ae04769ddb5e83ebac1
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Jul 7 12:18:44 2023 +0300

    all: upd tools

commit d28bf4cb42057b84e0c1325389db121a91f7c989
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Jul 6 19:35:48 2023 +0300

    all: upd chlog

commit 7df637b00331dff5810c3a76f4a7d2cee24148f1
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Jul 6 19:30:42 2023 +0300

    rulelist: fix tabs

commit 0598d0d43504b246570e9ee76d79dff0d86413c5
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Jul 6 19:06:18 2023 +0300

    all: add go-bench, go-fuzz; imp docs

commit a9ab7726048e216b13876a85991f3e3e8696a029
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Jul 6 18:18:14 2023 +0300

    filtering: add new parser
2023-07-07 18:27:33 +03:00
Ainar Garipov
f7dd832517 Pull request 1909: 5939-rm-healthcheck
Updates #5939.

Squashed commit of the following:

commit 087309b4ef100e97339f49cf1c2e90ba2fa4293f
Merge: 360df813d c21f958ea
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Jul 7 13:18:52 2023 +0300

    Merge branch 'master' into 5939-rm-healthcheck

commit 360df813d995f935c591aaea9c56fe4372ca2281
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Jul 5 14:16:18 2023 +0300

    all: rm docker healthcheck
2023-07-07 13:58:15 +03:00
Stanislav Chzhen
c21f958eaf Pull request 1878: AG-22597-imp-rdns
Squashed commit of the following:

commit ccad155c34989943d88a0a260c50845d1f4ece6b
Merge: 0cd889f6a 5a195b441
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Jul 6 17:00:58 2023 +0300

    Merge branch 'master' into AG-22597-imp-rdns

commit 0cd889f6a500f5616af0f8d8fdcde0403b87ad4f
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Jul 6 12:20:49 2023 +0300

    dnsforward: imp code

commit 1aaa1998b914b0d53142c21fa3bdcae502e4f3f6
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Jul 4 20:11:55 2023 +0300

    home: add todo

commit aed232fcf70ef546f373d5235b73abcb4fbb4b6c
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Tue Jul 4 13:25:28 2023 +0300

    all: imp code, tests

commit 5c028c2766ffb8ebdc358a245a249c6a55d9ad81
Merge: 83d6ae7f6 97af062f7
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Jul 3 18:54:42 2023 +0300

    Merge branch 'master' into AG-22597-imp-rdns

commit 83d6ae7f61a7b81a8d73cd6d747035278c64fb70
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Jul 3 18:53:05 2023 +0300

    home: imp code

commit 8153988dece0406e51a90a43eaffae59dba30a36
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Jun 30 18:06:09 2023 +0300

    all: imp code

commit 00d3cc11a9378318f176aae00ddf972f255d575c
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Jun 30 13:05:04 2023 +0300

    all: add tests

commit ffdc95f237bfdb780922b4390d82cdc0154b0621
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Jun 29 15:20:00 2023 +0300

    all: imp code, docs

commit 0dc60e2b355750ca701558927d22fb9ad187ea7e
Merge: 69dd56bdb d4a4bda64
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Thu Jun 29 15:13:19 2023 +0300

    Merge branch 'master' into AG-22597-imp-rdns

commit 69dd56bdb75056b0fa6bcf6538af7fff93383323
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Fri Jun 23 14:36:29 2023 +0300

    rdns: add tests

commit 16909b51adbe3a3f230291834cc9486dd8a0e8f8
Author: Stanislav Chzhen <s.chzhen@adguard.com>
Date:   Mon Jun 19 16:28:26 2023 +0300

    rdns: extract rdns
2023-07-06 17:10:06 +03:00
Dimitry Kolyshev
5a195b441c Pull request: log-yaml-conf
Updates #4897.

Squashed commit of the following:

commit 8a961157c9930bf4859ce2209e5016ce94987e12
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Wed Jul 5 10:13:24 2023 +0400

    home: imp code

commit 509c07eed06311d773bc3e34b3ca28d9f14186fe
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Tue Jul 4 17:33:31 2023 +0400

    all: fix

commit f032e28f98552f238721491c998fca2d7d4b9802
Merge: ee0113435 c46516475
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Tue Jul 4 17:31:29 2023 +0400

    Merge remote-tracking branch 'origin/master' into log-yaml-conf

    # Conflicts:
    #	CHANGELOG.md

commit ee011343512e82d4e21cb402759d0284523ba02a
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Tue Jul 4 12:31:42 2023 +0400

    all: changelog

commit 07f4c4a244b1b6200d3056cde5ebced6254084a7
Merge: 2042c0753 97af062f7
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Tue Jul 4 12:25:21 2023 +0400

    Merge remote-tracking branch 'origin/master' into log-yaml-conf

commit 2042c0753ec29de6045c3f1de6d075cb93d6ec27
Merge: a1d3a5130 8004b135b
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Mon Jul 3 16:25:26 2023 +0400

    Merge remote-tracking branch 'origin/master' into log-yaml-conf

commit a1d3a51307e80f9e509bd6f3bee1a7b17bf1ffe6
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Fri Jun 30 11:11:32 2023 +0400

    home: imp code

commit 2392a3b02620b8c38e88afb4d75988be85fe1338
Merge: 4224fbed7 39f5c50ac
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Thu Jun 29 16:46:47 2023 +0400

    home: imp code

commit 4224fbed7113e94bee44d0ab0272e8302a8086f3
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Thu Jun 29 16:39:35 2023 +0400

    home: imp code

commit 5ce708cc50bed83e64f062e599fc8b6143d0d44d
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Thu Jun 29 12:48:42 2023 +0400

    all: docs

commit 4b6d898a888818410f59b843c3ca1a685aafec82
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Thu Jun 29 12:20:54 2023 +0400

    home: imp code

commit 431b44eda71f488c747c3efa4da0a6c222b1cf06
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Thu Jun 29 12:03:17 2023 +0400

    home: imp tests

commit 53a5c31060018c37953beb27d80c46f92bbe14af
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Wed Jun 28 17:40:49 2023 +0400

    home: imp docs

commit bfa57d9f21e3326baafd3a52e91d54396d8e03fa
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Wed Jun 28 15:49:40 2023 +0400

    home: log conf

commit 49e06dca9dc2ceb2647b7e36dac145ccf78a6c3f
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Wed Jun 28 15:41:02 2023 +0400

    home: log conf

commit 9be432dea7cec8ae0c0d3ee1f73c58c76376d07e
Author: Dimitry Kolyshev <dkolyshev@adguard.com>
Date:   Wed Jun 28 10:45:01 2023 +0400

    home: log conf
2023-07-06 10:09:04 +03:00
Ainar Garipov
c46516475d Pull request 1906: 5896-safe-browsing-cpu-ram
Updates #5896.

Squashed commit of the following:

commit 81ac59e2f95ef3ad6ac5c4668c07c35d69570454
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Jul 4 16:07:33 2023 +0300

    hashprefix: fix cache coding
2023-07-04 16:17:18 +03:00
67 changed files with 2499 additions and 1487 deletions

View File

@@ -1,7 +1,7 @@
'name': 'build'
'env':
'GO_VERSION': '1.19.10'
'GO_VERSION': '1.19.11'
'NODE_VERSION': '14'
'on':

View File

@@ -1,7 +1,7 @@
'name': 'lint'
'env':
'GO_VERSION': '1.19.10'
'GO_VERSION': '1.19.11'
'on':
'push':

1
.gitignore vendored
View File

@@ -9,6 +9,7 @@
*.db
*.log
*.snap
*.test
/agh-backup/
/bin/
/build/*

View File

@@ -14,21 +14,118 @@ and this project adheres to
<!--
## [v0.108.0] - TBA
## [v0.107.34] - 2023-07-26 (APPROX.)
## [v0.107.35] - 2023-08-02 (APPROX.)
See also the [v0.107.34 GitHub milestone][ms-v0.107.34].
See also the [v0.107.35 GitHub milestone][ms-v0.107.35].
[ms-v0.107.34]: https://github.com/AdguardTeam/AdGuardHome/milestone/69?closed=1
[ms-v0.107.35]: https://github.com/AdguardTeam/AdGuardHome/milestone/70?closed=1
NOTE: Add new changes BELOW THIS COMMENT.
-->
### Fixed
- Occasional client information lookup failures that could lead to the DNS
server getting stuck ([#6006]).
- `bufio.Scanner: token too long` errors when trying to add filtering-rule lists
with lines over 1024 bytes long ([#6003]).
### Removed
- Default exposure of the non-standard ports 784 and 8853 for DNS-over-QUIC in
the `Dockerfile`.
[#6003]: https://github.com/AdguardTeam/AdGuardHome/issues/6003
[#6006]: https://github.com/AdguardTeam/AdGuardHome/issues/6006
<!--
NOTE: Add new changes ABOVE THIS COMMENT.
-->
## [v0.107.34] - 2023-07-12
See also the [v0.107.34 GitHub milestone][ms-v0.107.34].
### Security
- Go version has been updated to prevent the possibility of exploiting the
CVE-2023-29406 Go vulnerability fixed in [Go 1.19.11][go-1.19.11].
### Added
- Ability to ignore queries for the root domain, such as `NS .` queries
([#5990]).
### Changed
- Improved CPU and RAM consumption during updates of filtering-rule lists.
#### Configuration Changes
In this release, the schema version has changed from 23 to 24.
- Properties starting with `log_`, and `verbose` property, which used to set up
logging are now moved to the new object `log` containing new properties
`file`, `max_backups`, `max_size`, `max_age`, `compress`, `local_time`, and
`verbose`:
```yaml
# BEFORE:
'log_file': ""
'log_max_backups': 0
'log_max_size': 100
'log_max_age': 3
'log_compress': false
'log_localtime': false
'verbose': false
# AFTER:
'log':
'file': ""
'max_backups': 0
'max_size': 100
'max_age': 3
'compress': false
'local_time': false
'verbose': false
```
To rollback this change, remove the new object `log`, set back `log_` and
`verbose` properties and change the `schema_version` back to `23`.
### Deprecated
- Default exposure of the non-standard ports 784 and 8853 for DNS-over-QUIC in
the `Dockerfile`.
### Fixed
- Two unspecified IPs when a host is blocked in two filter lists ([#5972]).
- Incorrect setting of Parental Control cache size.
- Excessive RAM and CPU consumption by Safe Browsing and Parental Control
filters ([#5896]).
### Removed
- The `HEALTHCHECK` section and the use of `tini` in the `ENTRYPOINT` section in
`Dockerfile` ([#5939]). They caused a lot of issues, especially with tools
like `docker-compose` and `podman`.
**NOTE:** Some Docker tools may cache `ENTRYPOINT` sections, so some users may
be required to backup their configuration, stop the container, purge the old
image, and reload it from scratch.
[#5896]: https://github.com/AdguardTeam/AdGuardHome/issues/5896
[#5972]: https://github.com/AdguardTeam/AdGuardHome/issues/5972
[#5990]: https://github.com/AdguardTeam/AdGuardHome/issues/5990
[go-1.19.11]: https://groups.google.com/g/golang-announce/c/2q13H6LEEx0/m/sduSepLLBwAJ
[ms-v0.107.34]: https://github.com/AdguardTeam/AdGuardHome/milestone/69?closed=1
## [v0.107.33] - 2023-07-03
See also the [v0.107.33 GitHub milestone][ms-v0.107.33].
@@ -147,9 +244,9 @@ In this release, the schema version has changed from 20 to 23.
### Deprecated
- `HEALTHCHECK` and `ENTRYPOINT` sections in `Dockerfile` ([#5939]). They cause
a lot of issues, especially with tools like `docker-compose` and `podman`, and
will be removed in a future release.
- The `HEALTHCHECK` section and the use of `tini` in the `ENTRYPOINT` section in
`Dockerfile` ([#5939]). They cause a lot of issues, especially with tools
like `docker-compose` and `podman`, and will be removed in a future release.
- Flags `-h`, `--host`, `-p`, `--port` have been deprecated. The `-h` flag
will work as an alias for `--help`, instead of the deprecated `--host` in the
future releases.
@@ -2160,11 +2257,12 @@ See also the [v0.104.2 GitHub milestone][ms-v0.104.2].
<!--
[Unreleased]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.34...HEAD
[v0.107.34]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.33...v0.107.34
[Unreleased]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.35...HEAD
[v0.107.35]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.34...v0.107.35
-->
[Unreleased]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.33...HEAD
[Unreleased]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.34...HEAD
[v0.107.34]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.33...v0.107.34
[v0.107.33]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.32...v0.107.33
[v0.107.32]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.31...v0.107.32
[v0.107.31]: https://github.com/AdguardTeam/AdGuardHome/compare/v0.107.30...v0.107.31

View File

@@ -78,7 +78,7 @@ build: deps quick-build
quick-build: js-build go-build
ci: deps test
ci: deps test go-bench go-fuzz
deps: js-deps go-deps
lint: js-lint go-lint
@@ -104,8 +104,10 @@ js-deps:
js-lint: ; $(NPM) $(NPM_FLAGS) run lint
js-test: ; $(NPM) $(NPM_FLAGS) run test
go-bench: ; $(ENV) "$(SHELL)" ./scripts/make/go-bench.sh
go-build: ; $(ENV) "$(SHELL)" ./scripts/make/go-build.sh
go-deps: ; $(ENV) "$(SHELL)" ./scripts/make/go-deps.sh
go-fuzz: ; $(ENV) "$(SHELL)" ./scripts/make/go-fuzz.sh
go-lint: ; $(ENV) "$(SHELL)" ./scripts/make/go-lint.sh
go-tools: ; $(ENV) "$(SHELL)" ./scripts/make/go-tools.sh

View File

@@ -7,7 +7,7 @@
# Make sure to sync any changes with the branch overrides below.
'variables':
'channel': 'edge'
'dockerGo': 'adguard/golang-ubuntu:6.7'
'dockerGo': 'adguard/golang-ubuntu:6.8'
'stages':
- 'Build frontend':
@@ -272,7 +272,7 @@
# need to build a few of these.
'variables':
'channel': 'beta'
'dockerGo': 'adguard/golang-ubuntu:6.7'
'dockerGo': 'adguard/golang-ubuntu:6.8'
# release-vX.Y.Z branches are the branches from which the actual final
# release is built.
- '^release-v[0-9]+\.[0-9]+\.[0-9]+':
@@ -287,4 +287,4 @@
# are the ones that actually get released.
'variables':
'channel': 'release'
'dockerGo': 'adguard/golang-ubuntu:6.7'
'dockerGo': 'adguard/golang-ubuntu:6.8'

View File

@@ -10,7 +10,7 @@
# Make sure to sync any changes with the branch overrides below.
'variables':
'channel': 'edge'
'dockerGo': 'adguard/golang-ubuntu:6.7'
'dockerGo': 'adguard/golang-ubuntu:6.8'
'snapcraftChannel': 'edge'
'stages':
@@ -191,7 +191,7 @@
# need to build a few of these.
'variables':
'channel': 'beta'
'dockerGo': 'adguard/golang-ubuntu:6.7'
'dockerGo': 'adguard/golang-ubuntu:6.8'
'snapcraftChannel': 'beta'
# release-vX.Y.Z branches are the branches from which the actual final
# release is built.
@@ -207,5 +207,5 @@
# are the ones that actually get released.
'variables':
'channel': 'release'
'dockerGo': 'adguard/golang-ubuntu:6.7'
'dockerGo': 'adguard/golang-ubuntu:6.8'
'snapcraftChannel': 'candidate'

View File

@@ -5,7 +5,7 @@
'key': 'AHBRTSPECS'
'name': 'AdGuard Home - Build and run tests'
'variables':
'dockerGo': 'adguard/golang-ubuntu:6.7'
'dockerGo': 'adguard/golang-ubuntu:6.8'
'stages':
- 'Tests':

View File

@@ -444,7 +444,7 @@
"client_confirm_delete": "Are you sure you want to delete client \"{{key}}\"?",
"list_confirm_delete": "Are you sure you want to delete this list?",
"auto_clients_title": "Runtime clients",
"auto_clients_desc": "Devices not on the list of Persistent clients that may still use AdGuard Home",
"auto_clients_desc": "Information about IP addresses of devices that are using or may use AdGuard Home. This information is gathered from several sources, including hosts files, reverse DNS, etc.",
"access_title": "Access settings",
"access_desc": "Here you can configure access rules for the AdGuard Home DNS server",
"access_allowed_title": "Allowed clients",

View File

@@ -1,6 +1,6 @@
# A docker file for scripts/make/build-docker.sh.
FROM alpine:3.17
FROM alpine:3.18
ARG BUILD_DATE
ARG VERSION
@@ -25,8 +25,6 @@ RUN apk --no-cache add ca-certificates libcap tzdata && \
mkdir -p /opt/adguardhome/conf /opt/adguardhome/work && \
chown -R nobody: /opt/adguardhome
RUN apk --no-cache add tini
ARG DIST_DIR
ARG TARGETARCH
ARG TARGETOS
@@ -43,43 +41,18 @@ RUN setcap 'cap_net_bind_service=+eip' /opt/adguardhome/AdGuardHome
# 68 : UDP : DHCP (client)
# 80 : TCP : HTTP (main)
# 443 : TCP, UDP : HTTPS, DNS-over-HTTPS (incl. HTTP/3), DNSCrypt (main)
# 784 : UDP : DNS-over-QUIC (experimental)
# 853 : TCP, UDP : DNS-over-TLS, DNS-over-QUIC
# 3000 : TCP, UDP : HTTP(S) (alt, incl. HTTP/3)
# 3001 : TCP, UDP : HTTP(S) (beta, incl. HTTP/3)
# 5443 : TCP, UDP : DNSCrypt (alt)
# 6060 : TCP : HTTP (pprof)
# 8853 : UDP : DNS-over-QUIC (experimental)
#
# TODO(a.garipov): Remove the old, non-standard 784 and 8853 ports for
# DNS-over-QUIC in a future release.
EXPOSE 53/tcp 53/udp 67/udp 68/udp 80/tcp 443/tcp 443/udp 784/udp\
853/tcp 853/udp 3000/tcp 3000/udp 5443/tcp\
5443/udp 6060/tcp 8853/udp
EXPOSE 53/tcp 53/udp 67/udp 68/udp 80/tcp 443/tcp 443/udp 853/tcp\
853/udp 3000/tcp 3000/udp 5443/tcp 5443/udp 6060/tcp
WORKDIR /opt/adguardhome/work
# Install helpers for healthcheck.
COPY --chown=nobody:nogroup\
./${DIST_DIR}/docker/scripts\
/opt/adguardhome/scripts
HEALTHCHECK \
--interval=30s \
--timeout=10s \
--retries=3 \
CMD [ "/opt/adguardhome/scripts/healthcheck.sh" ]
# It seems that the healthckech script sometimes spawns zombie processes, so we
# need a way to handle them, since AdGuard Home doesn't know how to keep track
# of the processes delegated to it by the OS. Use tini as entry point because
# it needs the PID=1 to be the default parent for orphaned processes.
#
# See https://github.com/adguardTeam/adGuardHome/issues/3290.
ENTRYPOINT [ "/sbin/tini", "--" ]
ENTRYPOINT ["/opt/adguardhome/AdGuardHome"]
CMD [ \
"/opt/adguardhome/AdGuardHome", \
"--no-check-update", \
"-c", "/opt/adguardhome/conf/AdGuardHome.yaml", \
"-w", "/opt/adguardhome/work" \

View File

@@ -1,29 +0,0 @@
/^[^[:space:]]/ { is_dns = /^dns:/ }
/^[[:space:]]+bind_hosts:/ { if (is_dns) prev_line = FNR }
/^[[:space:]]+- .+/ {
if (FNR - prev_line == 1) {
addrs[$2] = true
prev_line = FNR
if ($2 == "0.0.0.0" || $2 == "'::'") {
# Drop all the other addresses.
delete addrs
addrs[""] = true
prev_line = -1
}
}
}
/^[[:space:]]+port:/ { if (is_dns) port = $2 }
END {
for (addr in addrs) {
if (match(addr, ":")) {
print "[" addr "]:" port
} else {
print addr ":" port
}
}
}

View File

@@ -1,107 +0,0 @@
#!/bin/sh
# AdGuard Home Docker healthcheck script
# Exit the script if a pipeline fails (-e), prevent accidental filename
# expansion (-f), and consider undefined variables as errors (-u).
set -e -f -u
# Function error_exit is an echo wrapper that writes to stderr and stops the
# script execution with code 1.
error_exit() {
echo "$1" 1>&2
exit 1
}
agh_dir="/opt/adguardhome"
readonly agh_dir
filename="${agh_dir}/conf/AdGuardHome.yaml"
readonly filename
if ! [ -f "$filename" ]
then
wget "http://127.0.0.1:3000" -O /dev/null -q || exit 1
exit 0
fi
help_dir="${agh_dir}/scripts"
readonly help_dir
# Parse web host
web_url="$( awk -f "${help_dir}/web-bind.awk" "$filename" )"
readonly web_url
if [ "$web_url" = '' ]
then
error_exit "no web bindings could be retrieved from $filename"
fi
# TODO(e.burkov): Deal with 0 port.
case "$web_url"
in
(*':0')
error_exit '0 in web port is not supported by healthcheck'
;;
(*)
# Go on.
;;
esac
# Parse DNS hosts
dns_hosts="$( awk -f "${help_dir}/dns-bind.awk" "$filename" )"
readonly dns_hosts
if [ "$dns_hosts" = '' ]
then
error_exit "no DNS bindings could be retrieved from $filename"
fi
first_dns="$( echo "$dns_hosts" | head -n 1 )"
readonly first_dns
# TODO(e.burkov): Deal with 0 port.
case "$first_dns"
in
(*':0')
error_exit '0 in DNS port is not supported by healthcheck'
;;
(*)
# Go on.
;;
esac
# Check
# Skip SSL certificate validation since there is no guarantee the container
# trusts the one used. It should be safe to drop the SSL validation since the
# current script intended to be used from inside the container and only checks
# the endpoint availability, ignoring the content of the response.
#
# See https://github.com/AdguardTeam/AdGuardHome/issues/5642.
wget --no-check-certificate "$web_url" -O /dev/null -q || exit 1
test_fqdn="healthcheck.adguardhome.test."
readonly test_fqdn
# The awk script currently returns only port prefixed with colon in case of
# unspecified address.
case "$first_dns"
in
(':'*)
nslookup -type=a "$test_fqdn" "127.0.0.1${first_dns}" > /dev/null ||\
nslookup -type=a "$test_fqdn" "[::1]${first_dns}" > /dev/null ||\
error_exit "nslookup failed for $host"
;;
(*)
echo "$dns_hosts" | while read -r host
do
nslookup -type=a "$test_fqdn" "$host" > /dev/null ||\
error_exit "nslookup failed for $host"
done
;;
esac

View File

@@ -1,5 +0,0 @@
# Don't consider the HTTPS hostname since the enforced HTTPS redirection should
# work if the SSL check skipped. See file docker/healthcheck.sh.
/^[^[:space:]]/ { is_http = /^http:/ }
/^[[:space:]]+address:/ { if (is_http) print "http://" $2 }

2
go.mod
View File

@@ -5,7 +5,7 @@ go 1.19
require (
// TODO(a.garipov): Update to a tagged version when it's released.
github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768
github.com/AdguardTeam/golibs v0.13.3
github.com/AdguardTeam/golibs v0.13.4
github.com/AdguardTeam/urlfilter v0.16.1
github.com/NYTimes/gziphandler v1.1.1
github.com/ameshkov/dnscrypt/v2 v2.2.7

4
go.sum
View File

@@ -2,8 +2,8 @@ github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768 h1:5Ia6wA+
github.com/AdguardTeam/dnsproxy v0.50.3-0.20230628054307-31e374065768/go.mod h1:CQhZTkqC8X0ID6glrtyaxgqRRdiYfn1gJulC1cZ5Dn8=
github.com/AdguardTeam/golibs v0.4.0/go.mod h1:skKsDKIBB7kkFflLJBpfGX+G8QFTx0WKUzB6TIgtUj4=
github.com/AdguardTeam/golibs v0.10.4/go.mod h1:rSfQRGHIdgfxriDDNgNJ7HmE5zRoURq8R+VdR81Zuzw=
github.com/AdguardTeam/golibs v0.13.3 h1:RT3QbzThtaLiFLkIUDS6/hlGEXrh0zYvdf4bd7UWpGo=
github.com/AdguardTeam/golibs v0.13.3/go.mod h1:wkJ6EUsN4np/9Gp7+9QeooY9E2U2WCLJYAioLCzkHsI=
github.com/AdguardTeam/golibs v0.13.4 h1:ACTwIR1pEENBijHcEWtiMbSh4wWQOlIHRxmUB8oBHf8=
github.com/AdguardTeam/golibs v0.13.4/go.mod h1:wkJ6EUsN4np/9Gp7+9QeooY9E2U2WCLJYAioLCzkHsI=
github.com/AdguardTeam/gomitmproxy v0.2.0/go.mod h1:Qdv0Mktnzer5zpdpi5rAwixNJzW2FN91LjKJCkVbYGU=
github.com/AdguardTeam/urlfilter v0.16.1 h1:ZPi0rjqo8cQf2FVdzo6cqumNoHZx2KPXj2yZa1A5BBw=
github.com/AdguardTeam/urlfilter v0.16.1/go.mod h1:46YZDOV1+qtdRDuhZKVPSSp7JWWes0KayqHrKAFBdEI=

43
internal/aghnet/addr.go Normal file
View File

@@ -0,0 +1,43 @@
package aghnet
import (
"fmt"
"strings"
"github.com/AdguardTeam/golibs/stringutil"
)
// NormalizeDomain returns a lowercased version of host without the final dot,
// unless host is ".", in which case it returns it unchanged. That is a special
// case that to allow matching queries like:
//
// dig IN NS '.'
func NormalizeDomain(host string) (norm string) {
if host == "." {
return host
}
return strings.ToLower(strings.TrimSuffix(host, "."))
}
// NewDomainNameSet returns nil and error, if list has duplicate or empty domain
// name. Otherwise returns a set, which contains domain names normalized using
// [NormalizeDomain].
func NewDomainNameSet(list []string) (set *stringutil.Set, err error) {
set = stringutil.NewSet()
for i, host := range list {
if host == "" {
return nil, fmt.Errorf("at index %d: hostname is empty", i)
}
host = NormalizeDomain(host)
if set.Has(host) {
return nil, fmt.Errorf("duplicate hostname %q at index %d", host, i)
}
set.Add(host)
}
return set, nil
}

View File

@@ -0,0 +1,59 @@
package aghnet_test
import (
"testing"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/golibs/testutil"
"github.com/stretchr/testify/assert"
)
func TestNewDomainNameSet(t *testing.T) {
t.Parallel()
testCases := []struct {
name string
wantErrMsg string
in []string
}{{
name: "nil",
wantErrMsg: "",
in: nil,
}, {
name: "success",
wantErrMsg: "",
in: []string{
"Domain.Example",
".",
},
}, {
name: "dups",
wantErrMsg: `duplicate hostname "domain.example" at index 1`,
in: []string{
"Domain.Example",
"domain.example",
},
}, {
name: "bad_domain",
wantErrMsg: "at index 0: hostname is empty",
in: []string{
"",
},
}}
for _, tc := range testCases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
set, err := aghnet.NewDomainNameSet(tc.in)
testutil.AssertErrorMsg(t, tc.wantErrMsg, err)
if err != nil {
return
}
for _, host := range tc.in {
assert.Truef(t, set.Has(aghnet.NormalizeDomain(host)), "%q not matched", host)
}
})
}
}

View File

@@ -1,12 +1,8 @@
package aghnet
import (
"fmt"
"net/netip"
"strings"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/stringutil"
)
// GenerateHostname generates the hostname from ip. In case of using IPv4 the
@@ -29,32 +25,8 @@ func GenerateHostname(ip netip.Addr) (hostname string) {
hostname = ip.StringExpanded()
if ip.Is4() {
return strings.Replace(hostname, ".", "-", -1)
return strings.ReplaceAll(hostname, ".", "-")
}
return strings.Replace(hostname, ":", "-", -1)
}
// NewDomainNameSet returns nil and error, if list has duplicate or empty
// domain name. Otherwise returns a set, which contains non-FQDN domain names,
// and nil error.
func NewDomainNameSet(list []string) (set *stringutil.Set, err error) {
set = stringutil.NewSet()
for i, v := range list {
host := strings.ToLower(strings.TrimSuffix(v, "."))
// TODO(a.garipov): Think about ignoring empty (".") names in the
// future.
if host == "" {
return nil, errors.Error("host name is empty")
}
if set.Has(host) {
return nil, fmt.Errorf("duplicate host name %q at index %d", host, i)
}
set.Add(host)
}
return set, nil
return strings.ReplaceAll(hostname, ":", "-")
}

View File

@@ -2,8 +2,8 @@ package aghtest
import (
"context"
"io"
"io/fs"
"net"
"github.com/AdguardTeam/AdGuardHome/internal/aghos"
"github.com/AdguardTeam/AdGuardHome/internal/next/agh"
@@ -19,23 +19,23 @@ import (
// Package fs
// type check
var _ fs.FS = &FS{}
// FS is a mock [fs.FS] implementation for tests.
// FS is a fake [fs.FS] implementation for tests.
type FS struct {
OnOpen func(name string) (fs.File, error)
}
// type check
var _ fs.FS = (*FS)(nil)
// Open implements the [fs.FS] interface for *FS.
func (fsys *FS) Open(name string) (fs.File, error) {
return fsys.OnOpen(name)
}
// type check
var _ fs.GlobFS = &GlobFS{}
var _ fs.GlobFS = (*GlobFS)(nil)
// GlobFS is a mock [fs.GlobFS] implementation for tests.
// GlobFS is a fake [fs.GlobFS] implementation for tests.
type GlobFS struct {
// FS is embedded here to avoid implementing all it's methods.
FS
@@ -48,9 +48,9 @@ func (fsys *GlobFS) Glob(pattern string) ([]string, error) {
}
// type check
var _ fs.StatFS = &StatFS{}
var _ fs.StatFS = (*StatFS)(nil)
// StatFS is a mock [fs.StatFS] implementation for tests.
// StatFS is a fake [fs.StatFS] implementation for tests.
type StatFS struct {
// FS is embedded here to avoid implementing all it's methods.
FS
@@ -62,47 +62,34 @@ func (fsys *StatFS) Stat(name string) (fs.FileInfo, error) {
return fsys.OnStat(name)
}
// Package net
// Package io
// type check
var _ net.Listener = (*Listener)(nil)
// Listener is a mock [net.Listener] implementation for tests.
type Listener struct {
OnAccept func() (conn net.Conn, err error)
OnAddr func() (addr net.Addr)
OnClose func() (err error)
// Writer is a fake [io.Writer] implementation for tests.
type Writer struct {
OnWrite func(b []byte) (n int, err error)
}
// Accept implements the [net.Listener] interface for *Listener.
func (l *Listener) Accept() (conn net.Conn, err error) {
return l.OnAccept()
}
var _ io.Writer = (*Writer)(nil)
// Addr implements the [net.Listener] interface for *Listener.
func (l *Listener) Addr() (addr net.Addr) {
return l.OnAddr()
}
// Close implements the [net.Listener] interface for *Listener.
func (l *Listener) Close() (err error) {
return l.OnClose()
// Write implements the [io.Writer] interface for *Writer.
func (w *Writer) Write(b []byte) (n int, err error) {
return w.OnWrite(b)
}
// Module adguard-home
// Package aghos
// type check
var _ aghos.FSWatcher = (*FSWatcher)(nil)
// FSWatcher is a mock [aghos.FSWatcher] implementation for tests.
// FSWatcher is a fake [aghos.FSWatcher] implementation for tests.
type FSWatcher struct {
OnEvents func() (e <-chan struct{})
OnAdd func(name string) (err error)
OnClose func() (err error)
}
// type check
var _ aghos.FSWatcher = (*FSWatcher)(nil)
// Events implements the [aghos.FSWatcher] interface for *FSWatcher.
func (w *FSWatcher) Events() (e <-chan struct{}) {
return w.OnEvents()
@@ -120,16 +107,16 @@ func (w *FSWatcher) Close() (err error) {
// Package agh
// type check
var _ agh.ServiceWithConfig[struct{}] = (*ServiceWithConfig[struct{}])(nil)
// ServiceWithConfig is a mock [agh.ServiceWithConfig] implementation for tests.
// ServiceWithConfig is a fake [agh.ServiceWithConfig] implementation for tests.
type ServiceWithConfig[ConfigType any] struct {
OnStart func() (err error)
OnShutdown func(ctx context.Context) (err error)
OnConfig func() (c ConfigType)
}
// type check
var _ agh.ServiceWithConfig[struct{}] = (*ServiceWithConfig[struct{}])(nil)
// Start implements the [agh.ServiceWithConfig] interface for
// *ServiceWithConfig.
func (s *ServiceWithConfig[_]) Start() (err error) {
@@ -152,10 +139,7 @@ func (s *ServiceWithConfig[ConfigType]) Config() (c ConfigType) {
// Package upstream
// type check
var _ upstream.Upstream = (*UpstreamMock)(nil)
// UpstreamMock is a mock [upstream.Upstream] implementation for tests.
// UpstreamMock is a fake [upstream.Upstream] implementation for tests.
//
// TODO(a.garipov): Replace with all uses of Upstream with UpstreamMock and
// rename it to just Upstream.
@@ -165,6 +149,9 @@ type UpstreamMock struct {
OnClose func() (err error)
}
// type check
var _ upstream.Upstream = (*UpstreamMock)(nil)
// Address implements the [upstream.Upstream] interface for *UpstreamMock.
func (u *UpstreamMock) Address() (addr string) {
return u.OnAddress()

View File

@@ -270,7 +270,10 @@ type ServerConfig struct {
UDPListenAddrs []*net.UDPAddr // UDP listen address
TCPListenAddrs []*net.TCPAddr // TCP listen address
UpstreamConfig *proxy.UpstreamConfig // Upstream DNS servers config
OnDNSRequest func(d *proxy.DNSContext)
// ClientIPs, if not nil, is used to send clients' IP addresses to other
// parts of AdGuard Home that may use it for resolving rDNS, WHOIS, etc.
ClientIPs chan netip.Addr
FilteringConfig
TLSConfig

View File

@@ -17,6 +17,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/dhcpd"
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/AdGuardHome/internal/querylog"
"github.com/AdguardTeam/AdGuardHome/internal/rdns"
"github.com/AdguardTeam/AdGuardHome/internal/stats"
"github.com/AdguardTeam/dnsproxy/proxy"
"github.com/AdguardTeam/dnsproxy/upstream"
@@ -98,6 +99,10 @@ type Server struct {
// must be a valid domain name plus dots on each side.
localDomainSuffix string
// ClientIPs, if not nil, is used to send clients' IP addresses to other
// parts of AdGuard Home that may use it for resolving rDNS, WHOIS, etc.
clientIPs chan<- netip.Addr
ipset ipsetCtx
privateNets netutil.SubnetSet
localResolvers *proxy.Proxy
@@ -277,17 +282,6 @@ func (s *Server) Resolve(host string) ([]net.IPAddr, error) {
return s.internalProxy.LookupIPAddr(host)
}
// RDNSExchanger is a resolver for clients' addresses.
type RDNSExchanger interface {
// Exchange tries to resolve the ip in a suitable way, i.e. either as local
// or as external.
Exchange(ip net.IP) (host string, err error)
// ResolvesPrivatePTR returns true if the RDNSExchanger is able to
// resolve PTR requests for locally-served addresses.
ResolvesPrivatePTR() (ok bool)
}
const (
// ErrRDNSNoData is returned by [RDNSExchanger.Exchange] when the answer
// section of response is either NODATA or has no PTR records.
@@ -299,10 +293,10 @@ const (
)
// type check
var _ RDNSExchanger = (*Server)(nil)
var _ rdns.Exchanger = (*Server)(nil)
// Exchange implements the RDNSExchanger interface for *Server.
func (s *Server) Exchange(ip net.IP) (host string, err error) {
// Exchange implements the [rdns.Exchanger] interface for *Server.
func (s *Server) Exchange(ip netip.Addr) (host string, err error) {
s.serverLock.RLock()
defer s.serverLock.RUnlock()
@@ -310,7 +304,7 @@ func (s *Server) Exchange(ip net.IP) (host string, err error) {
return "", nil
}
arpa, err := netutil.IPToReversedAddr(ip)
arpa, err := netutil.IPToReversedAddr(ip.AsSlice())
if err != nil {
return "", fmt.Errorf("reversing ip: %w", err)
}
@@ -328,14 +322,15 @@ func (s *Server) Exchange(ip net.IP) (host string, err error) {
Qclass: dns.ClassINET,
}},
}
ctx := &proxy.DNSContext{
dctx := &proxy.DNSContext{
Proto: "udp",
Req: req,
StartTime: time.Now(),
}
var resolver *proxy.Proxy
if s.privateNets.Contains(ip) {
if s.isPrivateIP(ip) {
if !s.conf.UsePrivateRDNS {
return "", nil
}
@@ -346,12 +341,16 @@ func (s *Server) Exchange(ip net.IP) (host string, err error) {
resolver = s.internalProxy
}
if err = resolver.Resolve(ctx); err != nil {
if err = resolver.Resolve(dctx); err != nil {
return "", err
}
return hostFromPTR(dctx.Res)
}
// hostFromPTR returns domain name from the PTR response or error.
func hostFromPTR(resp *dns.Msg) (host string, err error) {
// Distinguish between NODATA response and a failed request.
resp := ctx.Res
if resp.Rcode != dns.RcodeSuccess && resp.Rcode != dns.RcodeNameError {
return "", fmt.Errorf(
"received %s response: %w",
@@ -370,12 +369,25 @@ func (s *Server) Exchange(ip net.IP) (host string, err error) {
return "", ErrRDNSNoData
}
// ResolvesPrivatePTR implements the RDNSExchanger interface for *Server.
func (s *Server) ResolvesPrivatePTR() (ok bool) {
// isPrivateIP returns true if the ip is private.
func (s *Server) isPrivateIP(ip netip.Addr) (ok bool) {
return s.privateNets.Contains(ip.AsSlice())
}
// ShouldResolveClient returns false if ip is a loopback address, or ip is
// private and resolving of private addresses is disabled.
func (s *Server) ShouldResolveClient(ip netip.Addr) (ok bool) {
if ip.IsLoopback() {
return false
}
isPrivate := s.isPrivateIP(ip)
s.serverLock.RLock()
defer s.serverLock.RUnlock()
return s.conf.UsePrivateRDNS
return s.conf.ResolveClients &&
(s.conf.UsePrivateRDNS || !isPrivate)
}
// Start starts the DNS server.
@@ -548,6 +560,8 @@ func (s *Server) Prepare(conf *ServerConfig) (err error) {
s.recDetector.clear()
s.clientIPs = s.conf.ClientIPs
return nil
}
@@ -689,6 +703,9 @@ func (s *Server) Reconfigure(conf *ServerConfig) error {
// TODO(a.garipov): This whole piece of API is weird and needs to be remade.
if conf == nil {
conf = &s.conf
} else if s.clientIPs != nil {
close(s.clientIPs)
s.clientIPs = nil
}
err = s.Prepare(conf)

View File

@@ -39,11 +39,29 @@ func TestMain(m *testing.M) {
testutil.DiscardLogOutput(m)
}
// testTimeout is the common timeout for tests.
//
// TODO(a.garipov): Use more.
const testTimeout = 1 * time.Second
// testQuestionTarget is the common question target for tests.
//
// TODO(a.garipov): Use more.
const testQuestionTarget = "target.example"
const (
tlsServerName = "testdns.adguard.com"
testMessagesCount = 10
)
// testClientAddr is the common net.Addr for tests.
//
// TODO(a.garipov): Use more.
var testClientAddr net.Addr = &net.TCPAddr{
IP: net.IP{1, 2, 3, 4},
Port: 12345,
}
func startDeferStop(t *testing.T, s *Server) {
t.Helper()
@@ -53,6 +71,13 @@ func startDeferStop(t *testing.T, s *Server) {
testutil.CleanupAndRequireSuccess(t, s.Stop)
}
// packageUpstreamVariableMu is used to serialize access to the package-level
// variables of package upstream.
//
// TODO(s.chzhen): Move these parameters to upstream options and remove this
// crutch.
var packageUpstreamVariableMu = &sync.Mutex{}
func createTestServer(
t *testing.T,
filterConf *filtering.Config,
@@ -61,6 +86,9 @@ func createTestServer(
) (s *Server) {
t.Helper()
packageUpstreamVariableMu.Lock()
defer packageUpstreamVariableMu.Unlock()
rules := `||nxdomain.example.org
||NULL.example.org^
127.0.0.1 host.example.org
@@ -307,11 +335,9 @@ func TestServer(t *testing.T) {
}
func TestServer_timeout(t *testing.T) {
const timeout time.Duration = time.Second
t.Run("custom", func(t *testing.T) {
srvConf := &ServerConfig{
UpstreamTimeout: timeout,
UpstreamTimeout: testTimeout,
FilteringConfig: FilteringConfig{
BlockingMode: BlockingModeDefault,
EDNSClientSubnet: &EDNSClientSubnet{Enabled: false},
@@ -324,7 +350,7 @@ func TestServer_timeout(t *testing.T) {
err = s.Prepare(srvConf)
require.NoError(t, err)
assert.Equal(t, timeout, s.conf.UpstreamTimeout)
assert.Equal(t, testTimeout, s.conf.UpstreamTimeout)
})
t.Run("default", func(t *testing.T) {
@@ -545,7 +571,7 @@ func TestInvalidRequest(t *testing.T) {
// Send a DNS request without question.
_, _, err := (&dns.Client{
Timeout: 500 * time.Millisecond,
Timeout: testTimeout,
}).Exchange(&req, addr)
assert.NoErrorf(t, err, "got a response to an invalid query")
@@ -1273,11 +1299,11 @@ func TestServer_Exchange(t *testing.T) {
)
var (
onesIP = net.IP{1, 1, 1, 1}
localIP = net.IP{192, 168, 1, 1}
onesIP = netip.MustParseAddr("1.1.1.1")
localIP = netip.MustParseAddr("192.168.1.1")
)
revExtIPv4, err := netutil.IPToReversedAddr(onesIP)
revExtIPv4, err := netutil.IPToReversedAddr(onesIP.AsSlice())
require.NoError(t, err)
extUpstream := &aghtest.UpstreamMock{
@@ -1290,7 +1316,7 @@ func TestServer_Exchange(t *testing.T) {
},
}
revLocIPv4, err := netutil.IPToReversedAddr(localIP)
revLocIPv4, err := netutil.IPToReversedAddr(localIP.AsSlice())
require.NoError(t, err)
locUpstream := &aghtest.UpstreamMock{
@@ -1330,7 +1356,7 @@ func TestServer_Exchange(t *testing.T) {
want string
wantErr error
locUpstream upstream.Upstream
req net.IP
req netip.Addr
}{{
name: "external_good",
want: onesHost,
@@ -1354,7 +1380,7 @@ func TestServer_Exchange(t *testing.T) {
want: "",
wantErr: ErrRDNSNoData,
locUpstream: locUpstream,
req: net.IP{192, 168, 1, 2},
req: netip.MustParseAddr("192.168.1.2"),
}, {
name: "invalid_answer",
want: "",
@@ -1396,3 +1422,57 @@ func TestServer_Exchange(t *testing.T) {
assert.Empty(t, host)
})
}
func TestServer_ShouldResolveClient(t *testing.T) {
srv := &Server{
privateNets: netutil.SubnetSetFunc(netutil.IsLocallyServed),
}
testCases := []struct {
ip netip.Addr
want require.BoolAssertionFunc
name string
resolve bool
usePrivate bool
}{{
name: "default",
ip: netip.MustParseAddr("1.1.1.1"),
want: require.True,
resolve: true,
usePrivate: true,
}, {
name: "no_rdns",
ip: netip.MustParseAddr("1.1.1.1"),
want: require.False,
resolve: false,
usePrivate: true,
}, {
name: "loopback",
ip: netip.MustParseAddr("127.0.0.1"),
want: require.False,
resolve: true,
usePrivate: true,
}, {
name: "private_resolve",
ip: netip.MustParseAddr("192.168.0.1"),
want: require.True,
resolve: true,
usePrivate: true,
}, {
name: "private_no_resolve",
ip: netip.MustParseAddr("192.168.0.1"),
want: require.False,
resolve: true,
usePrivate: false,
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
srv.conf.ResolveClients = tc.resolve
srv.conf.UsePrivateRDNS = tc.usePrivate
ok := srv.ShouldResolveClient(tc.ip)
tc.want(t, ok)
})
}
}

View File

@@ -50,10 +50,10 @@ func (s *Server) beforeRequestHandler(
return true, nil
}
// getClientRequestFilteringSettings looks up client filtering settings using
// the client's IP address and ID, if any, from dctx.
func (s *Server) getClientRequestFilteringSettings(dctx *dnsContext) *filtering.Settings {
setts := s.dnsFilter.Settings()
// clientRequestFilteringSettings looks up client filtering settings using the
// client's IP address and ID, if any, from dctx.
func (s *Server) clientRequestFilteringSettings(dctx *dnsContext) (setts *filtering.Settings) {
setts = s.dnsFilter.Settings()
setts.ProtectionEnabled = dctx.protectionEnabled
if s.conf.FilterHandler != nil {
ip, _ := netutil.IPAndPortFromAddr(dctx.proxyCtx.Addr)

View File

@@ -21,6 +21,8 @@ func TestHandleDNSRequest_filterDNSResponse(t *testing.T) {
||cname.specific^$dnstype=~CNAME
||0.0.0.1^$dnstype=~A
||::1^$dnstype=~AAAA
0.0.0.0 duplicate.domain
0.0.0.0 duplicate.domain
`
forwardConf := ServerConfig{
@@ -137,6 +139,17 @@ func TestHandleDNSRequest_filterDNSResponse(t *testing.T) {
},
A: netutil.IPv4Zero(),
}},
}, {
req: createTestMessage("duplicate.domain."),
name: "duplicate_domain",
wantAns: []dns.RR{&dns.A{
Hdr: dns.RR_Header{
Name: "duplicate.domain.",
Rrtype: dns.TypeA,
Class: dns.ClassINET,
},
A: netutil.IPv4Zero(),
}},
}}
for _, tc := range testCases {

View File

@@ -26,11 +26,25 @@ func (s *Server) makeResponse(req *dns.Msg) (resp *dns.Msg) {
return resp
}
// ipsFromRules extracts non-IP addresses from the filtering result rules.
// containsIP returns true if the IP is already in the list.
func containsIP(ips []net.IP, ip net.IP) bool {
for _, a := range ips {
if a.Equal(ip) {
return true
}
}
return false
}
// ipsFromRules extracts unique non-IP addresses from the filtering result
// rules.
func ipsFromRules(resRules []*filtering.ResultRule) (ips []net.IP) {
for _, r := range resRules {
if r.IP != nil {
ips = append(ips, r.IP)
// len(resRules) and len(ips) are actually small enough for O(n^2) to do
// not raise performance questions.
if ip := r.IP; ip != nil && !containsIP(ips, ip) {
ips = append(ips, ip)
}
}

View File

@@ -30,6 +30,7 @@ type dnsContext struct {
setts *filtering.Settings
result *filtering.Result
// origResp is the response received from upstream. It is set when the
// response is modified by filters.
origResp *dns.Msg
@@ -48,13 +49,13 @@ type dnsContext struct {
// clientID is the ClientID from DoH, DoQ, or DoT, if provided.
clientID string
// startTime is the time at which the processing of the request has started.
startTime time.Time
// origQuestion is the question received from the client. It is set
// when the request is modified by rewrites.
origQuestion dns.Question
// startTime is the time at which the processing of the request has started.
startTime time.Time
// protectionEnabled shows if the filtering is enabled, and if the
// server's DNS filter is ready.
protectionEnabled bool
@@ -160,6 +161,22 @@ func (s *Server) processRecursion(dctx *dnsContext) (rc resultCode) {
return resultCodeSuccess
}
// mozillaFQDN is the domain used to signal the Firefox browser to not use its
// own DoH server.
//
// See https://support.mozilla.org/en-US/kb/canary-domain-use-application-dnsnet.
const mozillaFQDN = "use-application-dns.net."
// healthcheckFQDN is a reserved domain-name used for healthchecking.
//
// [Section 6.2 of RFC 6761] states that DNS Registries/Registrars must not
// grant requests to register test names in the normal way to any person or
// entity, making domain names under the test. TLD free to use in internal
// purposes.
//
// [Section 6.2 of RFC 6761]: https://www.rfc-editor.org/rfc/rfc6761.html#section-6.2
const healthcheckFQDN = "healthcheck.adguardhome.test."
// processInitial terminates the following processing for some requests if
// needed and enriches dctx with some client-specific information.
//
@@ -169,6 +186,8 @@ func (s *Server) processInitial(dctx *dnsContext) (rc resultCode) {
defer log.Debug("dnsforward: finished processing initial")
pctx := dctx.proxyCtx
s.processClientIP(pctx.Addr)
q := pctx.Req.Question[0]
qt := q.Qtype
if s.conf.AAAADisabled && qt == dns.TypeAAAA {
@@ -177,28 +196,13 @@ func (s *Server) processInitial(dctx *dnsContext) (rc resultCode) {
return resultCodeFinish
}
if s.conf.OnDNSRequest != nil {
s.conf.OnDNSRequest(pctx)
}
// Disable Mozilla DoH.
//
// See https://support.mozilla.org/en-US/kb/canary-domain-use-application-dnsnet.
if (qt == dns.TypeA || qt == dns.TypeAAAA) && q.Name == "use-application-dns.net." {
if (qt == dns.TypeA || qt == dns.TypeAAAA) && q.Name == mozillaFQDN {
pctx.Res = s.genNXDomain(pctx.Req)
return resultCodeFinish
}
// Handle a reserved domain healthcheck.adguardhome.test.
//
// [Section 6.2 of RFC 6761] states that DNS Registries/Registrars must not
// grant requests to register test names in the normal way to any person or
// entity, making domain names under test. TLD free to use in internal
// purposes.
//
// [Section 6.2 of RFC 6761]: https://www.rfc-editor.org/rfc/rfc6761.html#section-6.2
if q.Name == "healthcheck.adguardhome.test." {
if q.Name == healthcheckFQDN {
// Generate a NODATA negative response to make nslookup exit with 0.
pctx.Res = s.makeResponse(pctx.Req)
@@ -213,11 +217,33 @@ func (s *Server) processInitial(dctx *dnsContext) (rc resultCode) {
// Get the client-specific filtering settings.
dctx.protectionEnabled, _ = s.UpdatedProtectionStatus()
dctx.setts = s.getClientRequestFilteringSettings(dctx)
dctx.setts = s.clientRequestFilteringSettings(dctx)
return resultCodeSuccess
}
// processClientIP sends the client IP address to s.clientIPs, if needed.
func (s *Server) processClientIP(addr net.Addr) {
clientIP := netutil.NetAddrToAddrPort(addr).Addr()
if clientIP == (netip.Addr{}) {
log.Info("dnsforward: warning: bad client addr %q", addr)
return
}
// Do not assign s.clientIPs to a local variable to then use, since this
// lock also serializes the closure of s.clientIPs.
s.serverLock.RLock()
defer s.serverLock.RUnlock()
select {
case s.clientIPs <- clientIP:
// Go on.
default:
log.Debug("dnsforward: client ip channel is nil or full; len: %d", len(s.clientIPs))
}
}
func (s *Server) setTableHostToIP(t hostToIPTable) {
s.tableHostToIPLock.Lock()
defer s.tableHostToIPLock.Unlock()

View File

@@ -12,6 +12,7 @@ import (
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/testutil"
"github.com/AdguardTeam/urlfilter/rules"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -22,6 +23,95 @@ const (
ddrTestFQDN = ddrTestDomainName + "."
)
func TestServer_ProcessInitial(t *testing.T) {
t.Parallel()
testCases := []struct {
name string
target string
wantRCode rules.RCode
qType rules.RRType
aaaaDisabled bool
wantRC resultCode
}{{
name: "success",
target: testQuestionTarget,
wantRCode: -1,
qType: dns.TypeA,
aaaaDisabled: false,
wantRC: resultCodeSuccess,
}, {
name: "aaaa_disabled",
target: testQuestionTarget,
wantRCode: dns.RcodeSuccess,
qType: dns.TypeAAAA,
aaaaDisabled: true,
wantRC: resultCodeFinish,
}, {
name: "aaaa_disabled_a",
target: testQuestionTarget,
wantRCode: -1,
qType: dns.TypeA,
aaaaDisabled: true,
wantRC: resultCodeSuccess,
}, {
name: "mozilla_canary",
target: mozillaFQDN,
wantRCode: dns.RcodeNameError,
qType: dns.TypeA,
aaaaDisabled: false,
wantRC: resultCodeFinish,
}, {
name: "adguardhome_healthcheck",
target: healthcheckFQDN,
wantRCode: dns.RcodeSuccess,
qType: dns.TypeA,
aaaaDisabled: false,
wantRC: resultCodeFinish,
}}
for _, tc := range testCases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
clientIPs := make(chan netip.Addr, 1)
c := ServerConfig{
FilteringConfig: FilteringConfig{
AAAADisabled: tc.aaaaDisabled,
EDNSClientSubnet: &EDNSClientSubnet{Enabled: false},
},
ClientIPs: clientIPs,
}
s := createTestServer(t, &filtering.Config{}, c, nil)
dctx := &dnsContext{
proxyCtx: &proxy.DNSContext{
Req: createTestMessageWithType(tc.target, tc.qType),
Addr: testClientAddr,
RequestID: 1234,
},
}
gotRC := s.processInitial(dctx)
assert.Equal(t, tc.wantRC, gotRC)
gotAddr, _ := testutil.RequireReceive(t, clientIPs, testTimeout)
assert.Equal(t, netutil.NetAddrToAddrPort(testClientAddr).Addr(), gotAddr)
if tc.wantRCode > 0 {
gotResp := dctx.proxyCtx.Res
require.NotNil(t, gotResp)
assert.Equal(t, tc.wantRCode, gotResp.Rcode)
}
})
}
}
func TestServer_ProcessDDRQuery(t *testing.T) {
dohSVCB := &dns.SVCB{
Priority: 1,
@@ -64,7 +154,7 @@ func TestServer_ProcessDDRQuery(t *testing.T) {
}{{
name: "pass_host",
wantRes: resultCodeSuccess,
host: "example.net.",
host: testQuestionTarget,
qtype: dns.TypeSVCB,
ddrEnabled: true,
portDoH: 8043,
@@ -234,33 +324,33 @@ func TestServer_ProcessDetermineLocal(t *testing.T) {
func TestServer_ProcessDHCPHosts_localRestriction(t *testing.T) {
knownIP := netip.MustParseAddr("1.2.3.4")
testCases := []struct {
wantIP netip.Addr
name string
host string
wantIP netip.Addr
wantRes resultCode
isLocalCli bool
}{{
wantIP: knownIP,
name: "local_client_success",
host: "example.lan",
wantIP: knownIP,
wantRes: resultCodeSuccess,
isLocalCli: true,
}, {
wantIP: netip.Addr{},
name: "local_client_unknown_host",
host: "wronghost.lan",
wantIP: netip.Addr{},
wantRes: resultCodeSuccess,
isLocalCli: true,
}, {
wantIP: netip.Addr{},
name: "external_client_known_host",
host: "example.lan",
wantIP: netip.Addr{},
wantRes: resultCodeFinish,
isLocalCli: false,
}, {
wantIP: netip.Addr{},
name: "external_client_unknown_host",
host: "wronghost.lan",
wantIP: netip.Addr{},
wantRes: resultCodeFinish,
isLocalCli: false,
}}
@@ -332,52 +422,52 @@ func TestServer_ProcessDHCPHosts(t *testing.T) {
knownIP := netip.MustParseAddr("1.2.3.4")
testCases := []struct {
wantIP netip.Addr
name string
host string
suffix string
wantIP netip.Addr
wantRes resultCode
qtyp uint16
}{{
wantIP: netip.Addr{},
name: "success_external",
host: examplecom,
suffix: defaultLocalDomainSuffix,
wantIP: netip.Addr{},
wantRes: resultCodeSuccess,
qtyp: dns.TypeA,
}, {
wantIP: netip.Addr{},
name: "success_external_non_a",
host: examplecom,
suffix: defaultLocalDomainSuffix,
wantIP: netip.Addr{},
wantRes: resultCodeSuccess,
qtyp: dns.TypeCNAME,
}, {
wantIP: knownIP,
name: "success_internal",
host: examplelan,
suffix: defaultLocalDomainSuffix,
wantIP: knownIP,
wantRes: resultCodeSuccess,
qtyp: dns.TypeA,
}, {
wantIP: netip.Addr{},
name: "success_internal_unknown",
host: "example-new.lan",
suffix: defaultLocalDomainSuffix,
wantIP: netip.Addr{},
wantRes: resultCodeSuccess,
qtyp: dns.TypeA,
}, {
wantIP: netip.Addr{},
name: "success_internal_aaaa",
host: examplelan,
suffix: defaultLocalDomainSuffix,
wantIP: netip.Addr{},
wantRes: resultCodeSuccess,
qtyp: dns.TypeAAAA,
}, {
wantIP: knownIP,
name: "success_custom_suffix",
host: "example.custom",
suffix: "custom",
wantIP: knownIP,
wantRes: resultCodeSuccess,
qtyp: dns.TypeA,
}}
@@ -560,10 +650,8 @@ func TestServer_ProcessLocalPTR_usingResolvers(t *testing.T) {
var dnsCtx *dnsContext
setup := func(use bool) {
proxyCtx = &proxy.DNSContext{
Addr: &net.TCPAddr{
IP: net.IP{127, 0, 0, 1},
},
Req: createTestMessageWithType(reqAddr, dns.TypePTR),
Addr: testClientAddr,
Req: createTestMessageWithType(reqAddr, dns.TypePTR),
}
dnsCtx = &dnsContext{
proxyCtx: proxyCtx,

View File

@@ -2,9 +2,9 @@ package dnsforward
import (
"net"
"strings"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/AdGuardHome/internal/querylog"
"github.com/AdguardTeam/AdGuardHome/internal/stats"
@@ -24,7 +24,7 @@ func (s *Server) processQueryLogsAndStats(dctx *dnsContext) (rc resultCode) {
pctx := dctx.proxyCtx
q := pctx.Req.Question[0]
host := strings.ToLower(strings.TrimSuffix(q.Name, "."))
host := aghnet.NormalizeDomain(q.Name)
ip, _ := netutil.IPAndPortFromAddr(pctx.Addr)
ip = slices.Clone(ip)
@@ -139,11 +139,10 @@ func (s *Server) updateStats(
clientIP string,
) {
pctx := ctx.proxyCtx
e := stats.Entry{}
e.Domain = strings.ToLower(pctx.Req.Question[0].Name)
if e.Domain != "." {
// Remove last ".", but save the domain as is for "." queries.
e.Domain = e.Domain[:len(e.Domain)-1]
e := stats.Entry{
Domain: aghnet.NormalizeDomain(pctx.Req.Question[0].Name),
Result: stats.RNotFiltered,
Time: uint32(elapsed / 1000),
}
if clientID := ctx.clientID; clientID != "" {
@@ -152,9 +151,6 @@ func (s *Server) updateStats(
e.Client = clientIP
}
e.Time = uint32(elapsed / 1000)
e.Result = stats.RNotFiltered
switch res.Reason {
case filtering.FilteredSafeBrowsing:
e.Result = stats.RSafeBrowsing
@@ -162,7 +158,8 @@ func (s *Server) updateStats(
e.Result = stats.RParental
case filtering.FilteredSafeSearch:
e.Result = stats.RSafeSearch
case filtering.FilteredBlockList,
case
filtering.FilteredBlockList,
filtering.FilteredInvalid,
filtering.FilteredBlockedService:
e.Result = stats.RFiltered

View File

@@ -42,11 +42,13 @@ func (s *Server) loadUpstreams() (upstreams []string, err error) {
// prepareUpstreamSettings sets upstream DNS server settings.
func (s *Server) prepareUpstreamSettings() (err error) {
// We're setting a customized set of RootCAs. The reason is that Go default
// mechanism of loading TLS roots does not always work properly on some
// routers so we're loading roots manually and pass it here.
// Use a customized set of RootCAs, because Go's default mechanism of
// loading TLS roots does not always work properly on some routers so we're
// loading roots manually and pass it here.
//
// See [aghtls.SystemRootCAs].
//
// TODO(a.garipov): Investigate if that's true.
upstream.RootCAs = s.conf.TLSv12Roots
upstream.CipherSuites = s.conf.TLSCiphers
@@ -190,7 +192,7 @@ func (s *Server) resolveUpstreamsWithHosts(
// extractUpstreamHost returns the hostname of addr without port with an
// assumption that any address passed here has already been successfully parsed
// by [upstream.AddressToUpstream]. This function eesentially mirrors the logic
// by [upstream.AddressToUpstream]. This function essentially mirrors the logic
// of [upstream.AddressToUpstream], see TODO on [replaceUpstreamsWithHosts].
func extractUpstreamHost(addr string) (host string) {
var err error

View File

@@ -1,10 +1,7 @@
package filtering
import (
"bufio"
"bytes"
"fmt"
"hash/crc32"
"io"
"net/http"
"os"
@@ -14,6 +11,7 @@ import (
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
"github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/stringutil"
@@ -29,9 +27,9 @@ const filterDir = "filters"
// TODO(e.burkov): Use more deterministic approach.
var nextFilterID = time.Now().Unix()
// FilterYAML respresents a filter list in the configuration file.
// FilterYAML represents a filter list in the configuration file.
//
// TODO(e.burkov): Investigate if the field oredering is important.
// TODO(e.burkov): Investigate if the field ordering is important.
type FilterYAML struct {
Enabled bool
URL string // URL or a file path
@@ -213,7 +211,7 @@ func (d *DNSFilter) loadFilters(array []FilterYAML) {
err := d.load(filter)
if err != nil {
log.Error("Couldn't load filter %d contents due to %s", filter.ID, err)
log.Error("filtering: loading filter %d: %s", filter.ID, err)
}
}
}
@@ -338,7 +336,8 @@ func (d *DNSFilter) refreshFiltersArray(filters *[]FilterYAML, force bool) (int,
updateFlags = append(updateFlags, updated)
if err != nil {
nfail++
log.Printf("Failed to update filter %s: %s\n", uf.URL, err)
log.Info("filtering: updating filter from url %q: %s\n", uf.URL, err)
continue
}
}
@@ -367,7 +366,13 @@ func (d *DNSFilter) refreshFiltersArray(filters *[]FilterYAML, force bool) (int,
continue
}
log.Info("Updated filter #%d. Rules: %d -> %d", f.ID, f.RulesCount, uf.RulesCount)
log.Info(
"filtering: updated filter %d; rule count: %d (was %d)",
f.ID,
uf.RulesCount,
f.RulesCount,
)
f.Name = uf.Name
f.RulesCount = uf.RulesCount
f.checksum = uf.checksum
@@ -397,9 +402,10 @@ func (d *DNSFilter) refreshFiltersArray(filters *[]FilterYAML, force bool) (int,
//
// TODO(a.garipov, e.burkov): What the hell?
func (d *DNSFilter) refreshFiltersIntl(block, allow, force bool) (int, bool) {
log.Debug("filtering: updating...")
updNum := 0
log.Debug("filtering: starting updating")
defer func() { log.Debug("filtering: finished updating, %d updated", updNum) }()
var lists []FilterYAML
var toUpd []bool
isNetErr := false
@@ -437,131 +443,9 @@ func (d *DNSFilter) refreshFiltersIntl(block, allow, force bool) (int, bool) {
}
}
log.Debug("filtering: update finished: %d lists updated", updNum)
return updNum, false
}
// isPrintableText returns true if data is printable UTF-8 text with CR, LF, TAB
// characters.
//
// TODO(e.burkov): Investigate the purpose of this and improve the
// implementation. Perhaps, use something from the unicode package.
func isPrintableText(data string) (ok bool) {
for _, c := range []byte(data) {
if (c >= ' ' && c != 0x7f) || c == '\n' || c == '\r' || c == '\t' {
continue
}
return false
}
return true
}
// scanLinesWithBreak is essentially a [bufio.ScanLines] which keeps trailing
// line breaks.
func scanLinesWithBreak(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
return 0, nil, nil
}
if i := bytes.IndexByte(data, '\n'); i >= 0 {
return i + 1, data[0 : i+1], nil
}
if atEOF {
return len(data), data, nil
}
// Request more data.
return 0, nil, nil
}
// parseFilter copies filter's content from src to dst and returns the number of
// rules, number of bytes written, checksum, and title of the parsed list. dst
// must not be nil.
func (d *DNSFilter) parseFilter(
src io.Reader,
dst io.Writer,
) (rulesNum, written int, checksum uint32, title string, err error) {
scanner := bufio.NewScanner(src)
scanner.Split(scanLinesWithBreak)
titleFound := false
for n := 0; scanner.Scan(); written += n {
line := scanner.Text()
var isRule bool
var likelyTitle string
isRule, likelyTitle, err = d.parseFilterLine(line, !titleFound, written == 0)
if err != nil {
return 0, written, 0, "", err
}
if isRule {
rulesNum++
} else if likelyTitle != "" {
title, titleFound = likelyTitle, true
}
checksum = crc32.Update(checksum, crc32.IEEETable, []byte(line))
n, err = dst.Write([]byte(line))
if err != nil {
return 0, written, 0, "", fmt.Errorf("writing filter line: %w", err)
}
}
if err = scanner.Err(); err != nil {
return 0, written, 0, "", fmt.Errorf("scanning filter contents: %w", err)
}
return rulesNum, written, checksum, title, nil
}
// parseFilterLine returns true if the passed line is a rule. line is
// considered a rule if it's not a comment and contains no title.
func (d *DNSFilter) parseFilterLine(
line string,
lookForTitle bool,
testHTML bool,
) (isRule bool, title string, err error) {
if !isPrintableText(line) {
return false, "", errors.Error("filter contains non-printable characters")
}
line = strings.TrimSpace(line)
if line == "" || line[0] == '#' {
return false, "", nil
}
if testHTML && isHTML(line) {
return false, "", errors.Error("data is HTML, not plain text")
}
if line[0] == '!' && lookForTitle {
match := d.filterTitleRegexp.FindStringSubmatch(line)
if len(match) > 1 {
title = match[1]
}
return false, title, nil
}
return true, "", nil
}
// isHTML returns true if the line contains HTML tags instead of plain text.
// line shouldn have no leading space symbols.
//
// TODO(ameshkov): It actually gives too much false-positives. Perhaps, just
// check if trimmed string begins with angle bracket.
func isHTML(line string) (ok bool) {
line = strings.ToLower(line)
return strings.HasPrefix(line, "<html") || strings.HasPrefix(line, "<!doctype")
}
// update refreshes filter's content and a/mtimes of it's file.
func (d *DNSFilter) update(filter *FilterYAML) (b bool, err error) {
b, err = d.updateIntl(filter)
@@ -573,7 +457,7 @@ func (d *DNSFilter) update(filter *FilterYAML) (b bool, err error) {
filter.LastUpdated,
)
if chErr != nil {
log.Error("os.Chtimes(): %v", chErr)
log.Error("filtering: os.Chtimes(): %s", chErr)
}
}
@@ -582,14 +466,12 @@ func (d *DNSFilter) update(filter *FilterYAML) (b bool, err error) {
// finalizeUpdate closes and gets rid of temporary file f with filter's content
// according to updated. It also saves new values of flt's name, rules number
// and checksum if sucсeeded.
// and checksum if succeeded.
func (d *DNSFilter) finalizeUpdate(
file *os.File,
flt *FilterYAML,
updated bool,
name string,
rnum int,
cs uint32,
res *rulelist.ParseResult,
) (err error) {
tmpFileName := file.Name()
@@ -602,23 +484,24 @@ func (d *DNSFilter) finalizeUpdate(
}
if !updated {
log.Tracef("filter #%d from %s has no changes, skip", flt.ID, flt.URL)
log.Debug("filtering: filter %d from url %q has no changes, skipping", flt.ID, flt.URL)
return os.Remove(tmpFileName)
}
fltPath := flt.Path(d.DataDir)
log.Printf("saving contents of filter #%d into %s", flt.ID, fltPath)
log.Info("filtering: saving contents of filter %d into %q", flt.ID, fltPath)
// Don't use renamio or maybe packages, since those will require loading the
// whole filter content to the memory on Windows.
// Don't use renameio or maybe packages, since those will require loading
// the whole filter content to the memory on Windows.
err = os.Rename(tmpFileName, fltPath)
if err != nil {
return errors.WithDeferred(err, os.Remove(tmpFileName))
}
flt.Name, flt.checksum, flt.RulesCount = aghalg.Coalesce(flt.Name, name), cs, rnum
flt.Name = aghalg.Coalesce(flt.Name, res.Title)
flt.checksum, flt.RulesCount = res.Checksum, res.RulesCount
return nil
}
@@ -626,11 +509,9 @@ func (d *DNSFilter) finalizeUpdate(
// updateIntl updates the flt rewriting it's actual file. It returns true if
// the actual update has been performed.
func (d *DNSFilter) updateIntl(flt *FilterYAML) (ok bool, err error) {
log.Tracef("downloading update for filter %d from %s", flt.ID, flt.URL)
log.Debug("filtering: downloading update for filter %d from %q", flt.ID, flt.URL)
var name string
var rnum, n int
var cs uint32
var res *rulelist.ParseResult
var tmpFile *os.File
tmpFile, err = os.CreateTemp(filepath.Join(d.DataDir, filterDir), "")
@@ -638,9 +519,14 @@ func (d *DNSFilter) updateIntl(flt *FilterYAML) (ok bool, err error) {
return false, err
}
defer func() {
finErr := d.finalizeUpdate(tmpFile, flt, ok, name, rnum, cs)
finErr := d.finalizeUpdate(tmpFile, flt, ok, res)
if ok && finErr == nil {
log.Printf("updated filter %d: %d bytes, %d rules", flt.ID, n, rnum)
log.Info(
"filtering: updated filter %d: %d bytes, %d rules",
flt.ID,
res.BytesWritten,
res.RulesCount,
)
return
}
@@ -661,14 +547,14 @@ func (d *DNSFilter) updateIntl(flt *FilterYAML) (ok bool, err error) {
var resp *http.Response
resp, err = d.HTTPClient.Get(flt.URL)
if err != nil {
log.Printf("requesting filter from %s, skip: %s", flt.URL, err)
log.Info("filtering: requesting filter from %q: %s, skipping", flt.URL, err)
return false, err
}
defer func() { err = errors.WithDeferred(err, resp.Body.Close()) }()
if resp.StatusCode != http.StatusOK {
log.Printf("got status code %d from %s, skip", resp.StatusCode, flt.URL)
log.Info("filtering got status code %d from %q, skipping", resp.StatusCode, flt.URL)
return false, fmt.Errorf("got status code %d, want %d", resp.StatusCode, http.StatusOK)
}
@@ -685,16 +571,20 @@ func (d *DNSFilter) updateIntl(flt *FilterYAML) (ok bool, err error) {
r = f
}
rnum, n, cs, name, err = d.parseFilter(r, tmpFile)
bufPtr := d.bufPool.Get().(*[]byte)
defer d.bufPool.Put(bufPtr)
return cs != flt.checksum && err == nil, err
p := rulelist.NewParser()
res, err = p.Parse(tmpFile, r, *bufPtr)
return res.Checksum != flt.checksum && err == nil, err
}
// loads filter contents from the file in dataDir
func (d *DNSFilter) load(flt *FilterYAML) (err error) {
fileName := flt.Path(d.DataDir)
log.Debug("filtering: loading filter %d from %s", flt.ID, fileName)
log.Debug("filtering: loading filter %d from %q", flt.ID, fileName)
file, err := os.Open(fileName)
if errors.Is(err, os.ErrNotExist) {
@@ -710,14 +600,18 @@ func (d *DNSFilter) load(flt *FilterYAML) (err error) {
return fmt.Errorf("getting filter file stat: %w", err)
}
log.Debug("filtering: file %s, id %d, length %d", fileName, flt.ID, st.Size())
log.Debug("filtering: file %q, id %d, length %d", fileName, flt.ID, st.Size())
rulesCount, _, checksum, _, err := d.parseFilter(file, io.Discard)
bufPtr := d.bufPool.Get().(*[]byte)
defer d.bufPool.Put(bufPtr)
p := rulelist.NewParser()
res, err := p.Parse(io.Discard, file, *bufPtr)
if err != nil {
return fmt.Errorf("parsing filter file: %w", err)
}
flt.RulesCount, flt.checksum, flt.LastUpdated = rulesCount, checksum, st.ModTime()
flt.RulesCount, flt.checksum, flt.LastUpdated = res.RulesCount, res.Checksum, st.ModTime()
return nil
}
@@ -759,8 +653,9 @@ func (d *DNSFilter) enableFiltersLocked(async bool) {
})
}
if err := d.SetFilters(filters, allowFilters, async); err != nil {
log.Debug("enabling filters: %s", err)
err := d.setFilters(filters, allowFilters, async)
if err != nil {
log.Error("filtering: enabling filters: %s", err)
}
d.SetEnabled(d.FilteringEnabled)

View File

@@ -9,7 +9,6 @@ import (
"net/http"
"os"
"path/filepath"
"regexp"
"runtime"
"runtime/debug"
"strings"
@@ -18,6 +17,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/mathutil"
@@ -170,6 +170,15 @@ type Checker interface {
// DNSFilter matches hostnames and DNS requests against filtering rules.
type DNSFilter struct {
// bufPool is a pool of buffers used for filtering-rule list parsing.
bufPool *sync.Pool
rulesStorage *filterlist.RuleStorage
filteringEngine *urlfilter.DNSEngine
rulesStorageAllow *filterlist.RuleStorage
filteringEngineAllow *urlfilter.DNSEngine
safeSearch SafeSearch
// safeBrowsingChecker is the safe browsing hash-prefix checker.
@@ -178,12 +187,6 @@ type DNSFilter struct {
// parentalControl is the parental control hash-prefix checker.
parentalControlChecker Checker
rulesStorage *filterlist.RuleStorage
filteringEngine *urlfilter.DNSEngine
rulesStorageAllow *filterlist.RuleStorage
filteringEngineAllow *urlfilter.DNSEngine
engineLock sync.RWMutex
Config // for direct access by library users, even a = assignment
@@ -196,12 +199,6 @@ type DNSFilter struct {
refreshLock *sync.Mutex
// filterTitleRegexp is the regular expression to retrieve a name of a
// filter list.
//
// TODO(e.burkov): Don't use regexp for such a simple text processing task.
filterTitleRegexp *regexp.Regexp
hostCheckers []hostChecker
}
@@ -339,12 +336,12 @@ func cloneRewrites(entries []*LegacyRewrite) (clone []*LegacyRewrite) {
return clone
}
// SetFilters sets new filters, synchronously or asynchronously. When filters
// setFilters sets new filters, synchronously or asynchronously. When filters
// are set asynchronously, the old filters continue working until the new
// filters are ready.
//
// In this case the caller must ensure that the old filter files are intact.
func (d *DNSFilter) SetFilters(blockFilters, allowFilters []Filter, async bool) error {
func (d *DNSFilter) setFilters(blockFilters, allowFilters []Filter, async bool) error {
if async {
params := filtersInitializerParams{
allowFilters: allowFilters,
@@ -370,14 +367,7 @@ func (d *DNSFilter) SetFilters(blockFilters, allowFilters []Filter, async bool)
return nil
}
err := d.initFiltering(allowFilters, blockFilters)
if err != nil {
log.Error("filtering: can't initialize filtering subsystem: %s", err)
return err
}
return nil
return d.initFiltering(allowFilters, blockFilters)
}
// Starts initializing new filters by signal from channel
@@ -386,7 +376,8 @@ func (d *DNSFilter) filtersInitializer() {
params := <-d.filtersInitializerChan
err := d.initFiltering(params.allowFilters, params.blockFilters)
if err != nil {
log.Error("Can't initialize filtering subsystem: %s", err)
log.Error("filtering: initializing: %s", err)
continue
}
}
@@ -718,7 +709,7 @@ func newRuleStorage(filters []Filter) (rs *filterlist.RuleStorage, err error) {
}
// Initialize urlfilter objects.
func (d *DNSFilter) initFiltering(allowFilters, blockFilters []Filter) error {
func (d *DNSFilter) initFiltering(allowFilters, blockFilters []Filter) (err error) {
rulesStorage, err := newRuleStorage(blockFilters)
if err != nil {
return err
@@ -745,7 +736,8 @@ func (d *DNSFilter) initFiltering(allowFilters, blockFilters []Filter) error {
// Make sure that the OS reclaims memory as soon as possible.
debug.FreeOSMemory()
log.Debug("initialized filtering engine")
log.Debug("filtering: initialized filtering engine")
return nil
}
@@ -949,8 +941,14 @@ func InitModule() {
// be non-nil.
func New(c *Config, blockFilters []Filter) (d *DNSFilter, err error) {
d = &DNSFilter{
bufPool: &sync.Pool{
New: func() (buf any) {
bufVal := make([]byte, rulelist.DefaultRuleBufSize)
return &bufVal
},
},
refreshLock: &sync.Mutex{},
filterTitleRegexp: regexp.MustCompile(`^! Title: +(.*)$`),
safeBrowsingChecker: c.SafeBrowsingChecker,
parentalControlChecker: c.ParentalControlChecker,
}
@@ -1047,7 +1045,7 @@ func (d *DNSFilter) checkSafeBrowsing(
if log.GetLevel() >= log.DEBUG {
timer := log.StartTimer()
defer timer.LogElapsed("safebrowsing lookup for %q", host)
defer timer.LogElapsed("filtering: safebrowsing lookup for %q", host)
}
res = Result{
@@ -1079,7 +1077,7 @@ func (d *DNSFilter) checkParental(
if log.GetLevel() >= log.DEBUG {
timer := log.StartTimer()
defer timer.LogElapsed("parental lookup for %q", host)
defer timer.LogElapsed("filtering: parental lookup for %q", host)
}
res = Result{

View File

@@ -547,7 +547,7 @@ func TestWhitelist(t *testing.T) {
}}
d, setts := newForTest(t, nil, filters)
err := d.SetFilters(filters, whiteFilters, false)
err := d.setFilters(filters, whiteFilters, false)
require.NoError(t, err)
t.Cleanup(d.Close)

View File

@@ -25,7 +25,7 @@ func toCacheItem(data []byte) *cacheItem {
t := time.Unix(int64(binary.BigEndian.Uint64(data)), 0)
data = data[expirySize:]
hashes := make([]hostnameHash, len(data)/hashSize)
hashes := make([]hostnameHash, 0, len(data)/hashSize)
for i := 0; i < len(data); i += hashSize {
var hash hostnameHash
@@ -41,12 +41,13 @@ func toCacheItem(data []byte) *cacheItem {
// fromCacheItem encodes cacheItem into data.
func fromCacheItem(item *cacheItem) (data []byte) {
data = make([]byte, len(item.hashes)*hashSize+expirySize)
data = make([]byte, 0, len(item.hashes)*hashSize+expirySize)
expiry := item.expiry.Unix()
binary.BigEndian.PutUint64(data[:expirySize], uint64(expiry))
data = binary.BigEndian.AppendUint64(data, uint64(expiry))
for _, v := range item.hashes {
// nolint:looppointer // The subsilce is used for a copy.
// nolint:looppointer // The subslice of v is used for a copy.
data = append(data, v[:]...)
}
@@ -62,7 +63,7 @@ func (c *Checker) findInCache(
i := 0
for _, hash := range hashes {
// nolint:looppointer // The subsilce is used for a safe cache lookup.
// nolint:looppointer // The has subslice is used for a cache lookup.
data := c.cache.Get(hash[:prefixLen])
if data == nil {
hashes[i] = hash
@@ -97,34 +98,36 @@ func (c *Checker) storeInCache(hashesToRequest, respHashes []hostnameHash) {
for _, hash := range respHashes {
var pref prefix
// nolint:looppointer // The subsilce is used for a copy.
// nolint:looppointer // The hash subslice is used for a copy.
copy(pref[:], hash[:])
hashToStore[pref] = append(hashToStore[pref], hash)
}
for pref, hash := range hashToStore {
// nolint:looppointer // The subsilce is used for a safe cache lookup.
c.setCache(pref[:], hash)
c.setCache(pref, hash)
}
for _, hash := range hashesToRequest {
// nolint:looppointer // The subsilce is used for a safe cache lookup.
pref := hash[:prefixLen]
val := c.cache.Get(pref)
// nolint:looppointer // The hash subslice is used for a cache lookup.
val := c.cache.Get(hash[:prefixLen])
if val == nil {
var pref prefix
// nolint:looppointer // The hash subslice is used for a copy.
copy(pref[:], hash[:])
c.setCache(pref, nil)
}
}
}
// setCache stores hash in cache.
func (c *Checker) setCache(pref []byte, hashes []hostnameHash) {
func (c *Checker) setCache(pref prefix, hashes []hostnameHash) {
item := &cacheItem{
expiry: time.Now().Add(c.cacheTime),
hashes: hashes,
}
c.cache.Set(pref, fromCacheItem(item))
c.cache.Set(pref[:], fromCacheItem(item))
log.Debug("%s: stored in cache: %v", c.svc, pref)
}

View File

@@ -0,0 +1,44 @@
package hashprefix
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestCacheItem(t *testing.T) {
item := &cacheItem{
expiry: time.Unix(0x01_23_45_67_89_AB_CD_EF, 0),
hashes: []hostnameHash{{
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
}, {
0x01, 0x03, 0x05, 0x07, 0x02, 0x04, 0x06, 0x08,
0x01, 0x03, 0x05, 0x07, 0x02, 0x04, 0x06, 0x08,
0x01, 0x03, 0x05, 0x07, 0x02, 0x04, 0x06, 0x08,
0x01, 0x03, 0x05, 0x07, 0x02, 0x04, 0x06, 0x08,
}},
}
wantData := []byte{
0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF,
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
0x01, 0x03, 0x05, 0x07, 0x02, 0x04, 0x06, 0x08,
0x01, 0x03, 0x05, 0x07, 0x02, 0x04, 0x06, 0x08,
0x01, 0x03, 0x05, 0x07, 0x02, 0x04, 0x06, 0x08,
0x01, 0x03, 0x05, 0x07, 0x02, 0x04, 0x06, 0x08,
}
gotData := fromCacheItem(item)
assert.Equal(t, wantData, gotData)
newItem := toCacheItem(gotData)
gotData = fromCacheItem(newItem)
assert.Equal(t, wantData, gotData)
}

View File

@@ -173,7 +173,7 @@ func (c *Checker) getQuestion(hashes []hostnameHash) (q string) {
b := &strings.Builder{}
for _, hash := range hashes {
// nolint:looppointer // The subsilce is used for safe hex encoding.
// nolint:looppointer // The hash subslice is used for hex encoding.
stringutil.WriteToBuilder(b, hex.EncodeToString(hash[:prefixLen]), ".")
}

View File

@@ -95,7 +95,7 @@ func (d *DNSFilter) handleFilteringAddURL(w http.ResponseWriter, r *http.Request
r,
w,
http.StatusBadRequest,
"Couldn't fetch filter from url %s: %s",
"Couldn't fetch filter from URL %q: %s",
filt.URL,
err,
)

View File

@@ -122,7 +122,7 @@ func matchDomainWildcard(host, wildcard string) (ok bool) {
return isWildcard(wildcard) && strings.HasSuffix(host, wildcard[1:])
}
// legacyRewriteSortsBefore sorts rewirtes according to the following priority:
// legacyRewriteSortsBefore sorts rewrites according to the following priority:
//
// 1. A and AAAA > CNAME;
// 2. wildcard > exact;

View File

@@ -0,0 +1,9 @@
package rulelist
import "github.com/AdguardTeam/golibs/errors"
// ErrHTML is returned by [Parser.Parse] if the data is likely to be HTML.
//
// TODO(a.garipov): This error is currently returned to the UI. Stop that and
// make it all-lowercase.
const ErrHTML errors.Error = "data is HTML, not plain text"

View File

@@ -0,0 +1,198 @@
package rulelist
import (
"bufio"
"bytes"
"fmt"
"hash/crc32"
"io"
"unicode"
"unicode/utf8"
"github.com/AdguardTeam/golibs/errors"
)
// Parser is a filtering-rule parser that collects data, such as the checksum
// and the title, as well as counts rules and removes comments.
type Parser struct {
title string
rulesCount int
written int
checksum uint32
titleFound bool
}
// NewParser returns a new filtering-rule parser.
func NewParser() (p *Parser) {
return &Parser{}
}
// ParseResult contains information about the results of parsing a
// filtering-rule list by [Parser.Parse].
type ParseResult struct {
// Title is the title contained within the filtering-rule list, if any.
Title string
// RulesCount is the number of rules in the list. It excludes empty lines
// and comments.
RulesCount int
// BytesWritten is the number of bytes written to dst.
BytesWritten int
// Checksum is the CRC-32 checksum of the rules content. That is, excluding
// empty lines and comments.
Checksum uint32
}
// Parse parses data from src into dst using buf during parsing. r is never
// nil.
func (p *Parser) Parse(dst io.Writer, src io.Reader, buf []byte) (r *ParseResult, err error) {
s := bufio.NewScanner(src)
// Don't use [DefaultRuleBufSize] as the maximum size, since some
// filtering-rule lists compressed by e.g. HostlistsCompiler can have very
// large lines. The buffer optimization still works for the more common
// case of reasonably-sized lines.
//
// See https://github.com/AdguardTeam/AdGuardHome/issues/6003.
s.Buffer(buf, bufio.MaxScanTokenSize)
// Use a one-based index for lines and columns, since these errors end up in
// the frontend, and users are more familiar with one-based line and column
// indexes.
lineNum := 1
for s.Scan() {
var n int
n, err = p.processLine(dst, s.Bytes(), lineNum)
p.written += n
if err != nil {
// Don't wrap the error, because it's informative enough as is.
return p.result(), err
}
lineNum++
}
r = p.result()
err = s.Err()
return r, errors.Annotate(err, "scanning filter contents: %w")
}
// result returns the current parsing result.
func (p *Parser) result() (r *ParseResult) {
return &ParseResult{
Title: p.title,
RulesCount: p.rulesCount,
BytesWritten: p.written,
Checksum: p.checksum,
}
}
// processLine processes a single line. It may write to dst, and if it does, n
// is the number of bytes written.
func (p *Parser) processLine(dst io.Writer, line []byte, lineNum int) (n int, err error) {
trimmed := bytes.TrimSpace(line)
if p.written == 0 && isHTMLLine(trimmed) {
return 0, ErrHTML
}
badIdx, isRule := 0, false
if p.titleFound {
badIdx, isRule = parseLine(trimmed)
} else {
badIdx, isRule = p.parseLineTitle(trimmed)
}
if badIdx != -1 {
badRune, _ := utf8.DecodeRune(trimmed[badIdx:])
return 0, fmt.Errorf(
"line %d: character %d: non-printable character %q",
lineNum,
badIdx+bytes.Index(line, trimmed)+1,
badRune,
)
}
if !isRule {
return 0, nil
}
p.rulesCount++
p.checksum = crc32.Update(p.checksum, crc32.IEEETable, trimmed)
// Assume that there is generally enough space in the buffer to add a
// newline.
n, err = dst.Write(append(trimmed, '\n'))
return n, errors.Annotate(err, "writing rule line: %w")
}
// isHTMLLine returns true if line is likely an HTML line. line is assumed to
// be trimmed of whitespace characters.
func isHTMLLine(line []byte) (isHTML bool) {
return hasPrefixFold(line, []byte("<html")) || hasPrefixFold(line, []byte("<!doctype"))
}
// hasPrefixFold is a simple, best-effort prefix matcher. It may return
// incorrect results for some non-ASCII characters.
func hasPrefixFold(b, prefix []byte) (ok bool) {
l := len(prefix)
return len(b) >= l && bytes.EqualFold(b[:l], prefix)
}
// parseLine returns true if the parsed line is a filtering rule. line is
// assumed to be trimmed of whitespace characters. nonPrintIdx is the index of
// the first non-printable character, if any; if there are none, nonPrintIdx is
// -1.
//
// A line is considered a rule if it's not empty, not a comment, and contains
// only printable characters.
func parseLine(line []byte) (nonPrintIdx int, isRule bool) {
if len(line) == 0 || line[0] == '#' || line[0] == '!' {
return -1, false
}
nonPrintIdx = bytes.IndexFunc(line, isNotPrintable)
return nonPrintIdx, nonPrintIdx == -1
}
// isNotPrintable returns true if r is not a printable character that can be
// contained in a filtering rule.
func isNotPrintable(r rune) (ok bool) {
// Tab isn't included into Unicode's graphic symbols, so include it here
// explicitly.
return r != '\t' && !unicode.IsGraphic(r)
}
// parseLineTitle is like [parseLine] but additionally looks for a title. line
// is assumed to be trimmed of whitespace characters.
func (p *Parser) parseLineTitle(line []byte) (nonPrintIdx int, isRule bool) {
if len(line) == 0 || line[0] == '#' {
return -1, false
}
if line[0] != '!' {
nonPrintIdx = bytes.IndexFunc(line, isNotPrintable)
return nonPrintIdx, nonPrintIdx == -1
}
const titlePattern = "! Title: "
if !bytes.HasPrefix(line, []byte(titlePattern)) {
return -1, false
}
title := bytes.TrimSpace(line[len(titlePattern):])
if title != nil {
// Note that title can be a non-nil empty slice. Consider that normal
// and just stop looking for other titles.
p.title = string(title)
p.titleFound = true
}
return -1, false
}

View File

@@ -0,0 +1,258 @@
package rulelist_test
import (
"bufio"
"bytes"
"strings"
"testing"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/AdGuardHome/internal/filtering/rulelist"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/testutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestParser_Parse(t *testing.T) {
t.Parallel()
longRule := strings.Repeat("a", rulelist.DefaultRuleBufSize+1) + "\n"
tooLongRule := strings.Repeat("a", bufio.MaxScanTokenSize+1) + "\n"
testCases := []struct {
name string
in string
wantDst string
wantErrMsg string
wantTitle string
wantRulesNum int
wantWritten int
}{{
name: "empty",
in: "",
wantDst: "",
wantErrMsg: "",
wantTitle: "",
wantRulesNum: 0,
wantWritten: 0,
}, {
name: "html",
in: testRuleTextHTML,
wantErrMsg: rulelist.ErrHTML.Error(),
wantTitle: "",
wantRulesNum: 0,
wantWritten: 0,
}, {
name: "comments",
in: "# Comment 1\n" +
"! Comment 2\n",
wantErrMsg: "",
wantTitle: "",
wantRulesNum: 0,
wantWritten: 0,
}, {}, {
name: "rule",
in: testRuleTextBlocked,
wantDst: testRuleTextBlocked,
wantErrMsg: "",
wantRulesNum: 1,
wantTitle: "",
wantWritten: len(testRuleTextBlocked),
}, {
name: "html_in_rule",
in: testRuleTextBlocked + testRuleTextHTML,
wantDst: testRuleTextBlocked + testRuleTextHTML,
wantErrMsg: "",
wantTitle: "",
wantRulesNum: 2,
wantWritten: len(testRuleTextBlocked) + len(testRuleTextHTML),
}, {
name: "title",
in: "! Title: Test Title \n" +
"! Title: Bad, Ignored Title\n" +
testRuleTextBlocked,
wantDst: testRuleTextBlocked,
wantErrMsg: "",
wantTitle: "Test Title",
wantRulesNum: 1,
wantWritten: len(testRuleTextBlocked),
}, {
name: "bad_char",
in: "! Title: Test Title \n" +
testRuleTextBlocked +
">>>\x7F<<<",
wantDst: testRuleTextBlocked,
wantErrMsg: "line 3: " +
"character 4: " +
"non-printable character '\\x7f'",
wantTitle: "Test Title",
wantRulesNum: 1,
wantWritten: len(testRuleTextBlocked),
}, {
name: "too_long",
in: tooLongRule,
wantDst: "",
wantErrMsg: "scanning filter contents: bufio.Scanner: token too long",
wantTitle: "",
wantRulesNum: 0,
wantWritten: 0,
}, {
name: "longer_than_default",
in: longRule,
wantDst: longRule,
wantErrMsg: "",
wantTitle: "",
wantRulesNum: 1,
wantWritten: len(longRule),
}, {
name: "bad_tab_and_comment",
in: testRuleTextBadTab,
wantDst: testRuleTextBadTab,
wantErrMsg: "",
wantTitle: "",
wantRulesNum: 1,
wantWritten: len(testRuleTextBadTab),
}, {
name: "etc_hosts_tab_and_comment",
in: testRuleTextEtcHostsTab,
wantDst: testRuleTextEtcHostsTab,
wantErrMsg: "",
wantTitle: "",
wantRulesNum: 1,
wantWritten: len(testRuleTextEtcHostsTab),
}}
for _, tc := range testCases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
dst := &bytes.Buffer{}
buf := make([]byte, rulelist.DefaultRuleBufSize)
p := rulelist.NewParser()
r, err := p.Parse(dst, strings.NewReader(tc.in), buf)
require.NotNil(t, r)
testutil.AssertErrorMsg(t, tc.wantErrMsg, err)
assert.Equal(t, tc.wantDst, dst.String())
assert.Equal(t, tc.wantTitle, r.Title)
assert.Equal(t, tc.wantRulesNum, r.RulesCount)
assert.Equal(t, tc.wantWritten, r.BytesWritten)
if tc.wantWritten > 0 {
assert.NotZero(t, r.Checksum)
}
})
}
}
func TestParser_Parse_writeError(t *testing.T) {
t.Parallel()
dst := &aghtest.Writer{
OnWrite: func(b []byte) (n int, err error) {
return 1, errors.Error("test error")
},
}
buf := make([]byte, rulelist.DefaultRuleBufSize)
p := rulelist.NewParser()
r, err := p.Parse(dst, strings.NewReader(testRuleTextBlocked), buf)
require.NotNil(t, r)
testutil.AssertErrorMsg(t, "writing rule line: test error", err)
assert.Equal(t, 1, r.BytesWritten)
}
func TestParser_Parse_checksums(t *testing.T) {
t.Parallel()
const (
withoutComments = testRuleTextBlocked
withComments = "! Some comment.\n" +
" " + testRuleTextBlocked +
"# Another comment.\n"
)
buf := make([]byte, rulelist.DefaultRuleBufSize)
p := rulelist.NewParser()
r, err := p.Parse(&bytes.Buffer{}, strings.NewReader(withoutComments), buf)
require.NotNil(t, r)
require.NoError(t, err)
gotWithoutComments := r.Checksum
p = rulelist.NewParser()
r, err = p.Parse(&bytes.Buffer{}, strings.NewReader(withComments), buf)
require.NotNil(t, r)
require.NoError(t, err)
gotWithComments := r.Checksum
assert.Equal(t, gotWithoutComments, gotWithComments)
}
var (
resSink *rulelist.ParseResult
errSink error
)
func BenchmarkParser_Parse(b *testing.B) {
dst := &bytes.Buffer{}
src := strings.NewReader(strings.Repeat(testRuleTextBlocked, 1000))
buf := make([]byte, rulelist.DefaultRuleBufSize)
p := rulelist.NewParser()
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
resSink, errSink = p.Parse(dst, src, buf)
dst.Reset()
}
require.NoError(b, errSink)
require.NotNil(b, resSink)
}
func FuzzParser_Parse(f *testing.F) {
const n = 64
testCases := []string{
"",
"# Comment",
"! Comment",
"! Title ",
"! Title XXX",
testRuleTextEtcHostsTab,
testRuleTextHTML,
testRuleTextBlocked,
testRuleTextBadTab,
"1.2.3.4",
"1.2.3.4 etc-hosts.example",
">>>\x00<<<",
">>>\x7F<<<",
strings.Repeat("a", n+1),
}
for _, tc := range testCases {
f.Add(tc)
}
buf := make([]byte, n)
f.Fuzz(func(t *testing.T, input string) {
require.Eventually(t, func() (ok bool) {
dst := &bytes.Buffer{}
src := strings.NewReader(input)
p := rulelist.NewParser()
r, _ := p.Parse(dst, src, buf)
require.NotNil(t, r)
return true
}, testTimeout, testTimeout/100)
})
}

View File

@@ -0,0 +1,9 @@
// Package rulelist contains the implementation of the standard rule-list
// filter that wraps an urlfilter filtering-engine.
//
// TODO(a.garipov): Expand.
package rulelist
// DefaultRuleBufSize is the default length of a buffer used to read a line with
// a filtering rule, in bytes.
const DefaultRuleBufSize = 1024

View File

@@ -0,0 +1,14 @@
package rulelist_test
import "time"
// testTimeout is the common timeout for tests.
const testTimeout = 1 * time.Second
// Common texts for tests.
const (
testRuleTextHTML = "<!DOCTYPE html>\n"
testRuleTextBlocked = "||blocked.example^\n"
testRuleTextBadTab = "||bad-tab-and-comment.example^\t# A comment.\n"
testRuleTextEtcHostsTab = "0.0.0.0 tab..example^\t# A comment.\n"
)

View File

@@ -1505,6 +1505,7 @@ var blockedServices = []blockedService{{
"||aus.social^",
"||awscommunity.social^",
"||climatejustice.social^",
"||cupoftea.social^",
"||cyberplace.social^",
"||defcon.social^",
"||det.social^",
@@ -1530,6 +1531,7 @@ var blockedServices = []blockedService{{
"||masto.pt^",
"||mastodon.au^",
"||mastodon.bida.im^",
"||mastodon.com.tr^",
"||mastodon.eus^",
"||mastodon.green^",
"||mastodon.ie^",
@@ -1551,11 +1553,11 @@ var blockedServices = []blockedService{{
"||mastodont.cat^",
"||mastodontech.de^",
"||mastodontti.fi^",
"||mastouille.fr^",
"||mathstodon.xyz^",
"||metalhead.club^",
"||mindly.social^",
"||mstdn.ca^",
"||mstdn.jp^",
"||mstdn.party^",
"||mstdn.plus^",
"||mstdn.social^",
@@ -1567,7 +1569,6 @@ var blockedServices = []blockedService{{
"||nrw.social^",
"||o3o.ca^",
"||ohai.social^",
"||pewtix.com^",
"||piaille.fr^",
"||pol.social^",
"||ravenation.club^",
@@ -1582,20 +1583,19 @@ var blockedServices = []blockedService{{
"||social.linux.pizza^",
"||social.politicaconciencia.org^",
"||social.vivaldi.net^",
"||sself.co^",
"||stranger.social^",
"||sueden.social^",
"||tech.lgbt^",
"||techhub.social^",
"||theblower.au^",
"||tkz.one^",
"||todon.eu^",
"||toot.aquilenet.fr^",
"||toot.community^",
"||toot.funami.tech^",
"||toot.io^",
"||toot.wales^",
"||troet.cafe^",
"||twingyeo.kr^",
"||union.place^",
"||universeodon.com^",
"||urbanists.social^",

145
internal/home/clientaddr.go Normal file
View File

@@ -0,0 +1,145 @@
package home
import (
"context"
"net/netip"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/rdns"
"github.com/AdguardTeam/AdGuardHome/internal/whois"
"github.com/AdguardTeam/golibs/log"
)
// TODO(a.garipov): It is currently hard to add tests for this structure due to
// strong coupling between it and Context.dnsServer with Context.clients.
// Resolve this coupling and add proper testing.
// clientAddrProcessor processes incoming client addresses with rDNS and WHOIS,
// if configured.
type clientAddrProcessor struct {
rdns rdns.Interface
whois whois.Interface
}
const (
// defaultQueueSize is the size of queue of IPs for rDNS and WHOIS
// processing.
defaultQueueSize = 255
// defaultCacheSize is the maximum size of the cache for rDNS and WHOIS
// processing. It must be greater than zero.
defaultCacheSize = 10_000
// defaultIPTTL is the Time to Live duration for IP addresses cached by
// rDNS and WHOIS.
defaultIPTTL = 1 * time.Hour
)
// newClientAddrProcessor returns a new client address processor. c must not be
// nil.
func newClientAddrProcessor(c *clientSourcesConfig) (p *clientAddrProcessor) {
p = &clientAddrProcessor{
rdns: &rdns.Empty{},
whois: &whois.Empty{},
}
if c.RDNS {
p.rdns = rdns.New(&rdns.Config{
Exchanger: Context.dnsServer,
CacheSize: defaultCacheSize,
CacheTTL: defaultIPTTL,
})
}
if c.WHOIS {
// TODO(s.chzhen): Consider making configurable.
const (
// defaultTimeout is the timeout for WHOIS requests.
defaultTimeout = 5 * time.Second
// defaultMaxConnReadSize is an upper limit in bytes for reading from a
// net.Conn.
defaultMaxConnReadSize = 64 * 1024
// defaultMaxRedirects is the maximum redirects count.
defaultMaxRedirects = 5
// defaultMaxInfoLen is the maximum length of whois.Info fields.
defaultMaxInfoLen = 250
)
p.whois = whois.New(&whois.Config{
DialContext: customDialContext,
ServerAddr: whois.DefaultServer,
Port: whois.DefaultPort,
Timeout: defaultTimeout,
CacheSize: defaultCacheSize,
MaxConnReadSize: defaultMaxConnReadSize,
MaxRedirects: defaultMaxRedirects,
MaxInfoLen: defaultMaxInfoLen,
CacheTTL: defaultIPTTL,
})
}
return p
}
// process processes the incoming client IP-address information. It is intended
// to be used as a goroutine. Once clientIPs is closed, process exits.
func (p *clientAddrProcessor) process(clientIPs <-chan netip.Addr) {
defer log.OnPanic("clientAddrProcessor.process")
log.Info("home: processing client addresses")
for ip := range clientIPs {
p.processRDNS(ip)
p.processWHOIS(ip)
}
log.Info("home: finished processing client addresses")
}
// processRDNS resolves the clients' IP addresses using reverse DNS.
func (p *clientAddrProcessor) processRDNS(ip netip.Addr) {
start := time.Now()
log.Debug("home: processing client %s with rdns", ip)
defer func() {
log.Debug("home: finished processing client %s with rdns in %s", ip, time.Since(start))
}()
ok := Context.dnsServer.ShouldResolveClient(ip)
if !ok {
return
}
host, changed := p.rdns.Process(ip)
if host == "" || !changed {
return
}
ok = Context.clients.AddHost(ip, host, ClientSourceRDNS)
if ok {
return
}
log.Debug("dns: setting rdns info for client %q: already set with higher priority source", ip)
}
// processWHOIS looks up the information aobut clients' IP addresses in the
// WHOIS databases.
func (p *clientAddrProcessor) processWHOIS(ip netip.Addr) {
start := time.Now()
log.Debug("home: processing client %s with whois", ip)
defer func() {
log.Debug("home: finished processing client %s with whois in %s", ip, time.Since(start))
}()
// TODO(s.chzhen): Move the timeout logic from WHOIS configuration to the
// context.
info, changed := p.whois.Process(context.Background(), ip)
if info == nil || !changed {
return
}
Context.clients.setWHOISInfo(ip, info)
}

View File

@@ -141,7 +141,7 @@ func (clients *clientsContainer) handleHostsUpdates() {
}
}
// webHandlersRegistered prevents a [clientsContainer] from regisering its web
// webHandlersRegistered prevents a [clientsContainer] from registering its web
// handlers more than once.
//
// TODO(a.garipov): Refactor HTTP handler registration logic.

View File

@@ -30,32 +30,30 @@ import (
const dataDir = "data"
// logSettings are the logging settings part of the configuration file.
//
// TODO(a.garipov): Put them into a separate object.
type logSettings struct {
// File is the path to the log file. If empty, logs are written to stdout.
// If "syslog", logs are written to syslog.
File string `yaml:"log_file"`
File string `yaml:"file"`
// MaxBackups is the maximum number of old log files to retain.
//
// NOTE: MaxAge may still cause them to get deleted.
MaxBackups int `yaml:"log_max_backups"`
MaxBackups int `yaml:"max_backups"`
// MaxSize is the maximum size of the log file before it gets rotated, in
// megabytes. The default value is 100 MB.
MaxSize int `yaml:"log_max_size"`
MaxSize int `yaml:"max_size"`
// MaxAge is the maximum duration for retaining old log files, in days.
MaxAge int `yaml:"log_max_age"`
MaxAge int `yaml:"max_age"`
// Compress determines, if the rotated log files should be compressed using
// gzip.
Compress bool `yaml:"log_compress"`
Compress bool `yaml:"compress"`
// LocalTime determines, if the time used for formatting the timestamps in
// is the computer's local time.
LocalTime bool `yaml:"log_localtime"`
LocalTime bool `yaml:"local_time"`
// Verbose determines, if verbose (aka debug) logging is enabled.
Verbose bool `yaml:"verbose"`
@@ -142,7 +140,8 @@ type configuration struct {
// Keep this field sorted to ensure consistent ordering.
Clients *clientsConfig `yaml:"clients"`
logSettings `yaml:",inline"`
// Log is a block with log configuration settings.
Log logSettings `yaml:"log"`
OSConfig *osConfig `yaml:"os"`
@@ -241,6 +240,7 @@ type tlsConfigSettings struct {
type queryLogConfig struct {
// Ignored is the list of host names, which should not be written to log.
// "." is considered to be the root domain.
Ignored []string `yaml:"ignored"`
// Interval is the interval for query log's files rotation.
@@ -390,7 +390,7 @@ var config = &configuration{
HostsFile: true,
},
},
logSettings: logSettings{
Log: logSettings{
Compress: false,
LocalTime: false,
MaxBackups: 0,
@@ -421,19 +421,19 @@ func (c *configuration) getConfigFilename() string {
// separate method in order to configure logger before the actual configuration
// is parsed and applied.
func readLogSettings() (ls *logSettings) {
ls = &logSettings{}
conf := &configuration{}
yamlFile, err := readConfigFile()
if err != nil {
return ls
return &logSettings{}
}
err = yaml.Unmarshal(yamlFile, ls)
err = yaml.Unmarshal(yamlFile, conf)
if err != nil {
log.Error("Couldn't get logging settings from the configuration: %s", err)
}
return ls
return &conf.Log
}
// validateBindHosts returns error if any of binding hosts from configuration is

View File

@@ -18,8 +18,6 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/filtering"
"github.com/AdguardTeam/AdGuardHome/internal/querylog"
"github.com/AdguardTeam/AdGuardHome/internal/stats"
"github.com/AdguardTeam/AdGuardHome/internal/whois"
"github.com/AdguardTeam/dnsproxy/proxy"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/netutil"
@@ -153,90 +151,32 @@ func initDNSServer(
Context.clients.dnsServer = Context.dnsServer
dnsConf, err := generateServerConfig(tlsConf, httpReg)
dnsConf, err := newServerConfig(tlsConf, httpReg)
if err != nil {
closeDNSServer()
return fmt.Errorf("generateServerConfig: %w", err)
return fmt.Errorf("newServerConfig: %w", err)
}
err = Context.dnsServer.Prepare(&dnsConf)
err = Context.dnsServer.Prepare(dnsConf)
if err != nil {
closeDNSServer()
return fmt.Errorf("dnsServer.Prepare: %w", err)
}
if config.Clients.Sources.RDNS {
Context.rdns = NewRDNS(Context.dnsServer, &Context.clients, config.DNS.UsePrivateRDNS)
}
clientIPs := dnsConf.ClientIPs
addrProc := newClientAddrProcessor(config.Clients.Sources)
go addrProc.process(clientIPs)
initWHOIS()
const topClientsNumber = 100
for _, ip := range Context.stats.TopClientsIP(topClientsNumber) {
clientIPs <- ip
}
return nil
}
// initWHOIS initializes the WHOIS.
//
// TODO(s.chzhen): Consider making configurable.
func initWHOIS() {
const (
// defaultQueueSize is the size of queue of IPs for WHOIS processing.
defaultQueueSize = 255
// defaultTimeout is the timeout for WHOIS requests.
defaultTimeout = 5 * time.Second
// defaultCacheSize is the maximum size of the cache. If it's zero,
// cache size is unlimited.
defaultCacheSize = 10_000
// defaultMaxConnReadSize is an upper limit in bytes for reading from
// net.Conn.
defaultMaxConnReadSize = 64 * 1024
// defaultMaxRedirects is the maximum redirects count.
defaultMaxRedirects = 5
// defaultMaxInfoLen is the maximum length of whois.Info fields.
defaultMaxInfoLen = 250
// defaultIPTTL is the Time to Live duration for cached IP addresses.
defaultIPTTL = 1 * time.Hour
)
Context.whoisCh = make(chan netip.Addr, defaultQueueSize)
var w whois.Interface
if config.Clients.Sources.WHOIS {
w = whois.New(&whois.Config{
DialContext: customDialContext,
ServerAddr: whois.DefaultServer,
Port: whois.DefaultPort,
Timeout: defaultTimeout,
CacheSize: defaultCacheSize,
MaxConnReadSize: defaultMaxConnReadSize,
MaxRedirects: defaultMaxRedirects,
MaxInfoLen: defaultMaxInfoLen,
CacheTTL: defaultIPTTL,
})
} else {
w = whois.Empty{}
}
go func() {
defer log.OnPanic("whois")
for ip := range Context.whoisCh {
info, changed := w.Process(context.Background(), ip)
if info != nil && changed {
Context.clients.setWHOISInfo(ip, info)
}
}
}()
}
// parseSubnetSet parses a slice of subnets. If the slice is empty, it returns
// a subnet set that matches all locally served networks, see
// [netutil.IsLocallyServed].
@@ -267,21 +207,6 @@ func isRunning() bool {
return Context.dnsServer != nil && Context.dnsServer.IsRunning()
}
func onDNSRequest(pctx *proxy.DNSContext) {
ip := netutil.NetAddrToAddrPort(pctx.Addr).Addr()
if ip == (netip.Addr{}) {
// This would be quite weird if we get here.
return
}
srcs := config.Clients.Sources
if srcs.RDNS && !ip.IsLoopback() {
Context.rdns.Begin(ip)
}
Context.whoisCh <- ip
}
func ipsToTCPAddrs(ips []netip.Addr, port int) (tcpAddrs []*net.TCPAddr) {
if ips == nil {
return nil
@@ -308,19 +233,20 @@ func ipsToUDPAddrs(ips []netip.Addr, port int) (udpAddrs []*net.UDPAddr) {
return udpAddrs
}
func generateServerConfig(
func newServerConfig(
tlsConf *tlsConfigSettings,
httpReg aghhttp.RegisterFunc,
) (newConf dnsforward.ServerConfig, err error) {
) (newConf *dnsforward.ServerConfig, err error) {
dnsConf := config.DNS
hosts := aghalg.CoalesceSlice(dnsConf.BindHosts, []netip.Addr{netutil.IPv4Localhost()})
newConf = dnsforward.ServerConfig{
clientIPs := make(chan netip.Addr, defaultQueueSize)
newConf = &dnsforward.ServerConfig{
UDPListenAddrs: ipsToUDPAddrs(hosts, dnsConf.Port),
TCPListenAddrs: ipsToTCPAddrs(hosts, dnsConf.Port),
FilteringConfig: dnsConf.FilteringConfig,
ConfigModified: onConfigModified,
HTTPRegister: httpReg,
OnDNSRequest: onDNSRequest,
ClientIPs: clientIPs,
UseDNS64: config.DNS.UseDNS64,
DNS64Prefixes: config.DNS.DNS64Prefixes,
}
@@ -344,9 +270,9 @@ func generateServerConfig(
if tlsConf.PortDNSCrypt != 0 {
newConf.DNSCryptConfig, err = newDNSCrypt(hosts, *tlsConf)
if err != nil {
// Don't wrap the error, because it's already
// wrapped by newDNSCrypt.
return dnsforward.ServerConfig{}, err
// Don't wrap the error, because it's already wrapped by
// newDNSCrypt.
return nil, err
}
}
}
@@ -515,35 +441,26 @@ func startDNSServer() error {
Context.stats.Start()
Context.queryLog.Start()
const topClientsNumber = 100 // the number of clients to get
for _, ip := range Context.stats.TopClientsIP(topClientsNumber) {
srcs := config.Clients.Sources
if srcs.RDNS && !ip.IsLoopback() {
Context.rdns.Begin(ip)
}
Context.whoisCh <- ip
}
return nil
}
func reconfigureDNSServer() (err error) {
var newConf dnsforward.ServerConfig
tlsConf := &tlsConfigSettings{}
Context.tls.WriteDiskConfig(tlsConf)
newConf, err = generateServerConfig(tlsConf, httpRegister)
newConf, err := newServerConfig(tlsConf, httpRegister)
if err != nil {
return fmt.Errorf("generating forwarding dns server config: %w", err)
}
err = Context.dnsServer.Reconfigure(&newConf)
err = Context.dnsServer.Reconfigure(newConf)
if err != nil {
return fmt.Errorf("starting forwarding dns server: %w", err)
}
addrProc := newClientAddrProcessor(config.Clients.Sources)
go addrProc.process(newConf.ClientIPs)
return nil
}

View File

@@ -56,7 +56,6 @@ type homeContext struct {
stats stats.Interface // statistics module
queryLog querylog.QueryLog // query log module
dnsServer *dnsforward.Server // DNS module
rdns *RDNS // rDNS module
dhcpServer dhcpd.Interface // DHCP module
auth *Auth // HTTP authentication module
filters *filtering.DNSFilter // DNS filtering module
@@ -83,9 +82,6 @@ type homeContext struct {
client *http.Client
appSignalChannel chan os.Signal // Channel for receiving OS signals by the console app
// whoisCh is the channel for receiving IPs for WHOIS processing.
whoisCh chan netip.Addr
// tlsCipherIDs are the ID of the cipher suites that AdGuard Home must use.
tlsCipherIDs []uint16
@@ -468,7 +464,7 @@ func setupDNSFilteringConf(conf *filtering.Config) (err error) {
ServiceName: pcService,
TXTSuffix: pcTXTSuffix,
CacheTime: cacheTime,
CacheSize: conf.SafeBrowsingCacheSize,
CacheSize: conf.ParentalCacheSize,
})
conf.SafeSearchConf.CustomResolver = safeSearchResolver{}
@@ -632,10 +628,10 @@ func run(opts options, clientBuildFS fs.FS) {
Context.tls.start()
go func() {
sErr := startDNSServer()
if sErr != nil {
startErr := startDNSServer()
if startErr != nil {
closeDNSServer()
fatalOnError(sErr)
fatalOnError(startErr)
}
}()
@@ -829,20 +825,21 @@ func configureLogger(opts options) (err error) {
// getLogSettings returns a log settings object properly initialized from opts.
func getLogSettings(opts options) (ls *logSettings) {
ls = readLogSettings()
configLogSettings := config.Log
// Command-line arguments can override config settings.
if opts.verbose || config.Verbose {
if opts.verbose || configLogSettings.Verbose {
ls.Verbose = true
}
ls.File = stringutil.Coalesce(opts.logFile, config.File, ls.File)
ls.File = stringutil.Coalesce(opts.logFile, configLogSettings.File, ls.File)
// Handle default log settings overrides.
ls.Compress = config.Compress
ls.LocalTime = config.LocalTime
ls.MaxBackups = config.MaxBackups
ls.MaxSize = config.MaxSize
ls.MaxAge = config.MaxAge
ls.Compress = configLogSettings.Compress
ls.LocalTime = configLogSettings.LocalTime
ls.MaxBackups = configLogSettings.MaxBackups
ls.MaxSize = configLogSettings.MaxSize
ls.MaxAge = configLogSettings.MaxAge
if opts.runningAsService && ls.File == "" && runtime.GOOS == "windows" {
// When running as a Windows service, use eventlog by default if

View File

@@ -1,143 +0,0 @@
package home
import (
"encoding/binary"
"net/netip"
"sync/atomic"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/dnsforward"
"github.com/AdguardTeam/golibs/cache"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
)
// RDNS resolves clients' addresses to enrich their metadata.
type RDNS struct {
exchanger dnsforward.RDNSExchanger
clients *clientsContainer
// ipCh used to pass client's IP to rDNS workerLoop.
ipCh chan netip.Addr
// ipCache caches the IP addresses to be resolved by rDNS. The resolved
// address stays here while it's inside clients. After leaving clients the
// address will be resolved once again. If the address couldn't be
// resolved, cache prevents further attempts to resolve it for some time.
ipCache cache.Cache
// usePrivate stores the state of current private reverse-DNS resolving
// settings.
usePrivate atomic.Bool
}
// Default AdGuard Home reverse DNS values.
const (
revDNSCacheSize = 10000
// TODO(e.burkov): Make these values configurable.
revDNSCacheTTL = 24 * 60 * 60
revDNSFailureCacheTTL = 1 * 60 * 60
revDNSQueueSize = 256
)
// NewRDNS creates and returns initialized RDNS.
func NewRDNS(
exchanger dnsforward.RDNSExchanger,
clients *clientsContainer,
usePrivate bool,
) (rDNS *RDNS) {
rDNS = &RDNS{
exchanger: exchanger,
clients: clients,
ipCache: cache.New(cache.Config{
EnableLRU: true,
MaxCount: revDNSCacheSize,
}),
ipCh: make(chan netip.Addr, revDNSQueueSize),
}
rDNS.usePrivate.Store(usePrivate)
go rDNS.workerLoop()
return rDNS
}
// ensurePrivateCache ensures that the state of the RDNS cache is consistent
// with the current private client RDNS resolving settings.
//
// TODO(e.burkov): Clearing cache each time this value changed is not a perfect
// approach since only unresolved locally-served addresses should be removed.
// Implement when improving the cache.
func (r *RDNS) ensurePrivateCache() {
usePrivate := r.exchanger.ResolvesPrivatePTR()
if r.usePrivate.CompareAndSwap(!usePrivate, usePrivate) {
r.ipCache.Clear()
}
}
// isCached returns true if ip is already cached and not expired yet. It also
// caches it otherwise.
func (r *RDNS) isCached(ip netip.Addr) (ok bool) {
ipBytes := ip.AsSlice()
now := uint64(time.Now().Unix())
if expire := r.ipCache.Get(ipBytes); len(expire) != 0 {
return binary.BigEndian.Uint64(expire) > now
}
return false
}
// cache caches the ip address for ttl seconds.
func (r *RDNS) cache(ip netip.Addr, ttl uint64) {
ipData := ip.AsSlice()
ttlData := [8]byte{}
binary.BigEndian.PutUint64(ttlData[:], uint64(time.Now().Unix())+ttl)
r.ipCache.Set(ipData, ttlData[:])
}
// Begin adds the ip to the resolving queue if it is not cached or already
// resolved.
func (r *RDNS) Begin(ip netip.Addr) {
r.ensurePrivateCache()
if r.isCached(ip) || r.clients.clientSource(ip) > ClientSourceRDNS {
return
}
select {
case r.ipCh <- ip:
log.Debug("rdns: %q added to queue", ip)
default:
log.Debug("rdns: queue is full")
}
}
// workerLoop handles incoming IP addresses from ipChan and adds it into
// clients.
func (r *RDNS) workerLoop() {
defer log.OnPanic("rdns")
for ip := range r.ipCh {
ttl := uint64(revDNSCacheTTL)
host, err := r.exchanger.Exchange(ip.AsSlice())
if err != nil {
log.Debug("rdns: resolving %q: %s", ip, err)
if errors.Is(err, dnsforward.ErrRDNSFailed) {
// Cache failure for a less time.
ttl = revDNSFailureCacheTTL
}
}
r.cache(ip, ttl)
if host != "" {
_ = r.clients.AddHost(ip, host, ClientSourceRDNS)
}
}
}

View File

@@ -1,264 +0,0 @@
package home
import (
"bytes"
"encoding/binary"
"fmt"
"net"
"net/netip"
"sync"
"testing"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghalg"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/dnsproxy/upstream"
"github.com/AdguardTeam/golibs/cache"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/golibs/stringutil"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRDNS_Begin(t *testing.T) {
aghtest.ReplaceLogLevel(t, log.DEBUG)
w := &bytes.Buffer{}
aghtest.ReplaceLogWriter(t, w)
ip1234, ip1235 := netip.MustParseAddr("1.2.3.4"), netip.MustParseAddr("1.2.3.5")
testCases := []struct {
cliIDIndex map[string]*Client
customChan chan netip.Addr
name string
wantLog string
ip netip.Addr
wantCacheHit int
wantCacheMiss int
}{{
cliIDIndex: map[string]*Client{},
customChan: nil,
name: "cached",
wantLog: "",
ip: ip1234,
wantCacheHit: 1,
wantCacheMiss: 0,
}, {
cliIDIndex: map[string]*Client{},
customChan: nil,
name: "not_cached",
wantLog: "rdns: queue is full",
ip: ip1235,
wantCacheHit: 0,
wantCacheMiss: 1,
}, {
cliIDIndex: map[string]*Client{"1.2.3.5": {}},
customChan: nil,
name: "already_in_clients",
wantLog: "",
ip: ip1235,
wantCacheHit: 0,
wantCacheMiss: 1,
}, {
cliIDIndex: map[string]*Client{},
customChan: make(chan netip.Addr, 1),
name: "add_to_queue",
wantLog: `rdns: "1.2.3.5" added to queue`,
ip: ip1235,
wantCacheHit: 0,
wantCacheMiss: 1,
}}
for _, tc := range testCases {
w.Reset()
ipCache := cache.New(cache.Config{
EnableLRU: true,
MaxCount: revDNSCacheSize,
})
ttl := make([]byte, binary.Size(uint64(0)))
binary.BigEndian.PutUint64(ttl, uint64(time.Now().Add(100*time.Hour).Unix()))
rdns := &RDNS{
ipCache: ipCache,
exchanger: &rDNSExchanger{
ex: aghtest.NewErrorUpstream(),
},
clients: &clientsContainer{
list: map[string]*Client{},
idIndex: tc.cliIDIndex,
ipToRC: map[netip.Addr]*RuntimeClient{},
allTags: stringutil.NewSet(),
},
}
ipCache.Clear()
ipCache.Set(net.IP{1, 2, 3, 4}, ttl)
if tc.customChan != nil {
rdns.ipCh = tc.customChan
defer close(tc.customChan)
}
t.Run(tc.name, func(t *testing.T) {
rdns.Begin(tc.ip)
assert.Equal(t, tc.wantCacheHit, ipCache.Stats().Hit)
assert.Equal(t, tc.wantCacheMiss, ipCache.Stats().Miss)
assert.Contains(t, w.String(), tc.wantLog)
})
}
}
// rDNSExchanger is a mock dnsforward.RDNSExchanger implementation for tests.
type rDNSExchanger struct {
ex upstream.Upstream
usePrivate bool
}
// Exchange implements dnsforward.RDNSExchanger interface for *RDNSExchanger.
func (e *rDNSExchanger) Exchange(ip net.IP) (host string, err error) {
rev, err := netutil.IPToReversedAddr(ip)
if err != nil {
return "", fmt.Errorf("reversing ip: %w", err)
}
req := &dns.Msg{
Question: []dns.Question{{
Name: dns.Fqdn(rev),
Qclass: dns.ClassINET,
Qtype: dns.TypePTR,
}},
}
resp, err := e.ex.Exchange(req)
if err != nil {
return "", err
}
if len(resp.Answer) == 0 {
return "", nil
}
return resp.Answer[0].Header().Name, nil
}
// Exchange implements dnsforward.RDNSExchanger interface for *RDNSExchanger.
func (e *rDNSExchanger) ResolvesPrivatePTR() (ok bool) {
return e.usePrivate
}
func TestRDNS_ensurePrivateCache(t *testing.T) {
data := []byte{1, 2, 3, 4}
ipCache := cache.New(cache.Config{
EnableLRU: true,
MaxCount: revDNSCacheSize,
})
ex := &rDNSExchanger{
ex: aghtest.NewErrorUpstream(),
}
rdns := &RDNS{
ipCache: ipCache,
exchanger: ex,
}
rdns.ipCache.Set(data, data)
require.NotZero(t, rdns.ipCache.Stats().Count)
ex.usePrivate = !ex.usePrivate
rdns.ensurePrivateCache()
require.Zero(t, rdns.ipCache.Stats().Count)
}
func TestRDNS_WorkerLoop(t *testing.T) {
aghtest.ReplaceLogLevel(t, log.DEBUG)
w := &bytes.Buffer{}
aghtest.ReplaceLogWriter(t, w)
localIP := netip.MustParseAddr("192.168.1.1")
revIPv4, err := netutil.IPToReversedAddr(localIP.AsSlice())
require.NoError(t, err)
revIPv6, err := netutil.IPToReversedAddr(net.ParseIP("2a00:1450:400c:c06::93"))
require.NoError(t, err)
locUpstream := &aghtest.UpstreamMock{
OnAddress: func() (addr string) { return "local.upstream.example" },
OnExchange: func(req *dns.Msg) (resp *dns.Msg, err error) {
return aghalg.Coalesce(
aghtest.MatchedResponse(req, dns.TypePTR, revIPv4, "local.domain"),
aghtest.MatchedResponse(req, dns.TypePTR, revIPv6, "ipv6.domain"),
new(dns.Msg).SetRcode(req, dns.RcodeNameError),
), nil
},
}
errUpstream := aghtest.NewErrorUpstream()
testCases := []struct {
ups upstream.Upstream
cliIP netip.Addr
wantLog string
name string
wantClientSource clientSource
}{{
ups: locUpstream,
cliIP: localIP,
wantLog: "",
name: "all_good",
wantClientSource: ClientSourceRDNS,
}, {
ups: errUpstream,
cliIP: netip.MustParseAddr("192.168.1.2"),
wantLog: `rdns: resolving "192.168.1.2": test upstream error`,
name: "resolve_error",
wantClientSource: ClientSourceNone,
}, {
ups: locUpstream,
cliIP: netip.MustParseAddr("2a00:1450:400c:c06::93"),
wantLog: "",
name: "ipv6_good",
wantClientSource: ClientSourceRDNS,
}}
for _, tc := range testCases {
w.Reset()
cc := newClientsContainer(t)
ch := make(chan netip.Addr)
rdns := &RDNS{
exchanger: &rDNSExchanger{
ex: tc.ups,
},
clients: cc,
ipCh: ch,
ipCache: cache.New(cache.Config{
EnableLRU: true,
MaxCount: revDNSCacheSize,
}),
}
t.Run(tc.name, func(t *testing.T) {
var wg sync.WaitGroup
wg.Add(1)
go func() {
rdns.workerLoop()
wg.Done()
}()
ch <- tc.cliIP
close(ch)
wg.Wait()
if tc.wantLog != "" {
assert.Contains(t, w.String(), tc.wantLog)
}
assert.Equal(t, tc.wantClientSource, cc.clientSource(tc.cliIP))
})
}
}

View File

@@ -23,7 +23,7 @@ import (
)
// currentSchemaVersion is the current schema version.
const currentSchemaVersion = 23
const currentSchemaVersion = 24
// These aliases are provided for convenience.
type (
@@ -98,6 +98,7 @@ func upgradeConfigSchema(oldVersion int, diskConf yobj) (err error) {
upgradeSchema20to21,
upgradeSchema21to22,
upgradeSchema22to23,
upgradeSchema23to24,
}
n := 0
@@ -1325,6 +1326,110 @@ func upgradeSchema22to23(diskConf yobj) (err error) {
return nil
}
// upgradeSchema23to24 performs the following changes:
//
// # BEFORE:
// 'log_file': ""
// 'log_max_backups': 0
// 'log_max_size': 100
// 'log_max_age': 3
// 'log_compress': false
// 'log_localtime': false
// 'verbose': false
//
// # AFTER:
// 'log':
// 'file': ""
// 'max_backups': 0
// 'max_size': 100
// 'max_age': 3
// 'compress': false
// 'local_time': false
// 'verbose': false
func upgradeSchema23to24(diskConf yobj) (err error) {
log.Printf("Upgrade yaml: 23 to 24")
diskConf["schema_version"] = 24
logObj := yobj{}
err = coalesceError(
moveField[string](diskConf, logObj, "log_file", "file"),
moveField[int](diskConf, logObj, "log_max_backups", "max_backups"),
moveField[int](diskConf, logObj, "log_max_size", "max_size"),
moveField[int](diskConf, logObj, "log_max_age", "max_age"),
moveField[bool](diskConf, logObj, "log_compress", "compress"),
moveField[bool](diskConf, logObj, "log_localtime", "local_time"),
moveField[bool](diskConf, logObj, "verbose", "verbose"),
)
if err != nil {
// Don't wrap the error, because it's informative enough as is.
return err
}
if len(logObj) != 0 {
diskConf["log"] = logObj
}
delete(diskConf, "log_file")
delete(diskConf, "log_max_backups")
delete(diskConf, "log_max_size")
delete(diskConf, "log_max_age")
delete(diskConf, "log_compress")
delete(diskConf, "log_localtime")
delete(diskConf, "verbose")
return nil
}
// moveField gets field value for key from diskConf, and then set this value
// in newConf for newKey.
func moveField[T any](diskConf, newConf yobj, key, newKey string) (err error) {
ok, newVal, err := fieldValue[T](diskConf, key)
if !ok {
return err
}
switch v := newVal.(type) {
case int, bool, string:
newConf[newKey] = v
default:
return fmt.Errorf("invalid type of %s: %T", key, newVal)
}
return nil
}
// fieldValue returns the value of type T for key in diskConf object.
func fieldValue[T any](diskConf yobj, key string) (ok bool, field any, err error) {
fieldVal, ok := diskConf[key]
if !ok {
return false, new(T), nil
}
f, ok := fieldVal.(T)
if !ok {
return false, nil, fmt.Errorf("unexpected type of %s: %T", key, fieldVal)
}
return true, f, nil
}
// coalesceError returns the first non-nil error. It is named after function
// COALESCE in SQL. If all errors are nil, it returns nil.
//
// TODO(a.garipov): Consider a similar helper to group errors together to show
// as many errors as possible.
//
// TODO(a.garipov): Think of ways to merge with [aghalg.Coalesce].
func coalesceError(errors ...error) (res error) {
for _, err := range errors {
if err != nil {
return err
}
}
return nil
}
// TODO(a.garipov): Replace with log.Output when we port it to our logging
// package.
func funcName() string {

View File

@@ -1306,3 +1306,76 @@ func TestUpgradeSchema22to23(t *testing.T) {
})
}
}
func TestUpgradeSchema23to24(t *testing.T) {
const newSchemaVer = 24
testCases := []struct {
in yobj
want yobj
name string
wantErrMsg string
}{{
name: "empty",
in: yobj{},
want: yobj{
"schema_version": newSchemaVer,
},
wantErrMsg: "",
}, {
name: "ok",
in: yobj{
"log_file": "/test/path.log",
"log_max_backups": 1,
"log_max_size": 2,
"log_max_age": 3,
"log_compress": true,
"log_localtime": true,
"verbose": true,
},
want: yobj{
"log": yobj{
"file": "/test/path.log",
"max_backups": 1,
"max_size": 2,
"max_age": 3,
"compress": true,
"local_time": true,
"verbose": true,
},
"schema_version": newSchemaVer,
},
wantErrMsg: "",
}, {
name: "invalid",
in: yobj{
"log_file": "/test/path.log",
"log_max_backups": 1,
"log_max_size": 2,
"log_max_age": 3,
"log_compress": "",
"log_localtime": true,
"verbose": true,
},
want: yobj{
"log_file": "/test/path.log",
"log_max_backups": 1,
"log_max_size": 2,
"log_max_age": 3,
"log_compress": "",
"log_localtime": true,
"verbose": true,
"schema_version": newSchemaVer,
},
wantErrMsg: "unexpected type of log_compress: string",
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
err := upgradeSchema23to24(tc.in)
testutil.AssertErrorMsg(t, tc.wantErrMsg, err)
assert.Equal(t, tc.want, tc.in)
})
}
}

View File

@@ -6,13 +6,13 @@ import (
"sync/atomic"
"testing"
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/golibs/testutil/fakenet"
"github.com/stretchr/testify/assert"
)
func TestWaitListener_Accept(t *testing.T) {
var accepted atomic.Bool
var l net.Listener = &aghtest.Listener{
var l net.Listener = &fakenet.Listener{
OnAccept: func() (conn net.Conn, err error) {
accepted.Store(true)

View File

@@ -4,7 +4,6 @@ package querylog
import (
"fmt"
"os"
"strings"
"sync"
"time"
@@ -161,10 +160,7 @@ func (l *queryLog) clear() {
// newLogEntry creates an instance of logEntry from parameters.
func newLogEntry(params *AddParams) (entry *logEntry) {
q := params.Question.Question[0]
qHost := q.Name
if qHost != "." {
qHost = strings.ToLower(q.Name[:len(q.Name)-1])
}
qHost := aghnet.NormalizeDomain(q.Name)
entry = &logEntry{
// TODO(d.kolyshev): Export this timestamp to func params.

132
internal/rdns/rdns.go Normal file
View File

@@ -0,0 +1,132 @@
// Package rdns processes reverse DNS lookup queries.
package rdns
import (
"net/netip"
"time"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/bluele/gcache"
)
// Interface processes rDNS queries.
type Interface interface {
// Process makes rDNS request and returns domain name. changed indicates
// that domain name was updated since last request.
Process(ip netip.Addr) (host string, changed bool)
}
// Empty is an empty [Inteface] implementation which does nothing.
type Empty struct{}
// type check
var _ Interface = (*Empty)(nil)
// Process implements the [Interface] interface for Empty.
func (Empty) Process(_ netip.Addr) (host string, changed bool) {
return "", false
}
// Exchanger is a resolver for clients' addresses.
type Exchanger interface {
// Exchange tries to resolve the ip in a suitable way, i.e. either as local
// or as external.
Exchange(ip netip.Addr) (host string, err error)
}
// Config is the configuration structure for Default.
type Config struct {
// Exchanger resolves IP addresses to domain names.
Exchanger Exchanger
// CacheSize is the maximum size of the cache. It must be greater than
// zero.
CacheSize int
// CacheTTL is the Time to Live duration for cached IP addresses.
CacheTTL time.Duration
}
// Default is the default rDNS query processor.
type Default struct {
// cache is the cache containing IP addresses of clients. An active IP
// address is resolved once again after it expires. If IP address couldn't
// be resolved, it stays here for some time to prevent further attempts to
// resolve the same IP.
cache gcache.Cache
// exchanger resolves IP addresses to domain names.
exchanger Exchanger
// cacheTTL is the Time to Live duration for cached IP addresses.
cacheTTL time.Duration
}
// New returns a new default rDNS query processor. conf must not be nil.
func New(conf *Config) (r *Default) {
return &Default{
cache: gcache.New(conf.CacheSize).LRU().Build(),
exchanger: conf.Exchanger,
cacheTTL: conf.CacheTTL,
}
}
// type check
var _ Interface = (*Default)(nil)
// Process implements the [Interface] interface for Default.
func (r *Default) Process(ip netip.Addr) (host string, changed bool) {
fromCache, expired := r.findInCache(ip)
if !expired {
return fromCache, false
}
host, err := r.exchanger.Exchange(ip)
if err != nil {
log.Debug("rdns: resolving %q: %s", ip, err)
}
item := &cacheItem{
expiry: time.Now().Add(r.cacheTTL),
host: host,
}
err = r.cache.Set(ip, item)
if err != nil {
log.Debug("rdns: cache: adding item %q: %s", ip, err)
}
return host, fromCache == "" || host != fromCache
}
// findInCache finds domain name in the cache. expired is true if host is not
// valid anymore.
func (r *Default) findInCache(ip netip.Addr) (host string, expired bool) {
val, err := r.cache.Get(ip)
if err != nil {
if !errors.Is(err, gcache.KeyNotFoundError) {
log.Debug("rdns: cache: retrieving %q: %s", ip, err)
}
return "", true
}
item, ok := val.(*cacheItem)
if !ok {
log.Debug("rdns: cache: %q bad type %T", ip, val)
return "", true
}
return item.host, time.Now().After(item.expiry)
}
// cacheItem represents an item that we will store in the cache.
type cacheItem struct {
// expiry is the time when cacheItem will expire.
expiry time.Time
// host is the domain name of a runtime client.
host string
}

105
internal/rdns/rdns_test.go Normal file
View File

@@ -0,0 +1,105 @@
package rdns_test
import (
"net/netip"
"testing"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/rdns"
"github.com/AdguardTeam/golibs/netutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// fakeRDNSExchanger is a mock [rdns.Exchanger] implementation for tests.
type fakeRDNSExchanger struct {
OnExchange func(ip netip.Addr) (host string, err error)
}
// type check
var _ rdns.Exchanger = (*fakeRDNSExchanger)(nil)
// Exchange implements [rdns.Exchanger] interface for *fakeRDNSExchanger.
func (e *fakeRDNSExchanger) Exchange(ip netip.Addr) (host string, err error) {
return e.OnExchange(ip)
}
func TestDefault_Process(t *testing.T) {
ip1 := netip.MustParseAddr("1.2.3.4")
revAddr1, err := netutil.IPToReversedAddr(ip1.AsSlice())
require.NoError(t, err)
ip2 := netip.MustParseAddr("4.3.2.1")
revAddr2, err := netutil.IPToReversedAddr(ip2.AsSlice())
require.NoError(t, err)
localIP := netip.MustParseAddr("192.168.0.1")
localRevAddr1, err := netutil.IPToReversedAddr(localIP.AsSlice())
require.NoError(t, err)
config := &rdns.Config{
CacheSize: 100,
CacheTTL: time.Hour,
}
testCases := []struct {
name string
addr netip.Addr
want string
}{{
name: "first",
addr: ip1,
want: revAddr1,
}, {
name: "second",
addr: ip2,
want: revAddr2,
}, {
name: "empty",
addr: netip.MustParseAddr("0.0.0.0"),
want: "",
}, {
name: "private",
addr: localIP,
want: localRevAddr1,
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
hit := 0
onExchange := func(ip netip.Addr) (host string, err error) {
hit++
switch ip {
case ip1:
return revAddr1, nil
case ip2:
return revAddr2, nil
case localIP:
return localRevAddr1, nil
default:
return "", nil
}
}
exchanger := &fakeRDNSExchanger{
OnExchange: onExchange,
}
config.Exchanger = exchanger
r := rdns.New(config)
got, changed := r.Process(tc.addr)
require.True(t, changed)
assert.Equal(t, tc.want, got)
assert.Equal(t, 1, hit)
// From cache.
got, changed = r.Process(tc.addr)
require.False(t, changed)
assert.Equal(t, tc.want, got)
assert.Equal(t, 1, hit)
})
}
}

View File

@@ -86,7 +86,7 @@ func TestHandleStatsConfig(t *testing.T) {
},
},
wantCode: http.StatusUnprocessableEntity,
wantErr: "ignored: duplicate host name \"ignor.ed\" at index 1\n",
wantErr: "ignored: duplicate hostname \"ignor.ed\" at index 1\n",
}, {
name: "ignored_empty",
body: getConfigResp{
@@ -97,7 +97,7 @@ func TestHandleStatsConfig(t *testing.T) {
},
},
wantCode: http.StatusUnprocessableEntity,
wantErr: "ignored: host name is empty\n",
wantErr: "ignored: at index 0: hostname is empty\n",
}, {
name: "enabled_is_null",
body: getConfigResp{

View File

@@ -10,9 +10,10 @@ require (
github.com/kyoh86/looppointer v0.2.1
github.com/securego/gosec/v2 v2.16.0
github.com/uudashr/gocognit v1.0.6
golang.org/x/tools v0.10.0
golang.org/x/tools v0.11.0
golang.org/x/vuln v0.2.0
honnef.co/go/tools v0.4.3
// TODO(a.garipov): Return to tagged releases once a new one appears.
honnef.co/go/tools v0.5.0-0.dev.0.20230709092525-bc759185c5ee
mvdan.cc/gofumpt v0.5.0
mvdan.cc/unparam v0.0.0-20230610194454-9ea02bef9868
)
@@ -26,9 +27,9 @@ require (
github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
golang.org/x/exp/typeparams v0.0.0-20230626212559-97b1e661b5df // indirect
golang.org/x/mod v0.11.0 // indirect
golang.org/x/exp/typeparams v0.0.0-20230711023510-fffb14384f22 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.9.0 // indirect
golang.org/x/sys v0.10.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

View File

@@ -52,21 +52,21 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug=
golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/exp/typeparams v0.0.0-20230626212559-97b1e661b5df h1:jfUqBujZx2dktJVEmZpCkyngz7MWrVv1y9kLOqFNsqw=
golang.org/x/exp/typeparams v0.0.0-20230626212559-97b1e661b5df/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
golang.org/x/exp/typeparams v0.0.0-20230711023510-fffb14384f22 h1:e8iSCQYXZ4EB6q3kIfy2fgPFTvDbozqzRe4OuIOyrL4=
golang.org/x/exp/typeparams v0.0.0-20230711023510-fffb14384f22/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -82,8 +82,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220702020025-31831981b65f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -96,8 +96,8 @@ golang.org/x/tools v0.0.0-20201007032633-0806396f153e/go.mod h1:z6u4i615ZeAfBE4X
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4=
golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg=
golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM=
golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8=
golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8=
golang.org/x/vuln v0.2.0 h1:Dlz47lW0pvPHU7tnb10S8vbMn9GnV2B6eyT7Tem5XBI=
golang.org/x/vuln v0.2.0/go.mod h1:V0eyhHwaAaHrt42J9bgrN6rd12f6GU4T0Lu0ex2wDg4=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -107,8 +107,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.4.3 h1:o/n5/K5gXqk8Gozvs2cnL0F2S1/g1vcGCAx2vETjITw=
honnef.co/go/tools v0.4.3/go.mod h1:36ZgoUOrqOk1GxwHhyryEkq8FQWkUO2xGuSMhUCcdvA=
honnef.co/go/tools v0.5.0-0.dev.0.20230709092525-bc759185c5ee h1:mpyvMqtlVZTwEv78QL3S2ZDTMHMO1fgNwr2kC7+K7oU=
honnef.co/go/tools v0.5.0-0.dev.0.20230709092525-bc759185c5ee/go.mod h1:GUV+uIBCLpdf0/v6UhHHG/yzI/z6qPskBeQCjcNB96k=
mvdan.cc/gofumpt v0.5.0 h1:0EQ+Z56k8tXjj/6TQD25BFNKQXpCvT0rnansIc7Ug5E=
mvdan.cc/gofumpt v0.5.0/go.mod h1:HBeVDtMKRZpXyxFciAirzdKklDlGu8aAy1wEbH5Y9js=
mvdan.cc/unparam v0.0.0-20230610194454-9ea02bef9868 h1:F4Q7pXcrU9UiU1fq0ZWqSOxKjNAteRuDr7JDk7uVLRQ=

View File

@@ -18,7 +18,7 @@ Run `make init` from the project root.
## `make/`: Makefile Scripts
## `make/`: Makefile scripts
The release channels are: `development` (the default), `edge`, `beta`, and
`release`. If verbosity levels aren't documented here, there are only two: `0`,
@@ -26,7 +26,7 @@ don't print anything, and `1`, be verbose.
### `build-docker.sh`: Build A Multi-Architecture Docker Image
### `build-docker.sh`: Build a multi-architecture Docker image
Required environment:
@@ -51,7 +51,7 @@ Optional environment:
### `build-release.sh`: Build A Release For All Platforms
### `build-release.sh`: Build a release for all platforms
Required environment:
@@ -101,7 +101,22 @@ Required environment:
### `go-build.sh`: Build The Backend
### `go-bench.sh`: Run backend benchmarks
Optional environment:
* `GO`: set an alternative name for the Go compiler.
* `TIMEOUT_FLAGS`: set timeout flags for tests. The default value is
`--timeout=30s`.
* `VERBOSE`: verbosity level. `1` shows every command that is run and every
Go package that is processed. `2` also shows subcommands and environment.
The default value is `0`, don't be verbose.
### `go-build.sh`: Build the backend
Optional environment:
@@ -135,7 +150,7 @@ Required environment:
### `go-deps.sh`: Install Backend Dependencies
### `go-deps.sh`: Install backend dependencies
Optional environment:
@@ -147,7 +162,25 @@ Optional environment:
### `go-lint.sh`: Run Backend Static Analyzers
### `go-fuzz.sh`: Run backend fuzz tests
Optional environment:
* `GO`: set an alternative name for the Go compiler.
* `FUZZTIME_FLAGS`: set fuss flags for tests. The default value is
`--fuzztime=20s`.
* `TIMEOUT_FLAGS`: set timeout flags for tests. The default value is
`--timeout=30s`.
* `VERBOSE`: verbosity level. `1` shows every command that is run and every
Go package that is processed. `2` also shows subcommands and environment.
The default value is `0`, don't be verbose.
### `go-lint.sh`: Run backend static analyzers
Don't forget to run `make go-tools` once first!
@@ -163,7 +196,7 @@ Optional environment:
### `go-test.sh`: Run Backend Tests
### `go-test.sh`: Run backend tests
Optional environment:
@@ -173,7 +206,7 @@ Optional environment:
`1`, use the race detector.
* `TIMEOUT_FLAGS`: set timeout flags for tests. The default value is
`--timeout 30s`.
`--timeout=30s`.
* `VERBOSE`: verbosity level. `1` shows every command that is run and every
Go package that is processed. `2` also shows subcommands. The default
@@ -181,7 +214,7 @@ Optional environment:
### `go-tools.sh`: Install Backend Tooling
### `go-tools.sh`: Install backend tooling
Installs the Go static analysis and other tools into `${PWD}/bin`. Either add
`${PWD}/bin` to your `$PATH` before all other entries, or use the commands
@@ -236,25 +269,29 @@ Optional environment:
### Usage
* `go run main.go help`: print usage.
* `go run ./scripts/translations help`: print usage.
* `go run main.go download [-n <count>]`: download and save all translations.
`n` is optional flag where count is a number of concurrent downloads.
* `go run ./scripts/translations download [-n <count>]`: download and save
all translations. `n` is optional flag where count is a number of
concurrent downloads.
* `go run main.go upload`: upload the base `en` locale.
* `go run ./scripts/translations upload`: upload the base `en` locale.
* `go run main.go summary`: show the current locales summary.
* `go run ./scripts/translations summary`: show the current locales summary.
* `go run main.go unused`: show the list of unused strings.
* `go run ./scripts/translations unused`: show the list of unused strings.
* `go run main.go auto-add`: add locales with additions to the git and
restore locales with deletions.
* `go run ./scripts/translations auto-add`: add locales with additions to the
git and restore locales with deletions.
After the download you'll find the output locales in the `client/src/__locales/`
directory.
Optional environment:
* `DOWNLOAD_LANGUAGES`: set a list of specific languages to `download`. For
example `ar be bg`.
* `UPLOAD_LANGUAGE`: set an alternative language for `upload`.
* `TWOSKY_URI`: set an alternative URL for `download` or `upload`.

View File

@@ -107,18 +107,6 @@ cp "${dist_dir}/AdGuardHome_linux_arm_7/AdGuardHome/AdGuardHome"\
cp "${dist_dir}/AdGuardHome_linux_ppc64le/AdGuardHome/AdGuardHome"\
"${dist_docker}/AdGuardHome_linux_ppc64le_"
# Copy the helper scripts. See file docker/Dockerfile.
dist_docker_scripts="${dist_docker}/scripts"
readonly dist_docker_scripts
mkdir -p "$dist_docker_scripts"
cp "./docker/dns-bind.awk"\
"${dist_docker_scripts}/dns-bind.awk"
cp "./docker/web-bind.awk"\
"${dist_docker_scripts}/web-bind.awk"
cp "./docker/healthcheck.sh"\
"${dist_docker_scripts}/healthcheck.sh"
# Don't use quotes with $docker_version_tag and $docker_channel_tag, because we
# want word splitting and or an empty space if tags are empty.
#

55
scripts/make/go-bench.sh Normal file
View File

@@ -0,0 +1,55 @@
#!/bin/sh
verbose="${VERBOSE:-0}"
readonly verbose
# Verbosity levels:
# 0 = Don't print anything except for errors.
# 1 = Print commands, but not nested commands.
# 2 = Print everything.
if [ "$verbose" -gt '1' ]
then
set -x
v_flags='-v=1'
x_flags='-x=1'
elif [ "$verbose" -gt '0' ]
then
set -x
v_flags='-v=1'
x_flags='-x=0'
else
set +x
v_flags='-v=0'
x_flags='-x=0'
fi
readonly v_flags x_flags
set -e -f -u
if [ "${RACE:-1}" -eq '0' ]
then
race_flags='--race=0'
else
race_flags='--race=1'
fi
readonly race_flags
go="${GO:-go}"
count_flags='--count=1'
shuffle_flags='--shuffle=on'
timeout_flags="${TIMEOUT_FLAGS:---timeout=30s}"
readonly go count_flags shuffle_flags timeout_flags
"$go" test\
"$count_flags"\
"$shuffle_flags"\
"$race_flags"\
"$timeout_flags"\
"$x_flags"\
"$v_flags"\
--bench='.'\
--benchmem\
--benchtime=1s\
--run='^$'\
./...

58
scripts/make/go-fuzz.sh Normal file
View File

@@ -0,0 +1,58 @@
#!/bin/sh
verbose="${VERBOSE:-0}"
readonly verbose
# Verbosity levels:
# 0 = Don't print anything except for errors.
# 1 = Print commands, but not nested commands.
# 2 = Print everything.
if [ "$verbose" -gt '1' ]
then
set -x
v_flags='-v=1'
x_flags='-x=1'
elif [ "$verbose" -gt '0' ]
then
set -x
v_flags='-v=1'
x_flags='-x=0'
else
set +x
v_flags='-v=0'
x_flags='-x=0'
fi
readonly v_flags x_flags
set -e -f -u
if [ "${RACE:-1}" -eq '0' ]
then
race_flags='--race=0'
else
race_flags='--race=1'
fi
readonly race_flags
go="${GO:-go}"
count_flags='--count=1'
shuffle_flags='--shuffle=on'
timeout_flags="${TIMEOUT_FLAGS:---timeout=30s}"
fuzztime_flags="${FUZZTIME_FLAGS:---fuzztime=20s}"
readonly go count_flags shuffle_flags timeout_flags fuzztime_flags
# TODO(a.garipov): File an issue about using --fuzz with multiple packages.
"$go" test\
"$count_flags"\
"$shuffle_flags"\
"$race_flags"\
"$timeout_flags"\
"$x_flags"\
"$v_flags"\
"$fuzztime_flags"\
--fuzz='.'\
--run='^$'\
./internal/filtering/rulelist/\
;

View File

@@ -35,7 +35,7 @@ set -f -u
go_version="$( "${GO:-go}" version )"
readonly go_version
go_min_version='go1.19.10'
go_min_version='go1.19.11'
go_version_msg="
warning: your go version (${go_version}) is different from the recommended minimal one (${go_min_version}).
if you have the version installed, please set the GO environment variable.
@@ -176,10 +176,14 @@ run_linter gocognit --over 10\
./internal/aghchan/\
./internal/aghhttp/\
./internal/aghio/\
./internal/filtering/hashprefix/\
./internal/filtering/rulelist/\
./internal/next/\
./internal/rdns/\
./internal/tools/\
./internal/version/\
./internal/whois/\
./scripts/\
;
run_linter ineffassign ./...
@@ -210,6 +214,8 @@ run_linter gosec --quiet\
./internal/dhcpd\
./internal/dhcpsvc\
./internal/dnsforward\
./internal/filtering/hashprefix/\
./internal/filtering/rulelist/\
./internal/next\
./internal/schedule\
./internal/stats\
@@ -218,8 +224,7 @@ run_linter gosec --quiet\
./internal/whois\
;
# TODO(a.garipov): Enable --blank?
run_linter errcheck --asserts ./...
run_linter errcheck ./...
staticcheck_matrix='
darwin: GOOS=darwin

View File

@@ -0,0 +1,177 @@
package main
import (
"flag"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghio"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"golang.org/x/exp/slices"
)
// download and save all translations.
func (c *twoskyClient) download() (err error) {
var numWorker int
flagSet := flag.NewFlagSet("download", flag.ExitOnError)
flagSet.Usage = func() {
usage("download command error")
}
flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")
err = flagSet.Parse(os.Args[2:])
if err != nil {
// Don't wrap the error since it's informative enough as is.
return err
}
if numWorker < 1 {
usage("count must be positive")
}
downloadURI := c.uri.JoinPath("download")
client := &http.Client{
Timeout: 10 * time.Second,
}
wg := &sync.WaitGroup{}
failed := &sync.Map{}
uriCh := make(chan *url.URL, len(c.langs))
for i := 0; i < numWorker; i++ {
wg.Add(1)
go downloadWorker(wg, failed, client, uriCh)
}
for lang := range c.langs {
uri := translationURL(downloadURI, defaultBaseFile, c.projectID, lang)
uriCh <- uri
}
close(uriCh)
wg.Wait()
printFailedLocales(failed)
return nil
}
// printFailedLocales prints sorted list of failed downloads, if any.
func printFailedLocales(failed *sync.Map) {
keys := []string{}
failed.Range(func(k, _ any) bool {
s, ok := k.(string)
if !ok {
panic("unexpected type")
}
keys = append(keys, s)
return true
})
if len(keys) == 0 {
return
}
slices.Sort(keys)
log.Info("failed locales: %s", strings.Join(keys, " "))
}
// downloadWorker downloads translations by received urls and saves them.
// Where failed is a map for storing failed downloads.
func downloadWorker(
wg *sync.WaitGroup,
failed *sync.Map,
client *http.Client,
uriCh <-chan *url.URL,
) {
defer wg.Done()
for uri := range uriCh {
q := uri.Query()
code := q.Get("language")
err := saveToFile(client, uri, code)
if err != nil {
log.Error("download: worker: %s", err)
failed.Store(code, struct{}{})
}
}
}
// saveToFile downloads translation by url and saves it to a file, or returns
// error.
func saveToFile(client *http.Client, uri *url.URL, code string) (err error) {
data, err := getTranslation(client, uri.String())
if err != nil {
log.Info("%s", data)
return fmt.Errorf("getting translation: %s", err)
}
name := filepath.Join(localesDir, code+".json")
err = os.WriteFile(name, data, 0o664)
if err != nil {
return fmt.Errorf("writing file: %s", err)
}
fmt.Println(name)
return nil
}
// getTranslation returns received translation data and error. If err is not
// nil, data may contain a response from server for inspection.
func getTranslation(client *http.Client, url string) (data []byte, err error) {
resp, err := client.Get(url)
if err != nil {
return nil, fmt.Errorf("requesting: %w", err)
}
defer log.OnCloserError(resp.Body, log.ERROR)
if resp.StatusCode != http.StatusOK {
err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
// Go on and download the body for inspection.
}
limitReader, lrErr := aghio.LimitReader(resp.Body, readLimit)
if lrErr != nil {
// Generally shouldn't happen, since the only error returned by
// [aghio.LimitReader] is an argument error.
panic(fmt.Errorf("limit reading: %w", lrErr))
}
data, readErr := io.ReadAll(limitReader)
return data, errors.WithDeferred(err, readErr)
}
// translationURL returns a new url.URL with provided query parameters.
func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
uri = &url.URL{}
*uri = *oldURL
q := uri.Query()
q.Set("format", "json")
q.Set("filename", baseFile)
q.Set("project", projectID)
q.Set("language", string(lang))
uri.RawQuery = q.Encode()
return uri
}

View File

@@ -6,25 +6,16 @@ import (
"bufio"
"bytes"
"encoding/json"
"flag"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/AdGuardHome/internal/aghio"
"github.com/AdguardTeam/AdGuardHome/internal/aghos"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/httphdr"
"github.com/AdguardTeam/golibs/log"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
@@ -38,7 +29,8 @@ const (
srcDir = "./client/src"
twoskyURI = "https://twosky.int.agrd.dev/api/v1"
readLimit = 1 * 1024 * 1024
readLimit = 1 * 1024 * 1024
uploadTimeout = 10 * time.Second
)
// langCode is a language code.
@@ -62,31 +54,26 @@ func main() {
usage("")
}
uriStr := os.Getenv("TWOSKY_URI")
if uriStr == "" {
uriStr = twoskyURI
}
uri, err := url.Parse(uriStr)
conf, err := readTwoskyConfig()
check(err)
projectID := os.Getenv("TWOSKY_PROJECT_ID")
if projectID == "" {
projectID = defaultProjectID
}
conf, err := readTwoskyConf()
check(err)
var cli *twoskyClient
switch os.Args[1] {
case "summary":
err = summary(conf.Languages)
case "download":
err = download(uri, projectID, conf.Languages)
cli, err = conf.toClient()
check(err)
err = cli.download()
case "unused":
err = unused(conf.LocalizableFiles[0])
case "upload":
err = upload(uri, projectID, conf.BaseLangcode)
cli, err = conf.toClient()
check(err)
err = cli.upload()
case "auto-add":
err = autoAdd(conf.LocalizableFiles[0])
default:
@@ -133,51 +120,131 @@ Commands:
os.Exit(0)
}
// twoskyConf is the configuration structure for localization.
type twoskyConf struct {
// twoskyConfig is the configuration structure for localization.
type twoskyConfig struct {
Languages languages `json:"languages"`
ProjectID string `json:"project_id"`
BaseLangcode langCode `json:"base_locale"`
LocalizableFiles []string `json:"localizable_files"`
}
// readTwoskyConf returns configuration.
func readTwoskyConf() (t twoskyConf, err error) {
defer func() { err = errors.Annotate(err, "parsing twosky conf: %w") }()
// readTwoskyConfig returns twosky configuration.
func readTwoskyConfig() (t *twoskyConfig, err error) {
defer func() { err = errors.Annotate(err, "parsing twosky config: %w") }()
b, err := os.ReadFile(twoskyConfFile)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return twoskyConf{}, err
return nil, err
}
var tsc []twoskyConf
var tsc []twoskyConfig
err = json.Unmarshal(b, &tsc)
if err != nil {
err = fmt.Errorf("unmarshalling %q: %w", twoskyConfFile, err)
return twoskyConf{}, err
return nil, err
}
if len(tsc) == 0 {
err = fmt.Errorf("%q is empty", twoskyConfFile)
return twoskyConf{}, err
return nil, err
}
conf := tsc[0]
for _, lang := range conf.Languages {
if lang == "" {
return twoskyConf{}, errors.Error("language is empty")
return nil, errors.Error("language is empty")
}
}
if len(conf.LocalizableFiles) == 0 {
return twoskyConf{}, errors.Error("no localizable files specified")
return nil, errors.Error("no localizable files specified")
}
return conf, nil
return &conf, nil
}
// twoskyClient is the twosky client with methods for download and upload
// translations.
type twoskyClient struct {
// uri is the base URL.
uri *url.URL
// langs is the map of languages to download.
langs languages
// projectID is the name of the project.
projectID string
// baseLang is the base language code.
baseLang langCode
}
// toClient reads values from environment variables or defaults, validates
// them, and returns the twosky client.
func (t *twoskyConfig) toClient() (cli *twoskyClient, err error) {
defer func() { err = errors.Annotate(err, "filling config: %w") }()
uriStr := os.Getenv("TWOSKY_URI")
if uriStr == "" {
uriStr = twoskyURI
}
uri, err := url.Parse(uriStr)
if err != nil {
return nil, err
}
projectID := os.Getenv("TWOSKY_PROJECT_ID")
if projectID == "" {
projectID = defaultProjectID
}
baseLang := t.BaseLangcode
uLangStr := os.Getenv("UPLOAD_LANGUAGE")
if uLangStr != "" {
baseLang = langCode(uLangStr)
}
langs := t.Languages
dlLangStr := os.Getenv("DOWNLOAD_LANGUAGES")
if dlLangStr != "" {
var dlLangs languages
dlLangs, err = validateLanguageStr(dlLangStr, langs)
if err != nil {
return nil, err
}
langs = dlLangs
}
return &twoskyClient{
uri: uri,
projectID: projectID,
baseLang: baseLang,
langs: langs,
}, nil
}
// validateLanguageStr validates languages codes that contain in the str and
// returns language map, where key is language code and value is display name.
func validateLanguageStr(str string, all languages) (langs languages, err error) {
langs = make(languages)
codes := strings.Fields(str)
for _, k := range codes {
lc := langCode(k)
name, ok := all[lc]
if !ok {
return nil, fmt.Errorf("validating languages: unexpected language code %q", k)
}
langs[lc] = name
}
return langs, nil
}
// readLocales reads file with name fn and returns a map, where key is text
@@ -233,163 +300,33 @@ func summary(langs languages) (err error) {
return nil
}
// download and save all translations. uri is the base URL. projectID is the
// name of the project.
func download(uri *url.URL, projectID string, langs languages) (err error) {
var numWorker int
// unused prints unused text labels.
func unused(basePath string) (err error) {
defer func() { err = errors.Annotate(err, "unused: %w") }()
flagSet := flag.NewFlagSet("download", flag.ExitOnError)
flagSet.Usage = func() {
usage("download command error")
}
flagSet.IntVar(&numWorker, "n", 1, "number of concurrent downloads")
err = flagSet.Parse(os.Args[2:])
baseLoc, err := readLocales(basePath)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return err
}
if numWorker < 1 {
usage("count must be positive")
}
downloadURI := uri.JoinPath("download")
client := &http.Client{
Timeout: 10 * time.Second,
}
wg := &sync.WaitGroup{}
uriCh := make(chan *url.URL, len(langs))
for i := 0; i < numWorker; i++ {
wg.Add(1)
go downloadWorker(wg, client, uriCh)
}
for lang := range langs {
uri = translationURL(downloadURI, defaultBaseFile, projectID, lang)
uriCh <- uri
}
close(uriCh)
wg.Wait()
return nil
}
// downloadWorker downloads translations by received urls and saves them.
func downloadWorker(wg *sync.WaitGroup, client *http.Client, uriCh <-chan *url.URL) {
defer wg.Done()
for uri := range uriCh {
data, err := getTranslation(client, uri.String())
if err != nil {
log.Error("download worker: getting translation: %s", err)
log.Info("download worker: error response:\n%s", data)
continue
}
q := uri.Query()
code := q.Get("language")
// Fix some TwoSky weirdnesses.
//
// TODO(a.garipov): Remove when those are fixed.
code = strings.ToLower(code)
name := filepath.Join(localesDir, code+".json")
err = os.WriteFile(name, data, 0o664)
if err != nil {
log.Error("download worker: writing file: %s", err)
continue
}
fmt.Println(name)
}
}
// getTranslation returns received translation data and error. If err is not
// nil, data may contain a response from server for inspection.
func getTranslation(client *http.Client, url string) (data []byte, err error) {
resp, err := client.Get(url)
if err != nil {
return nil, fmt.Errorf("requesting: %w", err)
}
defer log.OnCloserError(resp.Body, log.ERROR)
if resp.StatusCode != http.StatusOK {
err = fmt.Errorf("url: %q; status code: %s", url, http.StatusText(resp.StatusCode))
// Go on and download the body for inspection.
}
limitReader, lrErr := aghio.LimitReader(resp.Body, readLimit)
if lrErr != nil {
// Generally shouldn't happen, since the only error returned by
// [aghio.LimitReader] is an argument error.
panic(fmt.Errorf("limit reading: %w", lrErr))
}
data, readErr := io.ReadAll(limitReader)
return data, errors.WithDeferred(err, readErr)
}
// translationURL returns a new url.URL with provided query parameters.
func translationURL(oldURL *url.URL, baseFile, projectID string, lang langCode) (uri *url.URL) {
uri = &url.URL{}
*uri = *oldURL
// Fix some TwoSky weirdnesses.
//
// TODO(a.garipov): Remove when those are fixed.
switch lang {
case "si-lk":
lang = "si-LK"
case "zh-hk":
lang = "zh-HK"
default:
// Go on.
}
q := uri.Query()
q.Set("format", "json")
q.Set("filename", baseFile)
q.Set("project", projectID)
q.Set("language", string(lang))
uri.RawQuery = q.Encode()
return uri
}
// unused prints unused text labels.
func unused(basePath string) (err error) {
baseLoc, err := readLocales(basePath)
if err != nil {
return fmt.Errorf("unused: %w", err)
}
locDir := filepath.Clean(localesDir)
js, err := findJS(locDir)
if err != nil {
return err
}
fileNames := []string{}
err = filepath.Walk(srcDir, func(name string, info os.FileInfo, err error) error {
return findUnused(js, baseLoc)
}
// findJS returns list of JavaScript and JSON files or error.
func findJS(locDir string) (fileNames []string, err error) {
walkFn := func(name string, _ os.FileInfo, err error) error {
if err != nil {
log.Info("warning: accessing a path %q: %s", name, err)
return nil
}
if info.IsDir() {
return nil
}
if strings.HasPrefix(name, locDir) {
return nil
}
@@ -400,13 +337,14 @@ func unused(basePath string) (err error) {
}
return nil
})
if err != nil {
return fmt.Errorf("filepath walking %q: %w", srcDir, err)
}
return findUnused(fileNames, baseLoc)
err = filepath.Walk(srcDir, walkFn)
if err != nil {
return nil, fmt.Errorf("filepath walking %q: %w", srcDir, err)
}
return fileNames, nil
}
// findUnused prints unused text labels from fileNames.
@@ -445,118 +383,6 @@ func findUnused(fileNames []string, loc locales) (err error) {
return nil
}
// upload base translation. uri is the base URL. projectID is the name of the
// project. baseLang is the base language code.
func upload(uri *url.URL, projectID string, baseLang langCode) (err error) {
defer func() { err = errors.Annotate(err, "upload: %w") }()
uploadURI := uri.JoinPath("upload")
lang := baseLang
langStr := os.Getenv("UPLOAD_LANGUAGE")
if langStr != "" {
lang = langCode(langStr)
}
basePath := filepath.Join(localesDir, defaultBaseFile)
formData := map[string]string{
"format": "json",
"language": string(lang),
"filename": defaultBaseFile,
"project": projectID,
}
buf, cType, err := prepareMultipartMsg(formData, basePath)
if err != nil {
return fmt.Errorf("preparing multipart msg: %w", err)
}
err = send(uploadURI.String(), cType, buf)
if err != nil {
return fmt.Errorf("sending multipart msg: %w", err)
}
return nil
}
// prepareMultipartMsg prepares translation data for upload.
func prepareMultipartMsg(
formData map[string]string,
basePath string,
) (buf *bytes.Buffer, cType string, err error) {
buf = &bytes.Buffer{}
w := multipart.NewWriter(buf)
var fw io.Writer
for k, v := range formData {
err = w.WriteField(k, v)
if err != nil {
return nil, "", fmt.Errorf("writing field: %w", err)
}
}
file, err := os.Open(basePath)
if err != nil {
return nil, "", fmt.Errorf("opening file: %w", err)
}
defer func() {
err = errors.WithDeferred(err, file.Close())
}()
h := make(textproto.MIMEHeader)
h.Set(httphdr.ContentType, aghhttp.HdrValApplicationJSON)
d := fmt.Sprintf("form-data; name=%q; filename=%q", "file", defaultBaseFile)
h.Set(httphdr.ContentDisposition, d)
fw, err = w.CreatePart(h)
if err != nil {
return nil, "", fmt.Errorf("creating part: %w", err)
}
_, err = io.Copy(fw, file)
if err != nil {
return nil, "", fmt.Errorf("copying: %w", err)
}
err = w.Close()
if err != nil {
return nil, "", fmt.Errorf("closing writer: %w", err)
}
return buf, w.FormDataContentType(), nil
}
// send POST request to uriStr.
func send(uriStr, cType string, buf *bytes.Buffer) (err error) {
var client http.Client
req, err := http.NewRequest(http.MethodPost, uriStr, buf)
if err != nil {
return fmt.Errorf("bad request: %w", err)
}
req.Header.Set(httphdr.ContentType, cType)
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("client post form: %w", err)
}
defer func() {
err = errors.WithDeferred(err, resp.Body.Close())
}()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode))
}
return nil
}
// autoAdd adds locales with additions to the git and restores locales with
// deletions.
func autoAdd(basePath string) (err error) {
@@ -572,28 +398,48 @@ func autoAdd(basePath string) (err error) {
return errors.Error("base locale contains deletions")
}
var (
args []string
code int
out []byte
)
if len(adds) > 0 {
args = append([]string{"add"}, adds...)
code, out, err = aghos.RunCommand("git", args...)
if err != nil || code != 0 {
return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err)
}
err = handleAdds(adds)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return nil
}
if len(dels) > 0 {
args = append([]string{"restore"}, dels...)
code, out, err = aghos.RunCommand("git", args...)
err = handleDels(dels)
if err != nil {
// Don't wrap the error since it's informative enough as is.
return nil
}
if err != nil || code != 0 {
return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err)
}
return nil
}
// handleAdds adds locales with additions to the git.
func handleAdds(locales []string) (err error) {
if len(locales) == 0 {
return nil
}
args := append([]string{"add"}, locales...)
code, out, err := aghos.RunCommand("git", args...)
if err != nil || code != 0 {
return fmt.Errorf("git add exited with code %d output %q: %w", code, out, err)
}
return nil
}
// handleDels restores locales with deletions.
func handleDels(locales []string) (err error) {
if len(locales) == 0 {
return nil
}
args := append([]string{"restore"}, locales...)
code, out, err := aghos.RunCommand("git", args...)
if err != nil || code != 0 {
return fmt.Errorf("git restore exited with code %d output %q: %w", code, out, err)
}
return nil

View File

@@ -0,0 +1,120 @@
package main
import (
"bytes"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/textproto"
"os"
"path/filepath"
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/httphdr"
"github.com/AdguardTeam/golibs/mapsutil"
)
// upload base translation.
func (c *twoskyClient) upload() (err error) {
defer func() { err = errors.Annotate(err, "upload: %w") }()
uploadURI := c.uri.JoinPath("upload")
basePath := filepath.Join(localesDir, defaultBaseFile)
formData := map[string]string{
"format": "json",
"language": string(c.baseLang),
"filename": defaultBaseFile,
"project": c.projectID,
}
buf, cType, err := prepareMultipartMsg(formData, basePath)
if err != nil {
return fmt.Errorf("preparing multipart msg: %w", err)
}
err = send(uploadURI.String(), cType, buf)
if err != nil {
return fmt.Errorf("sending multipart msg: %w", err)
}
return nil
}
// prepareMultipartMsg prepares translation data for upload.
func prepareMultipartMsg(
formData map[string]string,
basePath string,
) (buf *bytes.Buffer, cType string, err error) {
buf = &bytes.Buffer{}
w := multipart.NewWriter(buf)
var fw io.Writer
err = mapsutil.OrderedRangeError(formData, w.WriteField)
if err != nil {
return nil, "", fmt.Errorf("writing field: %w", err)
}
file, err := os.Open(basePath)
if err != nil {
return nil, "", fmt.Errorf("opening file: %w", err)
}
defer func() {
err = errors.WithDeferred(err, file.Close())
}()
h := make(textproto.MIMEHeader)
h.Set(httphdr.ContentType, aghhttp.HdrValApplicationJSON)
d := fmt.Sprintf("form-data; name=%q; filename=%q", "file", defaultBaseFile)
h.Set(httphdr.ContentDisposition, d)
fw, err = w.CreatePart(h)
if err != nil {
return nil, "", fmt.Errorf("creating part: %w", err)
}
_, err = io.Copy(fw, file)
if err != nil {
return nil, "", fmt.Errorf("copying: %w", err)
}
err = w.Close()
if err != nil {
return nil, "", fmt.Errorf("closing writer: %w", err)
}
return buf, w.FormDataContentType(), nil
}
// send POST request to uriStr.
func send(uriStr, cType string, buf *bytes.Buffer) (err error) {
client := http.Client{
Timeout: uploadTimeout,
}
req, err := http.NewRequest(http.MethodPost, uriStr, buf)
if err != nil {
return fmt.Errorf("bad request: %w", err)
}
req.Header.Set(httphdr.ContentType, cType)
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("client post form: %w", err)
}
defer func() {
err = errors.WithDeferred(err, resp.Body.Close())
}()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("status code is not ok: %q", http.StatusText(resp.StatusCode))
}
return nil
}