daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge: fix #822 - Whitelist filter rules

Browse Source 
Squashed commit of the following:

commit 350c6d5fadd77145b801df8887284bf4d64fbd19
Author: Ildar Kamalov <i.kamalov@adguard.com>
Date:   Wed Feb 26 15:43:29 2020 +0300

    * client: update translations

commit a884dffcd59f2259e2eee2c1e5a3270819bf8962
Author: Ildar Kamalov <i.kamalov@adguard.com>
Date:   Mon Feb 17 17:32:10 2020 +0300

    + client: handle whitelist filters

commit a586ec5bc614ffb0e01584a1fbdc7292b4865e68
Author: ArtemBaskal <a.baskal@adguard.com>
Date:   Wed Jan 29 18:16:59 2020 +0300

    + client: add whitelist

commit a52c3de62cf2fa34be6394771fb8bb56b4ee81e3
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Feb 20 17:50:44 2020 +0300

    * change /filtering/refresh

commit 7f8f2ecccb9f7fa65318c1717dc6a7bd61afccf4
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Thu Feb 20 16:17:07 2020 +0300

    * fix race-detector issue

commit ac4b64c4a52c5b364a4b154bf18dea0fdf45647f
Author: Simon Zolin <s.zolin@adguard.com>
Date:   Mon Jan 20 20:08:21 2020 +0300

    + whitelist filters
This commit is contained in:
Andrey Meshkov
2020-02-26 19:58:25 +03:00
parent 82aa38fce3
commit d839136fee
70 changed files with 1821 additions and 1132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
home/config.go
View File

@@ -58,11 +58,14 @@ type configuration struct {
// An active session is automatically refreshed once a day.
WebSessionTTLHours uint32 `yaml:"web_session_ttl"`
DNS dnsConfig `yaml:"dns"`
TLS tlsConfig `yaml:"tls"`
Filters []filter `yaml:"filters"`
UserRules []string `yaml:"user_rules"`
DHCP dhcpd.ServerConfig `yaml:"dhcp"`
DNS dnsConfig `yaml:"dns"`
TLS tlsConfig `yaml:"tls"`
Filters []filter `yaml:"filters"`
WhitelistFilters []filter `yaml:"whitelist_filters"`
UserRules []string `yaml:"user_rules"`
DHCP dhcpd.ServerConfig `yaml:"dhcp"`
// Note: this array is filled only before file read/write and then it's cleared
Clients []clientObject `yaml:"clients"`

84
home/control_filtering.go
View File

@@ -28,8 +28,9 @@ func IsValidURL(rawurl string) bool {
}
type filterAddJSON struct {
Name string `json:"name"`
URL string `json:"url"`
Name string `json:"name"`
URL string `json:"url"`
Whitelist bool `json:"whitelist"`
}
func handleFilteringAddURL(w http.ResponseWriter, r *http.Request) {
@@ -56,6 +57,7 @@ func handleFilteringAddURL(w http.ResponseWriter, r *http.Request) {
Enabled: true,
URL: fj.URL,
Name: fj.Name,
white: fj.Whitelist,
}
f.ID = assignUniqueFilterID()
@@ -82,7 +84,6 @@ func handleFilteringAddURL(w http.ResponseWriter, r *http.Request) {
}
// URL is deemed valid, append it to filters, update config, write new filter file and tell dns to reload it
// TODO: since we directly feed filters in-memory, revisit if writing configs is always necessary
if !filterAdd(f) {
httpError(w, http.StatusBadRequest, "Filter URL already added -- %s", f.URL)
return
@@ -100,7 +101,8 @@ func handleFilteringAddURL(w http.ResponseWriter, r *http.Request) {
func handleFilteringRemoveURL(w http.ResponseWriter, r *http.Request) {
type request struct {
URL string `json:"url"`
URL string `json:"url"`
Whitelist bool `json:"whitelist"`
}
req := request{}
err := json.NewDecoder(r.Body).Decode(&req)
@@ -117,7 +119,11 @@ func handleFilteringRemoveURL(w http.ResponseWriter, r *http.Request) {
// go through each element and delete if url matches
config.Lock()
newFilters := []filter{}
for _, filter := range config.Filters {
filters := &config.Filters
if req.Whitelist {
filters = &config.WhitelistFilters
}
for _, filter := range *filters {
if filter.URL != req.URL {
newFilters = append(newFilters, filter)
} else {
@@ -128,7 +134,7 @@ func handleFilteringRemoveURL(w http.ResponseWriter, r *http.Request) {
}
}
// Update the configuration after removing filter files
config.Filters = newFilters
*filters = newFilters
config.Unlock()
onConfigModified()
@@ -145,8 +151,9 @@ type filterURLJSON struct {
}
type filterURLReq struct {
URL string `json:"url"`
Data filterURLJSON `json:"data"`
URL string `json:"url"`
Whitelist bool `json:"whitelist"`
Data filterURLJSON `json:"data"`
}
func handleFilteringSetURL(w http.ResponseWriter, r *http.Request) {
@@ -167,7 +174,7 @@ func handleFilteringSetURL(w http.ResponseWriter, r *http.Request) {
Name: fj.Data.Name,
URL: fj.Data.URL,
}
status := filterSetProperties(fj.URL, f)
status := filterSetProperties(fj.URL, f, fj.Whitelist)
if (status & statusFound) == 0 {
http.Error(w, "URL doesn't exist", http.StatusBadRequest)
return
@@ -210,14 +217,27 @@ func handleFilteringSetRules(w http.ResponseWriter, r *http.Request) {
}
func handleFilteringRefresh(w http.ResponseWriter, r *http.Request) {
type Resp struct {
Updated int `json:"updated"`
}
resp := Resp{}
var err error
Context.controlLock.Unlock()
nUpdated, err := refreshFilters()
resp.Updated, err = refreshFilters()
Context.controlLock.Lock()
if err != nil {
httpError(w, http.StatusInternalServerError, "%s", err)
return
}
fmt.Fprintf(w, "OK %d filters updated\n", nUpdated)
js, err := json.Marshal(resp)
if err != nil {
httpError(w, http.StatusInternalServerError, "json encode: %s", err)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write(js)
}
type filterJSON struct {
@@ -230,10 +250,27 @@ type filterJSON struct {
}
type filteringConfig struct {
Enabled bool `json:"enabled"`
Interval uint32 `json:"interval"` // in hours
Filters []filterJSON `json:"filters"`
UserRules []string `json:"user_rules"`
Enabled bool `json:"enabled"`
Interval uint32 `json:"interval"` // in hours
Filters []filterJSON `json:"filters"`
WhitelistFilters []filterJSON `json:"whitelist_filters"`
UserRules []string `json:"user_rules"`
}
func filterToJSON(f filter) filterJSON {
fj := filterJSON{
ID: f.ID,
Enabled: f.Enabled,
URL: f.URL,
Name: f.Name,
RulesCount: uint32(f.RulesCount),
}
if !f.LastUpdated.IsZero() {
fj.LastUpdated = f.LastUpdated.Format(time.RFC3339)
}
return fj
}
// Get filtering configuration
@@ -243,20 +280,13 @@ func handleFilteringStatus(w http.ResponseWriter, r *http.Request) {
resp.Enabled = config.DNS.FilteringEnabled
resp.Interval = config.DNS.FiltersUpdateIntervalHours
for _, f := range config.Filters {
fj := filterJSON{
ID: f.ID,
Enabled: f.Enabled,
URL: f.URL,
Name: f.Name,
RulesCount: uint32(f.RulesCount),
}
if !f.LastUpdated.IsZero() {
fj.LastUpdated = f.LastUpdated.Format(time.RFC3339)
}
fj := filterToJSON(f)
resp.Filters = append(resp.Filters, fj)
}
for _, f := range config.WhitelistFilters {
fj := filterToJSON(f)
resp.WhitelistFilters = append(resp.WhitelistFilters, fj)
}
resp.UserRules = config.UserRules
config.RUnlock()

88
home/filter.go
View File

@@ -10,6 +10,7 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
@@ -26,9 +27,11 @@ var (
)
func initFiltering() {
loadFilters()
loadFilters(config.Filters)
loadFilters(config.WhitelistFilters)
deduplicateFilters()
updateUniqueFilterID(config.Filters)
updateUniqueFilterID(config.WhitelistFilters)
}
func startFiltering() {
@@ -55,6 +58,7 @@ type filter struct {
RulesCount int `yaml:"-"`
LastUpdated time.Time `yaml:"-"`
checksum uint32 // checksum of the file data
white bool
dnsfilter.Filter `yaml:",inline"`
}
@@ -78,13 +82,18 @@ const (
// Update properties for a filter specified by its URL
// Return status* flags.
func filterSetProperties(url string, newf filter) int {
func filterSetProperties(url string, newf filter, whitelist bool) int {
r := 0
config.Lock()
defer config.Unlock()
for i := range config.Filters {
f := &config.Filters[i]
filters := &config.Filters
if whitelist {
filters = &config.WhitelistFilters
}
for i := range *filters {
f := &(*filters)[i]
if f.URL != url {
continue
}
@@ -134,41 +143,44 @@ func filterExists(url string) bool {
return r
}
// Return TRUE if a filter with this URL exists
func filterExistsNoLock(url string) bool {
r := false
for i := range config.Filters {
if config.Filters[i].URL == url {
r = true
break
for _, f := range config.Filters {
if f.URL == url {
return true
}
}
return r
for _, f := range config.WhitelistFilters {
if f.URL == url {
return true
}
}
return false
}
// Add a filter
// Return FALSE if a filter with this URL exists
func filterAdd(f filter) bool {
config.Lock()
defer config.Unlock()
// Check for duplicates
for i := range config.Filters {
if config.Filters[i].URL == f.URL {
config.Unlock()
return false
}
if filterExistsNoLock(f.URL) {
return false
}
config.Filters = append(config.Filters, f)
config.Unlock()
if f.white {
config.WhitelistFilters = append(config.WhitelistFilters, f)
} else {
config.Filters = append(config.Filters, f)
}
return true
}
// Load filters from the disk
// And if any filter has zero ID, assign a new one
func loadFilters() {
for i := range config.Filters {
filter := &config.Filters[i] // otherwise we're operating on a copy
func loadFilters(array []filter) {
for i := range array {
filter := &array[i] // otherwise we're operating on a copy
if filter.ID == 0 {
filter.ID = assignUniqueFilterID()
}
@@ -223,8 +235,7 @@ func periodicallyRefreshFilters() {
intval := 5 // use a dynamically increasing time interval
for {
isNetworkErr := false
if config.DNS.FiltersUpdateIntervalHours != 0 && refreshStatus == 0 {
refreshStatus = 1
if config.DNS.FiltersUpdateIntervalHours != 0 && atomic.CompareAndSwapUint32(&refreshStatus, 0, 1) {
refreshLock.Lock()
_, isNetworkErr = refreshFiltersIfNecessary(false)
refreshLock.Unlock()
@@ -247,11 +258,10 @@ func periodicallyRefreshFilters() {
// Refresh filters
func refreshFilters() (int, error) {
if refreshStatus != 0 { // we could use atomic cmpxchg here, but it's not really required
if !atomic.CompareAndSwapUint32(&refreshStatus, 0, 1) {
return 0, fmt.Errorf("Filters update procedure is already running")
}
refreshStatus = 1
refreshLock.Lock()
nUpdated, _ := refreshFiltersIfNecessary(true)
refreshLock.Unlock()
@@ -561,21 +571,39 @@ func (filter *filter) LastTimeUpdated() time.Time {
}
func enableFilters(async bool) {
var filters map[int]string
var filters []dnsfilter.Filter
var whiteFilters []dnsfilter.Filter
if config.DNS.FilteringEnabled {
// convert array of filters
filters = make(map[int]string)
userFilter := userFilter()
filters[int(userFilter.ID)] = string(userFilter.Data)
f := dnsfilter.Filter{
ID: userFilter.ID,
Data: userFilter.Data,
}
filters = append(filters, f)
for _, filter := range config.Filters {
if !filter.Enabled {
continue
}
filters[int(filter.ID)] = filter.Path()
f := dnsfilter.Filter{
ID: filter.ID,
FilePath: filter.Path(),
}
filters = append(filters, f)
}
for _, filter := range config.WhitelistFilters {
if !filter.Enabled {
continue
}
f := dnsfilter.Filter{
ID: filter.ID,
FilePath: filter.Path(),
}
whiteFilters = append(whiteFilters, f)
}
}
_ = Context.dnsFilter.SetFilters(filters, async)
_ = Context.dnsFilter.SetFilters(filters, whiteFilters, async)
}