daxi20/AdGuardHome
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

all: sync with master

Browse Source
This commit is contained in:
Ainar Garipov
2022-11-02 16:18:02 +03:00
parent 16755c37d8
commit c9314610d4
173 changed files with 11539 additions and 6928 deletions
Download Patch File Download Diff File Expand all files Collapse all files

418
internal/filtering/blocked.go
View File

@@ -10,382 +10,6 @@ import (
"golang.org/x/exp/slices"
)
// svc represents a single blocked service.
type svc struct {
name string
rules []string
}
// servicesData contains raw blocked service data.
//
// Keep in sync with:
// - client/src/helpers/constants.js
// - client/src/components/ui/Icons.js
var servicesData = []svc{{
name: "whatsapp",
rules: []string{
"||wa.me^",
"||whatsapp.com^",
"||whatsapp.net^",
},
}, {
name: "facebook",
rules: []string{
"||facebook.com^",
"||facebook.net^",
"||fbcdn.net^",
"||accountkit.com^",
"||fb.me^",
"||fb.com^",
"||fb.gg^",
"||fbsbx.com^",
"||fbwat.ch^",
"||messenger.com^",
"||facebookcorewwwi.onion^",
"||fbcdn.com^",
"||fb.watch^",
},
}, {
name: "twitter",
rules: []string{
"||t.co^",
"||twimg.com^",
"||twitter.com^",
"||twttr.com^",
},
}, {
name: "youtube",
rules: []string{
"||googlevideo.com^",
"||wide-youtube.l.google.com^",
"||youtu.be^",
"||youtube",
"||youtube-nocookie.com^",
"||youtube.com^",
"||youtubei.googleapis.com^",
"||youtubekids.com^",
"||ytimg.com^",
},
}, {
name: "twitch",
rules: []string{
"||jtvnw.net^",
"||ttvnw.net^",
"||twitch.tv^",
"||twitchcdn.net^",
},
}, {
name: "netflix",
rules: []string{
"||nflxext.com^",
"||netflix.com^",
"||nflximg.net^",
"||nflxvideo.net^",
"||nflxso.net^",
},
}, {
name: "instagram",
rules: []string{"||instagram.com^", "||cdninstagram.com^"},
}, {
name: "snapchat",
rules: []string{
"||snapchat.com^",
"||sc-cdn.net^",
"||snap-dev.net^",
"||snapkit.co",
"||snapads.com^",
"||impala-media-production.s3.amazonaws.com^",
},
}, {
name: "discord",
rules: []string{
"||discord.gg^",
"||discordapp.net^",
"||discordapp.com^",
"||discord.com^",
"||discord.gift",
"||discord.media^",
},
}, {
name: "ok",
rules: []string{"||ok.ru^"},
}, {
name: "skype",
rules: []string{
"||edge-skype-com.s-0001.s-msedge.net^",
"||skype-edf.akadns.net^",
"||skype.com^",
"||skypeassets.com^",
"||skypedata.akadns.net^",
},
}, {
name: "vk",
rules: []string{
"||userapi.com^",
"||vk-cdn.net^",
"||vk.com^",
"||vkuservideo.net^",
},
}, {
name: "origin",
rules: []string{
"||accounts.ea.com^",
"||origin.com^",
"||signin.ea.com^",
},
}, {
name: "steam",
rules: []string{
"||steam.com^",
"||steampowered.com^",
"||steamcommunity.com^",
"||steamstatic.com^",
"||steamstore-a.akamaihd.net^",
"||steamcdn-a.akamaihd.net^",
},
}, {
name: "epic_games",
rules: []string{"||epicgames.com^", "||easyanticheat.net^", "||easy.ac^", "||eac-cdn.com^"},
}, {
name: "reddit",
rules: []string{"||reddit.com^", "||redditstatic.com^", "||redditmedia.com^", "||redd.it^"},
}, {
name: "mail_ru",
rules: []string{"||mail.ru^"},
}, {
name: "cloudflare",
rules: []string{
"||1.1.1.1^",
"||argotunnel.com^",
"||cloudflare-dns.com^",
"||cloudflare-ipfs.com^",
"||cloudflare-quic.com^",
"||cloudflare.cn^",
"||cloudflare.com^",
"||cloudflare.net^",
"||cloudflareaccess.com^",
"||cloudflareapps.com^",
"||cloudflarebolt.com^",
"||cloudflareclient.com^",
"||cloudflareinsights.com^",
"||cloudflareresolve.com^",
"||cloudflarestatus.com^",
"||cloudflarestream.com^",
"||cloudflarewarp.com^",
"||dns4torpnlfs2ifuz2s2yf3fc7rdmsbhm6rw75euj35pac6ap25zgqad.onion^",
"||one.one^",
"||pages.dev^",
"||trycloudflare.com^",
"||videodelivery.net^",
"||warp.plus^",
"||workers.dev^",
},
}, {
name: "amazon",
rules: []string{
"||amazon.com^",
"||media-amazon.com^",
"||primevideo.com^",
"||amazontrust.com^",
"||images-amazon.com^",
"||ssl-images-amazon.com^",
"||amazonpay.com^",
"||amazonpay.in^",
"||amazon-adsystem.com^",
"||a2z.com^",
"||amazon.ae^",
"||amazon.ca^",
"||amazon.cn^",
"||amazon.de^",
"||amazon.es^",
"||amazon.fr^",
"||amazon.in^",
"||amazon.it^",
"||amazon.nl^",
"||amazon.com.au^",
"||amazon.com.br^",
"||amazon.co.jp^",
"||amazon.com.mx^",
"||amazon.com.tr^",
"||amazon.co.uk^",
"||createspace.com^",
"||aws",
},
}, {
name: "ebay",
rules: []string{
"||ebay.com^",
"||ebayimg.com^",
"||ebaystatic.com^",
"||ebaycdn.net^",
"||ebayinc.com^",
"||ebay.at^",
"||ebay.be^",
"||ebay.ca^",
"||ebay.ch^",
"||ebay.cn^",
"||ebay.de^",
"||ebay.es^",
"||ebay.fr^",
"||ebay.ie^",
"||ebay.in^",
"||ebay.it^",
"||ebay.ph^",
"||ebay.pl^",
"||ebay.nl^",
"||ebay.com.au^",
"||ebay.com.cn^",
"||ebay.com.hk^",
"||ebay.com.my^",
"||ebay.com.sg^",
"||ebay.co.uk^",
},
}, {
name: "tiktok",
rules: []string{
"||amemv.com^",
"||bdurl.com^",
"||bytecdn.cn^",
"||bytedance.map.fastly.net^",
"||bytedapm.com^",
"||byteimg.com^",
"||byteoversea.com^",
"||douyin.com^",
"||douyincdn.com^",
"||douyinpic.com^",
"||douyinstatic.com^",
"||douyinvod.com^",
"||ixigua.com^",
"||ixiguavideo.com^",
"||muscdn.com^",
"||musical.ly^",
"||pstatp.com^",
"||snssdk.com^",
"||tiktok.com^",
"||tiktokcdn.com^",
"||tiktokv.com^",
"||toutiao.com^",
"||toutiaocloud.com^",
"||toutiaocloud.net^",
"||toutiaovod.com^",
},
}, {
name: "vimeo",
rules: []string{
"*vod-adaptive.akamaized.net^",
"||vimeo.com^",
"||vimeocdn.com^",
},
}, {
name: "pinterest",
rules: []string{
"||pinimg.com^",
"||pinterest.*^",
},
}, {
name: "imgur",
rules: []string{"||imgur.com^"},
}, {
name: "dailymotion",
rules: []string{
"||dailymotion.com^",
"||dm-event.net^",
"||dmcdn.net^",
},
}, {
name: "qq",
rules: []string{
// Block qq.com and subdomains excluding WeChat's domains.
"||qq.com^$denyallow=wx.qq.com|weixin.qq.com",
"||qqzaixian.com^",
"||qq-video.cdn-go.cn^",
"||url.cn^",
},
}, {
name: "wechat",
rules: []string{
"||wechat.com^",
"||weixin.qq.com.cn^",
"||weixin.qq.com^",
"||weixinbridge.com^",
"||wx.qq.com^",
},
}, {
name: "viber",
rules: []string{"||viber.com^"},
}, {
name: "weibo",
rules: []string{
"||weibo.cn^",
"||weibo.com^",
"||weibocdn.com^",
},
}, {
name: "9gag",
rules: []string{
"||9cache.com^",
"||9gag.com^",
},
}, {
name: "telegram",
rules: []string{
"||t.me^",
"||telegram.me^",
"||telegram.org^",
},
}, {
name: "disneyplus",
rules: []string{
"||disney-plus.net^",
"||disney.playback.edge.bamgrid.com^",
"||disneynow.com^",
"||disneyplus.com^",
"||hotstar.com^",
"||media.dssott.com^",
"||star.playback.edge.bamgrid.com^",
"||starplus.com^",
},
}, {
name: "hulu",
rules: []string{"||hulu.com^"},
}, {
name: "spotify",
rules: []string{
"/_spotify-connect._tcp.local/",
"||spotify.com^",
"||scdn.co^",
"||spotify.com.edgesuite.net^",
"||spotify.map.fastly.net^",
"||spotify.map.fastlylb.net^",
"||spotifycdn.net^",
"||audio-ak-spotify-com.akamaized.net^",
"||audio4-ak-spotify-com.akamaized.net^",
"||heads-ak-spotify-com.akamaized.net^",
"||heads4-ak-spotify-com.akamaized.net^",
},
}, {
name: "tinder",
rules: []string{
"||gotinder.com^",
"||tinder.com^",
"||tindersparks.com^",
},
}, {
name: "bilibili",
rules: []string{
"||b23.tv^",
"||biliapi.net^",
"||bilibili.com^",
"||bilicdn1.com^",
"||bilicdn2.com^",
"||biligame.com^",
"||bilivideo.cn^",
"||bilivideo.com^",
"||dreamcast.hk^",
"||hdslb.com^",
},
}}
// serviceRules maps a service ID to its filtering rules.
var serviceRules map[string][]*rules.NetworkRule
@@ -394,16 +18,16 @@ var serviceIDs []string
// initBlockedServices initializes package-level blocked service data.
func initBlockedServices() {
l := len(servicesData)
l := len(blockedServices)
serviceIDs = make([]string, l)
serviceRules = make(map[string][]*rules.NetworkRule, l)
for i, s := range servicesData {
netRules := make([]*rules.NetworkRule, 0, len(s.rules))
for _, text := range s.rules {
for i, s := range blockedServices {
netRules := make([]*rules.NetworkRule, 0, len(s.Rules))
for _, text := range s.Rules {
rule, err := rules.NewNetworkRule(text, BlockedSvcsListID)
if err != nil {
log.Error("parsing blocked service %q rule %q: %s", s.name, text, err)
log.Error("parsing blocked service %q rule %q: %s", s.ID, text, err)
continue
}
@@ -411,8 +35,8 @@ func initBlockedServices() {
netRules = append(netRules, rule)
}
serviceIDs[i] = s.name
serviceRules[s.name] = netRules
serviceIDs[i] = s.ID
serviceRules[s.ID] = netRules
}
slices.Sort(serviceIDs)
@@ -420,7 +44,7 @@ func initBlockedServices() {
log.Debug("filtering: initialized %d services", l)
}
// BlockedSvcKnown - return TRUE if a blocked service name is known
// BlockedSvcKnown returns true if a blocked service ID is known.
func BlockedSvcKnown(s string) (ok bool) {
_, ok = serviceRules[s]
@@ -452,14 +76,16 @@ func (d *DNSFilter) ApplyBlockedServices(setts *Settings, list []string) {
}
}
func (d *DNSFilter) handleBlockedServicesAvailableServices(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
err := json.NewEncoder(w).Encode(serviceIDs)
if err != nil {
aghhttp.Error(r, w, http.StatusInternalServerError, "encoding available services: %s", err)
func (d *DNSFilter) handleBlockedServicesIDs(w http.ResponseWriter, r *http.Request) {
_ = aghhttp.WriteJSONResponse(w, r, serviceIDs)
}
return
}
func (d *DNSFilter) handleBlockedServicesAll(w http.ResponseWriter, r *http.Request) {
_ = aghhttp.WriteJSONResponse(w, r, struct {
BlockedServices []blockedService `json:"blocked_services"`
}{
BlockedServices: blockedServices,
})
}
func (d *DNSFilter) handleBlockedServicesList(w http.ResponseWriter, r *http.Request) {
@@ -467,13 +93,7 @@ func (d *DNSFilter) handleBlockedServicesList(w http.ResponseWriter, r *http.Req
list := d.Config.BlockedServices
d.confLock.RUnlock()
w.Header().Set("Content-Type", "application/json")
err := json.NewEncoder(w).Encode(list)
if err != nil {
aghhttp.Error(r, w, http.StatusInternalServerError, "encoding services: %s", err)
return
}
_ = aghhttp.WriteJSONResponse(w, r, list)
}
func (d *DNSFilter) handleBlockedServicesSet(w http.ResponseWriter, r *http.Request) {
@@ -491,5 +111,5 @@ func (d *DNSFilter) handleBlockedServicesSet(w http.ResponseWriter, r *http.Requ
log.Debug("Updated blocked services list: %d", len(list))
d.ConfigModified()
d.Config.ConfigModified()
}

134
internal/filtering/filter.go
View File

@@ -54,97 +54,120 @@ func (filter *FilterYAML) Path(dataDir string) string {
}
const (
statusFound = 1 << iota
statusEnabledChanged
statusURLChanged
statusURLExists
statusUpdateRequired
// errFilterNotExist is returned from [filterSetProperties] when there are
// no lists with the desired URL to update.
//
// TODO(e.burkov): Use wherever the same error is needed.
errFilterNotExist errors.Error = "url doesn't exist"
// errFilterExists is returned from [filterSetProperties] when there is
// another filter having the same URL as the one updated.
//
// TODO(e.burkov): Use wherever the same error is needed.
errFilterExists errors.Error = "url already exists"
)
// Update properties for a filter specified by its URL
// Return status* flags.
func (d *DNSFilter) filterSetProperties(url string, newf FilterYAML, whitelist bool) int {
r := 0
// filterSetProperties searches for the particular filter list by url and sets
// the values of newList to it, updating afterwards if needed. It returns true
// if the update was performed and the filtering engine restart is required.
func (d *DNSFilter) filterSetProperties(
listURL string,
newList FilterYAML,
isAllowlist bool,
) (shouldRestart bool, err error) {
d.filtersMu.Lock()
defer d.filtersMu.Unlock()
filters := d.Filters
if whitelist {
if isAllowlist {
filters = d.WhitelistFilters
}
i := slices.IndexFunc(filters, func(filt FilterYAML) bool {
return filt.URL == url
})
i := slices.IndexFunc(filters, func(filt FilterYAML) bool { return filt.URL == listURL })
if i == -1 {
return 0
return false, errFilterNotExist
}
filt := &filters[i]
log.Debug(
"filtering: set name to %q, url to %s, enabled to %t for filter %s",
newList.Name,
newList.URL,
newList.Enabled,
filt.URL,
)
log.Debug("filter: set properties: %s: {%s %s %v}", filt.URL, newf.Name, newf.URL, newf.Enabled)
filt.Name = newf.Name
defer func(oldURL, oldName string, oldEnabled bool, oldUpdated time.Time) {
if err != nil {
filt.URL = oldURL
filt.Name = oldName
filt.Enabled = oldEnabled
filt.LastUpdated = oldUpdated
}
}(filt.URL, filt.Name, filt.Enabled, filt.LastUpdated)
if filt.URL != newf.URL {
r |= statusURLChanged | statusUpdateRequired
if d.filterExistsNoLock(newf.URL) {
return statusURLExists
filt.Name = newList.Name
if filt.URL != newList.URL {
if d.filterExistsLocked(newList.URL) {
return false, errFilterExists
}
filt.URL = newf.URL
filt.unload()
shouldRestart = true
filt.URL = newList.URL
filt.LastUpdated = time.Time{}
filt.checksum = 0
filt.RulesCount = 0
filt.unload()
}
if filt.Enabled != newf.Enabled {
r |= statusEnabledChanged
filt.Enabled = newf.Enabled
if filt.Enabled {
if (r & statusURLChanged) == 0 {
err := d.load(filt)
if err != nil {
// TODO(e.burkov): It seems the error is only returned when
// the file exists and couldn't be open. Investigate and
// improve.
log.Error("loading filter %d: %s", filt.ID, err)
if filt.Enabled != newList.Enabled {
filt.Enabled = newList.Enabled
shouldRestart = true
}
filt.LastUpdated = time.Time{}
filt.checksum = 0
filt.RulesCount = 0
r |= statusUpdateRequired
}
}
} else {
filt.unload()
if filt.Enabled {
if shouldRestart {
// Download the filter contents.
shouldRestart, err = d.update(filt)
}
} else {
// TODO(e.burkov): The validation of the contents of the new URL is
// currently skipped if the rule list is disabled. This makes it
// possible to set a bad rules source, but the validation should still
// kick in when the filter is enabled. Consider making changing this
// behavior to be stricter.
filt.unload()
}
return r | statusFound
return shouldRestart, err
}
// Return TRUE if a filter with this URL exists
func (d *DNSFilter) filterExists(url string) bool {
// filterExists returns true if a filter with the same url exists in d. It's
// safe for concurrent use.
func (d *DNSFilter) filterExists(url string) (ok bool) {
d.filtersMu.RLock()
defer d.filtersMu.RUnlock()
r := d.filterExistsNoLock(url)
r := d.filterExistsLocked(url)
return r
}
func (d *DNSFilter) filterExistsNoLock(url string) bool {
// filterExistsLocked returns true if d contains the filter with the same url.
// d.filtersMu is expected to be locked.
func (d *DNSFilter) filterExistsLocked(url string) (ok bool) {
for _, f := range d.Filters {
if f.URL == url {
return true
}
}
for _, f := range d.WhitelistFilters {
if f.URL == url {
return true
}
}
return false
}
@@ -155,7 +178,7 @@ func (d *DNSFilter) filterAdd(flt FilterYAML) bool {
defer d.filtersMu.Unlock()
// Check for duplicates
if d.filterExistsNoLock(flt.URL) {
if d.filterExistsLocked(flt.URL) {
return false
}
@@ -258,18 +281,6 @@ func (d *DNSFilter) tryRefreshFilters(block, allow, force bool) (updated int, is
return updated, isNetworkErr, ok
}
// refreshFilters updates the lists and returns the number of updated ones.
// It's safe for concurrent use, but blocks at least until the previous
// refreshing is finished.
func (d *DNSFilter) refreshFilters(block, allow, force bool) (updated int) {
d.refreshLock.Lock()
defer d.refreshLock.Unlock()
updated, _ = d.refreshFiltersIntl(block, allow, force)
return updated
}
// listsToUpdate returns the slice of filter lists that could be updated.
func (d *DNSFilter) listsToUpdate(filters *[]FilterYAML, force bool) (toUpd []FilterYAML) {
now := time.Now()
@@ -279,7 +290,6 @@ func (d *DNSFilter) listsToUpdate(filters *[]FilterYAML, force bool) (toUpd []Fi
for i := range *filters {
flt := &(*filters)[i] // otherwise we will be operating on a copy
log.Debug("checking list at index %d: %v", i, flt)
if !flt.Enabled {
continue

50
internal/filtering/filter_test.go
View File

@@ -4,40 +4,43 @@ import (
"io/fs"
"net"
"net/http"
"net/netip"
"net/url"
"os"
"path"
"path/filepath"
"testing"
"time"
"github.com/AdguardTeam/golibs/netutil"
"github.com/AdguardTeam/AdGuardHome/internal/aghnet"
"github.com/AdguardTeam/golibs/testutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const testFltsFileName = "1.txt"
func testStartFilterListener(t *testing.T, fltContent *[]byte) (l net.Listener) {
// serveFiltersLocally is a helper that concurrently listens on a free port to
// respond with fltContent. It also gracefully closes the listener when the
// test under t finishes.
func serveFiltersLocally(t *testing.T, fltContent []byte) (ipp netip.AddrPort) {
t.Helper()
h := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
n, werr := w.Write(*fltContent)
require.NoError(t, werr)
require.Equal(t, len(*fltContent), n)
pt := testutil.PanicT{}
n, werr := w.Write(fltContent)
require.NoError(pt, werr)
require.Equal(pt, len(fltContent), n)
})
var err error
l, err = net.Listen("tcp", ":0")
l, err := net.Listen("tcp", ":0")
require.NoError(t, err)
go func() {
_ = http.Serve(l, h)
}()
go func() { _ = http.Serve(l, h) }()
testutil.CleanupAndRequireSuccess(t, l.Close)
return l
addr := l.Addr()
require.IsType(t, new(net.TCPAddr), addr)
return netip.AddrPortFrom(aghnet.IPv4Localhost(), uint16(addr.(*net.TCPAddr).Port))
}
func TestFilters(t *testing.T) {
@@ -49,7 +52,7 @@ func TestFilters(t *testing.T) {
fltContent := []byte(content)
l := testStartFilterListener(t, &fltContent)
addr := serveFiltersLocally(t, fltContent)
tempDir := t.TempDir()
@@ -64,11 +67,7 @@ func TestFilters(t *testing.T) {
f := &FilterYAML{
URL: (&url.URL{
Scheme: "http",
Host: (&netutil.IPPort{
IP: net.IP{127, 0, 0, 1},
Port: l.Addr().(*net.TCPAddr).Port,
}).String(),
Path: path.Join(filterDir, testFltsFileName),
Host: addr.String(),
}).String(),
}
@@ -101,8 +100,15 @@ func TestFilters(t *testing.T) {
})
t.Run("refresh_actually", func(t *testing.T) {
fltContent = []byte(`||example.com^`)
t.Cleanup(func() { fltContent = []byte(content) })
anotherContent := []byte(`||example.com^`)
oldURL := f.URL
ipp := serveFiltersLocally(t, anotherContent)
f.URL = (&url.URL{
Scheme: "http",
Host: ipp.String(),
}).String()
t.Cleanup(func() { f.URL = oldURL })
updateAndAssert(t, require.True, 1)
})

16
internal/filtering/filtering.go
View File

@@ -345,27 +345,29 @@ func (d *DNSFilter) SetFilters(blockFilters, allowFilters []Filter, async bool)
blockFilters: blockFilters,
}
d.filtersInitializerLock.Lock() // prevent multiple writers from adding more than 1 task
d.filtersInitializerLock.Lock()
defer d.filtersInitializerLock.Unlock()
// remove all pending tasks
stop := false
for !stop {
// Remove all pending tasks.
removeLoop:
for {
select {
case <-d.filtersInitializerChan:
//
// Continue removing.
default:
stop = true
break removeLoop
}
}
d.filtersInitializerChan <- params
return nil
}
err := d.initFiltering(allowFilters, blockFilters)
if err != nil {
log.Error("Can't initialize filtering subsystem: %s", err)
log.Error("filtering: can't initialize filtering subsystem: %s", err)
return err
}

3
internal/filtering/filtering_test.go
View File

@@ -11,6 +11,7 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghtest"
"github.com/AdguardTeam/golibs/cache"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/testutil"
"github.com/AdguardTeam/urlfilter/rules"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
@@ -18,7 +19,7 @@ import (
)
func TestMain(m *testing.M) {
aghtest.DiscardLogOutput(m)
testutil.DiscardLogOutput(m)
}
const (

58
internal/filtering/http.go
View File

@@ -56,7 +56,6 @@ func (d *DNSFilter) handleFilteringAddURL(w http.ResponseWriter, r *http.Request
err = validateFilterURL(fj.URL)
if err != nil {
err = fmt.Errorf("invalid url: %s", err)
aghhttp.Error(r, w, http.StatusBadRequest, "%s", err)
return
@@ -75,8 +74,10 @@ func (d *DNSFilter) handleFilteringAddURL(w http.ResponseWriter, r *http.Request
URL: fj.URL,
Name: fj.Name,
white: fj.Whitelist,
Filter: Filter{
ID: assignUniqueFilterID(),
},
}
filt.ID = assignUniqueFilterID()
// Download the filter contents
ok, err := d.update(&filt)
@@ -216,32 +217,15 @@ func (d *DNSFilter) handleFilteringSetURL(w http.ResponseWriter, r *http.Request
Name: fj.Data.Name,
URL: fj.Data.URL,
}
status := d.filterSetProperties(fj.URL, filt, fj.Whitelist)
if (status & statusFound) == 0 {
aghhttp.Error(r, w, http.StatusBadRequest, "URL doesn't exist")
return
}
if (status & statusURLExists) != 0 {
aghhttp.Error(r, w, http.StatusBadRequest, "URL already exists")
restart, err := d.filterSetProperties(fj.URL, filt, fj.Whitelist)
if err != nil {
aghhttp.Error(r, w, http.StatusBadRequest, err.Error())
return
}
d.ConfigModified()
restart := (status & statusEnabledChanged) != 0
if (status&statusUpdateRequired) != 0 && fj.Data.Enabled {
// download new filter and apply its rules.
nUpdated := d.refreshFilters(!fj.Whitelist, fj.Whitelist, false)
// if at least 1 filter has been updated, refreshFilters() restarts the filtering automatically
// if not - we restart the filtering ourselves
restart = false
if nUpdated == 0 {
restart = true
}
}
if restart {
d.EnableFilters(true)
}
@@ -301,14 +285,7 @@ func (d *DNSFilter) handleFilteringRefresh(w http.ResponseWriter, r *http.Reques
return
}
w.Header().Set("Content-Type", "application/json")
err = json.NewEncoder(w).Encode(resp)
if err != nil {
aghhttp.Error(r, w, http.StatusInternalServerError, "json encode: %s", err)
return
}
_ = aghhttp.WriteJSONResponse(w, r, resp)
}
type filterJSON struct {
@@ -361,17 +338,7 @@ func (d *DNSFilter) handleFilteringStatus(w http.ResponseWriter, r *http.Request
resp.UserRules = d.UserRules
d.filtersMu.RUnlock()
jsonVal, err := json.Marshal(resp)
if err != nil {
aghhttp.Error(r, w, http.StatusInternalServerError, "json encode: %s", err)
return
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(jsonVal)
if err != nil {
aghhttp.Error(r, w, http.StatusInternalServerError, "http write: %s", err)
}
_ = aghhttp.WriteJSONResponse(w, r, resp)
}
// Set filtering configuration
@@ -473,11 +440,7 @@ func (d *DNSFilter) handleCheckHost(w http.ResponseWriter, r *http.Request) {
}
}
w.Header().Set("Content-Type", "application/json")
err = json.NewEncoder(w).Encode(resp)
if err != nil {
aghhttp.Error(r, w, http.StatusInternalServerError, "encoding response: %s", err)
}
_ = aghhttp.WriteJSONResponse(w, r, resp)
}
// RegisterFilteringHandlers - register handlers
@@ -503,7 +466,8 @@ func (d *DNSFilter) RegisterFilteringHandlers() {
registerHTTP(http.MethodPost, "/control/rewrite/add", d.handleRewriteAdd)
registerHTTP(http.MethodPost, "/control/rewrite/delete", d.handleRewriteDelete)
registerHTTP(http.MethodGet, "/control/blocked_services/services", d.handleBlockedServicesAvailableServices)
registerHTTP(http.MethodGet, "/control/blocked_services/services", d.handleBlockedServicesIDs)
registerHTTP(http.MethodGet, "/control/blocked_services/all", d.handleBlockedServicesAll)
registerHTTP(http.MethodGet, "/control/blocked_services/list", d.handleBlockedServicesList)
registerHTTP(http.MethodPost, "/control/blocked_services/set", d.handleBlockedServicesSet)

143
internal/filtering/http_test.go Normal file
View File

@@ -0,0 +1,143 @@
package filtering
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDNSFilter_handleFilteringSetURL(t *testing.T) {
filtersDir := t.TempDir()
var goodRulesEndpoint, anotherGoodRulesEndpoint, badRulesEndpoint string
for _, rulesSource := range []struct {
endpoint *string
content []byte
}{{
endpoint: &goodRulesEndpoint,
content: []byte(`||example.org^`),
}, {
endpoint: &anotherGoodRulesEndpoint,
content: []byte(`||example.com^`),
}, {
endpoint: &badRulesEndpoint,
content: []byte(`<html></html>`),
}} {
ipp := serveFiltersLocally(t, rulesSource.content)
*rulesSource.endpoint = (&url.URL{
Scheme: "http",
Host: ipp.String(),
}).String()
}
testCases := []struct {
name string
wantBody string
oldURL string
newName string
newURL string
initial []FilterYAML
}{{
name: "success",
wantBody: "",
oldURL: goodRulesEndpoint,
newName: "default_one",
newURL: anotherGoodRulesEndpoint,
initial: []FilterYAML{{
Enabled: true,
URL: goodRulesEndpoint,
Name: "default_one",
white: false,
}},
}, {
name: "non-existing",
wantBody: "url doesn't exist\n",
oldURL: anotherGoodRulesEndpoint,
newName: "default_one",
newURL: goodRulesEndpoint,
initial: []FilterYAML{{
Enabled: true,
URL: goodRulesEndpoint,
Name: "default_one",
white: false,
}},
}, {
name: "existing",
wantBody: "url already exists\n",
oldURL: goodRulesEndpoint,
newName: "default_one",
newURL: anotherGoodRulesEndpoint,
initial: []FilterYAML{{
Enabled: true,
URL: goodRulesEndpoint,
Name: "default_one",
white: false,
}, {
Enabled: true,
URL: anotherGoodRulesEndpoint,
Name: "another_default_one",
white: false,
}},
}, {
name: "bad_rules",
wantBody: "data is HTML, not plain text\n",
oldURL: goodRulesEndpoint,
newName: "default_one",
newURL: badRulesEndpoint,
initial: []FilterYAML{{
Enabled: true,
URL: goodRulesEndpoint,
Name: "default_one",
white: false,
}},
}}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
confModifiedCalled := false
d, err := New(&Config{
FilteringEnabled: true,
Filters: tc.initial,
HTTPClient: &http.Client{
Timeout: 5 * time.Second,
},
ConfigModified: func() { confModifiedCalled = true },
DataDir: filtersDir,
}, nil)
require.NoError(t, err)
t.Cleanup(d.Close)
d.Start()
reqData := &filterURLReq{
Data: &filterURLReqData{
// Leave the name of an existing list.
Name: tc.newName,
URL: tc.newURL,
Enabled: true,
},
URL: tc.oldURL,
Whitelist: false,
}
data, err := json.Marshal(reqData)
require.NoError(t, err)
r := httptest.NewRequest(http.MethodPost, "http://example.org", bytes.NewReader(data))
w := httptest.NewRecorder()
d.handleFilteringSetURL(w, r)
assert.Equal(t, tc.wantBody, w.Body.String())
// For the moment the non-empty response body only contains occurred
// error, so the configuration shouldn't be written.
assert.Equal(t, tc.wantBody == "", confModifiedCalled)
})
}
}

12
internal/filtering/rewrites.go
View File

@@ -13,8 +13,8 @@ import (
"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
"github.com/AdguardTeam/golibs/errors"
"github.com/AdguardTeam/golibs/log"
"github.com/AdguardTeam/golibs/netutil"
"github.com/miekg/dns"
"golang.org/x/exp/slices"
)
// LegacyRewrite is a single legacy DNS rewrite record.
@@ -41,7 +41,7 @@ func (rw *LegacyRewrite) clone() (cloneRW *LegacyRewrite) {
return &LegacyRewrite{
Domain: rw.Domain,
Answer: rw.Answer,
IP: netutil.CloneIP(rw.IP),
IP: slices.Clone(rw.IP),
Type: rw.Type,
}
}
@@ -240,13 +240,7 @@ func (d *DNSFilter) handleRewriteList(w http.ResponseWriter, r *http.Request) {
}
d.confLock.Unlock()
w.Header().Set("Content-Type", "application/json")
err := json.NewEncoder(w).Encode(arr)
if err != nil {
aghhttp.Error(r, w, http.StatusInternalServerError, "json.Encode: %s", err)
return
}
_ = aghhttp.WriteJSONResponse(w, r, arr)
}
func (d *DNSFilter) handleRewriteAdd(w http.ResponseWriter, r *http.Request) {

19
internal/filtering/safebrowsing.go
View File

@@ -5,7 +5,6 @@ import (
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"encoding/json"
"fmt"
"net"
"net/http"
@@ -381,17 +380,13 @@ func (d *DNSFilter) handleSafeBrowsingDisable(w http.ResponseWriter, r *http.Req
}
func (d *DNSFilter) handleSafeBrowsingStatus(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
err := json.NewEncoder(w).Encode(&struct {
resp := &struct {
Enabled bool `json:"enabled"`
}{
Enabled: d.Config.SafeBrowsingEnabled,
})
if err != nil {
aghhttp.Error(r, w, http.StatusInternalServerError, "Unable to write response json: %s", err)
return
}
_ = aghhttp.WriteJSONResponse(w, r, resp)
}
func (d *DNSFilter) handleParentalEnable(w http.ResponseWriter, r *http.Request) {
@@ -405,13 +400,11 @@ func (d *DNSFilter) handleParentalDisable(w http.ResponseWriter, r *http.Request
}
func (d *DNSFilter) handleParentalStatus(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
err := json.NewEncoder(w).Encode(&struct {
resp := &struct {
Enabled bool `json:"enabled"`
}{
Enabled: d.Config.ParentalEnabled,
})
if err != nil {
aghhttp.Error(r, w, http.StatusInternalServerError, "Unable to write response json: %s", err)
}
_ = aghhttp.WriteJSONResponse(w, r, resp)
}

15
internal/filtering/safesearch.go
View File

@@ -5,7 +5,6 @@ import (
"context"
"encoding/binary"
"encoding/gob"
"encoding/json"
"fmt"
"net"
"net/http"
@@ -146,21 +145,13 @@ func (d *DNSFilter) handleSafeSearchDisable(w http.ResponseWriter, r *http.Reque
}
func (d *DNSFilter) handleSafeSearchStatus(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
err := json.NewEncoder(w).Encode(&struct {
resp := &struct {
Enabled bool `json:"enabled"`
}{
Enabled: d.Config.SafeSearchEnabled,
})
if err != nil {
aghhttp.Error(
r,
w,
http.StatusInternalServerError,
"Unable to write response json: %s",
err,
)
}
_ = aghhttp.WriteJSONResponse(w, r, resp)
}
var safeSearchDomains = map[string]string{

471
internal/filtering/servicelist.go Normal file
View File

File diff suppressed because one or more lines are too long