all: sync with master

2022-09-07 18:03:18 +03:00
parent d23da1b757
commit faf2b32389
108 changed files with 2925 additions and 2390 deletions
--- a/internal/filtering/blocked.go
+++ b/internal/filtering/blocked.go
@@ -7,21 +7,24 @@ import (
 	"github.com/AdguardTeam/AdGuardHome/internal/aghhttp"
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/AdguardTeam/urlfilter/rules"
+	"golang.org/x/exp/slices"
 )

-var serviceRules map[string][]*rules.NetworkRule // service name -> filtering rules
-
+// svc represents a single blocked service.
 type svc struct {
 	name  string
 	rules []string
 }

+// servicesData contains raw blocked service data.
+//
 // Keep in sync with:
-// client/src/helpers/constants.js
-// client/src/components/ui/Icons.js
-var serviceRulesArray = []svc{{
+//   - client/src/helpers/constants.js
+//   - client/src/components/ui/Icons.js
+var servicesData = []svc{{
 	name: "whatsapp",
 	rules: []string{
+		"||wa.me^",
 		"||whatsapp.com^",
 		"||whatsapp.net^",
 	},
@@ -34,7 +37,9 @@ var serviceRulesArray = []svc{{
 		"||accountkit.com^",
 		"||fb.me^",
 		"||fb.com^",
+		"||fb.gg^",
 		"||fbsbx.com^",
+		"||fbwat.ch^",
 		"||messenger.com^",
 		"||facebookcorewwwi.onion^",
 		"||fbcdn.com^",
@@ -58,6 +63,7 @@ var serviceRulesArray = []svc{{
 		"||youtube-nocookie.com^",
 		"||youtube.com^",
 		"||youtubei.googleapis.com^",
+		"||youtubekids.com^",
 		"||ytimg.com^",
 	},
 }, {
@@ -97,20 +103,36 @@ var serviceRulesArray = []svc{{
 		"||discordapp.net^",
 		"||discordapp.com^",
 		"||discord.com^",
+		"||discord.gift",
 		"||discord.media^",
 	},
 }, {
 	name:  "ok",
 	rules: []string{"||ok.ru^"},
 }, {
-	name:  "skype",
-	rules: []string{"||skype.com^", "||skypeassets.com^"},
+	name: "skype",
+	rules: []string{
+		"||edge-skype-com.s-0001.s-msedge.net^",
+		"||skype-edf.akadns.net^",
+		"||skype.com^",
+		"||skypeassets.com^",
+		"||skypedata.akadns.net^",
+	},
 }, {
-	name:  "vk",
-	rules: []string{"||vk.com^", "||userapi.com^", "||vk-cdn.net^", "||vkuservideo.net^"},
+	name: "vk",
+	rules: []string{
+		"||userapi.com^",
+		"||vk-cdn.net^",
+		"||vk.com^",
+		"||vkuservideo.net^",
+	},
 }, {
-	name:  "origin",
-	rules: []string{"||origin.com^", "||signin.ea.com^", "||accounts.ea.com^"},
+	name: "origin",
+	rules: []string{
+		"||accounts.ea.com^",
+		"||origin.com^",
+		"||signin.ea.com^",
+	},
 }, {
 	name: "steam",
 	rules: []string{
@@ -133,20 +155,30 @@ var serviceRulesArray = []svc{{
 }, {
 	name: "cloudflare",
 	rules: []string{
-		"||cloudflare.com^",
-		"||cloudflare-dns.com^",
-		"||cloudflare.net^",
-		"||cloudflareinsights.com^",
-		"||cloudflarestream.com^",
-		"||cloudflareresolve.com^",
-		"||cloudflareclient.com^",
-		"||cloudflarebolt.com^",
-		"||cloudflarestatus.com^",
-		"||cloudflare.cn^",
-		"||one.one^",
-		"||warp.plus^",
 		"||1.1.1.1^",
+		"||argotunnel.com^",
+		"||cloudflare-dns.com^",
+		"||cloudflare-ipfs.com^",
+		"||cloudflare-quic.com^",
+		"||cloudflare.cn^",
+		"||cloudflare.com^",
+		"||cloudflare.net^",
+		"||cloudflareaccess.com^",
+		"||cloudflareapps.com^",
+		"||cloudflarebolt.com^",
+		"||cloudflareclient.com^",
+		"||cloudflareinsights.com^",
+		"||cloudflareresolve.com^",
+		"||cloudflarestatus.com^",
+		"||cloudflarestream.com^",
+		"||cloudflarewarp.com^",
 		"||dns4torpnlfs2ifuz2s2yf3fc7rdmsbhm6rw75euj35pac6ap25zgqad.onion^",
+		"||one.one^",
+		"||pages.dev^",
+		"||trycloudflare.com^",
+		"||videodelivery.net^",
+		"||warp.plus^",
+		"||workers.dev^",
 	},
 }, {
 	name: "amazon",
@@ -174,6 +206,7 @@ var serviceRulesArray = []svc{{
 		"||amazon.com.br^",
 		"||amazon.co.jp^",
 		"||amazon.com.mx^",
+		"||amazon.com.tr^",
 		"||amazon.co.uk^",
 		"||createspace.com^",
 		"||aws",
@@ -210,27 +243,31 @@ var serviceRulesArray = []svc{{
 }, {
 	name: "tiktok",
 	rules: []string{
-		"||tiktok.com^",
-		"||tiktokcdn.com^",
-		"||musical.ly^",
-		"||snssdk.com^",
 		"||amemv.com^",
-		"||toutiao.com^",
-		"||ixigua.com^",
-		"||pstatp.com^",
-		"||ixiguavideo.com^",
-		"||toutiaocloud.com^",
-		"||toutiaocloud.net^",
 		"||bdurl.com^",
 		"||bytecdn.cn^",
-		"||byteimg.com^",
-		"||ixigua.com^",
-		"||muscdn.com^",
 		"||bytedance.map.fastly.net^",
+		"||bytedapm.com^",
+		"||byteimg.com^",
+		"||byteoversea.com^",
 		"||douyin.com^",
-		"||tiktokv.com^",
-		"||toutiaovod.com^",
 		"||douyincdn.com^",
+		"||douyinpic.com^",
+		"||douyinstatic.com^",
+		"||douyinvod.com^",
+		"||ixigua.com^",
+		"||ixiguavideo.com^",
+		"||muscdn.com^",
+		"||musical.ly^",
+		"||pstatp.com^",
+		"||snssdk.com^",
+		"||tiktok.com^",
+		"||tiktokcdn.com^",
+		"||tiktokv.com^",
+		"||toutiao.com^",
+		"||toutiaocloud.com^",
+		"||toutiaocloud.net^",
+		"||toutiaovod.com^",
 	},
 }, {
 	name: "vimeo",
@@ -300,9 +337,13 @@ var serviceRulesArray = []svc{{
 	name: "disneyplus",
 	rules: []string{
 		"||disney-plus.net^",
-		"||disneyplus.com^",
 		"||disney.playback.edge.bamgrid.com^",
+		"||disneynow.com^",
+		"||disneyplus.com^",
+		"||hotstar.com^",
 		"||media.dssott.com^",
+		"||star.playback.edge.bamgrid.com^",
+		"||starplus.com^",
 	},
 }, {
 	name:  "hulu",
@@ -332,8 +373,11 @@ var serviceRulesArray = []svc{{
 }, {
 	name: "bilibili",
 	rules: []string{
+		"||b23.tv^",
 		"||biliapi.net^",
 		"||bilibili.com^",
+		"||bilicdn1.com^",
+		"||bilicdn2.com^",
 		"||biligame.com^",
 		"||bilivideo.cn^",
 		"||bilivideo.com^",
@@ -342,21 +386,38 @@ var serviceRulesArray = []svc{{
 	},
 }}

-// convert array to map
+// serviceRules maps a service ID to its filtering rules.
+var serviceRules map[string][]*rules.NetworkRule
+
+// serviceIDs contains service IDs sorted alphabetically.
+var serviceIDs []string
+
+// initBlockedServices initializes package-level blocked service data.
 func initBlockedServices() {
-	serviceRules = make(map[string][]*rules.NetworkRule)
-	for _, s := range serviceRulesArray {
-		netRules := []*rules.NetworkRule{}
+	l := len(servicesData)
+	serviceIDs = make([]string, l)
+	serviceRules = make(map[string][]*rules.NetworkRule, l)
+
+	for i, s := range servicesData {
+		netRules := make([]*rules.NetworkRule, 0, len(s.rules))
 		for _, text := range s.rules {
 			rule, err := rules.NewNetworkRule(text, BlockedSvcsListID)
 			if err != nil {
-				log.Error("rules.NewNetworkRule: %s  rule: %s", err, text)
+				log.Error("parsing blocked service %q rule %q: %s", s.name, text, err)
+
 				continue
 			}
+
 			netRules = append(netRules, rule)
 		}
+
+		serviceIDs[i] = s.name
 		serviceRules[s.name] = netRules
 	}
+
+	slices.Sort(serviceIDs)
+
+	log.Debug("filtering: initialized %d services", l)
 }

 // BlockedSvcKnown - return TRUE if a blocked service name is known
@@ -388,6 +449,16 @@ func (d *DNSFilter) ApplyBlockedServices(setts *Settings, list []string, global
 	}
 }

+func (d *DNSFilter) handleBlockedServicesAvailableServices(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	err := json.NewEncoder(w).Encode(serviceIDs)
+	if err != nil {
+		aghhttp.Error(r, w, http.StatusInternalServerError, "encoding available services: %s", err)
+
+		return
+	}
+}
+
 func (d *DNSFilter) handleBlockedServicesList(w http.ResponseWriter, r *http.Request) {
 	d.confLock.RLock()
 	list := d.Config.BlockedServices
@@ -396,7 +467,7 @@ func (d *DNSFilter) handleBlockedServicesList(w http.ResponseWriter, r *http.Req
 	w.Header().Set("Content-Type", "application/json")
 	err := json.NewEncoder(w).Encode(list)
 	if err != nil {
-		aghhttp.Error(r, w, http.StatusInternalServerError, "json.Encode: %s", err)
+		aghhttp.Error(r, w, http.StatusInternalServerError, "encoding services: %s", err)

 		return
 	}
@@ -422,6 +493,7 @@ func (d *DNSFilter) handleBlockedServicesSet(w http.ResponseWriter, r *http.Requ

 // registerBlockedServicesHandlers - register HTTP handlers
 func (d *DNSFilter) registerBlockedServicesHandlers() {
+	d.Config.HTTPRegister(http.MethodGet, "/control/blocked_services/services", d.handleBlockedServicesAvailableServices)
 	d.Config.HTTPRegister(http.MethodGet, "/control/blocked_services/list", d.handleBlockedServicesList)
 	d.Config.HTTPRegister(http.MethodPost, "/control/blocked_services/set", d.handleBlockedServicesSet)
 }
--- a/internal/filtering/dnsrewrite.go
+++ b/internal/filtering/dnsrewrite.go
@@ -34,8 +34,8 @@ func (d *DNSFilter) processDNSRewrites(dnsr []*rules.NetworkRule) (res Result) {
 			}}

 			return Result{
-				Reason:    RewrittenRule,
 				Rules:     rules,
+				Reason:    RewrittenRule,
 				CanonName: dr.NewCNAME,
 			}
 		}
@@ -60,16 +60,16 @@ func (d *DNSFilter) processDNSRewrites(dnsr []*rules.NetworkRule) (res Result) {
 			}

 			return Result{
-				Reason:           RewrittenRule,
-				Rules:            rules,
 				DNSRewriteResult: dnsrr,
+				Rules:            rules,
+				Reason:           RewrittenRule,
 			}
 		}
 	}

 	return Result{
-		Reason:           RewrittenRule,
-		Rules:            rules,
 		DNSRewriteResult: dnsrr,
+		Rules:            rules,
+		Reason:           RewrittenRule,
 	}
 }
--- a/internal/filtering/filtering.go
+++ b/internal/filtering/filtering.go
@@ -296,9 +296,11 @@ func cloneRewrites(entries []*LegacyRewrite) (clone []*LegacyRewrite) {
 	return clone
 }

-// SetFilters - set new filters (synchronously or asynchronously)
-// When filters are set asynchronously, the old filters continue working until the new filters are ready.
-//  In this case the caller must ensure that the old filter files are intact.
+// SetFilters sets new filters, synchronously or asynchronously.  When filters
+// are set asynchronously, the old filters continue working until the new
+// filters are ready.
+//
+// In this case the caller must ensure that the old filter files are intact.
 func (d *DNSFilter) SetFilters(blockFilters, allowFilters []Filter, async bool) error {
 	if async {
 		params := filtersInitializerParams{
@@ -388,18 +390,8 @@ type ResultRule struct {
 // TODO(a.garipov): Clarify relationships between fields.  Perhaps
 // replace with a sum type or an interface?
 type Result struct {
-	// IsFiltered is true if the request is filtered.
-	IsFiltered bool `json:",omitempty"`
-
-	// Reason is the reason for blocking or unblocking the request.
-	Reason Reason `json:",omitempty"`
-
-	// Rules are applied rules.  If Rules are not empty, each rule is not nil.
-	Rules []*ResultRule `json:",omitempty"`
-
-	// IPList is the lookup rewrite result.  It is empty unless Reason is set to
-	// Rewritten.
-	IPList []net.IP `json:",omitempty"`
+	// DNSRewriteResult is the $dnsrewrite filter rule result.
+	DNSRewriteResult *DNSRewriteResult `json:",omitempty"`

 	// CanonName is the CNAME value from the lookup rewrite result.  It is empty
 	// unless Reason is set to Rewritten or RewrittenRule.
@@ -409,8 +401,18 @@ type Result struct {
 	// Reason is set to FilteredBlockedService.
 	ServiceName string `json:",omitempty"`

-	// DNSRewriteResult is the $dnsrewrite filter rule result.
-	DNSRewriteResult *DNSRewriteResult `json:",omitempty"`
+	// IPList is the lookup rewrite result.  It is empty unless Reason is set to
+	// Rewritten.
+	IPList []net.IP `json:",omitempty"`
+
+	// Rules are applied rules.  If Rules are not empty, each rule is not nil.
+	Rules []*ResultRule `json:",omitempty"`
+
+	// Reason is the reason for blocking or unblocking the request.
+	Reason Reason `json:",omitempty"`
+
+	// IsFiltered is true if the request is filtered.
+	IsFiltered bool `json:",omitempty"`
 }

 // Matched returns true if any match at all was found regardless of
@@ -872,9 +874,9 @@ func makeResult(matchedRules []rules.Rule, reason Reason) (res Result) {
 	}

 	return Result{
-		IsFiltered: reason == FilteredBlockList,
-		Reason:     reason,
 		Rules:      resRules,
+		Reason:     reason,
+		IsFiltered: reason == FilteredBlockList,
 	}
 }

--- a/internal/filtering/rewrites.go
+++ b/internal/filtering/rewrites.go
@@ -130,10 +130,9 @@ func matchDomainWildcard(host, wildcard string) (ok bool) {
 //
 // The sorting priority:
 //
-//   A and AAAA > CNAME
-//   wildcard > exact
-//   lower level wildcard > higher level wildcard
-//
+//  1. A and AAAA > CNAME
+//  2. wildcard > exact
+//  3. lower level wildcard > higher level wildcard
 type rewritesSorted []*LegacyRewrite

 // Len implements the sort.Interface interface for legacyRewritesSorted.
--- a/internal/filtering/safebrowsing.go
+++ b/internal/filtering/safebrowsing.go
@@ -325,12 +325,12 @@ func (d *DNSFilter) checkSafeBrowsing(
 	}

 	res = Result{
-		IsFiltered: true,
-		Reason:     FilteredSafeBrowsing,
 		Rules: []*ResultRule{{
 			Text:         "adguard-malware-shavar",
 			FilterListID: SafeBrowsingListID,
 		}},
+		Reason:     FilteredSafeBrowsing,
+		IsFiltered: true,
 	}

 	return check(sctx, res, d.safeBrowsingUpstream)
@@ -359,12 +359,12 @@ func (d *DNSFilter) checkParental(
 	}

 	res = Result{
-		IsFiltered: true,
-		Reason:     FilteredParental,
 		Rules: []*ResultRule{{
 			Text:         "parental CATEGORY_BLACKLISTED",
 			FilterListID: ParentalListID,
 		}},
+		Reason:     FilteredParental,
+		IsFiltered: true,
 	}

 	return check(sctx, res, d.parentalUpstream)
--- a/internal/filtering/safesearch.go
+++ b/internal/filtering/safesearch.go
@@ -98,11 +98,11 @@ func (d *DNSFilter) checkSafeSearch(
 	}

 	res = Result{
-		IsFiltered: true,
-		Reason:     FilteredSafeSearch,
 		Rules: []*ResultRule{{
 			FilterListID: SafeSearchListID,
 		}},
+		Reason:     FilteredSafeSearch,
+		IsFiltered: true,
 	}

 	if ip := net.ParseIP(safeHost); ip != nil {