Pull request: 3972-hostlists-services

Updates #3972. Squashed commit of the following: commit 9dc0efe2453cb6c738d97d39b02c86eccb18a42c Merge: 239550f8 8a935d4f Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Thu Oct 27 14:42:38 2022 +0300 Merge branch 'master' into 3972-hostlists-services commit 239550f84228e7c7a6f4ae6b1cadcc47e01f54d5 Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Thu Oct 27 14:41:42 2022 +0300 filtering: upd service list commit b8bf3a6a4b1333059b886be95a1419612aebac39 Author: Ildar Kamalov <ik@adguard.com> Date: Thu Oct 27 13:41:09 2022 +0300 client: remove todo commit caa504b482befb804db2a1ca0b6d4834aa4da49a Author: Ildar Kamalov <ik@adguard.com> Date: Thu Oct 27 12:54:45 2022 +0300 fix build commit 511797c305d9eef84a20553dab795414e00da51a Author: Ildar Kamalov <ik@adguard.com> Date: Thu Oct 27 12:40:33 2022 +0300 client: add titles with service names to the clients table commit 79ed3157a85b489a0b13381cff867a8c73ba60e9 Author: Ildar Kamalov <ik@adguard.com> Date: Thu Oct 27 12:36:59 2022 +0300 client: fix empty icons commit ab69b95784de87665d5a1a3683f28e3b3df1c210 Author: Ildar Kamalov <ik@adguard.com> Date: Thu Oct 27 11:55:48 2022 +0300 client: use all blocked services commit 9a4a87665c8463224d8e93f1e162988107f6c7ca Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Tue Oct 25 19:25:20 2022 +0300 all: fix json response commit 86eb4493ce305cd5991176bd4cd8f7f5afdea330 Author: Ainar Garipov <A.Garipov@AdGuard.COM> Date: Tue Oct 25 19:09:44 2022 +0300 all: use hostslists registry for blocked svcs
2022-10-27 15:46:25 +03:00
parent 8a935d4ffb
commit 9951d861d1
27 changed files with 987 additions and 922 deletions
--- a/internal/filtering/blocked.go
+++ b/internal/filtering/blocked.go
@@ -10,382 +10,6 @@ import (
 	"golang.org/x/exp/slices"
 )

-// svc represents a single blocked service.
-type svc struct {
-	name  string
-	rules []string
-}
-
-// servicesData contains raw blocked service data.
-//
-// Keep in sync with:
-//   - client/src/helpers/constants.js
-//   - client/src/components/ui/Icons.js
-var servicesData = []svc{{
-	name: "whatsapp",
-	rules: []string{
-		"||wa.me^",
-		"||whatsapp.com^",
-		"||whatsapp.net^",
-	},
-}, {
-	name: "facebook",
-	rules: []string{
-		"||facebook.com^",
-		"||facebook.net^",
-		"||fbcdn.net^",
-		"||accountkit.com^",
-		"||fb.me^",
-		"||fb.com^",
-		"||fb.gg^",
-		"||fbsbx.com^",
-		"||fbwat.ch^",
-		"||messenger.com^",
-		"||facebookcorewwwi.onion^",
-		"||fbcdn.com^",
-		"||fb.watch^",
-	},
-}, {
-	name: "twitter",
-	rules: []string{
-		"||t.co^",
-		"||twimg.com^",
-		"||twitter.com^",
-		"||twttr.com^",
-	},
-}, {
-	name: "youtube",
-	rules: []string{
-		"||googlevideo.com^",
-		"||wide-youtube.l.google.com^",
-		"||youtu.be^",
-		"||youtube",
-		"||youtube-nocookie.com^",
-		"||youtube.com^",
-		"||youtubei.googleapis.com^",
-		"||youtubekids.com^",
-		"||ytimg.com^",
-	},
-}, {
-	name: "twitch",
-	rules: []string{
-		"||jtvnw.net^",
-		"||ttvnw.net^",
-		"||twitch.tv^",
-		"||twitchcdn.net^",
-	},
-}, {
-	name: "netflix",
-	rules: []string{
-		"||nflxext.com^",
-		"||netflix.com^",
-		"||nflximg.net^",
-		"||nflxvideo.net^",
-		"||nflxso.net^",
-	},
-}, {
-	name:  "instagram",
-	rules: []string{"||instagram.com^", "||cdninstagram.com^"},
-}, {
-	name: "snapchat",
-	rules: []string{
-		"||snapchat.com^",
-		"||sc-cdn.net^",
-		"||snap-dev.net^",
-		"||snapkit.co",
-		"||snapads.com^",
-		"||impala-media-production.s3.amazonaws.com^",
-	},
-}, {
-	name: "discord",
-	rules: []string{
-		"||discord.gg^",
-		"||discordapp.net^",
-		"||discordapp.com^",
-		"||discord.com^",
-		"||discord.gift",
-		"||discord.media^",
-	},
-}, {
-	name:  "ok",
-	rules: []string{"||ok.ru^"},
-}, {
-	name: "skype",
-	rules: []string{
-		"||edge-skype-com.s-0001.s-msedge.net^",
-		"||skype-edf.akadns.net^",
-		"||skype.com^",
-		"||skypeassets.com^",
-		"||skypedata.akadns.net^",
-	},
-}, {
-	name: "vk",
-	rules: []string{
-		"||userapi.com^",
-		"||vk-cdn.net^",
-		"||vk.com^",
-		"||vkuservideo.net^",
-	},
-}, {
-	name: "origin",
-	rules: []string{
-		"||accounts.ea.com^",
-		"||origin.com^",
-		"||signin.ea.com^",
-	},
-}, {
-	name: "steam",
-	rules: []string{
-		"||steam.com^",
-		"||steampowered.com^",
-		"||steamcommunity.com^",
-		"||steamstatic.com^",
-		"||steamstore-a.akamaihd.net^",
-		"||steamcdn-a.akamaihd.net^",
-	},
-}, {
-	name:  "epic_games",
-	rules: []string{"||epicgames.com^", "||easyanticheat.net^", "||easy.ac^", "||eac-cdn.com^"},
-}, {
-	name:  "reddit",
-	rules: []string{"||reddit.com^", "||redditstatic.com^", "||redditmedia.com^", "||redd.it^"},
-}, {
-	name:  "mail_ru",
-	rules: []string{"||mail.ru^"},
-}, {
-	name: "cloudflare",
-	rules: []string{
-		"||1.1.1.1^",
-		"||argotunnel.com^",
-		"||cloudflare-dns.com^",
-		"||cloudflare-ipfs.com^",
-		"||cloudflare-quic.com^",
-		"||cloudflare.cn^",
-		"||cloudflare.com^",
-		"||cloudflare.net^",
-		"||cloudflareaccess.com^",
-		"||cloudflareapps.com^",
-		"||cloudflarebolt.com^",
-		"||cloudflareclient.com^",
-		"||cloudflareinsights.com^",
-		"||cloudflareresolve.com^",
-		"||cloudflarestatus.com^",
-		"||cloudflarestream.com^",
-		"||cloudflarewarp.com^",
-		"||dns4torpnlfs2ifuz2s2yf3fc7rdmsbhm6rw75euj35pac6ap25zgqad.onion^",
-		"||one.one^",
-		"||pages.dev^",
-		"||trycloudflare.com^",
-		"||videodelivery.net^",
-		"||warp.plus^",
-		"||workers.dev^",
-	},
-}, {
-	name: "amazon",
-	rules: []string{
-		"||amazon.com^",
-		"||media-amazon.com^",
-		"||primevideo.com^",
-		"||amazontrust.com^",
-		"||images-amazon.com^",
-		"||ssl-images-amazon.com^",
-		"||amazonpay.com^",
-		"||amazonpay.in^",
-		"||amazon-adsystem.com^",
-		"||a2z.com^",
-		"||amazon.ae^",
-		"||amazon.ca^",
-		"||amazon.cn^",
-		"||amazon.de^",
-		"||amazon.es^",
-		"||amazon.fr^",
-		"||amazon.in^",
-		"||amazon.it^",
-		"||amazon.nl^",
-		"||amazon.com.au^",
-		"||amazon.com.br^",
-		"||amazon.co.jp^",
-		"||amazon.com.mx^",
-		"||amazon.com.tr^",
-		"||amazon.co.uk^",
-		"||createspace.com^",
-		"||aws",
-	},
-}, {
-	name: "ebay",
-	rules: []string{
-		"||ebay.com^",
-		"||ebayimg.com^",
-		"||ebaystatic.com^",
-		"||ebaycdn.net^",
-		"||ebayinc.com^",
-		"||ebay.at^",
-		"||ebay.be^",
-		"||ebay.ca^",
-		"||ebay.ch^",
-		"||ebay.cn^",
-		"||ebay.de^",
-		"||ebay.es^",
-		"||ebay.fr^",
-		"||ebay.ie^",
-		"||ebay.in^",
-		"||ebay.it^",
-		"||ebay.ph^",
-		"||ebay.pl^",
-		"||ebay.nl^",
-		"||ebay.com.au^",
-		"||ebay.com.cn^",
-		"||ebay.com.hk^",
-		"||ebay.com.my^",
-		"||ebay.com.sg^",
-		"||ebay.co.uk^",
-	},
-}, {
-	name: "tiktok",
-	rules: []string{
-		"||amemv.com^",
-		"||bdurl.com^",
-		"||bytecdn.cn^",
-		"||bytedance.map.fastly.net^",
-		"||bytedapm.com^",
-		"||byteimg.com^",
-		"||byteoversea.com^",
-		"||douyin.com^",
-		"||douyincdn.com^",
-		"||douyinpic.com^",
-		"||douyinstatic.com^",
-		"||douyinvod.com^",
-		"||ixigua.com^",
-		"||ixiguavideo.com^",
-		"||muscdn.com^",
-		"||musical.ly^",
-		"||pstatp.com^",
-		"||snssdk.com^",
-		"||tiktok.com^",
-		"||tiktokcdn.com^",
-		"||tiktokv.com^",
-		"||toutiao.com^",
-		"||toutiaocloud.com^",
-		"||toutiaocloud.net^",
-		"||toutiaovod.com^",
-	},
-}, {
-	name: "vimeo",
-	rules: []string{
-		"*vod-adaptive.akamaized.net^",
-		"||vimeo.com^",
-		"||vimeocdn.com^",
-	},
-}, {
-	name: "pinterest",
-	rules: []string{
-		"||pinimg.com^",
-		"||pinterest.*^",
-	},
-}, {
-	name:  "imgur",
-	rules: []string{"||imgur.com^"},
-}, {
-	name: "dailymotion",
-	rules: []string{
-		"||dailymotion.com^",
-		"||dm-event.net^",
-		"||dmcdn.net^",
-	},
-}, {
-	name: "qq",
-	rules: []string{
-		// Block qq.com and subdomains excluding WeChat's domains.
-		"||qq.com^$denyallow=wx.qq.com|weixin.qq.com",
-		"||qqzaixian.com^",
-		"||qq-video.cdn-go.cn^",
-		"||url.cn^",
-	},
-}, {
-	name: "wechat",
-	rules: []string{
-		"||wechat.com^",
-		"||weixin.qq.com.cn^",
-		"||weixin.qq.com^",
-		"||weixinbridge.com^",
-		"||wx.qq.com^",
-	},
-}, {
-	name:  "viber",
-	rules: []string{"||viber.com^"},
-}, {
-	name: "weibo",
-	rules: []string{
-		"||weibo.cn^",
-		"||weibo.com^",
-		"||weibocdn.com^",
-	},
-}, {
-	name: "9gag",
-	rules: []string{
-		"||9cache.com^",
-		"||9gag.com^",
-	},
-}, {
-	name: "telegram",
-	rules: []string{
-		"||t.me^",
-		"||telegram.me^",
-		"||telegram.org^",
-	},
-}, {
-	name: "disneyplus",
-	rules: []string{
-		"||disney-plus.net^",
-		"||disney.playback.edge.bamgrid.com^",
-		"||disneynow.com^",
-		"||disneyplus.com^",
-		"||hotstar.com^",
-		"||media.dssott.com^",
-		"||star.playback.edge.bamgrid.com^",
-		"||starplus.com^",
-	},
-}, {
-	name:  "hulu",
-	rules: []string{"||hulu.com^"},
-}, {
-	name: "spotify",
-	rules: []string{
-		"/_spotify-connect._tcp.local/",
-		"||spotify.com^",
-		"||scdn.co^",
-		"||spotify.com.edgesuite.net^",
-		"||spotify.map.fastly.net^",
-		"||spotify.map.fastlylb.net^",
-		"||spotifycdn.net^",
-		"||audio-ak-spotify-com.akamaized.net^",
-		"||audio4-ak-spotify-com.akamaized.net^",
-		"||heads-ak-spotify-com.akamaized.net^",
-		"||heads4-ak-spotify-com.akamaized.net^",
-	},
-}, {
-	name: "tinder",
-	rules: []string{
-		"||gotinder.com^",
-		"||tinder.com^",
-		"||tindersparks.com^",
-	},
-}, {
-	name: "bilibili",
-	rules: []string{
-		"||b23.tv^",
-		"||biliapi.net^",
-		"||bilibili.com^",
-		"||bilicdn1.com^",
-		"||bilicdn2.com^",
-		"||biligame.com^",
-		"||bilivideo.cn^",
-		"||bilivideo.com^",
-		"||dreamcast.hk^",
-		"||hdslb.com^",
-	},
-}}
-
 // serviceRules maps a service ID to its filtering rules.
 var serviceRules map[string][]*rules.NetworkRule

@@ -394,16 +18,16 @@ var serviceIDs []string

 // initBlockedServices initializes package-level blocked service data.
 func initBlockedServices() {
-	l := len(servicesData)
+	l := len(blockedServices)
 	serviceIDs = make([]string, l)
 	serviceRules = make(map[string][]*rules.NetworkRule, l)

-	for i, s := range servicesData {
-		netRules := make([]*rules.NetworkRule, 0, len(s.rules))
-		for _, text := range s.rules {
+	for i, s := range blockedServices {
+		netRules := make([]*rules.NetworkRule, 0, len(s.Rules))
+		for _, text := range s.Rules {
 			rule, err := rules.NewNetworkRule(text, BlockedSvcsListID)
 			if err != nil {
-				log.Error("parsing blocked service %q rule %q: %s", s.name, text, err)
+				log.Error("parsing blocked service %q rule %q: %s", s.ID, text, err)

 				continue
 			}
@@ -411,8 +35,8 @@ func initBlockedServices() {
 			netRules = append(netRules, rule)
 		}

-		serviceIDs[i] = s.name
-		serviceRules[s.name] = netRules
+		serviceIDs[i] = s.ID
+		serviceRules[s.ID] = netRules
 	}

 	slices.Sort(serviceIDs)
@@ -420,7 +44,7 @@ func initBlockedServices() {
 	log.Debug("filtering: initialized %d services", l)
 }

-// BlockedSvcKnown - return TRUE if a blocked service name is known
+// BlockedSvcKnown returns true if a blocked service ID is known.
 func BlockedSvcKnown(s string) (ok bool) {
 	_, ok = serviceRules[s]

@@ -452,10 +76,18 @@ func (d *DNSFilter) ApplyBlockedServices(setts *Settings, list []string) {
 	}
 }

-func (d *DNSFilter) handleBlockedServicesAvailableServices(w http.ResponseWriter, r *http.Request) {
+func (d *DNSFilter) handleBlockedServicesIDs(w http.ResponseWriter, r *http.Request) {
 	_ = aghhttp.WriteJSONResponse(w, r, serviceIDs)
 }

+func (d *DNSFilter) handleBlockedServicesAll(w http.ResponseWriter, r *http.Request) {
+	_ = aghhttp.WriteJSONResponse(w, r, struct {
+		BlockedServices []blockedService `json:"blocked_services"`
+	}{
+		BlockedServices: blockedServices,
+	})
+}
+
 func (d *DNSFilter) handleBlockedServicesList(w http.ResponseWriter, r *http.Request) {
 	d.confLock.RLock()
 	list := d.Config.BlockedServices
@@ -479,5 +111,5 @@ func (d *DNSFilter) handleBlockedServicesSet(w http.ResponseWriter, r *http.Requ

 	log.Debug("Updated blocked services list: %d", len(list))

-	d.ConfigModified()
+	d.Config.ConfigModified()
 }
--- a/internal/filtering/http.go
+++ b/internal/filtering/http.go
@@ -466,7 +466,8 @@ func (d *DNSFilter) RegisterFilteringHandlers() {
 	registerHTTP(http.MethodPost, "/control/rewrite/add", d.handleRewriteAdd)
 	registerHTTP(http.MethodPost, "/control/rewrite/delete", d.handleRewriteDelete)

-	registerHTTP(http.MethodGet, "/control/blocked_services/services", d.handleBlockedServicesAvailableServices)
+	registerHTTP(http.MethodGet, "/control/blocked_services/services", d.handleBlockedServicesIDs)
+	registerHTTP(http.MethodGet, "/control/blocked_services/all", d.handleBlockedServicesAll)
 	registerHTTP(http.MethodGet, "/control/blocked_services/list", d.handleBlockedServicesList)
 	registerHTTP(http.MethodPost, "/control/blocked_services/set", d.handleBlockedServicesSet)

--- a/internal/filtering/servicelist.go
+++ b/internal/filtering/servicelist.go