Pull request: 3972-hostlists-services

Updates #3972.

Squashed commit of the following:

commit 9dc0efe2453cb6c738d97d39b02c86eccb18a42c
Merge: 239550f8 8a935d4f
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Oct 27 14:42:38 2022 +0300

    Merge branch 'master' into 3972-hostlists-services

commit 239550f84228e7c7a6f4ae6b1cadcc47e01f54d5
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Oct 27 14:41:42 2022 +0300

    filtering: upd service list

commit b8bf3a6a4b1333059b886be95a1419612aebac39
Author: Ildar Kamalov <ik@adguard.com>
Date:   Thu Oct 27 13:41:09 2022 +0300

    client: remove todo

commit caa504b482befb804db2a1ca0b6d4834aa4da49a
Author: Ildar Kamalov <ik@adguard.com>
Date:   Thu Oct 27 12:54:45 2022 +0300

    fix build

commit 511797c305d9eef84a20553dab795414e00da51a
Author: Ildar Kamalov <ik@adguard.com>
Date:   Thu Oct 27 12:40:33 2022 +0300

    client: add titles with service names to the clients table

commit 79ed3157a85b489a0b13381cff867a8c73ba60e9
Author: Ildar Kamalov <ik@adguard.com>
Date:   Thu Oct 27 12:36:59 2022 +0300

    client: fix empty icons

commit ab69b95784de87665d5a1a3683f28e3b3df1c210
Author: Ildar Kamalov <ik@adguard.com>
Date:   Thu Oct 27 11:55:48 2022 +0300

    client: use all blocked services

commit 9a4a87665c8463224d8e93f1e162988107f6c7ca
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Oct 25 19:25:20 2022 +0300

    all: fix json response

commit 86eb4493ce305cd5991176bd4cd8f7f5afdea330
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Tue Oct 25 19:09:44 2022 +0300

    all: use hostslists registry for blocked svcs

scripts/README.md

@@ -208,10 +208,30 @@ code from [the repo][companiesrepo].
 
 
 
+##  `blocked-services/`: Blocked Services Updater
+
+A simple script that downloads and updates the blocked services index from
+AdGuard's [Hostlists Registry][reg].
+
+Optional environment:
+
+ *  `URL`: the URL of the index file.  By default it's
+    `https://adguardteam.github.io/HostlistsRegistry/assets/services.json`.
+
+ ###  Usage
+
+```sh
+go run ./scripts/blocked-services/main.go
+```
+
+[reg]: https://github.com/AdguardTeam/HostlistsRegistry
+
+
+
 ##  `vetted-filters/`: Vetted Filters Updater
 
-A simple script that downloads and updates the vetted filtering list data from
-AdGuard's [Hostlists Registry][reg].
+Similar to the one above, a script that downloads and updates the vetted
+filtering list data from AdGuard's [Hostlists Registry][reg].
 
 Optional environment:
 
@@ -223,5 +243,3 @@ Optional environment:
 ```sh
 go run ./scripts/vetted-filters/main.go
 ```
-
-[reg]: https://github.com/AdguardTeam/HostlistsRegistry

scripts/blocked-services/main.go

@@ -0,0 +1,117 @@
+// blocked services fetches the most recent Hostlists Registry blocked service
+// index and transforms the filters from it to AdGuard Home's data and code
+// formats.
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"text/template"
+	"time"
+
+	"github.com/AdguardTeam/golibs/log"
+	"golang.org/x/exp/slices"
+)
+
+func main() {
+	urlStr := "https://adguardteam.github.io/HostlistsRegistry/assets/services.json"
+	if v, ok := os.LookupEnv("URL"); ok {
+		urlStr = v
+	}
+
+	// Validate the URL.
+	_, err := url.Parse(urlStr)
+	check(err)
+
+	c := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+
+	resp, err := c.Get(urlStr)
+	check(err)
+	defer log.OnCloserError(resp.Body, log.ERROR)
+
+	if resp.StatusCode != http.StatusOK {
+		panic(fmt.Errorf("expected code %d, got %d", http.StatusOK, resp.StatusCode))
+	}
+
+	hlSvcs := &hlServices{}
+	err = json.NewDecoder(resp.Body).Decode(hlSvcs)
+	check(err)
+
+	// Sort all services and rules to make the output more predictable.
+	slices.SortStableFunc(hlSvcs.BlockedServices, func(a, b *hlServicesService) (less bool) {
+		return a.ID < b.ID
+	})
+	for _, s := range hlSvcs.BlockedServices {
+		slices.Sort(s.Rules)
+	}
+
+	// Use another set of delimiters to prevent them interfering with the Go
+	// code.
+	tmpl, err := template.New("main").Delims("<%", "%>").Funcs(template.FuncMap{
+		"isnotlast": func(idx, sliceLen int) (ok bool) { return idx != sliceLen-1 },
+	}).Parse(tmplStr)
+	check(err)
+
+	f, err := os.OpenFile(
+		"./internal/filtering/servicelist.go",
+		os.O_CREATE|os.O_TRUNC|os.O_WRONLY,
+		0o644,
+	)
+	check(err)
+	defer log.OnCloserError(f, log.ERROR)
+
+	err = tmpl.Execute(f, hlSvcs)
+	check(err)
+}
+
+// tmplStr is the template for the Go source file with the services.
+const tmplStr = `// Code generated by go run ./scripts/blocked-services/main.go; DO NOT EDIT.
+
+package filtering
+
+// blockedService represents a single blocked service.
+type blockedService struct {
+	ID      string   ` + "`" + `json:"id"` + "`" + `
+	Name    string   ` + "`" + `json:"name"` + "`" + `
+	IconSVG []byte   ` + "`" + `json:"icon_svg"` + "`" + `
+	Rules   []string ` + "`" + `json:"rules"` + "`" + `
+}
+
+// blockedServices contains raw blocked service data.
+var blockedServices = []blockedService{<% $l := len .BlockedServices %>
+	<%- range $i, $s := .BlockedServices %>{
+	ID:      <% printf "%q" $s.ID %>,
+	Name:    <% printf "%q" $s.Name %>,
+	IconSVG: []byte(<% printf "%q" $s.IconSVG %>),
+	Rules: []string{<% range $s.Rules %>
+		<% printf "%q" . %>,<% end %>
+	},
+}<% if isnotlast $i $l %>, <% end %><% end %>}
+`
+
+// check is a simple error-checking helper for scripts.
+func check(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+// hlServices is the JSON structure for the Hostlists Registry blocked service
+// index.
+type hlServices struct {
+	BlockedServices []*hlServicesService `json:"blocked_services"`
+}
+
+// hlServicesService is the JSON structure for a service in the Hostlists
+// Registry.
+type hlServicesService struct {
+	ID      string   `json:"id"`
+	Name    string   `json:"name"`
+	IconSVG string   `json:"icon_svg"`
+	Rules   []string `json:"rules"`
+}

scripts/vetted-filters/main.go

@@ -1,5 +1,5 @@
-// vetted-filters fetches the most recent Hostlists Registry index and
-// transforms the filters from it to AdGuard Home's format.
+// vetted-filters fetches the most recent Hostlists Registry filtering rule list
+// index and transforms the filters from it to AdGuard Home's format.
 package main
 
 import (
@@ -91,7 +91,7 @@ func main() {
 	check(err)
 }
 
-// jsHeader is the header for the generates JavaScript file.  It informs the
+// jsHeader is the header for the generated JavaScript file.  It informs the
 // reader that the file is generated and disables some style-related eslint
 // checks.
 const jsHeader = `// Code generated by go run ./scripts/vetted-filters/main.go; DO NOT EDIT.