Pull request: 3972-hostlists-registry

Updates #3972.

Squashed commit of the following:

commit 8341c13c39f38048796bc42c5f5337e1c8d3bcd4
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Fri Oct 21 13:06:55 2022 +0300

    all: imp docs

commit b7f961c879abf288c347e5bf55182e7dfa8e84e4
Merge: 607077ca 68d13fcc
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Oct 20 18:23:57 2022 +0300

    Merge branch 'master' into 3972-hostlists-registry

commit 607077ca8141f732d4678955c62cc2345958766b
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Thu Oct 20 18:07:09 2022 +0300

    all: use hostlists registry

scripts/README.md

@@ -205,3 +205,23 @@ code from [the repo][companiesrepo].
 ```
 
 [companiesrepo]: https://github.com/AdguardTeam/companiesdb
+
+
+
+##  `vetted-filters/`: Vetted Filters Updater
+
+A simple script that downloads and updates the vetted filtering list data from
+AdGuard's [Hostlists Registry][reg].
+
+Optional environment:
+
+ *  `URL`: the URL of the index file.  By default it's
+    `https://adguardteam.github.io/HostlistsRegistry/assets/filters.json`.
+
+ ###  Usage
+
+```sh
+go run ./scripts/vetted-filters/main.go
+```
+
+[reg]: https://github.com/AdguardTeam/HostlistsRegistry

scripts/make/go-lint.sh

@@ -229,7 +229,7 @@ gocyclo --over 13 ./internal/dhcpd ./internal/filtering/ ./internal/home/
 gocyclo --over 10 ./internal/aghio/ ./internal/aghnet/ ./internal/aghos/\
 	./internal/aghtest/ ./internal/dnsforward/ ./internal/stats/\
 	./internal/tools/ ./internal/updater/ ./internal/next/ ./internal/version/\
-	./main.go
+	./scripts/vetted-filters/ ./main.go
 
 ineffassign ./...
 

scripts/vetted-filters/main.go

@@ -0,0 +1,164 @@
+// vetted-filters fetches the most recent Hostlists Registry index and
+// transforms the filters from it to AdGuard Home's format.
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"time"
+
+	"github.com/AdguardTeam/golibs/log"
+	"github.com/google/renameio/maybe"
+)
+
+func main() {
+	urlStr := "https://adguardteam.github.io/HostlistsRegistry/assets/filters.json"
+	if v, ok := os.LookupEnv("URL"); ok {
+		urlStr = v
+	}
+
+	// Validate the URL.
+	_, err := url.Parse(urlStr)
+	check(err)
+
+	c := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+
+	resp, err := c.Get(urlStr)
+	check(err)
+	defer log.OnCloserError(resp.Body, log.ERROR)
+
+	if resp.StatusCode != http.StatusOK {
+		panic(fmt.Errorf("expected code %d, got %d", http.StatusOK, resp.StatusCode))
+	}
+
+	hlFlt := &hlFilters{}
+	err = json.NewDecoder(resp.Body).Decode(hlFlt)
+	check(err)
+
+	aghFlt := &aghFilters{
+		Categories: map[string]*aghFiltersCategory{
+			"general": {
+				Name:        "filter_category_general",
+				Description: "filter_category_general_desc",
+			},
+			"other": {
+				Name:        "filter_category_other",
+				Description: "filter_category_other_desc",
+			},
+			"regional": {
+				Name:        "filter_category_regional",
+				Description: "filter_category_regional_desc",
+			},
+			"security": {
+				Name:        "filter_category_security",
+				Description: "filter_category_security_desc",
+			},
+		},
+		Filters: map[string]*aghFiltersFilter{},
+	}
+
+	for i, f := range hlFlt.Filters {
+		id := f.FilterID
+		cat := f.category()
+		if cat == "" {
+			log.Info("warning: filter %s at index %d does not have a fitting category", id, i)
+		}
+
+		aghFlt.Filters[id] = &aghFiltersFilter{
+			Name:       f.Name,
+			CategoryID: cat,
+			Homepage:   f.Homepage,
+			Source:     f.SourceURL,
+		}
+	}
+
+	buf := &bytes.Buffer{}
+	_, _ = buf.WriteString(jsHeader)
+
+	enc := json.NewEncoder(buf)
+	enc.SetIndent("", "    ")
+
+	err = enc.Encode(aghFlt)
+	check(err)
+
+	err = maybe.WriteFile("client/src/helpers/filters/filters.js", buf.Bytes(), 0o644)
+	check(err)
+}
+
+// jsHeader is the header for the generates JavaScript file.  It informs the
+// reader that the file is generated and disables some style-related eslint
+// checks.
+const jsHeader = `// Code generated by go run ./scripts/vetted-filters/main.go; DO NOT EDIT.
+
+/* eslint quote-props: 'off', quotes: 'off', comma-dangle: 'off', semi: 'off' */
+
+export default `
+
+// check is a simple error-checking helper for scripts.
+func check(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+// hlFilters is the JSON structure for the Hostlists Registry rule list index.
+type hlFilters struct {
+	Filters []*hlFiltersFilter `json:"filters"`
+}
+
+// hlFiltersFilter is the JSON structure for a filter in the Hostlists Registry.
+type hlFiltersFilter struct {
+	FilterID  string   `json:"filterId"`
+	Name      string   `json:"name"`
+	Homepage  string   `json:"homepage"`
+	SourceURL string   `json:"sourceUrl"`
+	Tags      []string `json:"tags"`
+}
+
+// category returns the AdGuard Home category for this filter.  If there is no
+// fitting category, cat is empty.
+func (f *hlFiltersFilter) category() (cat string) {
+	for _, t := range f.Tags {
+		switch t {
+		case "purpose:general":
+			return "general"
+		case "purpose:other":
+			return "other"
+		case "purpose:regional":
+			return "regional"
+		case "purpose:security":
+			return "security"
+		}
+	}
+
+	return ""
+}
+
+// aghFilters is the JSON structure for AdGuard Home's list of vetted filtering
+// rule list in file client/src/helpers/filters/filters.js.
+type aghFilters struct {
+	Categories map[string]*aghFiltersCategory `json:"categories"`
+	Filters    map[string]*aghFiltersFilter   `json:"filters"`
+}
+
+// aghFiltersCategory is the JSON structure for a category in the vetted
+// filtering rule list file.
+type aghFiltersCategory struct {
+	Name        string `json:"name"`
+	Description string `json:"description"`
+}
+
+// aghFiltersFilter is the JSON structure for a filter in the vetted filtering
+// rule list file.
+type aghFiltersFilter struct {
+	Name       string `json:"name"`
+	CategoryID string `json:"categoryId"`
+	Homepage   string `json:"homepage"`
+	Source     string `json:"source"`
+}