all: sync with master

scripts/README.md

@@ -195,14 +195,51 @@ Optional environment:
 
 ##  `companiesdb/`: Whotracks.me Database Converter
 
-A simple script that downloads and updates the companies DB in the `client` 
+A simple script that downloads and updates the companies DB in the `client`
 code from [the repo][companiesrepo].
 
  ###  Usage
 
 ```sh
-cd scripts/companiesdb
-./download.sh
+( cd scripts/companiesdb && sh ./download.sh )
 ```
 
 [companiesrepo]: https://github.com/AdguardTeam/companiesdb
+
+
+
+##  `blocked-services/`: Blocked Services Updater
+
+A simple script that downloads and updates the blocked services index from
+AdGuard's [Hostlists Registry][reg].
+
+Optional environment:
+
+ *  `URL`: the URL of the index file.  By default it's
+    `https://adguardteam.github.io/HostlistsRegistry/assets/services.json`.
+
+ ###  Usage
+
+```sh
+go run ./scripts/blocked-services/main.go
+```
+
+[reg]: https://github.com/AdguardTeam/HostlistsRegistry
+
+
+
+##  `vetted-filters/`: Vetted Filters Updater
+
+Similar to the one above, a script that downloads and updates the vetted
+filtering list data from AdGuard's [Hostlists Registry][reg].
+
+Optional environment:
+
+ *  `URL`: the URL of the index file.  By default it's
+    `https://adguardteam.github.io/HostlistsRegistry/assets/filters.json`.
+
+ ###  Usage
+
+```sh
+go run ./scripts/vetted-filters/main.go
+```

scripts/blocked-services/main.go

@@ -0,0 +1,117 @@
+// blocked services fetches the most recent Hostlists Registry blocked service
+// index and transforms the filters from it to AdGuard Home's data and code
+// formats.
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"text/template"
+	"time"
+
+	"github.com/AdguardTeam/golibs/log"
+	"golang.org/x/exp/slices"
+)
+
+func main() {
+	urlStr := "https://adguardteam.github.io/HostlistsRegistry/assets/services.json"
+	if v, ok := os.LookupEnv("URL"); ok {
+		urlStr = v
+	}
+
+	// Validate the URL.
+	_, err := url.Parse(urlStr)
+	check(err)
+
+	c := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+
+	resp, err := c.Get(urlStr)
+	check(err)
+	defer log.OnCloserError(resp.Body, log.ERROR)
+
+	if resp.StatusCode != http.StatusOK {
+		panic(fmt.Errorf("expected code %d, got %d", http.StatusOK, resp.StatusCode))
+	}
+
+	hlSvcs := &hlServices{}
+	err = json.NewDecoder(resp.Body).Decode(hlSvcs)
+	check(err)
+
+	// Sort all services and rules to make the output more predictable.
+	slices.SortStableFunc(hlSvcs.BlockedServices, func(a, b *hlServicesService) (less bool) {
+		return a.ID < b.ID
+	})
+	for _, s := range hlSvcs.BlockedServices {
+		slices.Sort(s.Rules)
+	}
+
+	// Use another set of delimiters to prevent them interfering with the Go
+	// code.
+	tmpl, err := template.New("main").Delims("<%", "%>").Funcs(template.FuncMap{
+		"isnotlast": func(idx, sliceLen int) (ok bool) { return idx != sliceLen-1 },
+	}).Parse(tmplStr)
+	check(err)
+
+	f, err := os.OpenFile(
+		"./internal/filtering/servicelist.go",
+		os.O_CREATE|os.O_TRUNC|os.O_WRONLY,
+		0o644,
+	)
+	check(err)
+	defer log.OnCloserError(f, log.ERROR)
+
+	err = tmpl.Execute(f, hlSvcs)
+	check(err)
+}
+
+// tmplStr is the template for the Go source file with the services.
+const tmplStr = `// Code generated by go run ./scripts/blocked-services/main.go; DO NOT EDIT.
+
+package filtering
+
+// blockedService represents a single blocked service.
+type blockedService struct {
+	ID      string   ` + "`" + `json:"id"` + "`" + `
+	Name    string   ` + "`" + `json:"name"` + "`" + `
+	IconSVG []byte   ` + "`" + `json:"icon_svg"` + "`" + `
+	Rules   []string ` + "`" + `json:"rules"` + "`" + `
+}
+
+// blockedServices contains raw blocked service data.
+var blockedServices = []blockedService{<% $l := len .BlockedServices %>
+	<%- range $i, $s := .BlockedServices %>{
+	ID:      <% printf "%q" $s.ID %>,
+	Name:    <% printf "%q" $s.Name %>,
+	IconSVG: []byte(<% printf "%q" $s.IconSVG %>),
+	Rules: []string{<% range $s.Rules %>
+		<% printf "%q" . %>,<% end %>
+	},
+}<% if isnotlast $i $l %>, <% end %><% end %>}
+`
+
+// check is a simple error-checking helper for scripts.
+func check(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+// hlServices is the JSON structure for the Hostlists Registry blocked service
+// index.
+type hlServices struct {
+	BlockedServices []*hlServicesService `json:"blocked_services"`
+}
+
+// hlServicesService is the JSON structure for a service in the Hostlists
+// Registry.
+type hlServicesService struct {
+	ID      string   `json:"id"`
+	Name    string   `json:"name"`
+	IconSVG string   `json:"icon_svg"`
+	Rules   []string `json:"rules"`
+}

scripts/companiesdb/download.sh

@@ -10,5 +10,5 @@ adguard='https://raw.githubusercontent.com/AdguardTeam/companiesdb/main/dist/adg
 base_path='../../client/src/helpers/trackers'
 readonly whotracksme adguard base_path
 
-curl "$whotracksme" --output "${base_path}/whotracksme.json"
-curl "$adguard" --output "${base_path}/adguard.json"
+curl -o "${base_path}/whotracksme.json" -v "$whotracksme"
+curl -o "${base_path}/adguard.json"     -v "$adguard"

scripts/make/go-lint.sh

@@ -222,13 +222,13 @@ govulncheck ./...
 
 # Apply more lax standards to the code we haven't properly refactored yet.
 gocyclo --over 17 ./internal/querylog/
-gocyclo --over 15 ./internal/home/ ./internal/dhcpd
-gocyclo --over 13 ./internal/filtering/
+gocyclo --over 13 ./internal/dhcpd ./internal/filtering/ ./internal/home/
 
 # Apply stricter standards to new or somewhat refactored code.
 gocyclo --over 10 ./internal/aghio/ ./internal/aghnet/ ./internal/aghos/\
 	./internal/aghtest/ ./internal/dnsforward/ ./internal/stats/\
-	./internal/tools/ ./internal/updater/ ./internal/version/ ./main.go
+	./internal/tools/ ./internal/updater/ ./internal/version/\
+	./scripts/vetted-filters/ ./main.go
 
 ineffassign ./...
 

scripts/translations/package.json

@@ -2,8 +2,8 @@
   "name": "translations",
   "version": "0.2.0",
   "scripts": {
-    "locales:download": "TWOSKY_URI=https://twosky.adtidy.org/api/v1 TWOSKY_PROJECT_ID=home node download.js ; node count.js",
-    "locales:upload": "TWOSKY_URI=https://twosky.adtidy.org/api/v1 TWOSKY_PROJECT_ID=home node upload.js",
+    "locales:download": "TWOSKY_URI=https://twosky.int.agrd.dev/api/v1 TWOSKY_PROJECT_ID=home node download.js ; node count.js",
+    "locales:upload": "TWOSKY_URI=https://twosky.int.agrd.dev/api/v1 TWOSKY_PROJECT_ID=home node upload.js",
     "locales:summary": "node count.js",
     "locales:unused": "node unused.js"
   },

scripts/vetted-filters/main.go

@@ -0,0 +1,164 @@
+// vetted-filters fetches the most recent Hostlists Registry filtering rule list
+// index and transforms the filters from it to AdGuard Home's format.
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"time"
+
+	"github.com/AdguardTeam/golibs/log"
+	"github.com/google/renameio/maybe"
+)
+
+func main() {
+	urlStr := "https://adguardteam.github.io/HostlistsRegistry/assets/filters.json"
+	if v, ok := os.LookupEnv("URL"); ok {
+		urlStr = v
+	}
+
+	// Validate the URL.
+	_, err := url.Parse(urlStr)
+	check(err)
+
+	c := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+
+	resp, err := c.Get(urlStr)
+	check(err)
+	defer log.OnCloserError(resp.Body, log.ERROR)
+
+	if resp.StatusCode != http.StatusOK {
+		panic(fmt.Errorf("expected code %d, got %d", http.StatusOK, resp.StatusCode))
+	}
+
+	hlFlt := &hlFilters{}
+	err = json.NewDecoder(resp.Body).Decode(hlFlt)
+	check(err)
+
+	aghFlt := &aghFilters{
+		Categories: map[string]*aghFiltersCategory{
+			"general": {
+				Name:        "filter_category_general",
+				Description: "filter_category_general_desc",
+			},
+			"other": {
+				Name:        "filter_category_other",
+				Description: "filter_category_other_desc",
+			},
+			"regional": {
+				Name:        "filter_category_regional",
+				Description: "filter_category_regional_desc",
+			},
+			"security": {
+				Name:        "filter_category_security",
+				Description: "filter_category_security_desc",
+			},
+		},
+		Filters: map[string]*aghFiltersFilter{},
+	}
+
+	for i, f := range hlFlt.Filters {
+		id := f.FilterID
+		cat := f.category()
+		if cat == "" {
+			log.Info("warning: filter %s at index %d does not have a fitting category", id, i)
+		}
+
+		aghFlt.Filters[id] = &aghFiltersFilter{
+			Name:       f.Name,
+			CategoryID: cat,
+			Homepage:   f.Homepage,
+			Source:     f.SourceURL,
+		}
+	}
+
+	buf := &bytes.Buffer{}
+	_, _ = buf.WriteString(jsHeader)
+
+	enc := json.NewEncoder(buf)
+	enc.SetIndent("", "    ")
+
+	err = enc.Encode(aghFlt)
+	check(err)
+
+	err = maybe.WriteFile("client/src/helpers/filters/filters.js", buf.Bytes(), 0o644)
+	check(err)
+}
+
+// jsHeader is the header for the generated JavaScript file.  It informs the
+// reader that the file is generated and disables some style-related eslint
+// checks.
+const jsHeader = `// Code generated by go run ./scripts/vetted-filters/main.go; DO NOT EDIT.
+
+/* eslint quote-props: 'off', quotes: 'off', comma-dangle: 'off', semi: 'off' */
+
+export default `
+
+// check is a simple error-checking helper for scripts.
+func check(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+// hlFilters is the JSON structure for the Hostlists Registry rule list index.
+type hlFilters struct {
+	Filters []*hlFiltersFilter `json:"filters"`
+}
+
+// hlFiltersFilter is the JSON structure for a filter in the Hostlists Registry.
+type hlFiltersFilter struct {
+	FilterID  string   `json:"filterId"`
+	Name      string   `json:"name"`
+	Homepage  string   `json:"homepage"`
+	SourceURL string   `json:"sourceUrl"`
+	Tags      []string `json:"tags"`
+}
+
+// category returns the AdGuard Home category for this filter.  If there is no
+// fitting category, cat is empty.
+func (f *hlFiltersFilter) category() (cat string) {
+	for _, t := range f.Tags {
+		switch t {
+		case "purpose:general":
+			return "general"
+		case "purpose:other":
+			return "other"
+		case "purpose:regional":
+			return "regional"
+		case "purpose:security":
+			return "security"
+		}
+	}
+
+	return ""
+}
+
+// aghFilters is the JSON structure for AdGuard Home's list of vetted filtering
+// rule list in file client/src/helpers/filters/filters.js.
+type aghFilters struct {
+	Categories map[string]*aghFiltersCategory `json:"categories"`
+	Filters    map[string]*aghFiltersFilter   `json:"filters"`
+}
+
+// aghFiltersCategory is the JSON structure for a category in the vetted
+// filtering rule list file.
+type aghFiltersCategory struct {
+	Name        string `json:"name"`
+	Description string `json:"description"`
+}
+
+// aghFiltersFilter is the JSON structure for a filter in the vetted filtering
+// rule list file.
+type aghFiltersFilter struct {
+	Name       string `json:"name"`
+	CategoryID string `json:"categoryId"`
+	Homepage   string `json:"homepage"`
+	Source     string `json:"source"`
+}