AdGuardHome/scripts/vetted-filters/main.go

// vetted-filters fetches the most recent Hostlists Registry filtering rule list
// index and transforms the filters from it to AdGuard Home's format.
package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/AdguardTeam/golibs/log"
	"github.com/google/renameio/v2/maybe"
)

func main() {
	urlStr := "https://adguardteam.github.io/HostlistsRegistry/assets/filters.json"
	if v, ok := os.LookupEnv("URL"); ok {
		urlStr = v
	}

	// Validate the URL.
	_, err := url.Parse(urlStr)
	check(err)

	c := &http.Client{
		Timeout: 10 * time.Second,
	}

	resp, err := c.Get(urlStr)
	check(err)
	defer log.OnCloserError(resp.Body, log.ERROR)

	if resp.StatusCode != http.StatusOK {
		panic(fmt.Errorf("expected code %d, got %d", http.StatusOK, resp.StatusCode))
	}

	hlFlt := &hlFilters{}
	err = json.NewDecoder(resp.Body).Decode(hlFlt)
	check(err)

	aghFlt := &aghFilters{
		Categories: map[string]*aghFiltersCategory{
			"general": {
				Name:        "filter_category_general",
				Description: "filter_category_general_desc",
			},
			"other": {
				Name:        "filter_category_other",
				Description: "filter_category_other_desc",
			},
			"regional": {
				Name:        "filter_category_regional",
				Description: "filter_category_regional_desc",
			},
			"security": {
				Name:        "filter_category_security",
				Description: "filter_category_security_desc",
			},
		},
		Filters: map[string]*aghFiltersFilter{},
	}

	for i, f := range hlFlt.Filters {
		id := f.FilterID
		cat := f.category()
		if cat == "" {
			log.Info("warning: filter %s at index %d does not have a fitting category", id, i)
		}

		aghFlt.Filters[id] = &aghFiltersFilter{
			Name:       f.Name,
			CategoryID: cat,
			Homepage:   f.Homepage,
			// NOTE: The source URL in filters.json is not guaranteed to contain
			// the URL of the filtering rule list.  So, use our mirror for the
			// vetted blocklists, which are mostly guaranteed to be valid and
			// available lists.
			Source: f.DownloadURL,
		}
	}

	buf := &bytes.Buffer{}
	_, _ = buf.WriteString(jsHeader)

	enc := json.NewEncoder(buf)
	enc.SetIndent("", "    ")

	err = enc.Encode(aghFlt)
	check(err)

	err = maybe.WriteFile("client/src/helpers/filters/filters.ts", buf.Bytes(), 0o644)
	check(err)
}

// jsHeader is the header for the generated JavaScript file.  It informs the
// reader that the file is generated and disables some style-related eslint
// checks.
const jsHeader = `// Code generated by go run ./scripts/vetted-filters/main.go; DO NOT EDIT.

/* eslint quote-props: 'off', quotes: 'off', comma-dangle: 'off', semi: 'off' */

export default `

// check is a simple error-checking helper for scripts.
func check(err error) {
	if err != nil {
		panic(err)
	}
}

// hlFilters is the JSON structure for the Hostlists Registry rule list index.
type hlFilters struct {
	Filters []*hlFiltersFilter `json:"filters"`
}

// hlFiltersFilter is the JSON structure for a filter in the Hostlists Registry.
type hlFiltersFilter struct {
	DownloadURL string   `json:"downloadUrl"`
	FilterID    string   `json:"filterId"`
	Homepage    string   `json:"homepage"`
	Name        string   `json:"name"`
	Tags        []string `json:"tags"`
}

// category returns the AdGuard Home category for this filter.  If there is no
// fitting category, cat is empty.
func (f *hlFiltersFilter) category() (cat string) {
	for _, t := range f.Tags {
		switch t {
		case "purpose:general":
			return "general"
		case "purpose:other":
			return "other"
		case "purpose:regional":
			return "regional"
		case "purpose:security":
			return "security"
		}
	}

	return ""
}

// aghFilters is the JSON structure for AdGuard Home's list of vetted filtering
// rule list in file client/src/helpers/filters/filters.ts.
type aghFilters struct {
	Categories map[string]*aghFiltersCategory `json:"categories"`
	Filters    map[string]*aghFiltersFilter   `json:"filters"`
}

// aghFiltersCategory is the JSON structure for a category in the vetted
// filtering rule list file.
type aghFiltersCategory struct {
	Name        string `json:"name"`
	Description string `json:"description"`
}

// aghFiltersFilter is the JSON structure for a filter in the vetted filtering
// rule list file.
type aghFiltersFilter struct {
	Name       string `json:"name"`
	CategoryID string `json:"categoryId"`
	Homepage   string `json:"homepage"`
	Source     string `json:"source"`
}